ruby-stemmer-dimelo 0.9.3.dimelo1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 210c16040fce7de4293c411dd0139091debd99e8
+  data.tar.gz: 9a64fdd53a0c383fd286974a6cf57dbd06497e6a
+SHA512:
+  metadata.gz: 464276d630c2540b6db3f1e2c12f30a0e0a6d08195940673fbc01f8e6ec643bf5bc32778b081afcd1d2a3a950d94bacc5ff589865e156dddce619930aa1b5cea
+  data.tar.gz: d72d91388301b5835a92add0b298d9648082fe4d44a97e0a3360f1af828c31cc49f84586be7348dc9872dd6bcfec2fa4835697fa4428d5535f9d8e3725f59876

data/MIT-LICENSE

@@ -0,0 +1,21 @@
+Copyright (c) 2008-2011 Aurelian Oancea
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/README.rdoc

@@ -0,0 +1,117 @@
+= Ruby-Stemmer {Project Status}[http://stillmaintained.com/aurelian/ruby-stemmer.png]
+
+Ruby-Stemmer exposes SnowBall API to Ruby.
+
+This package includes libstemmer_c library released under BSD licence 
+and available for free {here}[http://snowball.tartarus.org/dist/libstemmer_c.tgz].
+
+Support for latin language is also included and it has been generated with the snowball compiler using 
+{schinke contribution}[http://snowball.tartarus.org/otherapps/schinke/intro.html].
+
+For more details about libstemmer_c please visit the {SnowBall website}[http://snowball.tartarus.org].
+
+== Usage
+
+  require 'rubygems'
+  require 'lingua/stemmer'
+
+  stemmer= Lingua::Stemmer.new(:language => "ro")
+  stemmer.stem("netăgăduit") #=> netăgădu
+
+=== Alternative
+
+  require 'rubygems'
+  require 'lingua/stemmer'
+
+  Lingua.stemmer( %w(incontestabil neîndoielnic), :language => "ro" ) #=> ["incontest", "neîndoieln"]
+  Lingua.stemmer("installation") #=> "instal"
+  Lingua.stemmer("installation", :language => "fr", :encoding => "ISO_8859_1") do | word |
+    puts "~> #{word}" #=> "instal"
+  end # => #<Lingua::Stemmer:0x102501e48>
+
+=== Rails
+  
+  # Rails2: -- config/environment.rb:
+  config.gem 'ruby-stemmer', :version => '>=0.6.2', :lib => 'lingua/stemmer'
+  
+  # Rails3: -- Gemfile
+  gem 'ruby-stemmer', '>=0.8.3', :require => 'lingua/stemmer'
+
+=== More details 
+
+* Complete API in {RDoc format}[http://rdoc.info/github/aurelian/ruby-stemmer/master/frames]
+* More usage on the {test file}[http://github.com/aurelian/ruby-stemmer/blob/master/test/lingua/test_stemmer.rb]
+
+== Install
+
+=== Standard install with:
+
+ gem install ruby-stemmer
+
+==== Windows
+
+There's also a Windows (Fat bin) compiled against ruby 1.9.3 and ruby 1.8.7.
+
+ gem install ruby-stemmer --platform=x86-mingw32
+
+As far as I know the above should work with {rubyinstaller}[http://rubyinstaller.org/]. If if fails, you could try with:
+
+ gem install ruby-stemmer --platform=x86-mswin32
+
+{It's known}[http://cl.ly/BX9o] to work under Windows XP.
+
+=== Development version
+
+  $ git clone git://github.com/aurelian/ruby-stemmer.git
+  $ cd ruby-stemmer
+  $ rake -T #<== see what we've got
+  $ rake compile #<== builds the extension do'h
+  $ rake test
+
+== NOT A BUG
+
+The stemming process is an algorithm to allow one to find the stem of an word (not the root of it).
+For further reference on stem vs. root, please check wikipedia articles on the topic:
+
+* http://en.wikipedia.org/wiki/Stem_%28linguistics%29
+* http://en.wikipedia.org/wiki/Root_%28linguistics%29
+
+== TODO
+
+* {Open issues}[http://github.com/aurelian/ruby-stemmer/issues]
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project from {github}[http://github.com/aurelian/ruby-stemmer]
+* Make your feature addition or {bug fix}[http://github.com/aurelian/ruby-stemmer/issues]
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+
+  if you want to have your own version, that is fine but
+  bump version in a commit by itself I can ignore when I pull
+* Send me a pull request. Bonus points for topic branches.
+
+== Alternative Stemmers for Ruby
+
+* {stemmer4r}[http://rubyforge.org/projects/stemmer4r] (ext)
+* {fast-stemmer}[http://github.com/romanbsd/fast-stemmer] (ext)
+* {uea-stemmer}[http://github.com/ealdent/uea-stemmer] (ext)
+* {stemmer}[http://rubyforge.org/projects/stemmer] (pure ruby)
+* add yours
+
+== Copyright
+
+Copyright (c) 2008-2011 {Aurelian Oancea}[http://locknet.ro]. See MIT-LICENSE for details.
+
+== Contributors
+
+* {Aurelian Oancea}[https://github.com/aurelian]
+* {Yury Korolev}[https://github.com/yury] - various bug fixes
+* {Aaron Patterson}[https://github.com/tenderlove] - rake compiler (windows support), code cleanup
+* {Damián Silvani}[https://github.com/munshkr] - Ruby 1.9 encoding
+
+== Real life usage
+* http://planet33.ru is using Ruby-Stemmer together with {Classifier}[http://github.com/yury/classifier] to automatically rate places based on users comments.
+
+# encoding: utf-8

data/Rakefile

@@ -0,0 +1,70 @@
+require 'rubygems'
+require 'rake'
+
+gem 'rake-compiler', '~>0.7'
+require 'rake/extensiontask'
+
+require 'jeweler'
+$jeweler = Jeweler::Tasks.new do |gem|
+  gem.name = "ruby-stemmer"
+  gem.version = File.read(File.expand_path(File.join(File.dirname(__FILE__),"VERSION"))).strip!
+  gem.summary = %Q{Expose libstemmer_c to Ruby.}
+  gem.description = %Q{Expose the bundled libstemmer_c library to Ruby.}
+  gem.email = "oancea@gmail.com"
+  gem.homepage = "http://github.com/aurelian/ruby-stemmer"
+  gem.authors = ["Aurelian Oancea", "Yury Korolev"]
+  gem.extensions = ["ext/lingua/extconf.rb"]
+  gem.rubyforge_project = "ruby-stemmer"
+  gem.files = FileList['lib/**/*.rb', 'README.rdoc', 'MIT-LICENSE', 'VERSION', 'Rakefile', 'libstemmer_c/**/*', 'ext/**/*', 'test/**/*']
+  %w(ext/lingua/*.so ext/lingua/*.bundle ext/lingua/Makefile ext/lingua/mkmf.log ext/lingua/*.o libstemmer_c/**/*.o libstemmer_c/stemwords).each do | f |
+    gem.files.exclude f
+  end
+end
+
+Jeweler::GemcutterTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: gem install rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+CLOBBER.include("libstemmer_c/**/*.o")
+
+Rake::ExtensionTask.new('ruby-stemmer', $jeweler.jeweler.gemspec) do |ext|
+  ext.lib_dir = File.join(*['lib', 'lingua', ENV['FAT_DIR']].compact)
+  ext.ext_dir = File.join 'ext', 'lingua'
+  ext.cross_compile = true
+  ext.cross_platform = ['i386-mswin32-60', 'i386-mingw32']
+  ext.name    = 'stemmer_native'
+end
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.options << '--charset' << 'utf-8'
+  rdoc.title = "Ruby-Stemmer #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+  rdoc.rdoc_files.include('ext/lingua/stemmer.c')
+  rdoc.rdoc_files.include('MIT-LICENSE')
+end

data/VERSION

@@ -0,0 +1 @@
+0.9.3

data/ext/lingua/extconf.rb

@@ -0,0 +1,40 @@
+ENV['RC_ARCHS'] = '' if RUBY_PLATFORM =~ /darwin/
+require "mkmf"
+
+ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+LIBSTEMMER = File.join(ROOT, 'libstemmer_c')
+
+# build libstemmer_c
+# FreeBSD make is gmake
+make= (RUBY_PLATFORM =~ /freebsd/)? 'gmake' : 'make'
+
+# MacOS architecture mess up
+if RUBY_PLATFORM =~ /darwin/
+  # see: #issue/3, #issue/5
+  begin
+    ENV['ARCHFLAGS']= "-arch " + %x[file #{File.expand_path(File.join(Config::CONFIG['bindir'], Config::CONFIG['RUBY_INSTALL_NAME']))}].strip!.match(/executable (.+)$/)[1] unless ENV['ARCHFLAGS'].nil?
+  rescue
+    $stderr << "Failed to get your ruby executable architecture.\n"
+    $stderr << "Please specify one using $ARCHFLAGS environment variable.\n"
+    exit
+  end
+  # see: #issue/9, #issue/6
+  # see: man compat
+  if ENV['COMMAND_MODE'] == 'legacy'
+    $stdout << "Setting compat mode to unix2003\n."
+    ENV['COMMAND_MODE']= 'unix2003'
+  end
+end
+
+# make libstemmer_c. unless we're cross-compiling.
+unless RUBY_PLATFORM =~ /i386-mingw32/
+  system "cd #{LIBSTEMMER}; #{make} libstemmer.o; cd #{ROOT};"
+  exit unless $? == 0
+end
+
+$CFLAGS  += " -I#{File.expand_path(File.join(LIBSTEMMER, 'include'))} "
+$libs    += " -L#{LIBSTEMMER} #{File.expand_path(File.join(LIBSTEMMER, 'libstemmer.o'))} "
+
+if have_header("libstemmer.h")
+  create_makefile("lingua/stemmer_native")
+end

data/ext/lingua/stemmer.c

@@ -0,0 +1,115 @@
+#include "ruby.h"
+#include <libstemmer.h>
+
+
+#ifdef HAVE_RUBY_ENCODING_H
+
+#include <ruby/encoding.h>
+
+#define ENCODED_STR_NEW2(str, encoding) \
+  ({ \
+   VALUE _string = rb_str_new2((const char *)str); \
+   int _enc = rb_enc_get_index(encoding); \
+   rb_enc_associate_index(_string, _enc); \
+   _string; \
+   })
+
+#else
+
+#define ENCODED_STR_NEW2(str, encoding) \
+  rb_str_new2((const char *)str)
+
+#endif
+
+
+VALUE rb_mLingua;
+VALUE rb_cStemmer;
+VALUE rb_eStemmerError;
+
+/*
+ * Document-method: new
+ * call-seq: Lingua::Stemmer.new
+ *
+ * Creates a new Stemmer, pass <tt>:language</tt> and <tt>:encoding</tt> as arguments 
+ * to change encoding or language, otherwise english with UTF_8 will be used
+ *   
+ *   require 'lingua/stemmer'
+ *   s = Lingua::Stemmer.new :language => 'fr'
+ */ 
+static VALUE 
+rb_stemmer_init(VALUE self, VALUE rlang, VALUE renc) { 
+  struct sb_stemmer * stemmer;
+
+  Data_Get_Struct(self, struct sb_stemmer, stemmer);
+
+  // In case someone sends() this method, free up the old one
+  if(stemmer) sb_stemmer_delete(stemmer);
+
+  stemmer = sb_stemmer_new( RSTRING_PTR(rlang), RSTRING_PTR(renc) );
+  if (!stemmer) {
+    if (!RTEST(renc)) {
+      rb_raise(rb_eStemmerError,
+          "Language %s not available for stemming", RSTRING_PTR(rlang));
+    } else {
+      rb_raise(rb_eStemmerError,
+          "Language %s not available for stemming in encoding %s", 
+                    RSTRING_PTR(rlang), RSTRING_PTR(renc));
+    }
+  }
+  
+  DATA_PTR(self) = stemmer;
+
+  return self;
+}
+
+/*
+ * Document-method: stem
+ * call-seq: stem
+ *
+ * Stems a word
+ *   
+ *   require 'lingua/stemmer'
+ *   s = Lingua::Stemmer.new
+ *   s.stem "installation" # ==> install
+ */ 
+static VALUE
+rb_stemmer_stem(VALUE self, VALUE word) {
+  struct sb_stemmer * stemmer;
+
+  Data_Get_Struct(self, struct sb_stemmer, stemmer);
+  if(!stemmer) rb_raise(rb_eRuntimeError, "Stemmer is not initialized");
+
+  VALUE s_word = rb_String(word);
+  const sb_symbol * stemmed = sb_stemmer_stem(stemmer,
+      (sb_symbol *)RSTRING_PTR(s_word),
+      RSTRING_LEN(s_word)
+  );
+
+  VALUE rb_enc = rb_iv_get(self, "@encoding");
+  return ENCODED_STR_NEW2((char *)stemmed, rb_enc);
+}
+
+static void
+sb_stemmer_free(struct sb_stemmer * stemmer)
+{
+  if(stemmer) sb_stemmer_delete(stemmer);
+}
+
+static VALUE
+sb_stemmer_alloc(VALUE klass)
+{
+  return Data_Wrap_Struct(klass, 0, sb_stemmer_free, 0);
+}
+
+/*
+ * Ruby-Stemmer, Ruby extension to SnowBall API using libstemmer_c
+ */
+void Init_stemmer_native() {
+  rb_mLingua = rb_define_module("Lingua");
+  rb_cStemmer = rb_define_class_under(rb_mLingua, "Stemmer", rb_cObject);
+  rb_define_alloc_func(rb_cStemmer, sb_stemmer_alloc);
+  rb_eStemmerError = rb_define_class_under(rb_mLingua, "StemmerError", rb_eException);
+  rb_define_private_method(rb_cStemmer, "native_init", rb_stemmer_init, 2);
+  rb_define_method(rb_cStemmer, "stem", rb_stemmer_stem, 1);
+}
+

data/lib/lingua/stemmer.rb

@@ -0,0 +1,60 @@
+if RUBY_PLATFORM =~/(mswin|mingw)/i
+  require "lingua/#{RUBY_VERSION.sub(/\.\d+$/, '')}/stemmer_native"
+else
+  require 'lingua/stemmer_native'
+end
+
+module Lingua
+  def self.stemmer(o, options={})
+    stemmer = Stemmer.new(options)
+
+    words = Array(o).map { |e| e.to_s }
+
+    results = []
+    words.each do |word|
+      result = stemmer.stem(word)
+      if block_given?
+        yield result
+      else
+        results << result
+      end
+    end
+
+    return stemmer if block_given?
+    results.length == 1 ? results[0] : results
+  end
+
+  class Stemmer
+    VERSION = File.read(File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "VERSION"))).strip!
+
+    attr_reader :language
+    attr_reader :encoding
+
+    # Creates a new Stemmer, pass <tt>:language</tt> and <tt>:encoding</tt>
+    # as arguments to change encoding or language, otherwise english with UTF_8
+    # will be used
+    #
+    #   require 'lingua/stemmer'
+    #   s = Lingua::Stemmer.new :language => 'fr'
+    #
+    def initialize(options={})
+      @language = (options[:language] || 'en').to_s
+      @encoding = (options[:encoding] || 'UTF_8').to_s
+
+      if RUBY_VERSION >= "1.9"
+        if not @encoding.is_a?(Encoding)
+          @encoding = Encoding.find(@encoding.gsub("_", "-"))
+        end
+      else
+        @encoding = @encoding.upcase.gsub("-", "_")
+      end
+
+      native_init(@language, native_encoding(@encoding))
+    end
+
+  private
+    def native_encoding(enc)
+      RUBY_VERSION >= "1.9" ? enc.name.gsub('-', '_') : enc
+    end
+  end
+end

data/libstemmer_c/MANIFEST

@@ -0,0 +1,72 @@
+README
+src_c/stem_ISO_8859_1_danish.c
+src_c/stem_ISO_8859_1_danish.h
+src_c/stem_ISO_8859_1_dutch.c
+src_c/stem_ISO_8859_1_dutch.h
+src_c/stem_ISO_8859_1_english.c
+src_c/stem_ISO_8859_1_english.h
+src_c/stem_ISO_8859_1_finnish.c
+src_c/stem_ISO_8859_1_finnish.h
+src_c/stem_ISO_8859_1_french.c
+src_c/stem_ISO_8859_1_french.h
+src_c/stem_ISO_8859_1_german.c
+src_c/stem_ISO_8859_1_german.h
+src_c/stem_ISO_8859_1_hungarian.c
+src_c/stem_ISO_8859_1_hungarian.h
+src_c/stem_ISO_8859_1_italian.c
+src_c/stem_ISO_8859_1_italian.h
+src_c/stem_ISO_8859_1_norwegian.c
+src_c/stem_ISO_8859_1_norwegian.h
+src_c/stem_ISO_8859_1_porter.c
+src_c/stem_ISO_8859_1_porter.h
+src_c/stem_ISO_8859_1_portuguese.c
+src_c/stem_ISO_8859_1_portuguese.h
+src_c/stem_ISO_8859_1_spanish.c
+src_c/stem_ISO_8859_1_spanish.h
+src_c/stem_ISO_8859_1_swedish.c
+src_c/stem_ISO_8859_1_swedish.h
+src_c/stem_ISO_8859_2_romanian.c
+src_c/stem_ISO_8859_2_romanian.h
+src_c/stem_KOI8_R_russian.c
+src_c/stem_KOI8_R_russian.h
+src_c/stem_UTF_8_danish.c
+src_c/stem_UTF_8_danish.h
+src_c/stem_UTF_8_dutch.c
+src_c/stem_UTF_8_dutch.h
+src_c/stem_UTF_8_english.c
+src_c/stem_UTF_8_english.h
+src_c/stem_UTF_8_finnish.c
+src_c/stem_UTF_8_finnish.h
+src_c/stem_UTF_8_french.c
+src_c/stem_UTF_8_french.h
+src_c/stem_UTF_8_german.c
+src_c/stem_UTF_8_german.h
+src_c/stem_UTF_8_hungarian.c
+src_c/stem_UTF_8_hungarian.h
+src_c/stem_UTF_8_italian.c
+src_c/stem_UTF_8_italian.h
+src_c/stem_UTF_8_norwegian.c
+src_c/stem_UTF_8_norwegian.h
+src_c/stem_UTF_8_porter.c
+src_c/stem_UTF_8_porter.h
+src_c/stem_UTF_8_portuguese.c
+src_c/stem_UTF_8_portuguese.h
+src_c/stem_UTF_8_romanian.c
+src_c/stem_UTF_8_romanian.h
+src_c/stem_UTF_8_russian.c
+src_c/stem_UTF_8_russian.h
+src_c/stem_UTF_8_spanish.c
+src_c/stem_UTF_8_spanish.h
+src_c/stem_UTF_8_swedish.c
+src_c/stem_UTF_8_swedish.h
+src_c/stem_UTF_8_turkish.c
+src_c/stem_UTF_8_turkish.h
+runtime/api.c
+runtime/api.h
+runtime/header.h
+runtime/utilities.c
+libstemmer/libstemmer.c
+libstemmer/libstemmer_utf8.c
+libstemmer/modules.h
+libstemmer/modules_utf8.h
+include/libstemmer.h