ruby-stemmer 0.8.5 → 0.9.0

data/README.rdoc

@@ -1,9 +1,10 @@
-= Ruby-Stemmer
+= Ruby-Stemmer {Project Status}[http://stillmaintained.com/aurelian/ruby-stemmer.png]
 
 Ruby-Stemmer exposes SnowBall API to Ruby.
 
 This package includes libstemmer_c library released under BSD licence 
-and available for free at: http://snowball.tartarus.org/dist/libstemmer_c.tgz.
+and available for free [here]{http://snowball.tartarus.org/dist/libstemmer_c.tgz}.
+
 Support for latin language is also included and it has been generated with the snowball compiler using 
 {schinke contribution}[http://snowball.tartarus.org/otherapps/schinke/intro.html]
 
@@ -88,7 +89,7 @@ For further reference on stem vs. root, please check wikipedia articles on the t
 * {fast-stemmer}[http://github.com/romanbsd/fast-stemmer] (ext)
 * {uea-stemmer}[http://github.com/ealdent/uea-stemmer] (ext)
 * {stemmer}[http://rubyforge.org/projects/stemmer] (pure ruby)
-* [add yours]
+* add yours
 
 == Copyright
 
@@ -96,11 +97,12 @@ Copyright (c) 2008-2011 {Aurelian Oancea}[http://locknet.ro]. See MIT-LICENSE fo
 
 == Contributors
 
-* Aurelian Oancea
-* Yury Korolev - various bug fixes
-* Aaron Patterson - rake compiler (windows support), code cleanup
+* {Aurelian Oancea}[https://github.com/aurelian]
+* {Yury Korolev}[https://github.com/yury] - various bug fixes
+* {Aaron Patterson}[https://github.com/tenderlove] - rake compiler (windows support), code cleanup
+* {Damián Silvani}[https://github.com/munshkr] - Ruby 1.9 encoding
 
 == Real life usage
-
 * http://planet33.ru is using Ruby-Stemmer together with {Classifier}[http://github.com/yury/classifier] to automatically rate places based on users comments.
-* {textamatch_rb}[http://github.com/dimus/taxamatch_rb] is using the Ruby-Stemmer to catch errors in suffixes while it discovers if two scientific names are actually the same.
+
+# encoding: utf-8

data/VERSION

@@ -1 +1 @@
-0.8.5
+0.9.0

data/ext/lingua/stemmer.c

@@ -1,6 +1,27 @@
 #include "ruby.h"
 #include <libstemmer.h>
 
+
+#ifdef HAVE_RUBY_ENCODING_H
+
+#include <ruby/encoding.h>
+
+#define ENCODED_STR_NEW2(str, encoding) \
+  ({ \
+   VALUE _string = rb_str_new2((const char *)str); \
+   int _enc = rb_enc_get_index(encoding); \
+   rb_enc_associate_index(_string, _enc); \
+   _string; \
+   })
+
+#else
+
+#define ENCODED_STR_NEW2(str, encoding) \
+  rb_str_new2((const char *)str)
+
+#endif
+
+
 VALUE rb_mLingua;
 VALUE rb_cStemmer;
 VALUE rb_eStemmerError;
@@ -63,7 +84,9 @@ rb_stemmer_stem(VALUE self, VALUE word) {
       (sb_symbol *)RSTRING_PTR(s_word),
       RSTRING_LEN(s_word)
   );
-  return rb_str_new2((char *)stemmed);
+
+  VALUE rb_enc = rb_iv_get(self, "@encoding");
+  return ENCODED_STR_NEW2((char *)stemmed, rb_enc);
 }
 
 static void
@@ -85,7 +108,7 @@ void Init_stemmer_native() {
   rb_mLingua = rb_define_module("Lingua");
   rb_cStemmer = rb_define_class_under(rb_mLingua, "Stemmer", rb_cObject);
   rb_define_alloc_func(rb_cStemmer, sb_stemmer_alloc);
-  rb_eStemmerError = rb_define_class_under(rb_mLingua, "StemmerError", rb_eException);  
+  rb_eStemmerError = rb_define_class_under(rb_mLingua, "StemmerError", rb_eException);
   rb_define_private_method(rb_cStemmer, "native_init", rb_stemmer_init, 2);
   rb_define_method(rb_cStemmer, "stem", rb_stemmer_stem, 1);
 }

data/lib/lingua/stemmer.rb

@@ -37,10 +37,24 @@ module Lingua
     #   require 'lingua/stemmer'
     #   s = Lingua::Stemmer.new :language => 'fr'
     #
-    def initialize options = {}
+    def initialize(options={})
       @language = (options[:language] || 'en').to_s
       @encoding = (options[:encoding] || 'UTF_8').to_s
-      native_init @language, @encoding
+
+      if RUBY_VERSION >= "1.9"
+        if not @encoding.is_a?(Encoding)
+          @encoding = Encoding.find(@encoding.gsub("_", "-"))
+        end
+      else
+        @encoding = @encoding.upcase.gsub("-", "_")
+      end
+
+      native_init(@language, native_encoding(@encoding))
+    end
+
+  private
+    def native_encoding(enc)
+      RUBY_VERSION >= "1.9" ? enc.name.gsub('-', '_') : enc
     end
   end
 end

data/test/lingua/test_stemmer.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require 'helper'
 
 class TestStemmer < Test::Unit::TestCase
@@ -35,7 +36,12 @@ class TestStemmer < Test::Unit::TestCase
       assert_equal word, "install"
     end
     assert_kind_of ::Lingua::Stemmer, stemmer
-    assert_equal stemmer.encoding, "UTF_8"
+
+    if RUBY_VERSION >= '1.9'
+      assert_equal stemmer.encoding, Encoding::UTF_8
+    else
+      assert_equal stemmer.encoding, "UTF_8"
+    end
   end
 
   def test_array_stemmer
@@ -51,4 +57,43 @@ class TestStemmer < Test::Unit::TestCase
       }.new.stem('cow')
     end
   end
+
+  def test_default_encoding_option
+    if RUBY_VERSION >= '1.9'
+      assert_equal ::Lingua::Stemmer.new.encoding, Encoding::UTF_8
+    else
+      assert_equal ::Lingua::Stemmer.new.encoding, "UTF_8"
+    end
+  end
+
+  def test_different_encoding_options
+    if RUBY_VERSION >= '1.9'
+      assert_equal ::Lingua::Stemmer.new(:encoding => "ISO_8859_1").encoding, Encoding::ISO_8859_1
+      assert_equal ::Lingua::Stemmer.new(:encoding => "UTF-8").encoding, Encoding::UTF_8
+      assert_equal ::Lingua::Stemmer.new(:encoding => "utf-8").encoding, Encoding::UTF_8
+      assert_equal ::Lingua::Stemmer.new(:encoding => :ISO_8859_1).encoding, Encoding::ISO_8859_1
+      assert_equal ::Lingua::Stemmer.new(:encoding => Encoding::UTF_8).encoding, Encoding::UTF_8
+    else
+      assert_equal ::Lingua::Stemmer.new(:encoding => "ISO_8859_1").encoding, "ISO_8859_1"
+      assert_equal ::Lingua::Stemmer.new(:encoding => "UTF-8").encoding, "UTF_8"
+      assert_equal ::Lingua::Stemmer.new(:encoding => "utf-8").encoding, "UTF_8"
+      assert_equal ::Lingua::Stemmer.new(:encoding => :ISO_8859_1).encoding, "ISO_8859_1"
+    end
+  end
+
+  if RUBY_VERSION >= '1.9'
+    def test_string_encoding
+      word = "așezare"
+
+      stem = ::Lingua.stemmer(word, :language => "ro", :encoding => "UTF_8")
+      assert_equal word.encoding, stem.encoding
+
+      s = ::Lingua::Stemmer.new(:language => "ro", :encoding => "UTF_8")
+      assert_equal s.stem(word).encoding, word.encoding
+
+      stem = ::Lingua.stemmer("installation", :language => "fr", :encoding => "ISO-8859-1")
+      assert_equal stem.encoding, Encoding::ISO_8859_1
+    end
+  end
+
 end

metadata

@@ -1,12 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: ruby-stemmer
 version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 0
-  - 8
-  - 5
-  version: 0.8.5
+  prerelease: 
+  version: 0.9.0
 platform: ruby
 authors: 
 - Aurelian Oancea
@@ -15,7 +11,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-08 00:00:00 +01:00
+date: 2011-03-12 00:00:00 +01:00
 default_executable: 
 dependencies: []
 
@@ -135,21 +131,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      segments: 
-      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      segments: 
-      - 0
       version: "0"
 requirements: []
 
 rubyforge_project: ruby-stemmer
-rubygems_version: 1.3.7
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Expose libstemmer_c to Ruby.