stemmer4r 0.5 → 0.6

data/ext/stemmer4r/stemmer4r.c

@@ -82,13 +82,25 @@ stemmer_free(struct sb_stemmer* stemmer)
  * You can stem:
  *
  * * a single word:
+ *
  *   puts stemmer.stem('�t�')
  *
  * * an array:
+ *
  *   puts stemmer.stem(%w{t�l�vision chapeau ordinateur})
  *
- * * a sentence:
- *   puts stemmer.stem("Une t�l�vision sur un chapeau d'ordinateur")
+ * * a string of words:
+ *
+ *   puts stemmer.stem("Une t�l�vision sur un chapeau d ordinateur")
+ *
+ * The fastest way to stem is to pass a string of words separated by spaces. You have to clean your string before stemming:
+ * remove all punctuation characters (! , ; : ! ? ...). All stemmed words in the string will be separated with one space.
+ *
+ *   puts stemmer.stem("Une    t�l�vision sur un chapeau        d ordinateur")
+ *
+ * will return the same stemmed string as 
+ *
+ *   puts stemmer.stem("Une t�l�vision sur un chapeau d ordinateur")
  * 
  */
 static VALUE
@@ -119,12 +131,16 @@ stemmer_allocate(VALUE klass, VALUE algorithm)
  * Document-method: stem
  * call-seq: stem(obj)
  *
- * Stems a word, a sentence or an array of words and returns the result (always in lowercase).
+ * Stems a word, a string of words separated by spaces or an array of words and returns the result (always in lowercase).
  * +obj+ is always converted to lowercase before stemming (mandatory for snowball algorithms to work).
  *
  * === Parameters
  *
- * +obj+:: word, sentence (string) or array of words to stem. All strings must be encoding the 'right' way (iso-8859-1 for french for example).
+ * +obj+:: word, string of words or array of words to stem.
+ *
+ * All strings must be encoding the 'right' way (iso-8859-1 for french for example).
+ * If you want to be able to 'transparently' stem UTF-8 characters, see the Stemmable_utf8 module.
+ *
  */
 static VALUE
 stemmer_stem(VALUE self, VALUE obj)
@@ -144,7 +160,7 @@ stemmer_stem(VALUE self, VALUE obj)
     case T_STRING:
       word_lowercase = rb_funcall2(obj, rb_intern("downcase"), 0, 0);
       cword = strdup(STR2CSTR(word_lowercase));
-      // Sentence
+      // String of words
       if (strchr(cword, ' '))
       {
         ret = rb_str_new2("");
@@ -172,7 +188,7 @@ stemmer_stem(VALUE self, VALUE obj)
         cword = strdup(STR2CSTR(word_lowercase));
         rb_ary_push(ret, rb_str_new2(sb_stemmer_stem(stemmer, cword, RSTRING(RARRAY(obj)->ptr[i])->len)));
       }
-      free(cword);
+      if (RARRAY(obj)->len) free(cword);
       break;
     default:
       rb_raise(rb_eTypeError, "not valid value");

data/lib/stemmer.rb

@@ -0,0 +1,70 @@
+#
+# == Stemmable module
+#
+# This module is automatically added to the String and Array classes when you:
+#
+#  require 'stemmer'
+#
+# It adds a +stem+ method to String and Array.
+#
+#  str = 'this is a string'
+#
+#  stemmed_str = str.stem
+#
+#  array = %w{this is an array}
+#
+#  stemmed_array = array.stem
+#
+# By default, stemming occurs in english. If you want to stem in another language, just give it as a parameter:
+#
+#  str = 'Cha�ne de caract�res fran�aise'
+#
+#  stemmed_str = str.stem('fr')
+#
+# Or you can change the default configuration:
+#
+#  Stemmable::stemmer_default_language = 'fr'
+#
+#  stemmed_str = str.stem
+#
+module Stemmable
+  begin
+    require 'rubygems'
+    require_gem 'stemmer4r'
+  rescue LoadError
+    require 'stemmer4r'
+  end
+
+  @@stemmer_default_language = 'en'
+  @@stemmer = Stemmer.new('en')
+  @@UTF8_MAP = {
+    'fr' => 'iso-8859-1'
+  }
+
+  def Stemmable.stemmer_default_language=(language)
+    @@stemmer_default_language = language
+    @@stemmer = Stemmer.new(language)
+    language
+  end
+
+  def Stemmable.stemmer_default_language
+    return @@stemmer_default_language
+  end
+
+  def stem(language = nil)
+    if (language.nil?)
+      @@stemmer.stem(self)
+    else
+      stemmer = Stemmer.new(language)
+      stemmer.stem(self)
+    end
+  end
+end
+
+class String
+  include Stemmable
+end
+
+class Array
+  include Stemmable
+end

data/lib/stemmer_utf8.rb

@@ -0,0 +1,66 @@
+#
+# == Stemmable_utf8 module
+#
+# This module is automatically added to the String and Array classes when you:
+#
+#  require 'stemmer_utf8'
+#
+# It adds a +stem_utf8+ method to String and Array.
+#
+#  str_utf8 = 'this is a UTF-8 encoded string'
+#
+#  stemmed_str_utf8 = str_utf8.stem_utf8
+#
+#  array_utf8 = %w{this is an array with utf8 caracters}
+#
+#  stemmed_array_utf8 = array_utf8.stem_utf8
+#
+# By default, stemming occurs in english. If you want to stem in another language, just give it as a parameter:
+#
+#  str_utf8 = 'Cha�ne de caract�res fran�aise en UTF-8'
+#
+#  stemmed_str_utf8 = str.stem_utf8('fr')
+#
+# Or you can change the default configuration:
+#
+#  Stemmable::stemmer_default_language = 'fr'
+#
+#  stemmed_str_utf8 = str_utf8.stem_utf8
+#
+module Stemmable_utf8
+  include Stemmable
+
+  def stem_utf8(language = nil)
+    require 'iconv'
+    if (language.nil?)
+      language = @@stemmer_default_language
+      stemmer = @@stemmer
+    else
+      stemmer = Stemmer.new(language)
+    end
+    language_encoding = @@UTF8_MAP[language] || 'iso-8859-1'
+    if self.is_a?(String)
+      Iconv.new('utf-8', language_encoding).iconv(stemmer.stem(Iconv.new(language_encoding, 'utf-8').iconv(self)))
+    elsif self.is_a?(Array)
+      temp = []
+      output = []
+      Iconv.open(language_encoding, 'utf-8') do |cd|
+        self.each { |s| temp << cd.iconv(s) + cd.iconv(nil) }
+      end
+      Iconv.open('utf-8', language_encoding) do |cd|
+        stemmer.stem(temp).each { |s| output << cd.iconv(s) + cd.iconv(nil) }
+      end
+      output
+    else
+      raise 'no valid type'
+    end
+  end
+end
+
+class String
+  include Stemmable_utf8
+end
+
+class Array
+  include Stemmable_utf8
+end

data/stemmer4r.gemspec

@@ -3,7 +3,7 @@ require 'rake'
 
 spec = Gem::Specification.new do |s|
   s.name    = 'stemmer4r'
-  s.version = '0.5'
+  s.version = '0.6'
   s.author            = "Fabien POTENCIER"
   s.email             = "fabien.potencier@gmail.com"
   s.homepage          = "http://stemmer4r.rubyforge.org"
@@ -14,9 +14,9 @@ spec = Gem::Specification.new do |s|
   s.description = <<-EOF
     Stemmer4r is a Ruby extension that wraps the snowball stemmer library (libstemmer).
   EOF
-  s.files = FileList['ext/**/*', 'test/**/*', 'stemmer4r.gemspec', 'README', 'LICENSE'].to_a
+  s.files = FileList['lib/**/*', 'ext/**/*', 'test/**/*', 'stemmer4r.gemspec', 'README', 'LICENSE'].to_a
   s.extensions << "ext/stemmer4r/extconf.rb"
-  s.require_path = '.'
+  s.require_paths << 'ext'
   s.autorequire = 'stemmer4r'
   s.has_rdoc = true
   s.rdoc_options = [

data/test/test.rb

@@ -7,18 +7,65 @@ rescue LoadError
   require 'stemmer4r'
 end
 
+require 'stemmer'
 require 'benchmark'
 require 'test/unit'
 
 class StemmerTest < Test::Unit::TestCase
+  begin
+    require 'iconv'
+    require 'stemmer_utf8'
+    @@has_iconv = true
+  rescue
+    @@has_iconv = false
+  end
+
+  def test_utf8
+    return unless @@has_iconv
+
+    utf8_str_input = Iconv.new('utf-8', 'iso-8859-1').iconv('aimera    t�l�vision')
+    utf8_str_output = Iconv.new('utf-8', 'iso-8859-1').iconv('aim t�l�vis')
+    assert_equal(utf8_str_output, utf8_str_input.stem_utf8('fr'))
+
+    utf8_str_input = Iconv.new('utf-8', 'iso-8859-1').iconv('love independant')
+    utf8_str_output = Iconv.new('utf-8', 'iso-8859-1').iconv('love independ')
+    assert_equal(utf8_str_output, utf8_str_input.stem_utf8)
+
+    utf8_ary_input = []
+    %w{aimera t�l�vision grandiose}.each { |k|
+      utf8_ary_input << Iconv.new('utf-8', 'iso-8859-1').iconv(k)
+    }
+    utf8_ary_output = []
+    %w{aim t�l�vis grandios}.each { |k|
+      utf8_ary_output << Iconv.new('utf-8', 'iso-8859-1').iconv(k)
+    }
+    assert_equal(utf8_ary_output, utf8_ary_input.stem_utf8('fr'))
+  end
+
+  def test_stemmable
+    assert_equal('il aim utilis le ordin dan le avion', 'il      aime utiliser les     ordinateurs dans les avions'.stem('fr'))
+    assert_equal('aim ordin', 'aime ordinateurs'.stem('fr'))
+    assert_equal('ordin jeux avion aim', %w{ordinateur jeux avion aimer}.stem('fr').join(' '))
+    assert_equal('ordin', %w{ordinateur}.stem('fr').join(' '))
+    assert_equal('aimera'.stem('fr'), 'AIMERA'.stem('fr'))
+    assert_equal('believable'.stem('en'), 'believable'.stem)
+    assert_equal('believ', 'believable'.stem)
+    assert_raise(ArgumentError) { 'test'.stem('notavalidlanguage') }
+    assert_equal('', ''.stem)
+    assert_equal([], [].stem)
+    assert_equal(['', '     '], ['', '     '].stem)
+  end
+
   def test_sentence
     s = Stemmer.new('fr')
-    assert_equal('il aim utilis le ordin dan le avion', s.stem('il aime utiliser les ordinateurs dans les avions'))
+    assert_equal('il aim utilis le ordin dan le avion', s.stem('il aime utiliser     les ordinateurs     dans les avions'))
+    assert_equal('aim ordin', s.stem('aime ordinateurs'))
   end
 
   def test_array
     s = Stemmer.new('fr')
     assert_equal('ordin jeux avion aim', s.stem(%w{ordinateur jeux avion aimer}).join(' '))
+    assert_equal('ordin', s.stem(%w{ordinateur}).join(' '))
   end
 
   def test_lowercase
@@ -52,17 +99,40 @@ class StemmerTest < Test::Unit::TestCase
         }
       }
       x.report {
-        %w{fr da de nl en es fi fr it no pt ru sv}.each { |language|
+        %w{da de nl en es fi fr it no pt ru sv}.each { |language|
           puts "Testing '#{language}' algorithm (a single array)..."
           assert_equal(output[language], s[language].stem(input[language]))
         }
       }
       x.report {
-        %w{fr da de nl en es fi fr it no pt ru sv}.each { |language|
+        %w{da de nl en es fi fr it no pt ru sv}.each { |language|
           puts "Testing '#{language}' algorithm (a single sentence)..."
           assert_equal(output[language].join(' '), s[language].stem(input[language].join(' ')))
         }
       }
+      x.report {
+        %w{da de nl en es fi fr it no pt ru sv}.each { |language|
+          puts "Testing '#{language}' algorithm (a single array - Array module)..."
+          assert_equal(output[language], input[language].stem(language))
+        }
+      }
+      x.report {
+        %w{da de nl en es fi fr it no pt ru sv}.each { |language|
+          puts "Testing '#{language}' algorithm (a single sentence - String module)..."
+          assert_equal(output[language].join(' '), input[language].join(' ').stem(language))
+        }
+      }
+
+      return unless @@has_iconv
+
+      x.report {
+        %w{da de nl en es fi fr it no pt ru sv}.each { |language|
+          utf8_input = Iconv.new('utf-8', 'iso-8859-1').iconv(input[language].join(' '))
+          utf8_output = Iconv.new('utf-8', 'iso-8859-1').iconv(output[language].join(' '))
+          puts "Testing '#{language}' algorithm (a single sentence - String module - utf8)..."
+          assert_equal(utf8_output, utf8_input.stem_utf8(language))
+        }
+      }
     }
   end
 end

metadata

@@ -3,11 +3,12 @@ rubygems_version: 0.8.10
 specification_version: 1
 name: stemmer4r
 version: !ruby/object:Gem::Version 
-  version: "0.5"
-date: 2005-05-12
+  version: "0.6"
+date: 2005-05-13
 summary: Stemmer4r is a Ruby extension that wraps the snowball stemmer library (libstemmer).
 require_paths: 
-  - "."
+  - lib
+  - ext
 email: fabien.potencier@gmail.com
 homepage: http://stemmer4r.rubyforge.org
 rubyforge_project: stemmer4r
@@ -27,6 +28,8 @@ platform: ruby
 authors: 
   - Fabien POTENCIER
 files: 
+  - lib/stemmer.rb
+  - lib/stemmer_utf8.rb
   - ext/stemmer4r
   - ext/stemmer4r/stemmer4r.c
   - ext/stemmer4r/libstemmer_c