stemmer4r 0.4 → 0.5

data/ext/stemmer4r/stemmer4r.c

@@ -74,12 +74,22 @@ stemmer_free(struct sb_stemmer* stemmer)
  *
  * === Examples
  *
- * stemmer = Stemmer.new('en')
- * puts stemmer.stem('testing')
+ *   stemmer = Stemmer.new('en')
+ *   puts stemmer.stem('testing')
  *
- * stemmer = Stemmer.new('fr')
- * puts stemmer.stem('�t�')
+ *   stemmer = Stemmer.new('fr')
  *
+ * You can stem:
+ *
+ * * a single word:
+ *   puts stemmer.stem('�t�')
+ *
+ * * an array:
+ *   puts stemmer.stem(%w{t�l�vision chapeau ordinateur})
+ *
+ * * a sentence:
+ *   puts stemmer.stem("Une t�l�vision sur un chapeau d'ordinateur")
+ * 
  */
 static VALUE
 stemmer_allocate(VALUE klass, VALUE algorithm)
@@ -107,30 +117,67 @@ stemmer_allocate(VALUE klass, VALUE algorithm)
 
 /*
  * Document-method: stem
- * call-seq: stem(str)
+ * call-seq: stem(obj)
  *
- * Stems string and returns the result.
+ * Stems a word, a sentence or an array of words and returns the result (always in lowercase).
+ * +obj+ is always converted to lowercase before stemming (mandatory for snowball algorithms to work).
  *
  * === Parameters
  *
- * +str+:: string to stem. String must be encoding the 'right' way (iso-8859-1 for french for example).
+ * +obj+:: word, sentence (string) or array of words to stem. All strings must be encoding the 'right' way (iso-8859-1 for french for example).
  */
 static VALUE
-stemmer_stem(VALUE self, VALUE word)
+stemmer_stem(VALUE self, VALUE obj)
 {
   VALUE ret;
+  VALUE word_lowercase;
   sb_symbol *cword;
+  sb_symbol *t1;
   const sb_symbol *rcword;
   struct sb_stemmer *stemmer;
+  int i;
 
-  Check_Type(word, T_STRING);
-
-  cword = strdup(STR2CSTR(word));
   Data_Get_Struct(self, struct sb_stemmer, stemmer);
-  rcword = sb_stemmer_stem(stemmer, cword, RSTRING(word)->len);
 
-  ret = rb_str_new2(rcword);
-  free(cword);
+  switch (TYPE(obj))
+  {
+    case T_STRING:
+      word_lowercase = rb_funcall2(obj, rb_intern("downcase"), 0, 0);
+      cword = strdup(STR2CSTR(word_lowercase));
+      // Sentence
+      if (strchr(cword, ' '))
+      {
+        ret = rb_str_new2("");
+        for (t1 = strtok(cword, " "); t1 != NULL; t1 = strtok(NULL, " "))
+        {
+          rb_str_cat2(ret, sb_stemmer_stem(stemmer, t1, strlen(t1)));
+          rb_str_cat2(ret, " ");
+        }
+        RSTRING(ret)->len--;
+        RSTRING(ret)->ptr[RSTRING(ret)->len] = '\0';
+      }
+      else
+      {
+        // A word
+        rcword = sb_stemmer_stem(stemmer, cword, RSTRING(obj)->len);
+        ret = rb_str_new2(rcword);
+      }
+      free(cword);
+      break;
+    case T_ARRAY:
+      ret = rb_ary_new2(RARRAY(obj)->len);
+      for (i = 0; i < RARRAY(obj)->len; i++)
+      {
+        word_lowercase = rb_funcall2(RARRAY(obj)->ptr[i], rb_intern("downcase"), 0, 0);
+        cword = strdup(STR2CSTR(word_lowercase));
+        rb_ary_push(ret, rb_str_new2(sb_stemmer_stem(stemmer, cword, RSTRING(RARRAY(obj)->ptr[i])->len)));
+      }
+      free(cword);
+      break;
+    default:
+      rb_raise(rb_eTypeError, "not valid value");
+      break;
+  }
 
   return ret;
 }

data/stemmer4r.gemspec

@@ -3,7 +3,7 @@ require 'rake'
 
 spec = Gem::Specification.new do |s|
   s.name    = 'stemmer4r'
-  s.version = '0.4'
+  s.version = '0.5'
   s.author            = "Fabien POTENCIER"
   s.email             = "fabien.potencier@gmail.com"
   s.homepage          = "http://stemmer4r.rubyforge.org"

data/test/test.rb

@@ -7,9 +7,25 @@ rescue LoadError
   require 'stemmer4r'
 end
 
+require 'benchmark'
 require 'test/unit'
 
 class StemmerTest < Test::Unit::TestCase
+  def test_sentence
+    s = Stemmer.new('fr')
+    assert_equal('il aim utilis le ordin dan le avion', s.stem('il aime utiliser les ordinateurs dans les avions'))
+  end
+
+  def test_array
+    s = Stemmer.new('fr')
+    assert_equal('ordin jeux avion aim', s.stem(%w{ordinateur jeux avion aimer}).join(' '))
+  end
+
+  def test_lowercase
+    s = Stemmer.new('fr')
+    assert_equal(s.stem('aimera'), s.stem('AIMERA'))
+  end
+
   def test_valid_language
     assert_not_nil(Stemmer.new('fr'))
     assert_nothing_raised() { Stemmer.new('fr') }
@@ -18,13 +34,34 @@ class StemmerTest < Test::Unit::TestCase
 
   def test_stemmer
     dir = File.dirname(__FILE__)
+    input = {}
+    output = {}
+    s = {}
     %w{da de nl en es fi fr it no pt ru sv}.each { |language|
-      puts "Testing '#{language}' algorithm..."
-      stemmer = Stemmer.new(language)
-      input = File.new("#{dir}/tests/#{language}/voc.txt")
-      output = File.new("#{dir}/tests/#{language}/output.txt")
-      input.each_line { |word|
-        assert_equal(output.gets.chomp!, stemmer.stem(word.chomp!))
+      input[language] = IO.read("#{dir}/tests/#{language}/voc.txt").split(/\s+/)
+      output[language] = IO.read("#{dir}/tests/#{language}/output.txt").split(/\s+/)
+      s[language] = Stemmer.new(language)
+    }
+    Benchmark.bm { |x|
+      x.report {
+        %w{da de nl en es fi fr it no pt ru sv}.each { |language|
+          puts "Testing '#{language}' algorithm (one word at a time)..."
+          input[language].length.times { |i|
+            assert_equal(output[language][i], s[language].stem(input[language][i]))
+          }
+        }
+      }
+      x.report {
+        %w{fr da de nl en es fi fr it no pt ru sv}.each { |language|
+          puts "Testing '#{language}' algorithm (a single array)..."
+          assert_equal(output[language], s[language].stem(input[language]))
+        }
+      }
+      x.report {
+        %w{fr da de nl en es fi fr it no pt ru sv}.each { |language|
+          puts "Testing '#{language}' algorithm (a single sentence)..."
+          assert_equal(output[language].join(' '), s[language].stem(input[language].join(' ')))
+        }
       }
     }
   end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.8.10
 specification_version: 1
 name: stemmer4r
 version: !ruby/object:Gem::Version 
-  version: "0.4"
-date: 2005-05-11
+  version: "0.5"
+date: 2005-05-12
 summary: Stemmer4r is a Ruby extension that wraps the snowball stemmer library (libstemmer).
 require_paths: 
   - "."