ruby-stemmer 0.7.0 → 0.8.1

data/README.rdoc

@@ -49,7 +49,7 @@ Please not that Windows is not supported at this time.
   $ git clone git://github.com/aurelian/ruby-stemmer.git
   $ cd ruby-stemmer
   $ rake -T #<== see what we've got
-  $ rake ext #<== builds the extension do'h
+  $ rake compile #<== builds the extension do'h
   $ rake test
 
 == NOT A BUG
@@ -63,7 +63,7 @@ For further reference on stem vs. root, please check wikipedia articles on the t
 == TODO
 
 * {Open issues}[http://github.com/aurelian/ruby-stemmer/issues]
-* Windows Support -> see rake-compiler to cross-compile a dll (with the gem and nokogiri Rakefile).
+* Release Windows Gem
 
 == Note on Patches/Pull Requests
  
@@ -92,9 +92,10 @@ Copyright (c) 2008,2009 {Aurelian Oancea}[http://locknet.ro]. See MIT-LICENSE fo
 == Contributors
 
 * Aurelian Oancea
-* Yury Korolev
+* Yury Korolev - various bug fixes
+* Aaron Patterson - rake compiler (windows support), code cleanup
 
 == Real life usage
 
-* http://planet33.ru is using Ruby-Stemmer together with {Classifier}[http://github.com/yury/classifier] to automatically rate places based on users comments
+* http://planet33.ru is using Ruby-Stemmer together with {Classifier}[http://github.com/yury/classifier] to automatically rate places based on users comments.
 

data/Rakefile

@@ -3,8 +3,9 @@ require 'rake'
 
 begin
   require 'jeweler'
-  Jeweler::Tasks.new do |gem|
+  JEWLER = Jeweler::Tasks.new do |gem|
     gem.name = "ruby-stemmer"
+    gem.version = File.read(File.expand_path(File.join(File.dirname(__FILE__),"VERSION"))).strip!
     gem.summary = %Q{Expose libstemmer_c to Ruby.}
     gem.description = %Q{Expose the bundled libstemmer_c library to Ruby.}
     gem.email = "oancea@gmail.com"
@@ -13,14 +14,14 @@ begin
     gem.extensions = ["ext/lingua/extconf.rb"]
     gem.rubyforge_project = "ruby-stemmer"
     gem.files = FileList['lib/**/*.rb', 'README.rdoc', 'MIT-LICENSE', 'VERSION', 'Rakefile', 'libstemmer_c/**/*', 'ext/**/*', 'test/**/*']
-    %w(ext/lingua/*.so ext/lingua/*.bundle ext/lingua/Makefile ext/lingua/mkmf.log ext/lingua/*.o libstemmer_c/**/*.o).each do | f |
+    %w(ext/lingua/*.so ext/lingua/*.bundle ext/lingua/Makefile ext/lingua/mkmf.log ext/lingua/*.o libstemmer_c/**/*.o libstemmer_c/stemwords).each do | f |
       gem.files.exclude f
     end
   end
   Jeweler::GemcutterTasks.new
   Jeweler::RubyforgeTasks.new do |rubyforge|
     rubyforge.doc_task = "rdoc"
-  end  
+  end
 rescue LoadError
   puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
 end
@@ -49,22 +50,23 @@ task :test => :check_dependencies
 
 task :default => :test
 
-desc "Cleans the project"
-task :clean do
-  `cd ext/lingua && rm -rf Makefile mkmf.log stemmer.o stemmer_native.bundle stemmer.so; cd ../../`
-  `cd libstemmer_c && make clean && cd ../`
-end
+gem 'rake-compiler', '>= 0.4.1'
+require "rake/extensiontask"
+
+CLOBBER.include("libstemmer_c/**/*.o")
 
-desc "Builds the extension"
-task :ext => :clean do
-  `cd ext/lingua/ && ruby extconf.rb && make && cd ../../`
+Rake::ExtensionTask.new(JEWLER.gemspec.name, JEWLER.gemspec) do |ext|
+  ext.lib_dir = File.join(*['lib', 'lingua', ENV['FAT_DIR']].compact)
+  ext.ext_dir = File.join 'ext', 'lingua'
+  ext.cross_compile = true
+  ext.name    = 'stemmer_native'
 end
 
 require 'rake/rdoctask'
 Rake::RDocTask.new do |rdoc|
   version = File.exist?('VERSION') ? File.read('VERSION') : ""
   rdoc.rdoc_dir = 'rdoc'
-  rdoc.options << '--charset' << 'utf-8' 
+  rdoc.options << '--charset' << 'utf-8'
   rdoc.title = "Ruby-Stemmer #{version}"
   rdoc.rdoc_files.include('README*')
   rdoc.rdoc_files.include('lib/**/*.rb')

data/VERSION

@@ -1 +1 @@
-0.7.0
+0.8.1

data/ext/lingua/extconf.rb

@@ -17,12 +17,15 @@ if RUBY_PLATFORM =~ /darwin/
     exit
   end
 end
-# make this stuff
-system "cd #{LIBSTEMMER}; #{make} libstemmer.o; cd #{ROOT};"
-exit unless $? == 0
 
-$CFLAGS  += " -I#{File.join(LIBSTEMMER, 'include')} "
-$libs    += " -L#{LIBSTEMMER} #{File.join(LIBSTEMMER, 'libstemmer.o')} "
+# make libstemmer_c. unless we're cross-compiling.
+unless RUBY_PLATFORM =~ /i386-mingw32/
+  system "cd #{LIBSTEMMER}; #{make} libstemmer.o; cd #{ROOT};"
+  exit unless $? == 0
+end
+
+$CFLAGS  += " -I#{File.expand_path(File.join(LIBSTEMMER, 'include'))} "
+$libs    += " -L#{LIBSTEMMER} #{File.expand_path(File.join(LIBSTEMMER, 'libstemmer.o'))} "
 
 if have_header("libstemmer.h")
   create_makefile("lingua/stemmer_native")

data/ext/lingua/stemmer.c

@@ -1,20 +1,10 @@
 #include "ruby.h"
 #include <libstemmer.h>
 
-#define GetStemmer(obj, sb_data) {\
-  Data_Get_Struct(obj, struct sb_stemmer_data, sb_data);\
-}
-
 VALUE rb_mLingua;
 VALUE rb_cStemmer;
 VALUE rb_eStemmerError;
 
-struct sb_stemmer_data {
-  struct sb_stemmer * stemmer;
-  const char * lang;
-  const char * enc;
-};
-
 /*
  * Document-method: new
  * call-seq: Lingua::Stemmer.new
@@ -26,48 +16,27 @@ struct sb_stemmer_data {
  *   s = Lingua::Stemmer.new :language => 'fr'
  */ 
 static VALUE 
-rb_stemmer_init(int argc, VALUE *argv, VALUE self) { 
-  VALUE roptions, rlang, renc;
-
+rb_stemmer_init(VALUE self, VALUE rlang, VALUE renc) { 
   struct sb_stemmer * stemmer;
-  struct sb_stemmer_data *sb_data;
 
-  rb_scan_args(argc, argv, "01", &roptions);
-  
-  if(argc > 0) {
-    Check_Type(roptions, T_HASH);
-    if((rlang = rb_hash_aref(roptions, ID2SYM(rb_intern("language")))) != Qnil) {
-      Check_Type(rlang, T_STRING);
-    } else {
-      rlang = rb_str_new2("en");
-    }
-    if((renc = rb_hash_aref(roptions, ID2SYM(rb_intern("encoding")))) != Qnil) {
-      Check_Type(renc, T_STRING);
-    } else {
-      renc = rb_str_new2("UTF_8");
-    }
-  } else {
-    rlang = rb_str_new2("en");
-    renc  = rb_str_new2("UTF_8");
-  }
-  
+  Data_Get_Struct(self, struct sb_stemmer, stemmer);
+
+  // In case someone sends() this method, free up the old one
+  if(stemmer) sb_stemmer_delete(stemmer);
+
   stemmer = sb_stemmer_new( RSTRING_PTR(rlang), RSTRING_PTR(renc) );
-  if (stemmer == 0) {
-    if (renc == 0 ) {
-      rb_raise(rb_eStemmerError, "Language %s not available for stemming", RSTRING_PTR(rlang));
-      exit(1);
+  if (!stemmer) {
+    if (!RTEST(renc)) {
+      rb_raise(rb_eStemmerError,
+          "Language %s not available for stemming", RSTRING_PTR(rlang));
     } else {
-      rb_raise(rb_eStemmerError, "Language %s not available for stemming in encoding %s", 
+      rb_raise(rb_eStemmerError,
+          "Language %s not available for stemming in encoding %s", 
                     RSTRING_PTR(rlang), RSTRING_PTR(renc));
-      exit(1);
     }
   }
   
-  sb_data = ALLOC(struct sb_stemmer_data);
-  DATA_PTR(self) = sb_data;
-  sb_data->stemmer= stemmer;
-  sb_data->lang   = RSTRING_PTR(rlang);
-  sb_data->enc    = RSTRING_PTR(renc);
+  DATA_PTR(self) = stemmer;
 
   return self;
 }
@@ -84,52 +53,23 @@ rb_stemmer_init(int argc, VALUE *argv, VALUE self) {
  */ 
 static VALUE
 rb_stemmer_stem(VALUE self, VALUE word) {
-  struct sb_stemmer_data * sb_data;
-  const  sb_symbol * stemmed;
-  VALUE s_word = rb_String(word);
-  GetStemmer(self, sb_data);
-  stemmed = sb_stemmer_stem(sb_data->stemmer, (sb_symbol *)RSTRING_PTR(s_word), RSTRING_LEN(s_word));
-  return rb_str_new2((char *)stemmed);
-}
+  struct sb_stemmer * stemmer;
 
-/*
- * Document-method: language
- * call-seq: language
- *
- * Gets the language for this stemmer
- *
- *   require 'lingua/stemmer'
- *   s = Lingua::Stemmer.new(:language => "fr")
- *   s.language #=> "fr"
- */ 
-static VALUE
-rb_stemmer_language(VALUE self) {
-  struct sb_stemmer_data * sb_data;
-  GetStemmer(self, sb_data);
-  return rb_str_new2(sb_data->lang);
-}
+  Data_Get_Struct(self, struct sb_stemmer, stemmer);
+  if(!stemmer) rb_raise(rb_eRuntimeError, "Stemmer is not initialized");
 
-/*
- * Document-method: encoding
- * call-seq: encoding
- *
- * Gets the encoding for this stemmer
- *
- *   require 'lingua/stemmer'
- *   s = Lingua::Stemmer.new(:language => "UTF_8")
- *   s.encoding #=> "UTF_8"
- */ 
-static VALUE
-rb_stemmer_encoding(VALUE self) {
-  struct sb_stemmer_data * sb_data;
-  GetStemmer(self, sb_data);
-  return rb_str_new2(sb_data->enc);
+  VALUE s_word = rb_String(word);
+  const sb_symbol * stemmed = sb_stemmer_stem(stemmer,
+      (sb_symbol *)RSTRING_PTR(s_word),
+      RSTRING_LEN(s_word)
+  );
+  return rb_str_new2((char *)stemmed);
 }
 
 static void
-sb_stemmer_free(struct sb_stemmer_data * sb_data)
+sb_stemmer_free(struct sb_stemmer * stemmer)
 {
-  sb_stemmer_delete(sb_data->stemmer);
+  if(stemmer) sb_stemmer_delete(stemmer);
 }
 
 static VALUE
@@ -146,9 +86,7 @@ void Init_stemmer_native() {
   rb_cStemmer = rb_define_class_under(rb_mLingua, "Stemmer", rb_cObject);
   rb_define_alloc_func(rb_cStemmer, sb_stemmer_alloc);
   rb_eStemmerError = rb_define_class_under(rb_mLingua, "StemmerError", rb_eException);  
-  rb_define_method(rb_cStemmer, "initialize", rb_stemmer_init, -1);
+  rb_define_private_method(rb_cStemmer, "native_init", rb_stemmer_init, 2);
   rb_define_method(rb_cStemmer, "stem", rb_stemmer_stem, 1);
-  rb_define_method(rb_cStemmer, "language", rb_stemmer_language, 0);
-  rb_define_method(rb_cStemmer, "encoding", rb_stemmer_encoding, 0);
 }
 

data/lib/lingua/stemmer.rb

@@ -1,24 +1,46 @@
-require 'lingua/stemmer_native'
+if RUBY_PLATFORM =~/(mswin|mingw)/i
+  require "lingua/#{RUBY_VERSION.sub(/\.\d+$/, '')}/stemmer_native"
+else
+  require 'lingua/stemmer_native'
+end
 
 module Lingua
-
   def self.stemmer(o, options={})
-    _stemmer= Stemmer.new({:language => "en", :encoding => "UTF_8"}.merge(options))
-    words= o.kind_of?(Array)? o.map{|e|e.to_s} : [o.to_s]
-    results = [] unless block_given?
-    words.each do | word |
-      result = _stemmer.stem(word)
+    stemmer = Stemmer.new(options)
+
+    words = Array(o).map { |e| e.to_s }
+
+    results = []
+    words.each do |word|
+      result = stemmer.stem(word)
       if block_given?
         yield result
       else
         results << result
       end
     end
-    return (results.length == 1)? results[0] : results unless block_given?
-    _stemmer
+
+    return stemmer if block_given?
+    results.length == 1 ? results[0] : results
   end
 
   class Stemmer
     VERSION = File.read(File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "VERSION"))).strip!
+
+    attr_reader :language
+    attr_reader :encoding
+
+    # Creates a new Stemmer, pass <tt>:language</tt> and <tt>:encoding</tt>
+    # as arguments to change encoding or language, otherwise english with UTF_8
+    # will be used
+    #
+    #   require 'lingua/stemmer'
+    #   s = Lingua::Stemmer.new :language => 'fr'
+    #
+    def initialize options = {}
+      @language = (options[:language] || 'en').to_s
+      @encoding = (options[:encoding] || 'UTF_8').to_s
+      native_init @language, @encoding
+    end
   end
 end

data/libstemmer_c/Makefile.windows

@@ -0,0 +1,14 @@
+include mkinc.mak
+
+AR=/usr/local/i386-mingw32-4.3.0/bin/i386-mingw32-ar
+CC=/usr/local/i386-mingw32-4.3.0/bin/i386-mingw32-gcc
+
+CFLAGS=-Iinclude
+
+all: libstemmer.o stemwords
+libstemmer.o: $(snowball_sources:.c=.o)
+	$(AR) -cru $@ $^
+stemwords: examples/stemwords.o libstemmer.o
+	$(CC) -o $@ $^
+clean:
+	rm -f stemwords *.o src_c/*.o runtime/*.o libstemmer/*.o

data/test/helper.rb

@@ -1,11 +1,3 @@
 require 'rubygems'
 require 'test/unit'
-
-%w(. ../lib ../ext).each do |path|
-  $LOAD_PATH.unshift(File.expand_path(File.join(File.dirname(__FILE__), path)))
-end
-
 require 'lingua/stemmer'
-
-class Test::Unit::TestCase
-end

data/test/lingua/test_stemmer.rb

@@ -38,4 +38,11 @@ class TestStemmer < Test::Unit::TestCase
     assert_kind_of Array, results
   end
 
+  def test_stemmer_subclass
+    assert_raises(RuntimeError) do
+      Class.new(Lingua::Stemmer) {
+        def native_init a, b; end
+      }.new.stem('cow')
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: ruby-stemmer
 version: !ruby/object:Gem::Version 
-  version: 0.7.0
+  version: 0.8.1
 platform: ruby
 authors: 
 - Aurelian Oancea
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-10-29 00:00:00 +01:00
+date: 2009-11-06 00:00:00 +01:00
 default_executable: 
 dependencies: []
 
@@ -32,6 +32,7 @@ files:
 - lib/lingua/stemmer.rb
 - libstemmer_c/MANIFEST
 - libstemmer_c/Makefile
+- libstemmer_c/Makefile.windows
 - libstemmer_c/README
 - libstemmer_c/examples/stemwords.c
 - libstemmer_c/include/libstemmer.h