rbtagger 0.0.1 → 0.2.0

data/Manifest.txt

@@ -18,6 +18,7 @@ ext/rule_tagger/lex.c
 ext/rule_tagger/lex.h
 ext/rule_tagger/memory.c
 ext/rule_tagger/memory.h
+ext/rule_tagger/mkmf.log
 ext/rule_tagger/rbtagger.c
 ext/rule_tagger/registry.c
 ext/rule_tagger/registry.h
@@ -31,12 +32,12 @@ ext/rule_tagger/tagger.h
 ext/rule_tagger/useful.c
 ext/rule_tagger/useful.h
 ext/word_tagger/extconf.rb
+ext/word_tagger/mkmf.log
 ext/word_tagger/porter_stemmer.c
 ext/word_tagger/porter_stemmer.h
 ext/word_tagger/rtagger.cc
 ext/word_tagger/tagger.cc
 ext/word_tagger/tagger.h
-ext/word_tagger/tagger.rb
 ext/word_tagger/test.rb
 ext/word_tagger/test/Makefile
 ext/word_tagger/test/doc.txt
@@ -44,6 +45,7 @@ ext/word_tagger/test/test.cc
 lib/brill/tagger.rb
 lib/rbtagger.rb
 lib/rbtagger/version.rb
+lib/word/tagger.rb
 script/console
 script/destroy
 script/generate
@@ -51,6 +53,9 @@ script/txt2html
 setup.rb
 tasks/deployment.rake
 tasks/environment.rake
+tasks/extconf.rake
+tasks/extconf/rule_tagger.rake
+tasks/extconf/word_tagger.rake
 tasks/website.rake
 test/CONTEXTUALRULEFILE
 test/LEXICALRULEFILE
@@ -65,8 +70,10 @@ test/docs/doc6.txt
 test/docs/doc7.txt
 test/docs/doc8.txt
 test/docs/doc9.txt
-test/tagger_test.rb
+test/fixtures/tags.txt
 test/test_helper.rb
+test/test_rule_tagger.rb
+test/test_word_tagger.rb
 tools/rakehelp.rb
 website/index.html
 website/index.txt

data/Rakefile

@@ -2,32 +2,3 @@ require 'config/requirements'
 require 'config/hoe' # setup Hoe + all gem configuration
 
 Dir['tasks/**/*.rake'].each { |rake| load rake }
-
-# redefine release
-
-desc 'Package and upload the release to rubyforge.'
-task :release_current => [:clean, :package] do |t|
-  require 'config/hoe'
-  version = ENV["VERSION"] or abort "Must supply VERSION=x.y.z"
-  name = $hoe.name
-  rubyforge_name = $hoe.rubyforge_name
-  description = $hoe.description
-  pkg = "pkg/#{name}-#{version}"
-  abort "Package doesn't exist => #{pkg}" if !File.exist?(pkg)
-
-  rf = RubyForge.new
-  puts "Logging in"
-  rf.login
-
-  c = rf.userconfig
-  c["release_notes"] = description if description
-  c["release_changes"] = $hoe.changes if $hoe.changes
-  c["preformatted"] = true
-
-  files = [(@need_tar ? "#{pkg}.tgz" : nil),
-           (@need_zip ? "#{pkg}.zip" : nil),
-           "#{pkg}.gem"].compact
-
-  puts "Releasing #{rubyforge_name} v. #{version}"
-  rf.add_release 'ruletagger', 'ruletagger', version, *files
-end

data/config/hoe.rb

@@ -4,7 +4,7 @@ AUTHOR = 'Todd A. Fisher'  # can also be an array of Authors
 EMAIL = 'todd.fisher@gmail.com'
 DESCRIPTION = "A Simple Ruby Rule-Based Part of Speech Tagger"
 GEM_NAME = 'rbtagger' # what ppl will type to install your gem
-RUBYFORGE_PROJECT = 'ruletagger' # The unix name for your project
+RUBYFORGE_PROJECT = 'rbtagger' # The unix name for your project
 HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
 DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
 EXTRA_DEPENDENCIES = [
@@ -57,7 +57,7 @@ $hoe = Hoe.new(GEM_NAME, VERS) do |p|
   p.description = DESCRIPTION
   p.summary = DESCRIPTION
   p.url = HOMEPATH
-  p.rubyforge_name = PATH #RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
+  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
   p.test_globs = ["test/**/test_*.rb"]
   p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']  #An array of file patterns to delete on clean.
 

data/ext/rule_tagger/mkmf.log

@@ -0,0 +1,46 @@
+have_header: checking for stdlib.h... -------------------- yes
+
+"gcc -E -I. -I/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/universal-darwin9.0 -I.    -Os -pipe -fno-common  conftest.c -o conftest.i"
+checked program was:
+/* begin */
+1: #include <stdlib.h>
+/* end */
+
+--------------------
+
+have_header: checking for string.h... -------------------- yes
+
+"gcc -E -I. -I/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/universal-darwin9.0 -I.    -Os -pipe -fno-common  conftest.c -o conftest.i"
+checked program was:
+/* begin */
+1: #include <string.h>
+/* end */
+
+--------------------
+
+have_library: checking for main() in -lc... -------------------- yes
+
+"gcc -o conftest -I. -I/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/universal-darwin9.0 -I.  -arch ppc -arch i386 -Os -pipe -fno-common conftest.c  -L"." -L"/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib" -L. -arch ppc -arch i386     -lruby -lc  -lpthread -ldl -lm  "
+checked program was:
+/* begin */
+1: /*top*/
+2: int main() { return 0; }
+3: int t() { void ((*volatile p)()); p = (void ((*)()))main; return 0; }
+/* end */
+
+--------------------
+
+have_func: checking for snprintf() in stdio.h... -------------------- yes
+
+"gcc -o conftest -I. -I/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/universal-darwin9.0 -I.  -arch ppc -arch i386 -Os -pipe -fno-common conftest.c  -L"." -L"/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib" -L. -arch ppc -arch i386    -lc  -lruby -lc  -lpthread -ldl -lm  "
+checked program was:
+/* begin */
+1: #include <stdio.h>
+2: 
+3: /*top*/
+4: int main() { return 0; }
+5: int t() { void ((*volatile p)()); p = (void ((*)()))snprintf; return 0; }
+/* end */
+
+--------------------
+

data/ext/word_tagger/mkmf.log

@@ -0,0 +1,24 @@
+have_library: checking for main() in -lc... -------------------- yes
+
+"gcc -o conftest -I. -I/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/universal-darwin9.0 -I.  -arch ppc -arch i386 -Os -pipe -fno-common conftest.c  -L"." -L"/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib" -L. -arch ppc -arch i386     -lruby -lc  -lpthread -ldl -lm  "
+checked program was:
+/* begin */
+1: /*top*/
+2: int main() { return 0; }
+3: int t() { void ((*volatile p)()); p = (void ((*)()))main; return 0; }
+/* end */
+
+--------------------
+
+have_library: checking for main() in -lstdc++... -------------------- yes
+
+"gcc -o conftest -I. -I/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/universal-darwin9.0 -I.  -arch ppc -arch i386 -Os -pipe -fno-common conftest.c  -L"." -L"/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib" -L. -arch ppc -arch i386    -lc  -lruby -lstdc++ -lc  -lpthread -ldl -lm  "
+checked program was:
+/* begin */
+1: /*top*/
+2: int main() { return 0; }
+3: int t() { void ((*volatile p)()); p = (void ((*)()))main; return 0; }
+/* end */
+
+--------------------
+

data/ext/word_tagger/rtagger.cc

@@ -70,10 +70,10 @@ VALUE Tagger_alloc(VALUE klass)
   return object;
 }
 
-extern "C" void Init_rtagger()
+extern "C" void Init_word_tagger()
 {
   rb_Tagger = rb_define_module( "Tagger" );
-  rb_NWordTagger = rb_define_class_under( rb_Tagger, "NWordTagger", rb_cObject );
+  rb_NWordTagger = rb_define_class_under( rb_Tagger, "WordTagger", rb_cObject );
 
   rb_define_alloc_func( rb_NWordTagger, Tagger_alloc );
 

data/ext/word_tagger/tagger.cc

@@ -33,12 +33,21 @@ static std::vector<std::string> word_split(const std::string& s)
   return std::vector<std::string>(std::istream_iterator<std::string>(is), std::istream_iterator<std::string>());
 }
 
+static void word_downcase( std::string &word )
+{
+  for( int j = 0; j < word.size(); ++j ) {
+    word[j] = tolower( word[j] );
+  }
+}
+
 NWordTagger::NWordTagger() 
  : nwords(2), stemmer(porter_stemmer_new()){
 }
 NWordTagger::~NWordTagger(){
   porter_stemmer_free(stemmer);
 }
+
+
 void NWordTagger::loadTags( const std::set<std::string> &tags )
 {
   for( std::set<std::string>::iterator i = tags.begin(); i != tags.end(); ++i ){
@@ -51,14 +60,14 @@ void NWordTagger::loadTags( const std::set<std::string> &tags )
         stemmed += this->stemWord(words[j]) + " ";
       }
       stemmed = stemmed.substr(0,stemmed.length()-1);
-      this->tags[stemmed] = word;
-      //printf( "word: %s -> %s\n", word.c_str(), stemmed.c_str() );
     }
     else{
       stemmed = this->stemWord(*i);
-      //printf( "word: %s -> %s\n", word.c_str(), stemmed.c_str() );
-      this->tags[stemmed] = word;
     }
+    // downcase stemmed
+    word_downcase( stemmed );
+    //printf( "word: %s -> %s\n", word.c_str(), stemmed.c_str() );
+    this->tags[stemmed] = word;
 
   }
 }
@@ -84,6 +93,7 @@ std::vector<std::string> NWordTagger::execute( const char *text, short max )cons
 
     // get the stemmed word at position i
     match_word = this->stemWord(words[i]);
+    word_downcase( match_word );
 
     // now scan ahead nwords positions searching our tags table for matches
     for( short j = 1; (j <= this->nwords) && ((i+j) < words.size()); ++j ) {

data/lib/rbtagger/version.rb

@@ -1,8 +1,8 @@
 module RbTagger #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
-    MINOR = 0
-    TINY  = 1
+    MINOR = 2
+    TINY  = 0
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/lib/rbtagger.rb

@@ -2,5 +2,12 @@ $:.unshift(File.dirname(__FILE__)) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
 module RbTagger
-  
+  class << self
+    def tags_from_file( file )
+      File.read(file).split("\n").map{|t| t.strip}
+    end
+  end
 end
+
+require 'word/tagger'
+require 'brill/tagger'

data/lib/word/tagger.rb

@@ -0,0 +1,18 @@
+module Word
+  require 'word_tagger'
+  class Tagger < Tagger::WordTagger
+    def initialize( tags, options = {} )
+      if tags.is_a?(String) and File.exist?(tags)
+        load_tags( RbTagger.tags_from_file( tags ) )
+      else
+        load_tags( tags )
+      end
+      set_words( options[:words] || 2 )
+    end
+
+    def execute( text )
+      # strip non alpha characters
+      super( text.gsub(/[^\w]/,' ') )
+    end
+  end
+end

data/script/txt2html

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
 GEM_NAME = 'rbtagger' # what ppl will type to install your gem
-RUBYFORGE_PROJECT = 'ruletagger'
+RUBYFORGE_PROJECT = 'rbtagger'
 
 require 'rubygems'
 begin

data/tasks/extconf/rule_tagger.rake

@@ -0,0 +1,43 @@
+namespace :extconf do
+  extension = File.basename(__FILE__, '.rake')
+
+  ext = "ext/#{extension}"
+  ext_so = "#{ext}/#{extension}.#{Config::CONFIG['DLEXT']}"
+  ext_files = FileList[
+    "#{ext}/*.c",
+    "#{ext}/*.h",
+    "#{ext}/*.rl",
+    "#{ext}/extconf.rb",
+    "#{ext}/Makefile",
+    # "lib"
+  ]
+
+
+  task :compile => extension do
+    if Dir.glob("**/#{extension}.{o,so,dll,bundle}").length == 0
+      STDERR.puts "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+      STDERR.puts "Gem actually failed to build.  Your system is"
+      STDERR.puts "NOT configured properly to build #{GEM_NAME}."
+      STDERR.puts "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+      exit(1)
+    end
+  end
+
+  desc "Builds just the #{extension} extension"
+  task extension.to_sym => ["#{ext}/Makefile", ext_so ]
+
+  file "#{ext}/Makefile" => ["#{ext}/extconf.rb"] do
+    Dir.chdir(ext) do ruby "extconf.rb" end
+  end
+
+  file ext_so => ext_files do
+    Dir.chdir(ext) do
+      sh(PLATFORM =~ /win32/ ? 'nmake' : 'make') do |ok, res|
+        if !ok
+          require "fileutils"
+          FileUtils.rm Dir.glob('*.{so,o,dll,bundle}')
+        end
+      end
+    end
+  end
+end

data/tasks/extconf/word_tagger.rake

@@ -0,0 +1,43 @@
+namespace :extconf do
+  extension = File.basename(__FILE__, '.rake')
+
+  ext = "ext/#{extension}"
+  ext_so = "#{ext}/#{extension}.#{Config::CONFIG['DLEXT']}"
+  ext_files = FileList[
+    "#{ext}/*.c",
+    "#{ext}/*.h",
+    "#{ext}/*.rl",
+    "#{ext}/extconf.rb",
+    "#{ext}/Makefile",
+    # "lib"
+  ]
+
+
+  task :compile => extension do
+    if Dir.glob("**/#{extension}.{o,so,dll,bundle}").length == 0
+      STDERR.puts "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+      STDERR.puts "Gem actually failed to build.  Your system is"
+      STDERR.puts "NOT configured properly to build #{GEM_NAME}."
+      STDERR.puts "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+      exit(1)
+    end
+  end
+
+  desc "Builds just the #{extension} extension"
+  task extension.to_sym => ["#{ext}/Makefile", ext_so ]
+
+  file "#{ext}/Makefile" => ["#{ext}/extconf.rb"] do
+    Dir.chdir(ext) do ruby "extconf.rb" end
+  end
+
+  file ext_so => ext_files do
+    Dir.chdir(ext) do
+      sh(PLATFORM =~ /win32/ ? 'nmake' : 'make') do |ok, res|
+        if !ok
+          require "fileutils"
+          FileUtils.rm Dir.glob('*.{so,o,dll,bundle}')
+        end
+      end
+    end
+  end
+end

data/tasks/extconf.rake

@@ -0,0 +1,18 @@
+namespace :extconf do
+  desc "Compiles the Ruby extension"
+  task :compile
+end
+
+BIN = "*.{bundle,jar,so,obj,pdb,lib,def,exp}"
+
+task :compile => "extconf:compile" do
+  Dir["ext/**/*.{bundle,so,dll}"].each do|lib|
+    sh "cp #{lib} lib/"
+  end
+end
+
+task :test => :compile
+
+$hoe.clean_globs |= ["ext/**/#{BIN}", "lib/**/#{BIN}", 'ext/word_tagger/Makefile', 'ext/rule_tagger/Makefile']
+$hoe.spec.require_paths = Dir['{lib,ext/*}']
+$hoe.spec.extensions = FileList["ext/**/extconf.rb"].to_a