rbtagger 0.3.2 → 0.4.0

data/README

@@ -1,3 +1,9 @@
+= rbtagger
+
+* http://github.com/taf2/rb-brill-tagger/tree/master
+
+== DESCRIPTION:
+
 A Simple Ruby Rule-Based Part of Speech Tagger
 
 This work is based on the work of Eric Brill
@@ -5,3 +11,41 @@ This work is based on the work of Eric Brill
 Credit for extracting Eric's work into a reusable library goes to Ken Williams as part of his work in creating the Lingua::BrillTagger perl module
 
 This software is made available under the MIT License, see LICENSE
+
+== FEATURES/PROBLEMS:
+
+* Rule based tagger
+* Word matching tagger
+
+== SYNOPSIS:
+
+ tagger = Brill::Tagger.new
+
+== INSTALL:
+
+* sudo gem install rbtagger
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2008 Todd A. Fisher
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -1,4 +1,78 @@
-require 'config/requirements'
-require 'config/hoe' # setup Hoe + all gem configuration
-
-Dir['tasks/**/*.rake'].each { |rake| load rake }
+require 'rake/clean'
+require 'rake/testtask'
+require 'rake/rdoctask'
+
+CLEAN.include '**/*.o'
+CLEAN.include "**/*.#{Config::MAKEFILE_CONFIG['DLEXT']}"
+CLOBBER.include 'doc'
+CLOBBER.include '**/*.log'
+CLOBBER.include '**/Makefile'
+CLOBBER.include '**/extconf.h'
+
+# Make tasks -----------------------------------------------------
+make_program = (/mswin/ =~ RUBY_PLATFORM) ? 'nmake' : 'make'
+MAKECMD = ENV['MAKE_CMD'] || make_program
+MAKEOPTS = ENV['MAKE_OPTS'] || ''
+
+RULE_SO = "ext/rule_tagger/rule_tagger.#{Config::MAKEFILE_CONFIG['DLEXT']}"
+WORD_SO = "ext/word_tagger/word_tagger.#{Config::MAKEFILE_CONFIG['DLEXT']}"
+
+def make(mod,target = '')
+  Dir.chdir("ext/#{mod}_tagger") do 
+    pid = system("#{MAKECMD} #{MAKEOPTS} #{target}")
+    $?.exitstatus
+  end    
+end
+
+file 'ext/word_tagger/Makefile' => 'ext/word_tagger/extconf.rb' do
+  Dir.chdir('ext/word_tagger') { ruby "extconf.rb #{ENV['EXTCONF_OPTS']}" }
+end
+
+file 'ext/rule_tagger/Makefile' => 'ext/rule_tagger/extconf.rb' do
+  Dir.chdir('ext/rule_tagger') { ruby "extconf.rb #{ENV['EXTCONF_OPTS']}" }
+end
+
+# Let make handle dependencies between c/o/so - we'll just run it. 
+file RULE_SO => (['ext/rule_tagger/Makefile','ext/rule_tagger/extconf.rb'] + Dir['ext/rule_tagger/*.cc'] + Dir['ext/rule_tagger/*.h']) do
+  m = make('rule')
+  fail "Make failed (status #{m})" unless m == 0
+end
+
+file WORD_SO => (['ext/word_tagger/Makefile','ext/rule_tagger/extconf.rb'] + Dir['ext/word_tagger/*.cc'] + Dir['ext/word_tagger/*.h']) do
+  m = make('word')
+  fail "Make failed (status #{m})" unless m == 0
+end
+
+desc "Compile the shared object"
+task :compile => [RULE_SO, WORD_SO]
+
+Rake::TestTask.new(:inttest) do |t|
+  t.test_files = FileList['test/test_*.rb']
+  t.verbose = false
+end
+
+desc "Test taggers"
+task :test => [:compile, :inttest]
+
+task :default => :test
+
+desc 'Generate gem specification'
+task :gemspec do
+  require 'erb'
+  tspec = ERB.new(File.read(File.join(File.dirname(__FILE__),'lib','rbtagger.gemspec.erb')))
+  File.open(File.join(File.dirname(__FILE__),'rbtagger.gemspec'),'wb') do|f|
+    f << tspec.result
+  end
+end
+require 'lib/rbtagger/version'
+CURRENT_VERSION = RbTagger::VERSION::STRING
+desc 'Build gem'
+task :package => :gemspec do
+  require 'rubygems/specification'
+  spec_source = File.read File.join(File.dirname(__FILE__),'rbtagger.gemspec')
+  spec = nil
+  # see: http://gist.github.com/16215
+  Thread.new { spec = eval("$SAFE = 3\n#{spec_source}") }.join
+  spec.validate
+  Gem::Builder.new(spec).build
+end

data/ext/rule_tagger/registry.c

@@ -251,12 +251,12 @@ NORET Registry_destroy(dir)
 
 unsigned int Registry_ptrhash(ptr, htsize)
      CONSTVOIDP ptr;
-     unsigned int htsize;
+     unsigned long htsize;
 {
-  unsigned int uns_int_ptr = (unsigned int)ptr;
+  unsigned long uns_int_ptr = (unsigned long)ptr;
 
-  if ((int)uns_int_ptr > 0)
-    return htsize ? (int)uns_int_ptr % (int)htsize : 0;
+  if ((long)uns_int_ptr > 0)
+    return htsize ? (long)uns_int_ptr % (int)htsize : 0;
   else
     return htsize ? uns_int_ptr % htsize : 0;
 }

data/ext/rule_tagger/registry.h

@@ -23,7 +23,7 @@ extern unsigned int Registry_entry_count(Registry);
 extern NORET Registry_fetch_contents(Registry, Darray, Darray);
 extern NORET Registry_destroy(Registry);
 extern int Registry_ptrcmp(CONSTVOIDP, CONSTVOIDP);
-extern unsigned int Registry_ptrhash(CONSTVOIDP, unsigned int);
+extern unsigned int Registry_ptrhash(CONSTVOIDP, unsigned long);
 extern int Registry_strcmp(CONSTVOIDP, CONSTVOIDP);
 extern unsigned int Registry_strhash(CONSTVOIDP, unsigned int);
 extern int Registry_strcasecmp(CONSTVOIDP, CONSTVOIDP);

data/ext/word_tagger/rtagger.cc

@@ -36,13 +36,35 @@ VALUE Tagger_execute( VALUE self, VALUE text )
   }
   return results;
 }
+/*
+ * call-seq:
+ *    tagger.freq(['word1','word2','word3',...,n])  => {'word1' => 5, 'word2' => 10, ... }
+ *
+ * run the tagger against a body of text either as an array or as a string.  extract words of interest
+ * given the predefined set of tags.
+ */
 VALUE Tagger_execute_freq( VALUE self, VALUE text )
 {
   NWordTagger *tagger;
   Data_Get_Struct( self, NWordTagger, tagger );
   int max_count = 0;
   std::map<std::string,int> tags;
-  tagger->execute_with_frequency( RSTRING_PTR(text), tags, max_count );
+  if( TYPE(text) == T_STRING) {
+    tagger->execute_with_frequency( RSTRING_PTR(text), tags, max_count );
+  }
+  else if( TYPE(text) == T_ARRAY ) {
+    // convert ruby array to std vector
+    std::vector<std::string> words;
+    unsigned int len = RARRAY_LEN(text);
+    words.reserve(len);
+    for( unsigned int i = 0; i < len; ++i ) {
+      words.push_back(std::string(RSTRING_PTR(rb_ary_entry(text,i))));
+    }
+    tagger->execute_with_frequency( words, tags, max_count );
+  }
+  else {
+    rb_raise(rb_eRuntimeError, "Must provide either a string or array as input");
+  }
   VALUE results = rb_hash_new();
   for( std::map<std::string,int>::const_iterator it = tags.begin(); it != tags.end(); ++it ) {
     rb_hash_aset( results, rb_str_new(it->first.c_str(), it->first.length()), rb_int_new(it->second) );

data/ext/word_tagger/tagger.cc

@@ -82,9 +82,8 @@ std::string NWordTagger::stemWord( const std::string &word )const
   return stemmed;
 }
   
-int NWordTagger::execute_with_frequency( const char *text, std::map<std::string,int> &matched_tags, int &max_count )const
+int NWordTagger::execute_with_frequency( const std::vector<std::string> &words, std::map<std::string,int> &matched_tags, int &max_count )const
 {
-  std::vector<std::string> words = word_split( text );
   std::string match_word;
   std::map<std::string,std::string>::const_iterator matched;
 
@@ -130,6 +129,11 @@ int NWordTagger::execute_with_frequency( const char *text, std::map<std::string,
   }
   return matched_tags.size();
 }
+int NWordTagger::execute_with_frequency( const char *text, std::map<std::string,int> &matched_tags, int &max_count )const
+{
+  std::vector<std::string> words = word_split( text );
+  return execute_with_frequency(words, matched_tags, max_count);
+}
 
 int NWordTagger::execute( std::vector<std::string> &reduced_tags, const char *text, unsigned short max )const
 {
@@ -159,12 +163,13 @@ int NWordTagger::execute( std::vector<std::string> &reduced_tags, const char *te
   // sort the tags in frequency order
   std::sort( sorted_tags.begin(), sorted_tags.end(), WordComparitor() );
 
+
   std::vector< std::pair<std::string, int> >::iterator mloc;
   do {
     for(mloc = sorted_tags.begin(); mloc != sorted_tags.end(); ++mloc ) {
       std::pair< std::string, int > word_freq = *mloc;
-//      printf( "word: %s, frequency: %d\n", word_freq.first.c_str(), word_freq.second );
-//      printf( "word: %s, frequency: %d\n", mloc->first.c_str(), mloc->second ); 
+      printf( "word: %s, frequency: %d\n", word_freq.first.c_str(), word_freq.second );
+      printf( "word: %s, frequency: %d\n", mloc->first.c_str(), mloc->second ); 
       if( word_freq.second < max_count ) { 
         sorted_tags.erase( mloc );
         break;

data/ext/word_tagger/tagger.h

@@ -18,10 +18,12 @@ struct NWordTagger {
   // fill results with matching tags in the text body
   // keep the number of tags returned within the threshold of max. reducing tags by least frequent
   int execute( std::vector<std::string> &reduced_tags, const char *text, unsigned short max = 10 )const;
+  int execute( std::vector<std::string> &reduced_tags, const std::vector<std::string> &words, unsigned short max = 10 )const;
 
   // return the number of matched tags
   // result is updated with a mapping of matched tags with their individual term frequency count
   int execute_with_frequency( const char *text, std::map<std::string,int> &matched_tags, int &max_count )const;
+  int execute_with_frequency( const std::vector<std::string> &words, std::map<std::string,int> &matched_tags, int &max_count )const;
 private:
   short nwords;
   struct stemmer *stemmer;

data/ext/word_tagger/test.rb

@@ -1,12 +1,12 @@
 if $0 == __FILE__
 require 'test/unit'
-require 'tagger'
+require 'word_tagger'
 
 class NWordTest < Test::Unit::TestCase
 
   def setup
     if !defined?($tagger)
-      $tagger = Tagger::SimpleTagger.new
+      $tagger = Tagger::WordTagger.new
       $tagger.load_tags( File.read('../../tags.txt').split("\n") )
       $tagger.set_words( 4 );
     end

data/lib/brill/tagger.rb

@@ -1,4 +1,4 @@
-require 'rule_tagger'
+require 'rule_tagger/rule_tagger'
 
 module Brill
   class Tagger

data/lib/rbtagger.rb

@@ -1,6 +1,3 @@
-$:.unshift(File.dirname(__FILE__)) unless
-  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
-
 module RbTagger
   class << self
     def tags_from_file( file )

data/lib/rbtagger/version.rb

@@ -1,8 +1,8 @@
 module RbTagger #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
-    MINOR = 3
-    TINY  = 2
+    MINOR = 4
+    TINY  = 0
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/lib/word/tagger.rb

@@ -1,5 +1,6 @@
+require 'word_tagger/word_tagger'
+
 module Word
-  require 'word_tagger'
   class Tagger < Tagger::WordTagger
     def initialize( tags, options = {} )
       if tags.is_a?(String) and File.exist?(tags)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: rbtagger
 version: !ruby/object:Gem::Version 
-  version: 0.3.2
+  version: 0.4.0
 platform: ruby
 authors: 
 - Todd A. Fisher
@@ -9,130 +9,69 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-07-07 00:00:00 -04:00
+date: 2010-01-24 00:00:00 -05:00
 default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: hoe
-  type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 2.3.2
-    version: 
+dependencies: []
+
 description: A Simple Ruby Rule-Based Part of Speech Tagger
-email: 
-- todd.fisher@gmail.com
+email: todd.fisher@gmail.com
 executables: []
 
 extensions: 
-- ext/word_tagger/extconf.rb
 - ext/rule_tagger/extconf.rb
+- ext/word_tagger/extconf.rb
 extra_rdoc_files: 
-- History.txt
-- License.txt
-- Manifest.txt
-- PostInstall.txt
-- README.txt
-- ext/word_tagger/test/doc.txt
-- website/index.txt
+- LICENSE
+- README
 files: 
-- COPYING
-- History.txt
 - LICENSE
-- License.txt
-- Manifest.txt
-- PostInstall.txt
 - README
-- README.txt
 - Rakefile
-- config/hoe.rb
-- config/requirements.rb
-- ext/rule_tagger/bool.h
+- lib/brill/brown/Lexicon.rb
+- lib/brill/tagger.rb
+- lib/rbtagger/version.rb
+- lib/rbtagger.rb
+- lib/word/tagger.rb
 - ext/rule_tagger/darray.c
-- ext/rule_tagger/darray.h
-- ext/rule_tagger/darrayP.h
-- ext/rule_tagger/extconf.rb
 - ext/rule_tagger/lex.c
-- ext/rule_tagger/lex.h
 - ext/rule_tagger/memory.c
-- ext/rule_tagger/memory.h
-- ext/rule_tagger/mkmf.log
 - ext/rule_tagger/rbtagger.c
 - ext/rule_tagger/registry.c
+- ext/rule_tagger/rules.c
+- ext/rule_tagger/tagger.c
+- ext/rule_tagger/useful.c
+- ext/word_tagger/porter_stemmer.c
+- ext/word_tagger/rtagger.cc
+- ext/word_tagger/tagger.cc
+- ext/word_tagger/test/test.cc
+- ext/rule_tagger/bool.h
+- ext/rule_tagger/darray.h
+- ext/rule_tagger/darrayP.h
+- ext/rule_tagger/lex.h
+- ext/rule_tagger/memory.h
 - ext/rule_tagger/registry.h
 - ext/rule_tagger/registryP.h
 - ext/rule_tagger/ruby-compat.h
-- ext/rule_tagger/rules.c
 - ext/rule_tagger/rules.h
 - ext/rule_tagger/sysdep.h
-- ext/rule_tagger/tagger.c
 - ext/rule_tagger/tagger.h
-- ext/rule_tagger/useful.c
 - ext/rule_tagger/useful.h
-- ext/word_tagger/extconf.rb
-- ext/word_tagger/mkmf.log
-- ext/word_tagger/porter_stemmer.c
 - ext/word_tagger/porter_stemmer.h
-- ext/word_tagger/rtagger.cc
-- ext/word_tagger/tagger.cc
 - ext/word_tagger/tagger.h
+- ext/rule_tagger/extconf.rb
+- ext/word_tagger/extconf.rb
 - ext/word_tagger/test.rb
-- ext/word_tagger/test/Makefile
-- ext/word_tagger/test/doc.txt
-- ext/word_tagger/test/test.cc
-- lib/brill/tagger.rb
-- lib/rbtagger.rb
-- lib/rbtagger/version.rb
-- lib/word/tagger.rb
-- lib/brill/brown/CONTEXTUALRULEFILE
-- lib/brill/brown/LEXICALRULEFILE
-- lib/brill/brown/LEXICON
-- script/console
-- script/destroy
-- script/generate
-- script/txt2html
-- setup.rb
-- tasks/deployment.rake
-- tasks/environment.rake
-- tasks/extconf.rake
-- tasks/extconf/rule_tagger.rake
-- tasks/extconf/word_tagger.rake
-- tasks/website.rake
-- test/docs/doc0.txt
-- test/docs/doc1.txt
-- test/docs/doc2.txt
-- test/docs/doc3.txt
-- test/docs/doc4.txt
-- test/docs/doc5.txt
-- test/docs/doc6.txt
-- test/docs/doc7.txt
-- test/docs/doc8.txt
-- test/docs/doc9.txt
-- test/fixtures/tags.txt
-- test/test_helper.rb
-- test/test_rule_tagger.rb
-- test/test_word_tagger.rb
-- tools/rakehelp.rb
-- website/index.html
-- website/index.txt
-- website/javascripts/rounded_corners_lite.inc.js
-- website/stylesheets/screen.css
-- website/template.html.erb
 has_rdoc: true
-homepage: http://rbtagger.rubyforge.org
-post_install_message: |
-  For more information on rbtagger, see http://rbtagger.rubyforge.org
+homepage: http://rbtagger.rubyforge.org/
+licenses: []
 
+post_install_message: 
 rdoc_options: 
 - --main
-- README.txt
+- README
 require_paths: 
 - lib
-- ext/word_tagger
-- ext/rule_tagger
+- ext
 required_ruby_version: !ruby/object:Gem::Requirement 
   requirements: 
   - - ">="
@@ -147,12 +86,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   version: 
 requirements: []
 
-rubyforge_project: rbtagger
-rubygems_version: 1.3.1
+rubyforge_project: curb
+rubygems_version: 1.3.5
 signing_key: 
-specification_version: 2
-summary: A Simple Ruby Rule-Based Part of Speech Tagger
-test_files: 
-- test/test_rule_tagger.rb
-- test/test_helper.rb
-- test/test_word_tagger.rb
+specification_version: 3
+summary: Ruby libcurl bindings
+test_files: []
+