rbtagger 0.2.8 → 0.2.9

data/ext/rule_tagger/mkmf.log

@@ -1,6 +1,6 @@
 have_header: checking for stdlib.h... -------------------- yes
 
-"gcc -E -I. -I/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/universal-darwin9.0 -I.    -Os -pipe -fno-common  conftest.c -o conftest.i"
+"gcc -E -I. -I/usr/lib64/ruby/1.8/x86_64-linux -I.   -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector --param=ssp-buffer-size=4 -m64 -mtune=generic -Wall  -fPIC  conftest.c -o conftest.i"
 checked program was:
 /* begin */
 1: #include <stdlib.h>
@@ -10,7 +10,7 @@ checked program was:
 
 have_header: checking for string.h... -------------------- yes
 
-"gcc -E -I. -I/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/universal-darwin9.0 -I.    -Os -pipe -fno-common  conftest.c -o conftest.i"
+"gcc -E -I. -I/usr/lib64/ruby/1.8/x86_64-linux -I.   -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector --param=ssp-buffer-size=4 -m64 -mtune=generic -Wall  -fPIC  conftest.c -o conftest.i"
 checked program was:
 /* begin */
 1: #include <string.h>
@@ -20,7 +20,7 @@ checked program was:
 
 have_library: checking for main() in -lc... -------------------- yes
 
-"gcc -o conftest -I. -I/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/universal-darwin9.0 -I.  -arch ppc -arch i386 -Os -pipe -fno-common conftest.c  -L"." -L"/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib" -L. -arch ppc -arch i386     -lruby -lc  -lpthread -ldl -lm  "
+"gcc -o conftest -I. -I/usr/lib64/ruby/1.8/x86_64-linux -I.   -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector --param=ssp-buffer-size=4 -m64 -mtune=generic -Wall  -fPIC conftest.c  -L. -L/usr/lib64 -L.  -rdynamic -Wl,-export-dynamic     -lruby-static -lc  -lpthread -ldl -lcrypt -lm   -lc"
 checked program was:
 /* begin */
 1: /*top*/
@@ -32,7 +32,7 @@ checked program was:
 
 have_func: checking for snprintf() in stdio.h... -------------------- yes
 
-"gcc -o conftest -I. -I/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/universal-darwin9.0 -I.  -arch ppc -arch i386 -Os -pipe -fno-common conftest.c  -L"." -L"/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib" -L. -arch ppc -arch i386    -lc  -lruby -lc  -lpthread -ldl -lm  "
+"gcc -o conftest -I. -I/usr/lib64/ruby/1.8/x86_64-linux -I.   -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector --param=ssp-buffer-size=4 -m64 -mtune=generic -Wall  -fPIC conftest.c  -L. -L/usr/lib64 -L.  -rdynamic -Wl,-export-dynamic    -lc  -lruby-static -lc  -lpthread -ldl -lcrypt -lm   -lc"
 checked program was:
 /* begin */
 1: #include <stdio.h>

data/ext/word_tagger/mkmf.log

@@ -1,6 +1,6 @@
 have_library: checking for main() in -lc... -------------------- yes
 
-"gcc -o conftest -I. -I/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/universal-darwin9.0 -I.  -arch ppc -arch i386 -Os -pipe -fno-common conftest.c  -L"." -L"/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib" -L. -arch ppc -arch i386     -lruby -lc  -lpthread -ldl -lm  "
+"gcc -o conftest -I. -I/usr/lib64/ruby/1.8/x86_64-linux -I.   -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector --param=ssp-buffer-size=4 -m64 -mtune=generic -Wall  -fPIC conftest.c  -L. -L/usr/lib64 -L.  -rdynamic -Wl,-export-dynamic     -lruby-static -lc  -lpthread -ldl -lcrypt -lm   -lc"
 checked program was:
 /* begin */
 1: /*top*/
@@ -12,7 +12,7 @@ checked program was:
 
 have_library: checking for main() in -lstdc++... -------------------- yes
 
-"gcc -o conftest -I. -I/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/universal-darwin9.0 -I.  -arch ppc -arch i386 -Os -pipe -fno-common conftest.c  -L"." -L"/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib" -L. -arch ppc -arch i386    -lc  -lruby -lstdc++ -lc  -lpthread -ldl -lm  "
+"gcc -o conftest -I. -I/usr/lib64/ruby/1.8/x86_64-linux -I.   -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector --param=ssp-buffer-size=4 -m64 -mtune=generic -Wall  -fPIC conftest.c  -L. -L/usr/lib64 -L.  -rdynamic -Wl,-export-dynamic    -lc  -lruby-static -lstdc++ -lc  -lpthread -ldl -lcrypt -lm   -lc"
 checked program was:
 /* begin */
 1: /*top*/

data/ext/word_tagger/tagger.cc

@@ -101,14 +101,15 @@ std::vector<std::string> NWordTagger::execute( const char *text, short max )cons
     for( short j = 1; (j <= this->nwords) && ((i+j) < words.size()); ++j ) {
       matched = this->tags.find( match_word );
       if( matched != this->tags.end() ){
-        //printf( "word: %d:(%s->%s)\n", i, match_word.c_str(), matched->second.c_str() );
         std::map<std::string, int>::iterator mloc = matched_tags.find( matched->second );
         if( mloc == matched_tags.end() ) {
           matched_tags[matched->second] = 1; // count 1
+          //printf( "word: %d:(%s->%s) %d, hits: 1\n", i, match_word.c_str(), matched->second.c_str(), j );
         }
         else {
           mloc->second++;
           if( max_count < mloc->second ) { max_count = mloc->second; }
+          //printf( "word: %d:(%s->%s) %d, hits: %d\n", i, match_word.c_str(), matched->second.c_str(), j, mloc->second );
         }
       }
       // stem each word and compare against our tag bank
@@ -129,11 +130,22 @@ std::vector<std::string> NWordTagger::execute( const char *text, short max )cons
       }
     }
   }
+ 
+  std::vector< std::string > reduced_tags;
+
+  // now we have a list of tags that match within the document text, check if we need to reduce the tags
+  if( matched_tags.size() < max ) {
+    // prepare the return vector
+    for( std::map<std::string, int>::iterator mloc = matched_tags.begin(); mloc != matched_tags.end(); ++mloc ){
+      reduced_tags.push_back( mloc->first );
+    }
+    return reduced_tags;
+  }
 
   // now that we have all the matched tags reduce to max using the tag frequency as a reduction measure
   std::vector< std::pair<std::string,int> > sorted_tags;
 
-  //printf( "max frequency: %d\n", max_count );
+  //printf( "max frequency: %d, total tagged: %d, reducing to %d\n", max_count, matched_tags.size(), max );
   for( std::map<std::string, int>::iterator mloc = matched_tags.begin(); mloc != matched_tags.end(); ++mloc ){
     //printf( "word: %s, frequency: %d\n", mloc->first.c_str(), mloc->second ); 
     sorted_tags.push_back(*mloc);
@@ -142,14 +154,13 @@ std::vector<std::string> NWordTagger::execute( const char *text, short max )cons
   // sort the tags in frequency order
   std::sort( sorted_tags.begin(), sorted_tags.end(), WordComparitor() );
 
-  std::vector< std::string > reduced_tags;
 
   std::vector< std::pair<std::string, int> >::iterator mloc;
   do {
     for(mloc = sorted_tags.begin(); mloc != sorted_tags.end(); ++mloc ) {
       std::pair< std::string, int > word_freq = *mloc;
-     // printf( "word: %s, frequency: %d\n", word_freq.first.c_str(), word_freq.second );
-      //printf( "word: %s, frequency: %d\n", mloc->first.c_str(), mloc->second ); 
+      printf( "word: %s, frequency: %d\n", word_freq.first.c_str(), word_freq.second );
+      printf( "word: %s, frequency: %d\n", mloc->first.c_str(), mloc->second ); 
       if( word_freq.second < max_count ) { 
         sorted_tags.erase( mloc );
         break;

data/lib/rbtagger/version.rb

@@ -2,7 +2,7 @@ module RbTagger #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
     MINOR = 2
-    TINY  = 8
+    TINY  = 9
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/test/test_rule_tagger.rb

@@ -32,9 +32,6 @@ The details of her visitation, however, are unclear.
 Asked by Us if she were happy with the court outcome, Spears (clutching an Ed Hardy purse) smiled and told Us, "Yes."
 Next up: A status hearing set for July 15.
 The couple last appeared in court May 6. Spears was granted extended visitation — three days a week from 9 a.m. to 5 p.m. — of Sean Preston, 2, and Jayden James, 20 months. 
-)
-SAMPLE_DOC3=%q(
-TMZ.com: Britney celebrated getting overnights with her kids by going on a wild shopping trip for herself.With L.A.'s finest at her service, it was a total clusterf**k outside of Fred Segal as Brit Brit made her way out. The scene was crazy -- and it was all... Read more
 )
   def setup
     if !defined?($tagger)
@@ -137,8 +134,6 @@ TMZ.com: Britney celebrated getting overnights with her kids by going on a wild
     assert results.include?(["Britney Spears", "NNP", 6])
     assert results.include?(["Jamie Spears", "NNP", 12])
 #    puts results.inspect
-    results = tagger.suggest( SAMPLE_DOC3, 5 )
-    puts results.inspect
   end
 
 private

data/test/test_word_tagger.rb

@@ -16,6 +16,14 @@ class TestWordTagger < Test::Unit::TestCase
     puts "Duration: #{Time.now - timer} sec"
   end
 
+  def test_sample_bug
+    tags = ["foo", "bar", "baz", "squishy", "yummy"]
+    txt = 'This is some sample text. Foo walked into a bar. The bartender said "What can I get you?" Foo said he wanted something yummy - like a baz.'
+    tagger = Word::Tagger.new tags, :words => 4
+    result_tags = tagger.execute( txt )
+    assert_equal ["bar", "baz", "foo", "yummy"], result_tags
+  end
+
   def test_ngram_size3
     timer = Time.now
     text = "This body of text contains something like ventricular septal defect"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: rbtagger
 version: !ruby/object:Gem::Version 
-  version: 0.2.8
+  version: 0.2.9
 platform: ruby
 authors: 
 - Todd A. Fisher
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-07-10 00:00:00 -04:00
+date: 2008-08-28 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -28,8 +28,8 @@ email:
 executables: []
 
 extensions: 
-- ext/rule_tagger/extconf.rb
 - ext/word_tagger/extconf.rb
+- ext/rule_tagger/extconf.rb
 extra_rdoc_files: 
 - History.txt
 - License.txt
@@ -142,8 +142,8 @@ rdoc_options:
 - README.txt
 require_paths: 
 - lib
-- ext/rule_tagger
 - ext/word_tagger
+- ext/rule_tagger
 required_ruby_version: !ruby/object:Gem::Requirement 
   requirements: 
   - - ">="
@@ -164,6 +164,6 @@ signing_key:
 specification_version: 2
 summary: A Simple Ruby Rule-Based Part of Speech Tagger
 test_files: 
-- test/test_helper.rb
 - test/test_rule_tagger.rb
+- test/test_helper.rb
 - test/test_word_tagger.rb