rbtagger 0.0.1 → 0.2.0

data/test/{tagger_test.rb → test_rule_tagger.rb}

@@ -1,16 +1,7 @@
-require 'test/unit'
-$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
-$:.unshift File.join(File.dirname(__FILE__), "..", "ext", "rule_tagger")
+require File.dirname(__FILE__) + '/test_helper'
 
-require 'brill/tagger'
 
-puts "loading tagger..."
-$tagger = Brill::Tagger.new( File.join(File.dirname(__FILE__),"LEXICON"),
-                             File.join(File.dirname(__FILE__),"LEXICALRULEFILE"),
-                             File.join(File.dirname(__FILE__),"CONTEXTUALRULEFILE") )
-puts "tagger loaded!"
-
-class TaggerTest < Test::Unit::TestCase
+class TestRuleTagger< Test::Unit::TestCase
 SAMPLE_DOC=%q(
 Take an active role in your care
 When it comes to making decisions about the goals and direction of treatment, don't sit back. Work closely and actively with your oncologist and the rest of your medical team.
@@ -33,13 +24,23 @@ Allow yourself time to discuss the emotional consequences of your illness and tr
 Stay connected
 Although many newly diagnosed patients fear they will not be able to keep working during treatment, this is usually not the case. Working, even at a reduced schedule, helps you maintain valuable social connections and weekly structure.
 )
+  def setup
+    if !defined?($tagger)
+      puts "loading tagger..."
+      $rtagger = Brill::Tagger.new( File.join(File.dirname(__FILE__),"LEXICON"),
+                                   File.join(File.dirname(__FILE__),"LEXICALRULEFILE"),
+                                   File.join(File.dirname(__FILE__),"CONTEXTUALRULEFILE") )
+      puts "tagger loaded!"
+    end
+  end
+
   def test_simple_tagger
     pairs = tagger.tag( SAMPLE_DOC )
     assert_equal [["", ")"], ["", ")"], ["Take", "VB"], ["an", "DT"], ["active", "JJ"], ["role", "NN"], ["in", "IN"], ["your", "PRP$"], ["care", "NN"], ["When", "WRB"], ["it", "PRP"], ["comes", "VBZ"], ["to", "TO"], ["making", "VBG"], ["decisions", "NNS"], ["about", "IN"], ["the", "DT"], ["goals", "NNS"], ["and", "CC"], ["direction", "NN"], ["of", "IN"], ["treatment", "NN"], [",", ","], ["", ")"], ["do", "VBP"], ["", ")"], ["n't", "RB"], ["sit", "VB"], ["back.", "CD"], ["Work", "NN"], ["closely", "RB"], ["and", "CC"], ["actively", "RB"], ["with", "IN"], ["your", "PRP$"], ["oncologist", "NN"], ["and", "CC"], ["the", "DT"], ["rest", "NN"], ["of", "IN"], ["your", "PRP$"], ["medical", "JJ"], ["team.", "NNP"], ["Dont", "NNP"], ["overlook", "VB"], ["clinical", "JJ"], ["trials", "NNS"], ["If", "IN"], ["youre", "NN"], ["eligible", "JJ"], ["to", "TO"], ["enroll", "VB"], ["in", "IN"], ["clinical", "JJ"], ["trials", "NNS"], [",", ","], ["", ")"], ["select", "VB"], ["an", "DT"], ["oncologist", "NN"], ["who", "WP"], ["participates", "VBZ"], ["in", "IN"], ["them.", "JJ"], ["Patients", "NNS"], ["who", "WP"], ["enroll", "VBP"], ["in", "IN"], ["clinical", "JJ"], ["studies", "NNS"], ["receive", "VBP"], ["closer", "JJR"], ["follow-up", "NN"], [",", ","], ["", ")"], ["the", "DT"], ["highest", "JJS"], ["standard-of-care", "JJ"], ["treatment", "NN"], ["and", "CC"], ["access", "NN"], ["to", "TO"], ["experimental", "JJ"], ["therapies", "NNS"], ["at", "IN"], ["no", "DT"], ["extra", "JJ"], ["cost.", "NNP"], ["Maximize", "NNP"], ["your", "PRP$"], ["nutrition", "NN"], ["strategy", "NN"], ["Doing", "NNP"], ["your", "PRP$"], ["best", "JJS"], ["to", "TO"], ["eat", "VB"], ["a", "DT"], ["healthy", "JJ"], [",", ","], ["", ")"], ["well-balanced", "JJ"], ["diet", "NN"], ["is", "VBZ"], ["vital", "JJ"], ["to", "TO"], ["prompt", "VB"], ["healing", "NN"], ["after", "IN"], ["surgery", "NN"], ["and", "CC"], ["for", "IN"], ["recovery", "NN"], ["from", "IN"], ["radiation", "NN"], ["or", "CC"], ["chemotherapy.", "JJ"], ["Many", "JJ"], ["oncology", "NN"], ["practices", "NNS"], ["employ", "VBP"], ["registered", "VBN"], ["dieticians", "NNS"], ["who", "WP"], ["can", "MD"], ["help", "VB"], ["you", "PRP"], ["optimize", "VB"], ["your", "PRP$"], ["nutrition.", "JJ"], ["Steer", "VB"], ["clear", "JJ"], ["of", "IN"], ["", ")"], ["``", "``"], ["natural", "JJ"], ["cures", "NNS"], ["''", "''"], ["", ")"], ["Before", "IN"], ["trying", "VBG"], ["nutritional", "JJ"], ["supplements", "NNS"], ["or", "CC"], ["herbal", "JJ"], ["remedies", "NNS"], [",", ","], ["", ")"], ["be", "VB"], ["sure", "JJ"], ["to", "TO"], ["discuss", "VB"], ["your", "PRP$"], ["plans", "NNS"], ["with", "IN"], ["a", "DT"], ["doctor.", "JJ"], ["Most", "JJS"], ["have", "VBP"], ["not", "RB"], ["been", "VBN"], ["tested", "VBN"], ["in", "IN"], ["clinical", "JJ"], ["studies", "NNS"], [",", ","], ["", ")"], ["and", "CC"], ["some", "DT"], ["may", "MD"], ["actually", "RB"], ["interfere", "VB"], ["with", "IN"], ["your", "PRP$"], ["treatment.", "JJ"], ["Build", "VB"], ["a", "DT"], ["stronger", "JJR"], ["body", "NN"], ["Even", "RB"], ["walking", "VBG"], ["regularly", "RB"], ["is", "VBZ"], ["can", "MD"], ["help", "VB"], ["you", "PRP"], ["minimize", "VB"], ["long-term", "JJ"], ["muscle", "NN"], ["weakness", "NN"], ["caused", "VBN"], ["by", "IN"], ["illness", "NN"], ["or", "CC"], ["de-conditioning.", "NNP"], ["Focus", "NNP"], ["on", "IN"], ["overall", "JJ"], ["health", "NN"], ["Patients", "NNS"], ["may", "MD"], ["be", "VB"], ["cured", "VBN"], ["of", "IN"], ["cancer", "NN"], ["but", "CC"], ["still", "JJ"], ["face", "NN"], ["life-threatening", "JJ"], ["medical", "JJ"], ["problems", "NNS"], ["that", "WDT"], ["are", "VBP"], ["underemphasized", "JJ"], ["during", "IN"], ["cancer", "NN"], ["treatments", "NNS"], [",", ","], ["", ")"], ["such", "JJ"], ["as", "IN"], ["diabetes", "NN"], [",", ","], ["", ")"], ["high", "JJ"], ["blood", "NN"], ["pressure", "NN"], ["and", "CC"], ["heart", "NN"], ["disease.", "JJ"], ["Continue", "VB"], ["to", "TO"], ["monitor", "VB"], ["your", "PRP$"], ["overall", "JJ"], ["health.", "JJ"], ["Put", "NN"], ["the", "DT"], ["fire", "NN"], ["out", "IN"], ["for", "IN"], ["good", "JJ"], ["Smoking", "NNP"], ["impairs", "NNS"], ["healing", "NN"], ["after", "IN"], ["surgery", "NN"], ["and", "CC"], ["radiation", "NN"], ["and", "CC"], ["increases", "NNS"], ["your", "PRP$"], ["risk", "NN"], ["of", "IN"], ["cardiovascular", "JJ"], ["disease", "NN"], ["and", "CC"], ["many", "JJ"], ["types", "NNS"], ["of", "IN"], ["cancers.", "CD"], ["Ask", "VB"], ["your", "PRP$"], ["doctor", "NN"], ["for", "IN"], ["help", "NN"], ["identifying", "VBG"], ["and", "CC"], ["obtaining", "VBG"], ["the", "DT"], ["most", "RBS"], ["appropriate", "JJ"], ["cessation", "NN"], ["aids.", "NNP"], ["Map", "NNP"], ["a", "DT"], ["healthy", "JJ"], ["future", "NN"], ["Once", "RB"], ["youve", "VBP"], ["completed", "VBN"], ["treatment", "NN"], [",", ","], ["", ")"], ["discuss", "VB"], ["appropriate", "JJ"], ["follow-up", "NN"], ["plans", "NNS"], ["with", "IN"], ["your", "PRP$"], ["doctor", "NN"], ["and", "CC"], ["keep", "VB"], ["track", "NN"], ["of", "IN"], ["them", "PRP"], ["yourself.", "CD"], ["Intensified", "JJ"], ["screening", "NN"], ["over", "IN"], ["many", "JJ"], ["years", "NNS"], ["is", "VBZ"], ["frequently", "RB"], ["recommended", "VBN"], ["to", "TO"], ["identify", "VB"], ["and", "CC"], ["treat", "VB"], ["a", "DT"], ["recurrence", "NN"], ["early", "JJ"], ["on.", "CD"], ["Share", "VB"], ["your", "PRP$"], ["feelings", "NNS"], ["Allow", "VB"], ["yourself", "PRP"], ["time", "NN"], ["to", "TO"], ["discuss", "VB"], ["the", "DT"], ["emotional", "JJ"], ["consequences", "NNS"], ["of", "IN"], ["your", "PRP$"], ["illness", "NN"], ["and", "CC"], ["treatment", "NN"], ["with", "IN"], ["family", "NN"], [",", ","], ["", ")"], ["friends", "NNS"], [",", ","], ["", ")"], ["your", "PRP$"], ["doctor", "NN"], ["and", "CC"], [",", ","], ["", ")"], ["if", "IN"], ["necessary", "JJ"], [",", ","], ["", ")"], ["a", "DT"], ["professional", "JJ"], ["therapist.", "JJ"], ["Many", "JJ"], ["patients", "NNS"], ["also", "RB"], ["find", "VBP"], ["antidepressants", "NNS"], ["helpful", "JJ"], ["during", "IN"], ["treatment.", "JJ"], ["Stay", "VB"], ["connected", "VBN"], ["Although", "IN"], ["many", "JJ"], ["newly", "RB"], ["diagnosed", "VBN"], ["patients", "NNS"], ["fear", "VBP"], ["they", "PRP"], ["will", "MD"], ["not", "RB"], ["be", "VB"], ["able", "JJ"], ["to", "TO"], ["keep", "VB"], ["working", "VBG"], ["during", "IN"], ["treatment", "NN"], [",", ","], ["", ")"], ["this", "DT"], ["is", "VBZ"], ["usually", "RB"], ["not", "RB"], ["the", "DT"], ["case.", "CD"], ["Working", "NNP"], [",", ","], ["", ")"], ["even", "RB"], ["at", "IN"], ["a", "DT"], ["reduced", "VBN"], ["schedule", "NN"], [",", ","], ["", ")"], ["helps", "VBZ"], ["you", "PRP"], ["maintain", "VBP"], ["valuable", "JJ"], ["social", "JJ"], ["connections", "NNS"], ["and", "CC"], ["weekly", "JJ"], ["structure", "NN"], [".", "."]], pairs 
     #puts pairs.inspect
     # enable these lines for memory leak testing
-    $tagger = nil
-    ObjectSpace.garbage_collect
+    #$tagger = nil
+    #ObjectSpace.garbage_collect
   end
 
   def test_multiple_docs
@@ -55,6 +56,6 @@ Although many newly diagnosed patients fear they will not be able to keep workin
 
 private
   def tagger
-    $tagger
+    $rtagger
   end
 end

data/test/test_word_tagger.rb

@@ -0,0 +1,33 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class TestWordTagger < Test::Unit::TestCase
+  
+  def setup
+    if !defined?($wtagger)
+      $wtagger = Word::Tagger.new( File.join(File.dirname(__FILE__),'fixtures','tags.txt'), :words => 4 )
+    end
+  end
+
+  def test_basic
+    timer = Time.now
+    text = "This is a sa'mple doc[]ument lets see how cancer ngrams 4 works out for this interesting text!"
+    tags = $wtagger.execute( text )
+    assert_equal ['cancer','work'], tags
+    puts "Duration: #{Time.now - timer} sec"
+  end
+
+  def test_ngram_size3
+    timer = Time.now
+    text = "This body of text contains something like ventricular septal defect"
+    tags = $wtagger.execute( text )
+    assert_equal ['ventricular septal defect'], tags
+    puts "Duration: #{Time.now - timer} sec"
+  end
+
+  def test_cat_and_the_hat
+    tagger = Word::Tagger.new( ['Cat','hat'], :words => 4 )
+    tags = tagger.execute( 'the cAt and the hat' )
+    assert_equal( ["Cat", "hat"], tags )
+  end
+
+end

data/website/index.html

@@ -33,7 +33,7 @@
     <h1>rbtagger</h1>
     <div id="version" class="clickable" onclick='document.location = "http://rubyforge.org/projects/ruletagger"; return false'>
       <p>Get Version</p>
-      <a href="http://rubyforge.org/projects/ruletagger" class="numbers">0.0.1</a>
+      <a href="http://rubyforge.org/projects/ruletagger" class="numbers">0.2.0</a>
     </div>
     <h4 style="float:right;padding-right:10px;"> &#x2192; &#8216;rb-brill-tagger&#8217;</h4>
 
@@ -49,19 +49,29 @@
 	<h2>Installing</h2>
 
 
-	<p><pre class='syntax'>sudo gem install rbtagger</pre></p>
+	<p><pre class='syntax'>sudo gem install ruletagger</pre></p>
 
 
 	<h2>The basics</h2>
 
 
 	<p><pre class='syntax'>
+<span class="ident">require</span> <span class="punct">'</span><span class="string">rbtagger</span><span class="punct">'</span>
+
+<span class="comment"># Using the rule tagger</span>
 <span class="ident">tagger</span> <span class="punct">=</span> <span class="constant">Brill</span><span class="punct">::</span><span class="constant">Tagger</span><span class="punct">.</span><span class="ident">new</span><span class="punct">(</span> <span class="constant">File</span><span class="punct">.</span><span class="ident">join</span><span class="punct">(</span><span class="constant">File</span><span class="punct">.</span><span class="ident">dirname</span><span class="punct">(</span><span class="constant">__FILE__</span><span class="punct">),&quot;</span><span class="string">LEXICON</span><span class="punct">&quot;),</span>
                             <span class="constant">File</span><span class="punct">.</span><span class="ident">join</span><span class="punct">(</span><span class="constant">File</span><span class="punct">.</span><span class="ident">dirname</span><span class="punct">(</span><span class="constant">__FILE__</span><span class="punct">),&quot;</span><span class="string">LEXICALRULEFILE</span><span class="punct">&quot;),</span>
                             <span class="constant">File</span><span class="punct">.</span><span class="ident">join</span><span class="punct">(</span><span class="constant">File</span><span class="punct">.</span><span class="ident">dirname</span><span class="punct">(</span><span class="constant">__FILE__</span><span class="punct">),&quot;</span><span class="string">CONTEXTUALRULEFILE</span><span class="punct">&quot;)</span> <span class="punct">)</span>
 <span class="ident">docs</span><span class="punct">.</span><span class="ident">each</span> <span class="keyword">do</span><span class="punct">|</span><span class="ident">doc</span><span class="punct">|</span>
   <span class="ident">tagger</span><span class="punct">.</span><span class="ident">tag</span><span class="punct">(</span> <span class="constant">File</span><span class="punct">.</span><span class="ident">read</span><span class="punct">(</span> <span class="ident">doc</span> <span class="punct">)</span> <span class="punct">)</span>
 <span class="keyword">end</span>
+
+<span class="comment"># Using the word tagger</span>
+<span class="ident">tagger</span> <span class="punct">=</span> <span class="constant">Word</span><span class="punct">::</span><span class="constant">Tagger</span><span class="punct">.</span><span class="ident">new</span><span class="punct">(</span> <span class="punct">['</span><span class="string">cat</span><span class="punct">','</span><span class="string">hat</span><span class="punct">'],</span> <span class="symbol">:words</span> <span class="punct">=&gt;</span> <span class="number">4</span> <span class="punct">)</span>
+<span class="ident">tags</span> <span class="punct">=</span> <span class="ident">tagger</span><span class="punct">.</span><span class="ident">execute</span><span class="punct">(</span> <span class="punct">'</span><span class="string">the cat and the hat</span><span class="punct">'</span> <span class="punct">)</span>
+<span class="ident">assert_equal</span><span class="punct">(</span> <span class="punct">[&quot;</span><span class="string">cat</span><span class="punct">&quot;,</span> <span class="punct">&quot;</span><span class="string">hat</span><span class="punct">&quot;],</span> <span class="ident">tags</span> <span class="punct">)</span>
+
+
 </pre></p>
 
 

data/website/index.txt

@@ -11,17 +11,27 @@ This work is based on the work of Eric Brill
 
 h2. Installing
 
-<pre syntax="bash">sudo gem install rbtagger</pre>
+<pre syntax="bash">sudo gem install ruletagger</pre>
 
 h2. The basics
 
 <pre syntax="ruby">
+require 'rbtagger'
+
+# Using the rule tagger
 tagger = Brill::Tagger.new( File.join(File.dirname(__FILE__),"LEXICON"),
                             File.join(File.dirname(__FILE__),"LEXICALRULEFILE"),
                             File.join(File.dirname(__FILE__),"CONTEXTUALRULEFILE") )
 docs.each do|doc|
   tagger.tag( File.read( doc ) )
 end
+
+# Using the word tagger
+tagger = Word::Tagger.new( ['cat','hat'], :words => 4 )
+tags = tagger.execute( 'the cat and the hat' )
+assert_equal( ["cat", "hat"], tags )
+
+
 </pre>
 
 h2. Forum

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: rbtagger
 version: !ruby/object:Gem::Version 
-  version: 0.0.1
+  version: 0.2.0
 platform: ruby
 authors: 
 - Todd A. Fisher
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-05-14 00:00:00 -04:00
+date: 2008-05-15 00:00:00 -04:00
 default_executable: 
 dependencies: []
 
@@ -38,6 +38,7 @@ extra_rdoc_files:
 - test/docs/doc7.txt
 - test/docs/doc8.txt
 - test/docs/doc9.txt
+- test/fixtures/tags.txt
 - website/index.txt
 files: 
 - COPYING
@@ -60,6 +61,7 @@ files:
 - ext/rule_tagger/lex.h
 - ext/rule_tagger/memory.c
 - ext/rule_tagger/memory.h
+- ext/rule_tagger/mkmf.log
 - ext/rule_tagger/rbtagger.c
 - ext/rule_tagger/registry.c
 - ext/rule_tagger/registry.h
@@ -73,12 +75,12 @@ files:
 - ext/rule_tagger/useful.c
 - ext/rule_tagger/useful.h
 - ext/word_tagger/extconf.rb
+- ext/word_tagger/mkmf.log
 - ext/word_tagger/porter_stemmer.c
 - ext/word_tagger/porter_stemmer.h
 - ext/word_tagger/rtagger.cc
 - ext/word_tagger/tagger.cc
 - ext/word_tagger/tagger.h
-- ext/word_tagger/tagger.rb
 - ext/word_tagger/test.rb
 - ext/word_tagger/test/Makefile
 - ext/word_tagger/test/doc.txt
@@ -86,6 +88,7 @@ files:
 - lib/brill/tagger.rb
 - lib/rbtagger.rb
 - lib/rbtagger/version.rb
+- lib/word/tagger.rb
 - script/console
 - script/destroy
 - script/generate
@@ -93,6 +96,9 @@ files:
 - setup.rb
 - tasks/deployment.rake
 - tasks/environment.rake
+- tasks/extconf.rake
+- tasks/extconf/rule_tagger.rake
+- tasks/extconf/word_tagger.rake
 - tasks/website.rake
 - test/CONTEXTUALRULEFILE
 - test/LEXICALRULEFILE
@@ -107,8 +113,10 @@ files:
 - test/docs/doc7.txt
 - test/docs/doc8.txt
 - test/docs/doc9.txt
-- test/tagger_test.rb
+- test/fixtures/tags.txt
 - test/test_helper.rb
+- test/test_rule_tagger.rb
+- test/test_word_tagger.rb
 - tools/rakehelp.rb
 - website/index.html
 - website/index.txt
@@ -116,7 +124,7 @@ files:
 - website/stylesheets/screen.css
 - website/template.html.erb
 has_rdoc: true
-homepage: http://ruletagger.rubyforge.org
+homepage: http://rbtagger.rubyforge.org
 post_install_message: |+
   
   For more information on rb-brill-tagger, see http://rb-brill-tagger.rubyforge.org
@@ -146,10 +154,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   version: 
 requirements: []
 
-rubyforge_project: ruletagger/rbtagger
+rubyforge_project: rbtagger
 rubygems_version: 1.1.1
 signing_key: 
 specification_version: 2
 summary: A Simple Ruby Rule-Based Part of Speech Tagger
 test_files: 
 - test/test_helper.rb
+- test/test_rule_tagger.rb
+- test/test_word_tagger.rb

data/ext/word_tagger/tagger.rb

@@ -1,8 +0,0 @@
-module Tagger
-  require 'rtagger'
-  class SimpleTagger < Tagger::NWordTagger
-    def execute( text )
-      super( text.gsub(/[^\w]/,' ') )
-    end
-  end
-end