DRMacIver-term-extractor 0.0.0

data/test/term_extractor_spec.rb

@@ -0,0 +1,141 @@
+require "term-extractor"
+require "rubygems"
+require "rake"
+
+PE = TermExtractor.new
+
+def each_tag_in(file)
+  PE.extract_terms_from_text(IO.read(file)).each do |tag|
+    yield(tag)
+  end
+end
+
+def each_tag(&blk) 
+  FileList["test/files/*"].each { |f| each_tag_in(f, &blk) }
+end
+
+describe TermExtractor do
+  it "should only return themes ending in nouns" do
+    each_tag do |tag|
+      tag.pos.should =~ /(^|-)(#{PE.required_ending})$/
+    end
+  end
+
+  it "must not return themes starting with proscribed parts of speech" do
+    each_tag do  |tag|
+      tag.pos.should_not =~ /^(#{PE.proscribed_start})($|-)/
+    end
+  end
+
+  it "should produce at least as many tags as words" do
+    each_tag do |tag|
+      tag.pos.split("-").length.should be >= tag.to_s.split.length
+    end
+  end 
+
+  it "should correctly identify the subterms of a known term" do
+    PE.extract_terms_from_text("I am a big fan of kitties").map{|x| x.to_s}.sort.should == ["big fan", "big fan of kitties", "kitties"]
+  end
+
+  it "should allow terms ending in numerals" do
+    PE.extract_terms_from_text("I think Enterprise 2.0 is neato").map{|x| x.to_s}.sort.should == ["Enterprise 2.0"]  
+  end
+
+  it "should not concatenate words" do 
+    internalconfig = <<PC
+knowing their
+internal network config
+PC
+
+    (PE.extract_terms_from_text(internalconfig).join(" ") =~ /theirinternal/).should be(nil)
+  
+  end
+
+  it "should not concatenate words, even after ellipses" do 
+    oukc = "Oracle University Knowledge Center...               http://www.oracle.com/education/oln"
+
+  (PE.extract_terms_from_text(oukc).join(" ") =~ /Centerhttp/).should be(nil)
+  end
+  
+  it "should not split contractions" do 
+    terms = PE.extract_terms_from_sentence("It is my considered opinion that Jon should've liked the puppies").map{|x| x.to_s }
+
+    terms.should_not include("ve")
+    terms.should_not include("ve liked the puppies")
+  end
+
+  it "shouldn 't leave spaces in terms containing contractions" do
+    terms = PE.extract_terms_from_sentence("Kittens aren't villains, they're cute").map{|x| x.to_s }
+
+    terms.should include("Kittens aren't villains")
+    terms.should_not include("Kittens aren 't villains")
+  end
+
+  def number_of_sentences(text, n)
+    counts = [0] * n
+    PE.extract_terms_from_text(text).each{|p| counts[p.sentence] += 1 }
+    counts.should_not include(0)
+  end
+
+  it "should correctly attribute terms to sentences" do
+    number_of_sentences("I like kitties", 1)
+    number_of_sentences("I like kitties. They are cute creatures", 2)
+  end
+
+  it "should not start terms with contractions" do 
+    terms = PE.extract_terms_from_sentence("But I don't have time for such a drastic rewrite right now, I'm thinking it would take at least two weeks for someone who is experienced with Eclipe editors").map{|x| x.to_s}
+    
+    terms.should_not include("don't have time")
+  end
+
+  it "should not produce terms which consist entirely of numbers" do
+    text = <<BINARYSOLO    
+Binary solo
+0000001
+00000011
+0000001
+00000011
+0000001
+0000001
+0000001
+0000001
+BINARYSOLO
+
+    PE.extract_terms_from_text(text).each{|p| p.to_s.should_not match(/^[\s\d]*$/) }
+  end
+
+  it "should pick out interesting nouns which follow a possessive" do 
+    PE.extract_terms_from_sentence("You know, you could always have asked me to change your password...").map{|x| x.to_s}.should include("password")
+  end
+
+  it "should never generate stopwords" do 
+    PE.extract_terms_from_sentence('A "Today Only" or "Sneak Preview" special tied to a specific day or time frame will encourage many recipients to open the message right away instead of passing it over for another one in the inbox.').map{|x| x.to_s}.should_not include("A")
+  end
+
+  it "should never generate URLs" do
+    PE.extract_terms_from_text("I like http://www.google.com for searching").map{|t| t.to_s }.should_not include("http://www.google.com")
+  end
+
+  it "should not generate verb terms" do 
+    PE.extract_terms_from_text("Do you think it makes sense to be the very model of a modern major general?").map{|t| t.to_s }.should_not include("makes sense")
+  end
+
+
+  it "should not allow verb terms internally" do
+    PE.extract_terms_from_text("Please consider the environment before printing this email").map{|t| t.to_s }.should_not include("environment before printing this email")
+  end
+
+  it "should not start terms with comparison adjectives" do
+    terms = PE.extract_terms_from_sentence("European policymakers urged the U.S. Senate on Wednesday to approve a revised $700 billion financial rescue plan aimed at tackling the worst financial crisis since the 1930s.").map{|t| t.to_s}
+    terms.should_not include("worst financial crisis") 
+    terms.should include("financial crisis") 
+
+  end
+
+  it "should not be confused by smart apostrophes" do
+    PE.extract_terms_from_sentence("By training I’m a mathematician, but I seem to have drifted away from that and become a programmer.").each { |term|
+      term.to_s.should_not =~ /’|'/
+    }
+  end
+
+end

metadata

@@ -0,0 +1,83 @@
+--- !ruby/object:Gem::Specification 
+name: DRMacIver-term-extractor
+version: !ruby/object:Gem::Version 
+  version: 0.0.0
+platform: ruby
+authors: 
+- David R. MacIver
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-08-06 00:00:00 -07:00
+default_executable: terms.rb
+dependencies: []
+
+description: 
+email: david.maciver@gmail.com
+executables: 
+- terms.rb
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.markdown
+files: 
+- LICENSE
+- README.markdown
+- Rakefile
+- VERSION
+- bin/terms.rb
+- lib/term-extractor.rb
+- lib/term-extractor/maxent-2.5.2.jar
+- lib/term-extractor/nlp.rb
+- lib/term-extractor/opennlp-tools.jar
+- lib/term-extractor/snowball.jar
+- lib/term-extractor/trove.jar
+- licenses/Maxent
+- licenses/OpenNLP
+- licenses/Trove
+- licenses/snowball.php
+- models/chunk.bin.gz
+- models/sd.bin.gz
+- models/stopwords
+- models/tag.bin.gz
+- models/tagdict
+- models/tok.bin.gz
+- term-extractor.gemspec
+- test/examples_spec.rb
+- test/files/1.email
+- test/files/juries_seg_8_v1
+- test/nlp_spec.rb
+- test/term_extractor_spec.rb
+has_rdoc: false
+homepage: http://github.com/david.maciver@gmail.com/term-extractor
+licenses: 
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: A library for extracting useful terms from text
+test_files: 
+- test/term_extractor_spec.rb
+- test/nlp_spec.rb
+- test/examples_spec.rb