DRMacIver-term-extractor 0.0.0

data/term-extractor.gemspec

@@ -0,0 +1,66 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{term-extractor}
+  s.version = "0.0.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["David R. MacIver"]
+  s.date = %q{2009-08-06}
+  s.default_executable = %q{terms.rb}
+  s.email = %q{david.maciver@gmail.com}
+  s.executables = ["terms.rb"]
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.markdown"
+  ]
+  s.files = [
+    "LICENSE",
+     "README.markdown",
+     "Rakefile",
+     "VERSION",
+     "bin/terms.rb",
+     "lib/term-extractor.rb",
+     "lib/term-extractor/maxent-2.5.2.jar",
+     "lib/term-extractor/nlp.rb",
+     "lib/term-extractor/opennlp-tools.jar",
+     "lib/term-extractor/snowball.jar",
+     "lib/term-extractor/trove.jar",
+     "licenses/Maxent",
+     "licenses/OpenNLP",
+     "licenses/Trove",
+     "licenses/snowball.php",
+     "models/chunk.bin.gz",
+     "models/sd.bin.gz",
+     "models/stopwords",
+     "models/tag.bin.gz",
+     "models/tagdict",
+     "models/tok.bin.gz",
+     "term-extractor.gemspec",
+     "test/examples_spec.rb",
+     "test/files/1.email",
+     "test/files/juries_seg_8_v1",
+     "test/nlp_spec.rb",
+     "test/term_extractor_spec.rb"
+  ]
+  s.homepage = %q{http://github.com/david.maciver@gmail.com/term-extractor}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.4}
+  s.summary = %q{A library for extracting useful terms from text}
+  s.test_files = [
+    "test/term_extractor_spec.rb",
+     "test/nlp_spec.rb",
+     "test/examples_spec.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end

data/test/examples_spec.rb

@@ -0,0 +1,131 @@
+require "term-extractor"
+
+PE = TermExtractor.new
+
+Diagrams = <<DIAGRAMS
+I think having nice standardised diagrams of stuff like that is REALLY
+useful. One OO architect drops dead and your replacement walks in and
+can pick up the documents and read them because they already speak
+that language. That's a great thing. I sort of wish it had been pushed
+as being that -- a lingua franca for documenting designs.
+DIAGRAMS
+
+
+describe "Diagram terms" do
+
+
+end
+
+Murray = <<MURRAY
+The MCHS Department of Music is one of the most distinguished music programs in the State, having an award-winning choral and band program. The Marching Indians, under the direction of Mr. Mike Weaver, have performed all over the country, most recently at Universal Studios in Orlando, Disney World and the St. Patrick's Day Parade in New York City. Since 1958, the Marching Indians have been entreating fans with exciting, visually stimulating shows and their trademark deep, loud sound. Recently the Marching Indians received the Grand Championship at the 2008 Golden River Music Festival and won the first ever US101 radio battle of the bands receiving a concert by the Eli Young Band. Many students from MCHS Department of Bands have been involved with All District and All State bands as well as various summer clinics, orchestras and even the Georgia Lions All State Band.
+MURRAY
+MurrayTerms = PE.extract_terms_from_text(Murray).map{|x| x.to_s}
+
+describe "Murray terms" do
+  it "should get Mike's name right" do
+    MurrayTerms.should_not include("Mr . Mike Weaver")
+    MurrayTerms.should include("Mr. Mike Weaver")
+  end
+end
+
+Chromosome = <<CHROM
+Humans have 23 pairs of chromosomes packed with genes that dictate every aspect of our biological functioning. Of these pairs, the sex chromosomes are different; women have two X chromosomes and men have an X and a Y chromosome. The Y chromosome contains essential blueprints for the male reproductive system, in particular those for sperm development.
+
+But the Y chromosome, which once contained as many genes as the X chromosome, has deteriorated over time and now contains less than 80 functional genes compared to its partner, which contains more than 1,000 genes. Geneticists and evolutionary biologists determined that the Y chromosome's deterioration is due to accumulated mutations, deletions and anomalies that have nowhere to go because the chromosome doesn't swap genes with the X chromosome like every other chromosomal pair in our cells do. 
+CHROM
+
+ChromosomeTerms = PE.extract_terms_from_text(Chromosome).map{|x| x.to_s}
+
+describe "Chromosome terms" do
+  it "should say nothing about what humans have" do
+    ChromosomeTerms.should_not include("Humans have 23 pairs")
+  end
+
+  it "knows about the male reproductive system, if you know what I mean" do 
+    ChromosomeTerms.should include("male reproductive system")
+    ChromosomeTerms.should include("sperm development")
+  end
+
+  it "is about humans" do 
+    ChromosomeTerms.should include("Humans")
+  end
+end
+
+Environment = "Please consider the environment before printing this e-mail"
+EnvironmentTerms = PE.extract_terms_from_text(Environment).map{|x| x.to_s}.sort
+
+describe "Environment terms" do
+  it "is about email" do 
+    EnvironmentTerms.should include("e-mail")
+  end
+end
+
+Apollo = <<APOLLO
+Fate has ordained that the men who went to the moon to explore in peace will stay on the moon to rest in peace.
+
+These brave men, Neil Armstrong and Edwin Aldrin, know that there is no hope for their recovery. But they also know that there is hope for mankind in their sacrifice.
+
+These two men are laying down their lives in mankind's most noble goal: the search for truth and understanding.
+
+They will be mourned by their families and friends; they will be mourned by their nation; they will be mourned by the people of the world; they will be mourned by a Mother Earth that dared send two of her sons into the unknown.
+
+In their exploration, they stirred the people of the world to feel as one; in their sacrifice, they bind more tightly the brotherhood of man.
+
+In ancient days, men looked at stars and saw their heroes in the constellations. In modern times, we do much the same, but our heroes are epic men of flesh and blood.
+
+Others will follow, and surely find their way home. Man's search will not be denied. But these men were the first, and they will remain the foremost in our hearts.
+APOLLO
+
+ApolloTerms = PE.extract_terms_from_text(Apollo).map{|x| x.to_s}.sort.uniq
+
+describe "Apollo terms" do
+  it "knows of Neil and Buzz" do
+    ApolloTerms.should include("Neil Armstrong")
+    ApolloTerms.should include("Edwin Aldrin")
+  end
+
+  it "knows of where they've been" do
+    ApolloTerms.should include("moon")
+  end
+
+  it "knows of times past and present" do
+    ApolloTerms.should include("ancient days")
+    ApolloTerms.should include("modern times")
+  end
+
+  it "knows of destiny" do
+    ApolloTerms.should include("Fate")
+  end
+
+  it "knows of searching" do
+    ApolloTerms.should include("exploration")
+    ApolloTerms.should include("search")
+    ApolloTerms.should include("Man's search")
+  end
+
+  it "knows not of mourning, but of courage and sacrifice" do
+    ApolloTerms.should_not include("mourned")
+    ApolloTerms.should include("brave men")
+    ApolloTerms.should include("sacrifice")
+  end
+
+  it "knows of brotherhood" do 
+    ApolloTerms.should include("brotherhood of man")
+  end
+
+  it "knows of mankind, and of its heroes" do
+    ApolloTerms.should include("man")
+    ApolloTerms.should include("men")
+    ApolloTerms.should include("mankind")
+    ApolloTerms.should include("heroes")
+    ApolloTerms.should include("epic men")
+  end
+
+  it "looks to the stars from the earth" do
+    ApolloTerms.should include("stars")
+    ApolloTerms.should include("constellations")
+    ApolloTerms.should include("Mother Earth")
+    ApolloTerms.should include("world")
+  end
+
+end

data/test/files/1.email

@@ -0,0 +1,37 @@
+what's the point of children if you can't cut down on rickshaw charges ?
+
+
+On 8 Aug 2008, at 10:48, Roderick Parks wrote:
+
+> Now there’s a horror movie in the making. You wake up in the morning  
+> and find a little wire growing out of your nose.....
+>
+> I was quite impressed by those tricycles, given that they have two  
+> seats each.
+>
+> It would seem that rickshaw drivers start their training at an early  
+> age :D
+>
+> ~R
+>
+> From: lab-bounces@trampolinesystems.com [mailto:lab-bounces@trampolinesystems.com 
+> ] On Behalf Of Peter Biddle
+> Sent: 08 August 2008 10:32
+> To: lab@trampolinesystems.com
+> Subject: RE: The Case for Wireless
+>
+> It looks like cables just reached down and grabbed those tricycles.
+>
+> Procreation and then sentience cannot be far off…
+>
+> P
+>
+>
+> _______________________________________________
+> Lab mailing list
+> Lab@trampolinesystems.com
+> http://zimbra.trampolinesystems.com/mailman/listinfo/lab
+
+
+
+

data/test/files/juries_seg_8_v1

@@ -0,0 +1,20 @@
+OLATI JOHNSON [01:54:39]
+THERE ARE LAWS IN THIS COUNTRY THAT WE HAVE ALL VOTED ON EITHER DIRECTLY OR THROUGH OUR REPRESENTATIVES.  AND THE PROSECUTOR REPRESENTS THE PEOPLE OR THE UNITED STATES GOVERNMENT AND IS CHARGED WITH ENFORCING THOSE LAWS. AND SO THAT IS THE PROSECUTOR�S DUTY IS TO CHARGE PEOPLE AND BRING WORTHY CASES TO TRIAL WHEN PEOPLE HAVE VIOLATED THESE LAWS THAT WE�VE ALL AGREED SHOULD BE IN -- IN PLACE.
+Here are some things prosecutors would like jurors to know.
+JAMES TIERNEY  [01:28:57]:
+PROSECUTOR REALLY HAS TWO JOBS.  ONE JOB A JUROR WILL SEE IN THE COURTROOM AND THAT IS TO MAKE THE ARGUMENT THAT THE PERSON WHO IS ACCUSED OF CRIME IS -- IS GUILTY AND SHOULD BE APPROPRIATELY PUNISHED.  PROSECUTOR HAS ANOTHER IMPORTANT JOB, WHICH OCCURS OUTSIDE THE COURTROOM, AND THAT IS TO MAKE SURE THAT EVERYONE PLAYS BY THE RULES.  // [03:51:18] BECAUSE IT'S NOT A PROSECUTOR'S JOB JUST TO SECURE GUILT. A PROSECUTOR HAS TO STAND UP FOR JUSTICE
+PAUL RADVANY [4:40:15]
+THE ROLE OF THE PROSECUTOR, UM, IN OUR CRIMINAL JUSTICE SYSTEM BEGINS WITH THE INVESTIGATION OFTEN.  AND SO THEY WORK WITH LAW ENFORCEMENT, UM, TO TRY AND INVESTIGATE THE FACTS.  AND DETERMINE WHETHER OR NOT SOMEONE SHOULD BE CHARGED WITH A CRIME.
+So during the trial, just like the judge and the defense, the prosecution wants a juror�s undivided attention.
+JAMES TIERNEY [4:24:40]
+WHEN THE PROSECUTOR PRESENTS EVIDENCE, UH, HE OR SHE WANTS THE JURORS TO PAY ATTENTION AND TO LISTEN.
+RADVANY [4:25:53]
+AS YOU PRESENT EVIDENCE YOU WANNA MAKE SURE THAT THE JURORS ARE LISTENING VERY CAREFULLY.  THAT THEY KEEP AN OPEN MIND.  THAT THEY�RE JUDGING WITNESSES� CREDIBILITY.  AND THAT, UM, ULTIMATELY THEY�LL BE DE-- DETERMINE, UM, INFERENCES THAT CAN BE DRAWN THROUGH THE -- FROM THE EVIDENCE.
+Once jurors are ready to deliberate, prosecutors would hope that jurors have committed their full attention to the evidence presented during the trial.
+JAMES TIERNEY [04:04:44]
+SO THE FIRST THING A JUROR HAS TO DO IS LISTEN VERY, VERY CLOSELY TO THE JUDGE BEFORE THEY RETIRE//  BUT, UH, EITHER A PROSECUTOR OR A DEFENSE COUNSEL I'M SURE WOULD WANT THE JURORS TO CAREFULLY GO OVER THE EVIDENCE, TO DISCUSS THE ISSUE AMONG THEMSELVES, IF THEY HAVE QUESTIONS, TO SUBMIT QUESTIONS BACK TO THE JUDGE DURING THE COURSE OF THEIR DELIBERATIONS, AND THEN TO WORK HARD TO COME UP WITH THE RIGHT ANSWER.  IT'S -- IT'S IMPORTANT THAT THE JURORS FEEL COMFORTABLE WITH THEIR DECISION.
+And just like the judge and the defense attorney, the prosecutor, would also like to remind jurors that every defendant must be presumed innocent until proven guilty.
+JAMES TIERNEY [04:00:05]
+OUR CONSTITUTION IS BASED ON A BILL OF RIGHTS, AND IT'S BASED ON -- ON A PRESUMPTION, THAT A GOVERNMENT SHOULD NOT DEPRIVE SOMEONE OF THEIR LIBERTY UNLESS THEY CAN TRULY PROVE WHAT, UH, WHAT OCCURRED, AND //ONE OF THE ELEMENTS OF THAT IS TO PRESUME THAT THAT PERSON WHO SITS THERE IN THE COURTROOM ON THE FIRST DAY OF TRIAL IS JUST AS INNOCENT AS ANY JUROR OR THE JUDGE OR THE PROSECUTOR OR DEFENSE COUNSEL.  SO YOU PRESUME THE INNOCENCE, AND THEN IT'S UP TO THE STATE, THE GOVERNMENT T-- Y-- WORKING THROUGH THE PROSECUTOR TO PROVE THAT THAT IS NOT TRUE.
+RADVANY [4:11:15]
+THE ROLE OF THE PROSECUTOR IS VERY IMPORTANT IN SOCIETY OBVIOUSLY.  // AND YOUR GOAL IS TO SEEK JUSTICE -- NOT NECESSARILY CONVICTIONS.  BECAUSE YOU LEAVE IT UP TO THE JURY UNDER OUR SYSTEM TO DETERMINE WHETHER SOMEONE // IS GUILTY BEYOND A REASONABLE DOUBT.

data/test/nlp_spec.rb

@@ -0,0 +1,231 @@
+require "term-extractor/nlp"
+require "rubygems"
+require "rake"
+
+NLP = TermExtractor::NLP
+MyNLP = NLP.new("#{File.dirname(__FILE__)}/../models")
+
+def dont_split(text)
+  # This might not be quite right. We're currently allowing stripping punctuation
+  # from the beginning and end. Maybe we shouldn't?
+  NLP.tokenize_sentence(text).map{|x| x.to_s}.should == [text] 
+end
+
+
+def one_sentence(text)
+  MyNLP.sentences(text).should == [text]
+end
+
+def n_sentences(n, text)
+  s = MyNLP.sentences(text)
+  s.should have_exactly(n).sentences
+end
+
+describe "sentence splitting" do
+  it "should not split sentences around URLs" do 
+    one_sentence("If you go to http://www.google.com and type 'kitties' you will get lots of kitties")
+  end
+
+  it "should split sentences with abbreviations sensibly" do
+    one_sentence("Dr. Smith likes kitties")
+  end
+
+  it "should produce two sentences when there are line breaks" do
+
+    n_sentences 2, "Posting Date: November 8, 2008 \r\n Release Date: January, 1998\r\n \r\n Language: English"
+
+    n_sentences 2, <<KITTIES
+I like kitties
+
+I like puppies
+KITTIES
+
+
+  end
+end
+
+describe "url removal" do
+  it "should replace URLs in the middle of sentences" do
+    NLP.remove_urls("I like the links you find at http://www.google.com when searching for kitties").should == "I like the links you find at <URL> when searching for kitties"
+  end
+
+  it "should replace URLs at the beginning of sentences" do
+    NLP.remove_urls("http://www.google.com is your number one source for kitties").should == "<URL> is your number one source for kitties"
+  end
+
+  it "should replace URLs at the end of sentences" do 
+    NLP.remove_urls("When I want kitties I go to http://www.google.com").should == "When I want kitties I go to <URL>"
+  end
+
+  it "shuold replace URLs between sentences" do
+    NLP.remove_urls("The number one kitty finding service is http://www.google.com. Accept no substitutes").should == "The number one kitty finding service is <URL>. Accept no substitutes"
+  end
+end
+
+describe "path removal" do 
+  it "should remove windows style paths" do
+    path ="C:\\Home\\Windows\\Nonsense\\Kitties is where windows people store kitty related material"
+
+    NLP.remove_paths(path).should == "<PATH> is where windows people store kitty related material" 
+  end
+
+  it "should remove windows style paths with spaces in them" do 
+    path = "C:\\Documents and Settings\\Kitty is the kitty's home directory"
+
+    NLP.remove_paths(path).should == "<PATH> is the kitty's home directory"    
+  end
+
+  it "should remove unix style paths" do 
+    NLP.remove_paths("/home/david/kitties is where *I* store kitty related material").should == "<PATH> is where *I* store kitty related material"
+  end
+end
+
+describe "extracting embedded terms" do
+  it "should replace quotes with <QUOTE>" do 
+    quote = "\"I like kitties\", she declared"
+
+    main, embedded = NLP.extract_embedded_sentences(quote)
+  
+    main.should == "<QUOTE>, she declared"
+    embedded.should == "I like kitties" 
+  end
+
+
+  it "should replace parenthetical comments with an empty string" do 
+    main, embedded = NLP.extract_embedded_sentences("I like kitties (especially fuzzy ones)")
+
+    main.should == "I like kitties "
+    embedded.should == "especially fuzzy ones"
+  end
+
+  it "should correctly deal with multiple nested parenthetical comments" do
+    main, e1, e2 =  NLP.extract_embedded_sentences("I like kitties (especially fuzzy ones (but the long haired ones are kinda ugly))")
+
+    main.should == "I like kitties "
+    e1.should == "but the long haired ones are kinda ugly"
+    e2.should == "especially fuzzy ones "
+  end
+
+  it "should correctly deal with multiple non nested parenthetical comments" do 
+    main, e1, e2 =  NLP.extract_embedded_sentences("I like kitties (especially fuzzy ones)(but the long haired ones are kinda ugly)")
+
+    main.should == "I like kitties "
+    e1.should == "especially fuzzy ones"
+    e2.should == "but the long haired ones are kinda ugly"
+  end
+
+  it "should not extract a subterm when it is not matched" do 
+    NLP.extract_embedded_sentences("She declared \" I like kitties").should have(1).fragment
+  end  
+
+  it "should not extract a subterm when it would have to span multiple lines to do so " do 
+kitties = <<KITTIES    
+I like kitties (they are
+the best)  
+KITTIES
+
+    NLP.extract_embedded_sentences(kitties).should == [kitties]
+  end
+
+end
+
+describe "tokenization" do
+ 
+  it "should not split up URLs" do
+    dont_split("http://www.theonion.com/content/news/female_serial_killer_has_to_work")
+  end
+
+  it "should not split up URLs with -s in them" do 
+    dont_split("http://www.amazon.com/Fierce-Conversations-Acheiving-Success-Conversation/dp/0670031240")
+  end
+
+  it "should not split up emails" do
+    dont_split("david.maciver@trampolinesystems.com")
+  end
+
+  it "should split up contractions" do
+    NLP.tokenize_sentence("I'm the very model of a modern major general").should == ["I", "'m", "the", "very", "model", "of", "a", "modern", "major", "general"]
+  end
+
+  it "should split sentences around ellipses" do
+    NLP.tokenize_sentence("I like kitties...puppies are ok too").should == ["I", "like", "kitties", ",", "puppies", "are", "ok", "too"]
+  end
+
+  it "shouldn't split paths containing .." do
+    dont_split("/home/david/cute/puppies/../kitties/pictures")
+  end
+
+  it "should pull a sentence terminator into its own token" do 
+    NLP.tokenize_sentence("You don't like kitties?!?")[-1].should == "?!?"
+  end
+
+  it "should detach punctuation as a separate token" do
+    NLP.tokenize_sentence("babies... the other white meat")[1].should == "..."
+  end
+
+  def dont_produce_token(text, term)  
+    tokens = NLP.tokenize_sentence(text)  
+    tokens.should_not include(term)
+  end
+
+  it "should not split numbers around commas" do
+    dont_produce_token("the reasons for selecting opengl rather than prefuse were to visualise >10,000 nodes and do 3d", "000")
+  end 
+
+  it "should pull commas off the ends of tokens" do
+    dont_produce_token("kitties, puppies and birdies are all cute", "kitties,")
+  end
+
+end
+
+describe "cleaning" do
+  it "should remove stars trailing or leading a word" do
+    NLP.clean_sentence("Should that really take 5 minutes *over a network*").should == "Should that really take 5 minutes , over a network"
+  end
+
+
+  it "should turn quotes into commas" do
+    NLP.clean_sentence("I read \"Why kitties are cute\" over the summer").should == "I read , Why kitties are cute , over the summer"
+  end
+
+  it "should remove all new lines" do 
+    (NLP.clean_sentence("
+      This sentence
+      has lots of 
+      line breaks
+      in it
+    ") =~ /\n|\./).should == nil
+  end
+end
+
+def equate(foo, bar)
+  MyNLP.canonicalize(foo).should == MyNLP.canonicalize(bar)
+end
+
+describe "canonicalization" do
+  it "should identify plurals" do
+    equate("kitties", "kitty")
+  end
+
+  it "should identify strings that differ only in non alphanumeric characters" do
+    equate("foo/bar/baz", "foo bar baz")
+  end
+
+  it "should be insensitive to order" do
+    equate("foo bar baz", "bar foo baz")
+  end
+
+  it "should ignore stopwords" do 
+    equate("programming in java", "java programming")
+  end
+end
+
+describe "stopword detection" do
+  it "should mark a as a stopword" do
+    MyNLP.stopword?("a").should be(true)
+  end
+
+  it "should not be fooled by capitalisation" do
+    MyNLP.stopword?("A").should be(true)
+  end
+end