term-extract 0.4.0 → 0.5.0

data/README.markdown

@@ -37,6 +37,12 @@ By default, the term extractor attempts to extract both ordinary nouns and prope
 
     terms = TermExtract.extract(content, :types => :nnp)
 
+## Command Line Tool
+
+There is a command line tool that can be used for testing the term extractor.  It is best used in conjunction with another tool to extract the relevent content (e.g. pismo) :
+
+    pismo http://www.bbc.co.uk/news/uk-politics-12085506 body | ruby -rubygems -e 'puts YAML.parse($stdin.read)[:body].value' | ./term-extract nnp | ruby -rubygems -e 'puts YAML.load($stdin.read)'
+
 ## Note on Patches/Pull Requests
 
 * Fork the project.

data/VERSION

@@ -1 +1 @@
-0.4.0
+0.5.0

data/bin/term-extract

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require 'yaml'
+require 'rubygems'
+$:.unshift(File.dirname(__FILE__) + "/../lib")
+require 'term-extract'
+
+types = :nnp
+types = ARGV.shift.to_sym if ARGV.length > 0
+
+pp TermExtract.extract($stdin.readlines.join(" "), :types => types).to_yaml

data/lib/term-extract.rb

@@ -74,7 +74,7 @@ class TermExtract
         # Allow preposition : "Secretary of State"
         # Only use when in NNP mode
         multiterm << [term,tag]
-      elsif state == @@NOUN and tag =~ /#{pos}/
+      elsif state == @@NOUN and tag =~ /^NN/
         # In noun mode, found a noun, add a multiterm noun
         add_term(term, tag, multiterm, terms)
       elsif state == @@NOUN and tag !=~ /#{pos}/
@@ -96,11 +96,14 @@ class TermExtract
     terms.each_key do |term|
       occur = terms[term][:occurances]
       strength = term.split(/ /).length
+      terms.delete(term) if occur < 1
       terms.delete(term) unless ((strength == 1 and occur >= @min_occurance) or (strength >= @min_terms))
     end
 
     # Remove shorter terms that form part of larger terms
     # This typically removes surname references when we already have a full name
+    # This doesn't test that the larger term has more occurrences than the smaller
+    # term as testing has shown issues with this approach
     if @collapse_terms
       terms.each_key do |term1|
         terms.each_key do |term2|
@@ -151,12 +154,16 @@ class TermExtract
     multiterm.each_with_index do |term, index|
       if (multiterm[index] == multiterm.last && term[1] == 'POS')
         # Don't add a final 's if it's the last term
+      elsif (multiterm[index] == multiterm.last && term[1] == 'IN' ||
+             multiterm[index] == multiterm.last && term[1] == 'JJ')
+        # Don't add a final preposition if it's the last term
       else
         # Don't require a space for POS type concats
         word+= term[1] == 'POS' ? term[0] : " #{term[0]}"
       end
     end
     word.lstrip!
+    # Add the term
     increment_term(word, 'NNP', terms)
   end
 

data/term-extract.gemspec

@@ -5,12 +5,14 @@
 
 Gem::Specification.new do |s|
   s.name = %q{term-extract}
-  s.version = "0.4.0"
+  s.version = "0.5.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["rattle"]
-  s.date = %q{2010-12-24}
+  s.date = %q{2010-12-29}
+  s.default_executable = %q{term-extract}
   s.email = %q{robl@rjlee.net}
+  s.executables = ["term-extract"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
     "README.markdown"
@@ -23,6 +25,7 @@ Gem::Specification.new do |s|
     "README.markdown",
     "Rakefile",
     "VERSION",
+    "bin/term-extract",
     "lib/term-extract.rb",
     "term-extract.gemspec",
     "test/helper.rb",

data/test/test_term-extract.rb

@@ -1,5 +1,4 @@
 require 'helper'
-require 'pp'
 
 class TestTermExtract < Test::Unit::TestCase
 
@@ -14,7 +13,10 @@ DOC1
 
   @@DOC2 = <<DOC2
 Secretary of State Owen Paterson has appointed Peter Osborne as Chair of the
-Parades Commission for Northern Ireland and six new Commission members.
+Parades Commission for Northern Ireland and six new Commission members.  Owen Paterson
+said Osborne was an excellent choice for the position.  Osborne was reported as
+saying he was delighted at the appointment and thanked Owen Paterson for the honour.
+Osborne then went on to outline his plans for the position.
 DOC2
 
   @@DOCUMENT = <<SOURCE
@@ -174,5 +176,4 @@ SOURCE
 
   end
 
-end
-
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: term-extract
 version: !ruby/object:Gem::Version 
-  hash: 15
+  hash: 11
   prerelease: false
   segments: 
   - 0
-  - 4
+  - 5
   - 0
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors: 
 - rattle
@@ -15,8 +15,8 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-24 00:00:00 +00:00
-default_executable: 
+date: 2010-12-29 00:00:00 +00:00
+default_executable: term-extract
 dependencies: 
 - !ruby/object:Gem::Dependency 
   prerelease: false
@@ -94,8 +94,8 @@ dependencies:
   type: :development
 description: 
 email: robl@rjlee.net
-executables: []
-
+executables: 
+- term-extract
 extensions: []
 
 extra_rdoc_files: 
@@ -109,6 +109,7 @@ files:
 - README.markdown
 - Rakefile
 - VERSION
+- bin/term-extract
 - lib/term-extract.rb
 - term-extract.gemspec
 - test/helper.rb