term-extract 0.3.0 → 0.4.0

data/README.markdown

@@ -25,6 +25,7 @@ The #extract method takes an (optional) options hash, that allows the term extra
 * min_terms - Always include multiword terms that comprise more than @min_terms words, default 2
 * types - Extract proper nouns (:nnp) or nouns (:nn) or both (:all), default :all
 * include_tags - Include the extracted POS tags in the results, default false
+* collapse_terms - Remove shorter terms that are part of larger ones, default true
 
 Sample usage:
 

data/VERSION

@@ -1 +1 @@
-0.3.0
+0.4.0

data/lib/term-extract.rb

@@ -27,6 +27,8 @@ class TermExtract
     @types = options.key?(:types) ? options.delete(:types) : :all
     # Include the extracted POS tags in the results
     @include_tags = options.key?(:include_tags) ? options.delete(:include_tags) : false
+    # Remove shorter terms that are part of larger ones
+    @collapse_terms = options.key?(:collapse_terms) ? options.delete(:collapse_terms) : true
     #@lazy = options.key?(:lazy) ? options.delete(:lazy) : false
   end
 
@@ -97,6 +99,16 @@ class TermExtract
       terms.delete(term) unless ((strength == 1 and occur >= @min_occurance) or (strength >= @min_terms))
     end
 
+    # Remove shorter terms that form part of larger terms
+    # This typically removes surname references when we already have a full name
+    if @collapse_terms
+      terms.each_key do |term1|
+        terms.each_key do |term2|
+          terms.delete(term2) if term1.length > term2.length && (term1 =~ /[^A-Za-z0-9]#{term2}$/ || term1 =~ /^#{term2}[^A-Za-z0-9]/)
+        end
+      end
+    end
+
     # Filter out tags unless required
     unless @include_tags
       terms.each_key { |term| terms[term] = terms[term][:occurances] }

data/term-extract.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{term-extract}
-  s.version = "0.3.0"
+  s.version = "0.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["rattle"]

data/test/test_term-extract.rb

@@ -1,4 +1,5 @@
 require 'helper'
+require 'pp'
 
 class TestTermExtract < Test::Unit::TestCase
 
@@ -72,7 +73,7 @@ SOURCE
     'Palestinians hope',
     'Richard Makepeace',
     'court order',
-    'event',
+    #'event',
     'literature festival',
     'peace accords',
     'police notice',
@@ -108,17 +109,19 @@ SOURCE
       assert terms.keys.include?("St Paul's Cathedral")
     end
 
-    should "extract terms with joining words" do
+    should "extract terms with prepositions" do
       terms = @te.extract(@@DOC2)
       assert terms.keys.include?("Secretary of State Owen Paterson")
     end
 
-    should "extract terms and include pos tags when configured to" do
-      @te.include_tags = true
-      terms = @te.extract(@@DOCUMENT)
-      term = terms.keys.first
-      assert terms[term].key?(:tag)
-      assert terms[term][:tag]
+    should "extract terms with long prepositions" do
+      terms = @te.extract(@@DOC2)
+      assert terms.keys.include?("Chair of the Parades Commission for Northern Ireland")
+    end
+
+    should "collapse duplicate terms" do
+      terms = @te.extract(@@DOC2)
+      assert !terms.keys.include?("event")
     end
 
     should "extract common nouns when configured to" do
@@ -162,8 +165,9 @@ SOURCE
 
       should "include pos tags in the results" do
         terms = @te.extract(@@DOCUMENT)
-        assert terms.keys.include?("Jerusalem")
-        assert terms['Jerusalem'][:tag] == 'NNP'
+        term = terms.keys.first
+        assert terms[term].key?(:tag)
+        assert terms[term][:tag]
       end
 
     end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: term-extract
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 15
   prerelease: false
   segments: 
   - 0
-  - 3
+  - 4
   - 0
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors: 
 - rattle