phrasie 0.1.3

data/lib/phrasie/extractor.rb

@@ -0,0 +1,68 @@
+SEARCH = 0
+NOUN = 1
+
+module Phrasie
+  class Extractor
+    attr_accessor :tagger, :filter
+
+    def initialize(options={})
+      self.tagger = Tagger.new
+      self.filter = options[:filter] || {:strength => 2, :occur => 3}
+    end
+    
+    def to_s
+      "#<Phrasie::Extractor>"
+    end
+
+    def phrases(input, min_occur=3)
+      if input.is_a? String
+        taggedTerms = self.tagger.tag(input)
+      elsif input.is_a? Array
+        taggedTerms = input
+      else
+        return []
+      end
+
+      terms = {}
+      multiterm = []
+      state = SEARCH
+
+      while taggedTerms.size > 0
+        term, tag, norm = taggedTerms.shift
+        if state == SEARCH && tag[0] == "N"
+          state = NOUN
+          add(term, norm, multiterm, terms)
+        elsif state == SEARCH && tag == 'JJ' && term[0].upcase == term[0]
+          state = NOUN
+          add(term, norm, multiterm, terms)
+        elsif state == NOUN && tag[0] == "N"
+          add(term, norm, multiterm, terms)
+        elsif state == NOUN && tag[0] != "N"
+          state = SEARCH
+          if multiterm.size > 1
+            word = multiterm.map(&:first).join(' ')
+            terms[word] ||= 0
+            terms[word] += 1
+          end
+          multiterm = []
+        end
+      end
+
+      return terms \
+              .map{|phrase, occurance| [phrase, occurance, phrase.split.size] } \
+              .keep_if{|arr| self.validate(*arr)} \
+              .sort_by{|phrase, occurance, strength|  occurance + ((occurance/5.0)*strength) }.reverse
+    end
+
+    protected
+    def validate(word, occur, strength)
+      occur >= self.filter[:occur] || (occur >= 2 && strength >= self.filter[:strength])
+    end
+
+    def add(term, norm, multiterm, terms)
+      multiterm << [term, norm]
+      terms[norm] ||= 0
+      terms[norm] += 1
+    end
+  end
+end

data/lib/phrasie/rules.rb

@@ -0,0 +1,82 @@
+module Phrasie
+  module Rules
+    # Determine whether a default noun is plural or singular.
+    def correctDefaultNounTag(id, tagged_term, tagged_terms)
+      term, tag, norm = tagged_term
+      if tag == 'NND'
+        if term[-1] == 's'
+          tagged_term[1] = 'NNS'
+          tagged_term[2] = term[0..-2]
+        else
+          tagged_term[1] = 'NN'
+        end
+      end
+      return [id, tagged_term, tagged_terms]
+    end
+  
+    # Verify that noun at sentence start is truly proper.
+    def verifyProperNounAtSentenceStart(id, tagged_term, tagged_terms)
+      term, tag, norm = tagged_term
+      if ['NNP', 'NNPS'].include?(tag) && (id == 0 || tagged_terms[id-1][1] == '.')
+        lower_term = term.downcase
+        lower_tag = self.tags_by_term[lower_term]
+        if ['NN', 'NNS'].include?(lower_tag)
+          tagged_term[0] = tagged_term[2] = lower_term
+          tagged_term[1] = lower_tag
+        end
+      end
+      return [id, tagged_term, tagged_terms]
+    end
+  
+    # Determine the verb after a modal verb to avoid accidental noun detection.
+    def determineVerbAfterModal(id, tagged_term, tagged_terms)
+      term, tag, norm = tagged_term
+      return [id, tagged_term, tagged_terms] if tag != 'MD'
+      len_terms = tagged_terms.size
+      i = id
+      i += 1
+      while i < len_terms      
+        if tagged_terms[i][1] == 'RB'
+          i += 1
+          next
+        end
+      
+        if tagged_terms[i][1] == 'NN'
+          tagged_terms[i][1] = 'VB'
+        end
+      
+        break
+      end
+        
+      return [id, tagged_term, tagged_terms]
+    end
+  
+   
+    def normalizePluralForms(id, tagged_term, tagged_terms)
+      term, tag, norm = tagged_term
+      if ['NNS', 'NNPS'].include?(tag) && term == norm
+        # Plural form ends in "s"
+        singular = term[0..-2]
+        if term[-1] && !self.tags_by_term[singular].nil?
+          tagged_term[2] = singular
+          return [id, tagged_term, tagged_terms]
+        end
+      
+        # Plural form ends in "es"
+        singular = term[0..-3]
+        if term[-2..-1] && !self.tags_by_term[singular].nil?
+          tagged_term[2] = singular
+          return [id, tagged_term, tagged_terms]
+        end
+      
+        # Plural form ends in "ies" (from "y")
+        singular = term[0..-4]+'y'
+        if term[-3..-1] && !self.tags_by_term[singular].nil?
+          tagged_term[2] = singular
+          return [id, tagged_term, tagged_terms]
+        end
+      end
+      return [id, tagged_term, tagged_terms]
+    end       
+  end
+end

data/lib/phrasie/tag.rb

@@ -0,0 +1,62 @@
+module Phrasie
+  class Tagger
+    include Phrasie::Rules
+    TERM_SPEC = /([^a-zA-Z]*)([a-zA-Z\-\.]*[a-zA-Z])([^a-zA-Z]*[a-zA-Z]*)/
+    attr_accessor :language, :tags_by_term, :lexicon
+  
+    def initialize(options={})
+      self.language = options[:language] || 'english'
+      self.lexicon = options[:lexicon] || File.expand_path("#{__FILE__}/../data/#{self.language}-lexicon.txt")
+      file = File.read(self.lexicon)
+      self.tags_by_term = Hash[file.split("\n").map{|line| line.split.first(2)}]
+    end
+  
+    def tokenize(text)
+      terms = []
+      text.split(/\s/).each do |term|
+        next if term == ''
+        match = TERM_SPEC.match(term).to_a
+        match.shift
+        if match.size == 0
+          terms << term
+          next
+        end
+      
+        match.each do |sub_term|
+          terms << sub_term if sub_term != ''
+        end
+      end
+      return terms
+    end
+  
+    def tag(input)
+      if input.is_a? String
+        terms = self.tokenize(input)
+      elsif input.is_a? Array
+        terms = input
+      else
+        return []
+      end
+    
+      tagged_terms = []
+      terms.each do |term|
+        tag = self.tags_by_term[term] || "NND"
+        tagged_terms << [term, tag, term]
+      end
+        
+      rules = [
+        'correctDefaultNounTag',
+        'verifyProperNounAtSentenceStart',
+        'determineVerbAfterModal',
+        'normalizePluralForms'
+      ]
+        
+      tagged_terms.each_with_index do |tagged_term, id|
+        rules.each do |rule|
+          id, tagged_terms[id], tagged_terms = self.send(rule.to_sym, id, tagged_term, tagged_terms)
+        end
+      end
+      return tagged_terms
+    end
+  end
+end

data/phrasie.gemspec

@@ -0,0 +1,17 @@
+Gem::Specification.new do |s|
+  s.name        = "phrasie"
+  s.version     = '0.1.3'
+  s.authors     = ["Ashley Williams"]
+  s.email       = ["hi@ashleyw.co.uk"]
+  s.summary     = "Determines important terms within a given piece of content."
+  s.homepage         = "https://github.com/ashleyw/Phrasie"
+  s.description = "Determines important terms within a given piece of content. It
+  uses linguistic tools such as Parts-Of-Speech (POS) and some simple
+  statistical analysis to determine the terms and their strength."
+ 
+  s.required_rubygems_version = ">= 1.3.6"
+  s.rubyforge_project         = "phrasie"
+  
+  s.files        = File.read("Manifest.txt").split("\n")
+  s.require_path = 'lib'
+end

data/script/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# File: script/console
+irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+
+libs =  " -r irb/completion"
+# Perhaps use a console_lib to store any extra methods I may want available in the cosole
+# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}"
+libs <<  " -r #{File.dirname(__FILE__) + '/../lib/phrasie.rb'}"
+puts "Loading phrasie gem"
+exec "#{irb} #{libs} --simple-prompt"

data/script/destroy

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Destroy.new.run(ARGV)

data/script/generate

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Generate.new.run(ARGV)

data/test/test_helper.rb

@@ -0,0 +1,3 @@
+require 'stringio'
+require 'test/unit'
+require File.dirname(__FILE__) + '/../lib/phrasie'

data/test/test_phrasie.rb

@@ -0,0 +1,104 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class TestPhrasie < Test::Unit::TestCase
+  def setup
+    @text = 'The British consul of Boston resides in Newton. The British consul is awesome.'
+    @long_text = %(Police shut Palestinian theatre in Jerusalem.
+    
+    Israeli police have shut down a Palestinian theatre in East Jerusalem.
+    
+    The action, on Thursday, prevented the closing event of an international
+    literature festival from taking place.
+    
+    Police said they were acting on a court order, issued after intelligence
+    indicated that the Palestinian Authority was involved in the event.
+    
+    Israel has occupied East Jerusalem since 1967 and has annexed the
+    area. This is not recognised by the international community.
+    
+    The British consul-general in Jerusalem , Richard Makepeace, was
+    attending the event.
+    
+    "I think all lovers of literature would regard this as a very
+    regrettable moment and regrettable decision," he added.
+    
+    Mr Makepeace said the festival's closing event would be reorganised to
+    take place at the British Council in Jerusalem.
+    
+    The Israeli authorities often take action against events in East
+    Jerusalem they see as connected to the Palestinian Authority.
+    
+    Saturday's opening event at the same theatre was also shut down.
+    
+    A police notice said the closure was on the orders of Israel's internal
+    security minister on the grounds of a breach of interim peace accords
+    from the 1990s.
+    
+    These laid the framework for talks on establishing a Palestinian state
+    alongside Israel, but left the status of Jerusalem to be determined by
+    further negotiation.
+    
+    Israel has annexed East Jerusalem and declares it part of its eternal
+    capital.
+    
+    Palestinians hope to establish their capital in the area.)
+    @extractor = Phrasie::Extractor.new
+  end
+  
+  def test_extractor
+    expected = [["British consul", 2, 2]]
+    assert_equal expected, @extractor.phrases(@text).sort_by{|a| a[1]}
+  end
+  
+  def test_non_words
+    text = %(LONDON - WikiLeaks founder Julian Assange was refused bail and jailed for a week by a British court Tuesday, pending an extradition hearing over alleged sex offenses in Sweden.
+    Assange turned himself in to U.K. police earlier in the day in the latest blow to his WikiLeaks organization, which faces legal, financial and technological challenges after releasing hundreds of secret U.S. diplomatic cables.
+    Swedish prosecutors had issued an arrest warrant for the 39-year-old Australian, who is accused of sexual misconduct with two women.
+    Assange surrendered at 9:30 a.m. local time (4:30 a.m. ET) Tuesday. The U.K.'s Guardian newspaper reported that Assange later arrived at a London court accompanied by British lawyers Mark Stephens and Jennifer Robinson.
+    During his court appearance, Assange said he would fight extradition to Sweden and he provided the court with an Australian address. Britain's Sky News reported that Assange was receiving consular assistance from officials at the Australian High Commission.
+    The next court hearing is scheduled for next Tuesday, and Assange will remain in custody until then because he was deemed to be a flight risk.
+    Judge Howard Riddle asked Assange whether he understood that he could agree to be extradited to Sweden. Assange, dressed in a navy blue suit, cleared his throat and said: "I understand that and I do not consent."
+    The judge said he had grounds to believe that the former computer hacker - a self-described homeless refugee - might not show up to his next hearing if he were granted bail.
+    Arguments during the hour-long hearing detailed the sex accusations against Assange, all of which he has denied.
+    Australian journalist John Pilger, British film director Ken Loach and Jemima Khan, former wife of Pakistani cricketer and politician Imran Khan, all offered to put up sureties to persuade the court Assange would not flee.)
+    assert_equal 7, @extractor.phrases(text).size
+  end
+  
+  #   [["Jerusalem", 8, 1],
+  #    ["event", 6, 1],
+  #    ["Palestinian", 6, 1],
+  #    ["East Jerusalem", 4, 2],
+  #    ["East", 4, 1],
+  #    ["police", 4, 1],
+  #    ["Israel", 4, 1],
+  #    ["theatre", 3, 1],
+  #    ["Palestinian theatre", 2, 2],
+  #    ["Palestinian Authority", 2, 2],
+  #    ["opening event", 1, 2],
+  #    ["Israeli authorities", 1, 2],
+  #    ["Richard Makepeace", 1, 2],
+  #    ["court order", 1, 2],
+  #    ["literature festival", 1, 2],
+  #    ["British consul-general", 1, 2],
+  #    ["police notice", 1, 2],
+  #    ["security minister", 1, 2],
+  #    ["Israeli police", 1, 2],
+  #    ["peace accords", 1, 2],
+  #    ["Mr Makepeace", 1, 2],
+  #    ["British Council", 1, 2],
+  #    ["Palestinian state", 1, 2],
+  #    ["Palestinians hope", 1, 2]]
+  
+  def test_long_text
+    assert_equal 10, @extractor.phrases(@long_text).size
+  end
+  
+  def test_filter_options
+    @extractor.filter = {:occur => 4, :strength => 3}
+    assert_equal 7, @extractor.phrases(@long_text).size
+  end
+  
+  def test_extractor_to_s
+    assert @extractor.to_s == "#<Phrasie::Extractor>"
+  end
+end

metadata

@@ -0,0 +1,83 @@
+--- !ruby/object:Gem::Specification 
+name: phrasie
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 3
+  version: 0.1.3
+platform: ruby
+authors: 
+- Ashley Williams
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-01-17 00:00:00 +00:00
+default_executable: 
+dependencies: []
+
+description: |-
+  Determines important terms within a given piece of content. It
+    uses linguistic tools such as Parts-Of-Speech (POS) and some simple
+    statistical analysis to determine the terms and their strength.
+email: 
+- hi@ashleyw.co.uk
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- Manifest.txt
+- README.md
+- Rakefile
+- lib/phrasie.rb
+- lib/phrasie/extractor.rb
+- lib/phrasie/rules.rb
+- lib/phrasie/tag.rb
+- lib/phrasie/data/english-lexicon.txt
+- script/console
+- script/destroy
+- script/generate
+- phrasie.gemspec
+- test/test_helper.rb
+- test/test_phrasie.rb
+has_rdoc: true
+homepage: https://github.com/ashleyw/Phrasie
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 1
+      - 3
+      - 6
+      version: 1.3.6
+requirements: []
+
+rubyforge_project: phrasie
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Determines important terms within a given piece of content.
+test_files: []
+