entifier 0.0.0

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/Gemfile

@@ -0,0 +1,13 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "shoulda", ">= 0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.5.2"
+  gem "rcov", ">= 0"
+end

data/Gemfile.lock

@@ -0,0 +1,20 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    git (1.2.5)
+    jeweler (1.5.2)
+      bundler (~> 1.0.0)
+      git (>= 1.2.5)
+      rake
+    rake (0.8.7)
+    rcov (0.9.9)
+    shoulda (2.11.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.0)
+  jeweler (~> 1.5.2)
+  rcov
+  shoulda

data/LICENSE.txt

@@ -0,0 +1,5 @@
+This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with this program. If not, see <www.gnu.org/licenses/>

data/README.rdoc

@@ -0,0 +1,33 @@
+= entifier
+
+Extract named entities from text.
+
+== Installation
+
+gem install entifier
+
+== Usage
+
+ require 'rubygems'
+ require 'entifier'
+ require 'pp'
+
+ text = "Gordon Brown went London to see Tony Blair."
+ pp Entifier.extract(text)
+
+
+== Contributing to entifier
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2011 Frankie Roberto. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,53 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "entifier"
+  gem.homepage = "http://github.com/rattle/entifier"
+  gem.license = "GPLv3"
+  gem.summary = %Q{Entifier gem to extract named entities}
+  #gem.description = %Q{TODO: longer description of your gem}
+  gem.email = "frankie[at]rattlecentral.com"
+  gem.authors = ["robl"]
+  # Include your dependencies below. Runtime dependencies are required when using your gem,
+  # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
+  #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
+  #  gem.add_development_dependency 'rspec', '> 1.2.3'
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+require 'rcov/rcovtask'
+Rcov::RcovTask.new do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "entifier #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.0

data/entifier.gemspec

@@ -0,0 +1,69 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{entifier}
+  s.version = "0.0.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["robl"]
+  s.date = %q{2011-01-06}
+  s.email = %q{frankie[at]rattlecentral.com}
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "entifier.gemspec",
+    "lib/entifier.rb",
+    "lib/entifier/extensions.rb",
+    "lib/entifier/init.rb",
+    "test/errors/errors.rb",
+    "test/helper.rb",
+    "test/known_failures/known_failures_test.rb",
+    "test/test_entifier.rb"
+  ]
+  s.homepage = %q{http://github.com/rattle/entifier}
+  s.licenses = ["GPLv3"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Entifier gem to extract named entities}
+  s.test_files = [
+    "test/errors/errors.rb",
+    "test/helper.rb",
+    "test/known_failures/known_failures_test.rb",
+    "test/test_entifier.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+    else
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+  end
+end
+

data/lib/entifier/extensions.rb

@@ -0,0 +1,21 @@
+class Hash
+  # File merb/core_ext/hash.rb, line 166
+  def nested_symbolize_keys!
+    each do |k,v|
+      sym = k.respond_to?(:to_sym) ? k.to_sym : k
+      self[sym] = Hash === v ? v.nested_symbolize_keys! : v
+      delete(k) unless k == sym
+    end
+    self
+  end
+
+  def nested_stringify_keys!
+    each do |k,v|
+      s = k.respond_to?(:to_s) ? k.to_s : k
+      self[s] = Hash === v ? v.nested_stringify_keys! : v
+      delete(k) unless k == s
+    end
+    self
+  end
+
+end

data/lib/entifier/init.rb

@@ -0,0 +1 @@
+# Added for gem compatibility

data/lib/entifier.rb

@@ -0,0 +1,110 @@
+%w(extensions).each do |file|
+  require File.join(File.dirname(__FILE__), 'entifier', file)
+end
+
+class Entifier
+
+  DAY_NAMES = ["Monday","Tuesday","Wednesday","Thursday","Friday","Saturday","Sunday"]
+  MONTH_NAMES = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"]
+  INDEXICALS_PRECEDING_APOSTROPHE_S = ["Here", "There", "He", "She", "It", "Now", "Who"]
+
+  def self.extract(string, options = {})
+    options.nested_stringify_keys!
+    
+    entities = []
+    
+    # HACK! This allows comma separated entities to be picked up.
+    # string.gsub!(/\,\s/, ",  ")  
+
+    # HACK! This allows entities to be at the end and start of conjoining sentences.
+    # string.gsub!(/\./, ".")   No longer needed
+
+    # HACK! Remove extra spaces between sentences.
+    #string.gsub!(/([\.\?\!])\s\s+/, "\1")    
+
+    # Pre-processor: remove extra spaces (this shouldn't affect which entities are detected, 
+    # it just makes the output look better)
+    string.gsub!(/[[:blank:]]+/, "\s")
+
+    # HACK! This allows parenthesized entity following other entity to be picked up.
+    string.gsub!(/\s\(/, "  (")
+     
+    
+     capitalised_word = /[ÄÅÖA-Z](?:[a-zA-ZÄÅÖÜàâæçéèêëîïôøöûùüÿñ\-\d\&]+|\.(?:[A-Z]\.)*)/
+     capitalised_word_phrase = %r{
+      (?:\d{4}\s|Dr\.\s)?
+      #{capitalised_word}
+      (?:
+        (?:
+          (?:\s+(?:of|for|on|of\sthe|\&|d\'|du|de)|\'s)
+        )?
+        \s+#{capitalised_word})*
+      (?:\s\d+)?
+    }x
+     
+    
+    regex = %r{
+      (?:
+        (?:
+        (?:\A|[\.\?\!\:][\"\']?\s+|\n)             # At start of string, or starting new sentence...
+          (?:[\"\'\(])?                       # ...optionally started with quote marks.
+          )
+          (
+            (?:In\s(?:\d{4}\s)?)? 
+            #{capitalised_word_phrase}(?:\'s)? 
+          )
+        |                                   # --- OR ---
+
+        [^\.\n\?\!\:\"][[:blank:]][\"\'\(]?           # After any non-full-stop followed by a space...                     
+
+          (#{capitalised_word_phrase})
+      )
+    }x
+
+    
+    #[\,\'\s\.\Z]
+    
+    string.scan(regex) do |match|
+      #entity = match
+      if match[0]
+        word_count = match[0].split(" ").size
+        if word_count > 1
+          entity = match[0].gsub(/\A(In(?:\s\d{4})?|The|If|But|Two|(?:One|Two)\sof)\s/, "").gsub(/\'s\Z/, "")
+        elsif match[0][-2,2] == "'s"
+          entity = match[0].gsub(/\'s\Z/, "")  
+        elsif match[0] =~ /\A[A-Z]+\Z/        
+          entity = match[0]
+        else
+          entity = nil
+        end
+      else
+        entity = match[1]
+      end
+      
+      # HACK: These should really be filtered out by the regex.
+      if entity
+       # entity = entity.strip.gsub(/\'s\Z/, "").gsub(/\AIn\s/, "").gsub(/\AIf\s/, "")
+      end
+      entity = nil if DAY_NAMES.include?(entity)
+      entity = nil if MONTH_NAMES.include?(entity)
+      entity = nil if INDEXICALS_PRECEDING_APOSTROPHE_S.include?(entity)
+   
+      
+      if entity
+        entity.gsub!( /((I|i)n\s)(January|Feburary|March|April|May|June|July|August|September|October|November|December)/, "")
+        entity.gsub!( /(January|Feburary|March|April|May|June|July|August|September|October|November|December)\s(\d{4}|\d{2})/, "")
+        entity.gsub!(/(O|\so)n\s(Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday)/, "")
+
+        entity.gsub!(/\A\d+\Z/, "")   # Make string blank if it's just numbers.
+        entity = nil if entity == ""  # If there's nothing left, make it nil.
+
+      end
+      
+      
+      entities << entity unless entity.nil?  # Don't collect the entity if it's nil
+    end
+    entities.uniq!
+    return entities
+  end
+
+end

data/test/errors/errors.rb

@@ -0,0 +1,59 @@
+require 'test_helper'
+require 'pp'
+class PAErrorsTest < Test::Unit::TestCase
+
+
+
+  # PA File : eAP-D979UPS01.eAP-NA-US-GM-Labor-Costs.nitf.xml
+  should "extract 'Treasury Department'" do
+    assert Entifier.extract("to the Treasury Department Tuesday").include?('Treasury Department')
+  end
+
+  # PA File : eAP-D979UI0O1.eAP-EU-Obama-Queen-apos-s-Song-List.nitf.xml
+  should "treat quotes as one string ? i.e. we shouldn't extract Dolly here" do
+    assert !Entifier.extract('Hello, Dolly!" Carol Channing').include?('Dolly')
+  end
+
+  should "not treat quote mark as ending a sentence" do
+    assert_equal "Frankie", Entifier.extract("\"This should work\" Frankie said.").first
+  end
+
+# ----
+
+  should "not extract Feb" do
+    assert !Entifier.extract('Nazir had previously allied with the Pakistani government to fight Uzbeks partnered with the international terrorist group.  The Feb. 22 communique, a copy of which was obtained by The Associated Press, announced the ').include?('Feb')
+  end
+
+  should "not extract Die" do
+    assert !Entifier.extract('FunnyOrDie.com, the comedy video Web site co-founded by Will Ferrell, announced that it had been bought by country star Reba McEntire. The site was temporarily renamed "Reba or Die" and its home page was populated entirely with videos featuring McEntire').include?('Die')
+  end
+  
+
+text = <<PA
+President Barack Obama's gift of an iPod to Queen Elizabeth II came loaded with 40 songs from popular Broadway productions, including "The King and I," "West Side Story" and "Dreamgirls." The iPod was given to accompany a rare coffee table book of songs by composers Richard Rodgers and Lorenz Hart, which Obama also gave the queen.  Songs on the iPod are:  "Oklahoma!"  "If I Loved You," Jan Clayton, "Carousel"  "You'll Never Walk Alone," Jan Clayton, "Carousel"  "There's No Business Like Show Business," Ethel Merman, "Annie Get Your Gun"  "Once in Love with Amy (Where's Charley?)," Ray Bolger  "Some Enchanted Evening," "South Pacific"  "Diamonds Are a Girl's Best Friend," Carol Channing, "Gentlemen Prefer Blondes"  "Getting to Know You," Gertrude Lawrence, "The King and I"  "Shall We Dance?" Gertrude Lawrence, "The King and I"  "I Could Have Danced All Night," Julie Andrews, "My Fair Lady"  "I've Grown Accustomed to Her Face," Rex Harrison, "My Fair Lady"  "The Party's Over (Bells Are Ringing)," Judy Holliday  "Maria," "West Side Story"  "Tonight," "West Side Story"  "Seventy Six Trombones," "The Music Man"  "Everything's Coming up Roses," Ethel Merman, "Gypsy"  "The Sound of Music"  "Try to Remember," Jerry Orbach, "The Fantasticks"  "Camelot," Richard Burton  "If Ever I Would Leave You," Robert Goulet, "Camelot"  "Hello, Dolly!" Carol Channing  "If I Were a Rich Man," Zero Mostel, "Fiddler on the Roof"  "People," Barbra Streisand, "Funny Girl"  "On a Clear Day (You Can See Forever)," John Cullum  "The Impossible Dream," Richard Kiley, "Man of La Mancha"  "Mame," Charles Braswell  "Cabaret," Liza Minnelli  "Aquarius, Ronald Dyson, "Hair'  "Send in the Clowns," Judy Collins, "A Little Night Music"  "All That Jazz," Chita Rivera, "Chicago"  "One," "A Chorus Line"  "Tomorrow," Andrea McArdle, "Annie"  "Don't Cry for Me Argentina," Patti LuPone, "Evita"  "And I Am Telling You I'm Not Going," Jennifer Holliday, "Dreamgirls"  "Memory," Elaine Paige, "Cats"  "The Best of Times," George Hearn, "La Cage Aux Folles"  "I Dreamed a Dream," Aretha Franklin, "Les Miserables"  "The Music of the Night," Michael Crawford, "The Phantom of the Opera"  "As If We Never Said Goodbye," Elaine Paige, "Sunset Blvd."  "Seasons of Love," "Rent"
+PA
+
+  should "extract 'Ethel Merman'" do
+    assert Entifier.extract(text).include?('Ethel Merman')
+  end
+
+
+  should "extract 'Christine Lord'" do
+    assert Entifier.extract('This would be an added extra, outside their legal framework." But he added it could also be a question of money. "It may well be that it could be done but for it to be done the coroners would have to be given more resources than they are presently being given" Christine Lord, whose son died nearly two years ago of vCJD, urged coroners to carry out the tests as it had the potential to "save lives".').include?('Christine Lord')
+  end
+
+
+  #The video, which is on YouTube and the website of the multimedia magazine Don't Panic, which Prowse edits, has become a hit. It is entitled "Pound Force — Alan Duncan MP gets a new garden feature".
+  should "extract 'Don't Panic'" do
+    assert Entifier.extract('The video, which is on YouTube and the website of the multimedia magazine Don\'t Panic, which Prowse edits, has become a hit.').include?("Don't Panic")
+  end
+
+  should "extract 'Pound Force'" do
+    assert Entifier.extract('has become a hit. It is entitled \"Pound Force — Alan Duncan MP gets a new garden feature Panic').include?('Pound Force')
+  end
+
+  should "extract 'Ian Trow" do
+    assert Entifier.extract('. Ian Trow, 42, of Deanshanger, Milton Keynes, Buckinghamshire, and a').include?('Ian Trow')
+  end
+
+end

data/test/helper.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'entifier'
+
+class Test::Unit::TestCase
+end

data/test/known_failures/known_failures_test.rb

@@ -0,0 +1,13 @@
+require 'test_helper'
+
+class KnownFailuresTest < Test::Unit::TestCase
+  
+  should "not extract 'Hawaii' as we don't know that this is a proper known" do
+    assert !Entifier.extract("Hawaii basketball coach Bob Nash has agreed to a one-year").include?('Hawaii')
+  end  
+  
+  should "not extract Vietnam" do
+    assert !Entifier.extract("Around the country, Masses at Catholic churches are heavily attended.  Vietnam has often come under international criticism for its record on religious and human rights.").include?('Vietnam')
+  end  
+
+end

data/test/test_entifier.rb

@@ -0,0 +1,319 @@
+require 'helper'
+
+class TestEntifier < Test::Unit::TestCase
+
+
+  should "test_simple_name_at_start_of_string" do
+    assert_equal "Frankie Roberto", Entifier.extract("Frankie Roberto did blah.").first
+  end
+  
+  def test_simple_name_at_end_of_sentence
+    assert_equal "Frankie Roberto", Entifier.extract("It was Frankie Roberto.").first
+  end
+
+  def test_simple_name_in_middle_of_string
+    assert_equal "Frankie Roberto", Entifier.extract("If it was Frankie Roberto who did it.").first
+  end
+
+  def test_name_with_apostrophe
+    assert_equal "Samantha Harvey", Entifier.extract("Samantha Harvey's first novel is...").first
+  end
+  
+  def test_name_before_comma
+    assert_equal "Frankie Roberto", Entifier.extract("Call me Frankie Roberto, said the...").first
+  end
+  
+  def test_single_word_name_at_start_with_apostrophe
+    assert_equal "Britain", Entifier.extract("Britain's nuclear capabilities...").first
+  end
+  
+  def test_name_at_start_of_sentence
+    assert_equal "Frankie Roberto", Entifier.extract("First sentence. Frankie Roberto is fun.").first
+  end
+  
+  def test_single_word_name_at_start_of_sentence
+    assert_equal "Britain", Entifier.extract("First sentence. Britain's nuclear capabilities...").first
+  end
+    
+  def test_list_of_countries
+    entities = Entifier.extract("The US, Canada, Spain, Britain and Israel have confirmed cases of the virus, but no deaths have been reported outside Mexico.")
+    assert_equal 6, entities.size
+    assert_equal "US", entities.first
+    assert_equal "Mexico", entities.last
+  end
+  
+  def test_WHO_example
+    entities = Entifier.extract("World Health Organization deputy chief Keiji Fukuda was speaking...")
+    assert_equal 2, entities.size
+    assert_equal "World Health Organization", entities.first
+    assert_equal "Keiji Fukuda", entities.last
+  end
+  
+  def test_three_word_name
+    assert_equal "Dr Keiji Fukuda", Entifier.extract("The expert Dr Keiji Fukuda said it").first
+  end
+  
+  def test_four_word_name_at_start_of_string
+    assert_equal "Mr Bob Jones Robert", Entifier.extract("Mr Bob Jones Robert said...").first
+    
+  end
+  
+  def test_name_containing_umlaut
+    assert_equal "Albrecht Dürer", Entifier.extract("Albrecht Dürer is a famous artist.").first
+  end
+  
+  def test_dont_include_capitalised_sentence_starts
+    assert_equal 0, Entifier.extract("How now brown cow. Knees up mother brown.").size
+  end
+  
+  def test_dont_include_capitalised_new_paragraph_starts
+    assert_equal 0, Entifier.extract("How now brown cow.\n\nKnees up mother brown.").size
+  end
+  
+  def test_name_containing_of
+    assert_equal "Department of Health", Entifier.extract("If Department of Health is looking").first
+  end
+  
+  def test_name_containing_of_in_middle_of_string
+    assert_equal "Department of Health", Entifier.extract("It has been reported that the Department of Health is looking").first
+  end
+  
+  def test_name_continaing_in_in_middle_of_string
+    assert_equal "Intergovernmental Panel on Climate Change", Entifier.extract("At the Intergovernmental Panel on Climate Change, experts said...").first
+  end
+  
+  def test_sentence_starting_with_in
+    assert_equal "Northern Ireland", Entifier.extract("In Northern Ireland the chief").first
+  end
+  
+  def test_name_including_hypen
+    assert_equal "Mr Rory-Jones", Entifier.extract("He said to Mr Rory-Jones that").first
+  end
+
+  def test_dont_include_day_names
+    assert_equal 0, Entifier.extract("She was born on Thursday.").size
+  end
+
+  def test_dont_include_month_names
+    assert_equal 0, Entifier.extract("She was born in September.").size
+  end
+  
+  def test_dont_include_its
+    assert_equal 0, Entifier.extract("It's not too late to...").size
+  end
+  
+  def test_names_at_beginning_and_end_of_sentences
+    entities = Entifier.extract("He lived in London. Frankie Roberto is his name.")
+    assert_equal 2, entities.size
+    assert_equal "London", entities.first
+    assert_equal "Frankie Roberto", entities.last
+
+  end
+  
+  def test_abbreviation_with_numeral
+    assert_equal "G8", Entifier.extract("At the G8 meeting last week...").first
+  end
+  
+  def test_event_preceeded_by_year
+    assert_equal "2012 Olympics", Entifier.extract("At the 2012 Olympics, someone will win.").first
+  end
+  
+  def dont_test_abbreviation_at_start_of_string_yet
+    assert_equal "US", Entifier.extract("USA officials have urged...").first
+  end  
+  
+  def test_name_with_of_followed_by_two_words
+    assert_equal "Department of Clinical Health", Entifier.extract("At the Department of Clinical Health, people were...").first
+  end
+  
+  should "double quoted name" do
+    assert_equal "Demon Crossroads", Entifier.extract("At the place they call the \"Demon Crossroads\", two people met...").first
+  end
+
+  should "single quoted name" do
+    assert_equal "Demon Crossroads", Entifier.extract("At the place they call the 'Demon Crossroads', two people met...").first
+  end
+  
+  should "name starting paragraph after header" do
+    assert_equal "Britain", Entifier.extract("Worrying times\n\nBritain's nuclear capabilities...").first
+  end
+
+  should "name after sentence ending in question mark" do
+    assert_equal "Frankie Roberto", Entifier.extract("Who's the daddy? Frankie Roberto is.").first
+  end
+  
+  should "don't include capitalised word after questions mark" do
+    assert_equal 0, Entifier.extract("Who's the daddy? What a tricky question.").size
+  end
+
+  should "don't include capitalised word after exclaimation mark" do
+    assert_equal 0, Entifier.extract("Oh look! How peculiar.").size
+  end
+  
+  should "name including 'of the'" do
+    assert_equal "Horseman of the Apocalypse", Entifier.extract("In the Horseman of the Apocalypse...").first
+  end
+  
+  should "name included twice" do 
+    assert_equal 1, Entifier.extract("Frankie Roberto was a man. Frankie Roberto was a mouse.").size
+  end
+    
+  should "ignore extra spaces betwen capitalised words" do
+    assert_equal "Schools Secretary", Entifier.extract("the Schools  Secretary is...").first
+  end
+
+  should "ignore space before two newlines" do
+    assert_nil Entifier.extract("A sentence.\s\n\nA new sentence.").first
+  end
+  
+  should "allow entity with 'for' in it" do
+    assert_equal "Deputy Mayor for Government", Entifier.extract("The office of Deputy Mayor for Government is important.").first
+  end
+    
+  should "ignore quote marks at the start of a sentence" do
+    assert_nil Entifier.extract("\"Blindly, he walked forwards\"").first
+  end    
+    
+  should "ignore 'if' at the beginning of a sentence" do
+    assert_equal "Frankie Roberto", Entifier.extract("If Frankie Roberto can do something.").first
+  end  
+    
+  should "allow entities to end in a number" do
+    assert_equal "BBC 1", Entifier.extract("And now, on BBC 1, it's...").first
+  end
+    
+  should "treat a double line break as starting a new sentence" do
+    assert_nil Entifier.extract("the end\n\nThe start").first
+  end
+
+  should "treat a single line break as starting a new sentence" do
+    assert_nil Entifier.extract("the end\nThe start").first
+  end
+
+  should "treat full-stop followed by quote mark as starting a new sentence" do
+    assert_nil Entifier.extract("process.\"  'Emerging threats").first
+  end
+
+  should "not treat quote mark followed by a comma as ending a sentence" do
+    assert_equal "Frankie", Entifier.extract("\"This should work\", Frankie said.").first
+  end
+
+  should "include words with accented e character" do
+    assert_equal "Béarn", Entifier.extract("I live in Béarn.").first
+  end
+  
+  should "include words with tilded n character" do 
+    assert_equal "El Niño", Entifier.extract("He's called El Niño.").first
+  end
+  
+  should "include words with funny characters" do
+    assert_equal "Bjørn Dæhlie", Entifier.extract("He's called Bjørn Dæhlie").first
+  end
+  
+  should "include words starting with haloed A" do
+    assert_equal "Anders Jonas Ångström", Entifier.extract("His name is Anders Jonas Ångström.").first
+  end
+  
+  should "include words starting with umlauted O" do
+    assert_equal "Öland", Entifier.extract("I live in Öland.").first
+  end
+  
+  should "pick out a phrases with 'for' in it at the start of a sentence" do
+    assert_equal "Department for Transport", Entifier.extract("The Department for Transport has finished...").first
+  end
+  
+  should "ignore but at the start of sentence" do 
+    assert_equal "Foreign Secretary David Miliband", Entifier.extract("But Foreign Secretary David Miliband said that...").first
+  end
+  
+  should "ignore months" do
+    assert_nil Entifier.extract("In the month of April 2008 bad things happened.").first
+  end
+  
+  should "pick out abbreviations including ampersand" do
+    assert_equal "C&A", Entifier.extract("I used to shop at C&A.").first
+  end
+    
+  should "not include 'on Tuesday' as part of a phrase" do 
+    assert_equal "Commons", Entifier.extract("In the Commons on Tuesday, the...").first
+  end  
+  
+  should "not include 'On Friday' as an entity" do
+    assert_nil Entifier.extract("On Friday, something happened.").first
+  end
+    
+  should "detect single word entities after 'If' at the start of a sentence" do
+    assert_equal "Frankie", Entifier.extract("If Frankie can do something.").first
+  end    
+  
+  should "extract 'Florida'" do
+    assert Entifier.extract("In Florida, Lynn Orr was waiting").include?('Florida')
+  end  
+
+  # PA File : eAP-D979UQDG0.eAP-Officers-Indicted-1st-Ld-Writethru.nitf.xml
+  should "extract 'Baltimore'" do
+    assert_equal 'Baltimore', Entifier.extract("Two Baltimore police officers beat").first
+  end
+
+  # PA File : eAP-D979UPS01.eAP-NA-US-GM-Labor-Costs.nitf.xml
+  should "extract 'GM'" do
+    assert_equal "GM", Entifier.extract("GM submitted a progress report").first
+  end
+
+  # PA File : eAP-D979UHQ01.eAP-CB-Puerto-Rico-Agitated-Passenger.nitf.xml
+  should "extract FBI" do
+    assert_equal "FBI", Entifier.extract('The FBI has arrested').first
+  end
+
+  # PA File : eAP-D979U4P03.eAP-Wall-Street-19th-Ld-Writethru.nitf.xml
+  should "extract Russell 2000" do
+    assert_equal ["Russell 2000"], Entifier.extract("The Russell 2000 index of smaller companies")
+  end
+
+  # PA File : PRN-3869490en-1.PRN-GXX-HEALTH-HPV-Testing-Study.nitf.xml
+  should "extract 'Bill & Melinda Gates Foundation'" do
+    assert_equal ["Bill & Melinda Gates Foundation"], Entifier.extract("Funded by the Bill & Melinda Gates Foundation.")
+  end
+
+  should "not include year after 'in'" do
+    assert_equal "Appleton", Entifier.extract("In 1924 Appleton began...").first
+  end
+
+  should "extract April Fool's Day" do
+    assert_equal ["April Fool's Day"], Entifier.extract(" was being tracked throughout April Fool's Day, more ")
+  end
+  
+  should "ignore 'One of' at start of sentence" do
+    assert_equal ["Britain"], Entifier.extract("One of Britain's best-known...")
+  end
+  
+  should "not extract numbers" do
+    assert_equal [], Entifier.extract("The 2-1 victory over...")
+  end  
+  
+  should "extract 'Cap d' Antibes" do
+    assert_equal ["Cap d' Antibes"], Entifier.extract("In Cap d' Antibes, ...")
+  end  
+  
+  should "find entities at start and end of sentences" do
+    assert_equal ["Frankie Roberto", "United Kingdom"], Entifier.extract("name was Frankie Roberto. United Kingdom was...")
+  end
+
+  should "not extract 'Dr'" do
+    assert_equal ["Dr. Christopher Ziebell"], Entifier.extract("\"It's a pretty significant issue,\" said Dr. Christopher Ziebell, chief of the emergency department...")
+  end
+    
+  should "extract 'B.J. Upton'" do
+    assert_equal ["B.J. Upton"], Entifier.extract("The move won't be made until Sunday, when injured outfielders B.J. Upton and ...")
+  end    
+  
+  should "extract Richard E. Grant" do
+    assert_equal ["Richard E. Grant"], Entifier.extract("The film starred Richard E. Grant as...")
+  end
+
+  should "extract 'Fourie du Prez'" do
+    assert_equal ["Fourie du Prez"], Entifier.extract("...looking forward to pitting his wits against experienced opposite number Fourie du Preez when the...")
+  end
+
+    
+end

metadata

@@ -0,0 +1,144 @@
+--- !ruby/object:Gem::Specification 
+name: entifier
+version: !ruby/object:Gem::Version 
+  hash: 31
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 0
+  version: 0.0.0
+platform: ruby
+authors: 
+- robl
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-01-06 00:00:00 +00:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  name: shoulda
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id001
+  type: :development
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  name: bundler
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  requirement: *id002
+  type: :development
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  name: jeweler
+  version_requirements: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 1
+        - 5
+        - 2
+        version: 1.5.2
+  requirement: *id003
+  type: :development
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  name: rcov
+  version_requirements: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id004
+  type: :development
+description: 
+email: frankie[at]rattlecentral.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE.txt
+- README.rdoc
+files: 
+- .document
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- entifier.gemspec
+- lib/entifier.rb
+- lib/entifier/extensions.rb
+- lib/entifier/init.rb
+- test/errors/errors.rb
+- test/helper.rb
+- test/known_failures/known_failures_test.rb
+- test/test_entifier.rb
+has_rdoc: true
+homepage: http://github.com/rattle/entifier
+licenses: 
+- GPLv3
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Entifier gem to extract named entities
+test_files: 
+- test/errors/errors.rb
+- test/helper.rb
+- test/known_failures/known_failures_test.rb
+- test/test_entifier.rb