namor 0.5.4 → 0.6.0

data/lib/namor.rb

@@ -1,5 +1,6 @@
 require_relative "namor/version"
 require_relative "namor/namor"
+require_relative "namor/comparator"
 
 module Namor
 end

data/lib/namor/comparator.rb

@@ -0,0 +1,85 @@
+# MULTI-MATCHING via components
+# go through all users
+# group by distinct sets of components
+# pick a (small) subset of component-keys, say <10.  Maybe random sample?
+# build a set of matching rules
+# run the subset * the full corpus * the matching rules
+
+class Namor::Comparator
+  attr_reader :corpus
+
+  def initialize(corpus)
+    @corpus = corpus
+
+    prep_missing_initials
+  end
+
+  def crunch(record)
+    (@corpus - [record]).each_with_object([]) do |candidate,matches|
+      if evaluate(record, candidate)
+        matches << candidate
+      end
+    end
+  end
+
+  def evaluate(record, candidate)
+    [:missing_initials].each do |rule|
+      return true if send(rule, record, candidate)
+    end
+    false
+  end
+
+  # don't need an 'identical' test - assuming that the input record does not appear in the corpus
+  # def identical(a,b)
+  #   a == b
+  # end
+
+  # must have at least 2 long (non-initial-only) components in each
+  # those long parts must be identical
+  # only one of the names can have any initials
+  def missing_initials(a,b)
+    longnames_a = a.select {|s| s.length > 1}
+    longnames_b = b.select {|s| s.length > 1}
+    inits_a = a.select {|s| s.length == 1}
+    inits_b = b.select {|s| s.length == 1}
+
+    longnames_a.count >= 2 && longnames_b.count >= 2 && longnames_a == longnames_b && (inits_a.empty? || inits_b.empty?)
+  end
+
+  def prep_missing_initials
+    @corpus_missing_initials = corpus.each_with_object(Set.new) do |rec,set|
+      without_initials = rec.select {|s| s.length > 1}
+      if without_initials.count >= 2
+        set << without_initials
+      end
+    end
+  end
+
+  # must have at least 1 long (non-initial-only) component in each
+  # those long parts must be identical
+  # all initials should correspond to non-matched longnames in the other input
+  def matching_initials(a,b)
+    longnames_a = a.select {|s| s.length > 1}
+    longnames_b = b.select {|s| s.length > 1}
+    inits_a = a.select {|s| s.length == 1}
+    inits_b = b.select {|s| s.length == 1}
+
+    return false unless longnames_a.count >= 1 && longnames_b.count >= 1
+
+    unmatched_longnames_a = longnames_a - longnames_b
+    unmatched_longnames_b = longnames_b - longnames_a
+    unmatched_inits_a = unmatched_longnames_a.map {|s| s[0]}
+    unmatched_inits_b = unmatched_longnames_b.map {|s| s[0]}
+
+    inits_a == unmatched_inits_b && inits_b == unmatched_inits_a
+  end
+
+  # ignore any initials. look for cases where there is exactly one name component that differs between the inputs.
+  def matching_all_but_one(a,b)
+    longnames_a = a.select {|s| s.length > 1}
+    longnames_b = b.select {|s| s.length > 1}
+
+    ((longnames_a | longnames_b) - (longnames_a & longnames_b)).count == 1
+  end
+
+end

data/lib/namor/namor.rb

@@ -20,6 +20,11 @@ class Namor::Namor
     # Regexp.new(bits.join('|'))
   end
 
+  def suppress(name, supplist)
+    @re_cache[supplist] ||= suppression_re(supplist)
+    name && name.upcase.gsub(@re_cache[supplist], '')
+  end
+
   # clean up a single name component
   # * output all converted to uppercase
   # * strip leading ZZ+ or XX+ (frequently used as invalid-account prefixes)
@@ -31,7 +36,7 @@ class Namor::Namor
   def scrub(name, opts = {})
     @re_cache[opts[:suppress]] ||= suppression_re(opts[:suppress])
 
-    name && name.upcase.gsub(/^[ZX]{2,}/, '').gsub(@re_cache[opts[:suppress]], '').gsub(/\b(JR|SR|II|III|IV)\b/i, '').gsub(/\([^\(]*\)/, '').gsub(/\./, ' ').gsub(/[_'\&]/, '').gsub(/,\s*$/, '').gsub(/ +/, ' ').strip
+    name && name.upcase.gsub(/^[ZX]{2,}/, '').gsub(@re_cache[opts[:suppress]], '').gsub(/\b(JR|SR|II|III|IV)\b/i, '').gsub(/\([^\)]*\)/, '').gsub(/\[[^\]]*\]/, '').gsub(/\./, ' ').gsub(/[_'\&]/, '').gsub(/,\s*$/, '').gsub(/ +/, ' ').strip
   end
 
   def fullscrub(name, opts = {})
@@ -44,8 +49,9 @@ class Namor::Namor
     s && s.gsub(/[- ]/, '')
   end
 
-  def demaiden(lastname)
+  def demaiden(lastname, opts = {})
     return [nil,nil] unless lastname && !lastname.empty?
+    lastname = suppress(lastname, opts[:suppress]) if opts[:suppress]
     if lastname =~ /\-/
       [lastname.upcase.gsub(/ /, ''), lastname.split(/\-/).last.gsub(/ /, '')]
     else
@@ -114,17 +120,17 @@ class Namor::Namor
     ary << ary[4].gsub(/\W/, '_')
   end
 
-  def extract_from_pieces(hash)
+  def extract_from_pieces(hash, opts = {})
     assemble(
-      scrub(hash[:first]),
-      scrub(hash[:middle]),
-      scrub_and_squash(hash[:last]),
-      scrub_and_squash((s = demaiden(hash[:last])) && s.last)
+      scrub(hash[:first], opts),
+      scrub(hash[:middle], opts),
+      scrub_and_squash(hash[:last], opts),
+      scrub_and_squash((s = demaiden(hash[:last], opts)) && s.last, opts)
     )
   end
 
-  def extract_from_pieces_with_cluster(hash)
-    ary = extract_from_pieces(hash)
+  def extract_from_pieces_with_cluster(hash, opts = {})
+    ary = extract_from_pieces(hash, opts)
     ary << ary[3].gsub(/\W/, '_')
     ary << ary[4].gsub(/\W/, '_')
   end

data/lib/namor/version.rb

@@ -1,3 +1,3 @@
 module Namor
-  VERSION = "0.5.4"
+  VERSION = "0.6.0"
 end

data/spec/lib/comparator_spec.rb

@@ -0,0 +1,59 @@
+require "spec_helper"
+
+describe "Comparator" do
+  def explode(name)
+    name.gsub(/[,._-]/, ' ').split.map(&:upcase).sort
+  end
+
+  def try(rule, name1, name2)
+    @comp.send(rule, explode(name1), explode(name2)).should be_true
+  end
+
+  def bust(rule, name1, name2)
+    @comp.send(rule, explode(name1), explode(name2)).should be_false
+  end
+
+  before :all do
+    names = [
+      "michael g palmer",
+      "francis l palmer",
+      "michael palmer"
+    ]
+    corpus = names.map {|name| explode(name)}
+    @comp = Namor::Comparator.new(corpus)
+  end
+
+  it "finds names that match without initials" do
+    try(:missing_initials, "michael palmer", "michael g palmer")
+    try(:missing_initials, "michael palmer", "Q michael palmer")
+    try(:missing_initials, "michael palmer", "Michael N Palmer x")
+    bust(:missing_initials, "michael palmer", "Michael P")
+    bust(:missing_initials, "michael palmer", "Michael John Palmer")
+
+    matches = @comp.crunch(explode("michael palmer"))
+    matches.should == [explode("michael g palmer")]
+    matches = @comp.crunch(explode("palmer michael"))
+    matches.should == [explode("michael g palmer")]
+    matches = @comp.crunch(explode("michael g palmer"))
+    matches.should == [explode("michael palmer")]
+  end
+
+  it "finds names that match initials to names" do
+    try(:matching_initials, "fred jones", "f jones")
+    try(:matching_initials, "fred jones", "jones f")
+    try(:matching_initials, "fred jones", "fred j")
+    try(:matching_initials, "fred xavier jones", "fred x jones")
+    try(:matching_initials, "fred xavier jones", "xavier jones f")
+    bust(:matching_initials, "fred xavier jones", "fred jones")
+    bust(:matching_initials, "fred xavier jones", "fred q jones")
+    bust(:matching_initials, "fred x jones", "fred q jones")
+    bust(:matching_initials, "fred xavier jones", "homer simpson")
+  end
+
+  it "finds names that match on all but one long names" do
+    try(:matching_all_but_one, "john philip sousa", "john sousa")
+    try(:matching_all_but_one, "philip sousa", "philip john sousa")
+    bust(:matching_all_but_one, "john philip sousa", "philip john sousa")
+    try(:matching_all_but_one, "Helen Q. Glorpworth-Smythe", "helen smythe")
+  end
+end

data/spec/lib/namor_spec.rb

@@ -28,6 +28,10 @@ describe "name extract" do
     @namor.extract("SMITH, JOHN (Jacko) R").should == ['JOHN', 'R', 'SMITH', 'SMITH,JOHN R', 'SMITH,JOHN R']
   end
 
+  it "should strip elements within square brackets" do
+    @namor.extract("SMITH, JOHN [Jacko] R").should == ['JOHN', 'R', 'SMITH', 'SMITH,JOHN R', 'SMITH,JOHN R']
+  end
+
   it "should drop periods" do
     @namor.extract("John R. Smith").should == ['JOHN', 'R', 'SMITH', 'SMITH,JOHN R', 'SMITH,JOHN R']
   end
@@ -56,6 +60,7 @@ describe "name extract" do
   it "should excise terms from optional suppression list" do
     @namor.extract("Smith Jr, Edward M MD DDS").should == ['EDWARD', 'M', 'SMITH', 'SMITH,EDWARD M', 'SMITH,EDWARD M']
     @namor.extract("Smith Jr, Edward M M.D.").should == ['EDWARD', 'M', 'SMITH', 'SMITH,EDWARD M', 'SMITH,EDWARD M']
+    @namor.extract("Smith Jr, Edward M M.D. [Oph,Ped Orth]").should == ['EDWARD', 'M', 'SMITH', 'SMITH,EDWARD M', 'SMITH,EDWARD M']
     @namor.extract("Smith Jr, Edward III MD PHD").should == ['EDWARD', 'PHD', 'SMITH', 'SMITH,EDWARD PHD', 'SMITH,EDWARD PHD']
   end
 
@@ -88,6 +93,8 @@ describe "name extract" do
     @namor.extract_from_pieces_with_cluster(:first => 'John', :middle => 'M', :last => 'Smith').should == ['JOHN', 'M', 'SMITH', 'SMITH,JOHN M', 'SMITH,JOHN M', 'SMITH_JOHN_M', 'SMITH_JOHN_M']
     @namor.extract_from_pieces_with_cluster(:first => 'Susan', :last => 'Smith-Jones').should == ['SUSAN', nil, 'SMITHJONES', 'SMITHJONES,SUSAN', 'JONES,SUSAN', 'SMITHJONES_SUSAN', 'JONES_SUSAN']
 
+    @namor.extract_from_pieces_with_cluster({:first => 'Susan', :last => 'Smith-Jones MD PHD'}, {:suppress => ['MD', 'PHD']}).should == ['SUSAN', nil, 'SMITHJONES', 'SMITHJONES,SUSAN', 'JONES,SUSAN', 'SMITHJONES_SUSAN', 'JONES_SUSAN']
+
     @namor.extract_from_pieces(:last => 'Smith').should == [nil,nil, 'SMITH', 'SMITH', 'SMITH']
 
     @namor.extract_from_pieces(:first => 'Mary', :last => 'Smith Jones').should == ['MARY',nil, 'SMITHJONES', 'SMITHJONES,MARY', 'JONES,MARY']

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: namor
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.6.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-12 00:00:00.000000000 Z
+date: 2012-10-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -89,9 +89,11 @@ files:
 - README.md
 - Rakefile
 - lib/namor.rb
+- lib/namor/comparator.rb
 - lib/namor/namor.rb
 - lib/namor/version.rb
 - namor.gemspec
+- spec/lib/comparator_spec.rb
 - spec/lib/namor_spec.rb
 - spec/spec_helper.rb
 homepage: ''
@@ -108,7 +110,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2394129238240412470
+      hash: -145397464691454724
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -117,7 +119,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2394129238240412470
+      hash: -145397464691454724
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24
@@ -125,5 +127,6 @@ signing_key:
 specification_version: 3
 summary: Parse & extract pieces of names
 test_files:
+- spec/lib/comparator_spec.rb
 - spec/lib/namor_spec.rb
 - spec/spec_helper.rb