fuzzy_match 1.3.1 → 1.3.2

data/lib/fuzzy_match/result.rb

@@ -28,7 +28,7 @@ ERB
     attr_accessor :haystack
     attr_accessor :options
     attr_accessor :normalizers
-    attr_accessor :blockings
+    attr_accessor :groupings
     attr_accessor :identities
     attr_accessor :stop_words
     attr_accessor :winner

data/lib/fuzzy_match/rule.rb

@@ -0,0 +1,14 @@
+class FuzzyMatch
+  # A rule characterized by a regexp. Abstract.
+  class Rule
+    attr_reader :regexp
+    
+    def initialize(regexp_or_str)
+      @regexp = regexp_or_str.to_regexp
+    end
+    
+    def ==(other)
+      regexp == other.regexp
+    end
+  end
+end

data/lib/fuzzy_match/rule/grouping.rb

@@ -0,0 +1,32 @@
+class FuzzyMatch
+  class Rule
+    # "Record linkage typically involves two main steps: grouping and scoring..."
+    # http://en.wikipedia.org/wiki/Record_linkage
+    #
+    # Groupings effectively divide up the haystack into groups that match a pattern
+    #
+    # A grouping (formerly known as a blocking) comes into effect when a str matches.
+    # Then the needle must also match the grouping's regexp.
+    class Grouping < Rule
+      def match?(str)
+        !!(regexp.match(str))
+      end
+
+      # If a grouping "joins" two strings, that means they both fit into it.
+      #
+      # Returns false if they certainly don't fit this grouping.
+      # Returns nil if the grouping doesn't apply, i.e. str2 doesn't fit the grouping.
+      def join?(str1, str2)
+        if str2_match_data = regexp.match(str2)
+          if str1_match_data = regexp.match(str1)
+            str2_match_data.captures.join.downcase == str1_match_data.captures.join.downcase
+          else
+            false
+          end
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/fuzzy_match/rule/identity.rb

@@ -0,0 +1,19 @@
+class FuzzyMatch
+  class Rule
+    # Identities take effect when needle and haystack both match a regexp
+    # Then the captured part of the regexp has to match exactly
+    class Identity < Rule
+      # Two strings are "identical" if they both match this identity and the captures are equal.
+      #
+      # Only returns true/false if both strings match the regexp.
+      # Otherwise returns nil.
+      def identical?(str1, str2)
+        if str1_match_data = regexp.match(str1) and match_data = regexp.match(str2)
+          str1_match_data.captures.join.downcase == match_data.captures.join.downcase
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/fuzzy_match/rule/normalizer.rb

@@ -0,0 +1,20 @@
+class FuzzyMatch
+  class Rule
+    # A normalizer just strips a string down to its core
+    class Normalizer < Rule
+      # A normalizer applies when its regexp matches and captures a new (shorter) string
+      def apply?(str)
+        !!(regexp.match(str))
+      end
+    
+      # The result of applying a normalizer is just all the captures put together.
+      def apply(str)
+        if match_data = regexp.match(str)
+          match_data.captures.join
+        else
+          str
+        end
+      end
+    end
+  end
+end

data/lib/fuzzy_match/rule/stop_word.rb

@@ -0,0 +1,11 @@
+class FuzzyMatch
+  class Rule
+    # A stop word is ignored
+    class StopWord < Rule
+      # Destructively remove stop words from the string
+      def apply!(str)
+        str.gsub! regexp, ''
+      end
+    end
+  end
+end

data/lib/fuzzy_match/version.rb

@@ -1,3 +1,3 @@
 class FuzzyMatch
-  VERSION = '1.3.1'
+  VERSION = '1.3.2'
 end

data/test/helper.rb

@@ -3,7 +3,9 @@ require 'bundler'
 Bundler.setup
 require 'minitest/spec'
 require 'minitest/autorun'
-require 'stringio'
+require 'minitest/reporters'
+MiniTest::Unit.runner = MiniTest::SuiteRunner.new
+MiniTest::Unit.runner.reporters << MiniTest::Reporters::SpecReporter.new
 
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))

data/test/test_fuzzy_match.rb

@@ -2,98 +2,126 @@
 require 'helper'
 
 class TestFuzzyMatch < MiniTest::Spec
-  it %{identify the best match based on string similarity} do
-    d = FuzzyMatch.new %w{ RATZ CATZ }
-    d.find('RITZ').must_equal 'RATZ'
-    d.find('RíTZ').must_equal 'RATZ'
-
-    d = FuzzyMatch.new [ 'X' ]
-    d.find('X').must_equal 'X'
-    d.find('A').must_be_nil
-  end
+  describe '#find' do
+    it %{identifies the best match based on string similarity} do
+      d = FuzzyMatch.new %w{ RATZ CATZ }
+      d.find('RITZ').must_equal 'RATZ'
+      d.find('RíTZ').must_equal 'RATZ'
 
-  it %{not gather metadata about the last result by default} do
-    d = FuzzyMatch.new %w{ NISSAN HONDA }
-    d.find('MISSAM')
-    lambda do
-      d.last_result
-    end.must_raise ::RuntimeError, /gather_last_result/
+      d = FuzzyMatch.new [ 'X' ]
+      d.find('X').must_equal 'X'
+      d.find('A').must_be_nil
+    end
+  
+    it %{not return any result if the maximum score is zero} do
+      FuzzyMatch.new(['a']).find('b').must_be_nil
+    end
   end
-
-  it %{optionally gather metadata about the last result} do
-    d = FuzzyMatch.new %w{ NISSAN HONDA }
-    d.find 'MISSAM', :gather_last_result => true
-    d.last_result.score.must_equal 0.6
-    d.last_result.winner.must_equal 'NISSAN'
+  
+  describe '#find_all' do
+    it %{return all records in sorted order} do
+      d = FuzzyMatch.new [ 'X', 'X22', 'Y', 'Y4' ], :groupings => [ /X/, /Y/ ], :must_match_grouping => true
+      d.find_all('X').must_equal ['X', 'X22' ]
+      d.find_all('A').must_equal []
+    end
   end
 
-  it %{use NORMALIZERS} do
-    d = FuzzyMatch.new ['BOEING 737-100/200', 'BOEING 737-900']
-    d.find('BOEING 737100 number 900').must_equal 'BOEING 737-900'
-
-    normalizers = [
-      %r{(7\d)(7|0)-?(\d{1,3})} # tighten 737-100/200 => 737100, which will cause it to win over 737-900
-    ]
-    d = FuzzyMatch.new ['BOEING 737-100/200', 'BOEING 737-900'], :normalizers => normalizers
-    d.find('BOEING 737100 number 900').must_equal 'BOEING 737-100/200'
+  describe '#explain' do
+    before do
+      require 'stringio'
+      @capture = StringIO.new
+      @old_stdout = $stdout
+      $stdout = @capture
+    end
+    after do
+      $stdout = @old_stdout
+    end
+      
+    it %{print a basic explanation to stdout} do
+      d = FuzzyMatch.new %w{ RATZ CATZ }
+      d.explain('RITZ')
+      @capture.rewind
+      @capture.read.must_include 'CATZ'
+    end
+    
+    it %{explains match failures} do
+      FuzzyMatch.new(['aaa']).explain('bbb')
+      @capture.rewind
+      @capture.read.must_match %r{No winner assigned.*aaa.*bbb}
+    end
   end
 
-  it %{use IDENTITIES} do
-    # false positive without identity
-    d = FuzzyMatch.new %w{ foo bar }
-    d.find('baz').must_equal 'bar'
-
-    d = FuzzyMatch.new %w{ foo bar }, :identities => [ /ba(.)/ ]
-    d.find('baz').must_be_nil
-  end
+  describe "normalizers" do
+    it %{sometimes gets false results without them} do
+      d = FuzzyMatch.new ['BOEING 737-100/200', 'BOEING 737-900']
+      d.find('BOEING 737100 number 900').must_equal 'BOEING 737-900'
+    end
 
-  # TODO this is not very helpful
-  it %{use BLOCKINGS} do
-    d = FuzzyMatch.new [ 'X' ], :blockings => [ /X/, /Y/ ]
-    d.find('X').must_equal 'X'
-    d.find('A').must_be_nil
+    it %{can be used to improve results} do
+      normalizers = [
+        %r{(7\d)(7|0)-?(\d{1,3})} # tighten 737-100/200 => 737100, which will cause it to win over 737-900
+      ]
+      d = FuzzyMatch.new ['BOEING 737-100/200', 'BOEING 737-900'], :normalizers => normalizers
+      d.find('BOEING 737100 number 900').must_equal 'BOEING 737-100/200'
+    end
   end
 
-  # TODO this is not very helpful
-  it %{optionally only attempt matches with records that fit into a blocking} do
-    d = FuzzyMatch.new [ 'X' ], :blockings => [ /X/, /Y/ ], :must_match_blocking => true
-    d.find('X').must_equal 'X'
-    d.find('A').must_be_nil
+  describe "identities" do
+    it %{sometimes gets false results without them} do
+      # false positive without identity
+      d = FuzzyMatch.new %w{ foo bar }
+      d.find('baz').must_equal 'bar'
+    end
 
-    d = FuzzyMatch.new [ 'X' ], :blockings => [ /X/, /Y/ ]
-    d.find('X', :must_match_blocking => true).must_equal 'X'
-    d.find('A', :must_match_blocking => true).must_be_nil
+    it %{can be used to improve results} do
+      d = FuzzyMatch.new %w{ foo bar }, :identities => [ /ba(.)/ ]
+      d.find('baz').must_be_nil
+    end
   end
 
-  it %{receive the deprecated FuzzyMatch#free method without complaint} do
-    d = FuzzyMatch.new %w{ A B }
-    d.free
-    d.find('A').wont_be_nil
+  describe 'groupings' do
+    it %{sometimes gets false results without them} do
+      d = FuzzyMatch.new [ 'Barack Obama', 'George Bush' ]
+      d.find('Barack Bush').must_equal 'Barack Obama' # luke i am your father
+      d.find('George Obama').must_equal 'George Bush' # nooooooooooooooooooo
+    end
+    
+    it %{can be used to improve results} do
+      d = FuzzyMatch.new [ 'Barack Obama', 'George Bush' ], :groupings => [ /Obama/, /Bush/ ]
+      d.find('Barack Bush').must_equal 'George Bush'
+      d.find('George Obama').must_equal 'Barack Obama'
+    end
   end
-
-  it %{return all records in sorted order} do
-    d = FuzzyMatch.new [ 'X', 'X22', 'Y', 'Y4' ], :blockings => [ /X/, /Y/ ], :must_match_blocking => true
-    d.find_all('X').must_equal ['X', 'X22' ]
-    d.find_all('A').must_equal []
+  
+  describe "the :must_match_grouping option" do
+    it %{optionally only attempt matches with records that fit into a grouping} do
+      d = FuzzyMatch.new [ 'Barack Obama', 'George Bush' ], :groupings => [ /Obama/, /Bush/ ], :must_match_grouping => true
+      d.find('George Clinton').must_be_nil
+
+      d = FuzzyMatch.new [ 'Barack Obama', 'George Bush' ], :groupings => [ /Obama/, /Bush/ ]
+      d.find('George Clinton', :must_match_grouping => true).must_be_nil
+    end
   end
+  
+  describe "the :first_grouping_decides option" do
+    it %{optionally force the first grouping to decide} do
+      d = FuzzyMatch.new [ 'Boeing 747', 'Boeing 747SR', 'Boeing ER6' ], :groupings => [ /(boeing \d{3})/i, /boeing/i ]
+      d.find_all('Boeing 747').must_equal [ 'Boeing 747', 'Boeing 747SR', 'Boeing ER6' ]
 
-  it %{optionally force the first blocking to decide} do
-    d = FuzzyMatch.new [ 'Boeing 747', 'Boeing 747SR', 'Boeing ER6' ], :blockings => [ /(boeing \d{3})/i, /boeing/i ]
-    d.find_all('Boeing 747').must_equal [ 'Boeing 747', 'Boeing 747SR', 'Boeing ER6' ]
-
-    d = FuzzyMatch.new [ 'Boeing 747', 'Boeing 747SR', 'Boeing ER6' ], :blockings => [ /(boeing \d{3})/i, /boeing/i ], :first_blocking_decides => true
-    d.find_all('Boeing 747').must_equal [ 'Boeing 747', 'Boeing 747SR' ]
+      d = FuzzyMatch.new [ 'Boeing 747', 'Boeing 747SR', 'Boeing ER6' ], :groupings => [ /(boeing \d{3})/i, /boeing/i ], :first_grouping_decides => true
+      d.find_all('Boeing 747').must_equal [ 'Boeing 747', 'Boeing 747SR' ]
 
-    # first_blocking_decides refers to the needle
-    d = FuzzyMatch.new [ 'Boeing 747', 'Boeing 747SR', 'Boeing ER6' ], :blockings => [ /(boeing \d{3})/i, /boeing/i ], :first_blocking_decides => true
-    d.find_all('Boeing ER6').must_equal ["Boeing ER6", "Boeing 747", "Boeing 747SR"]
+      # first_grouping_decides refers to the needle
+      d = FuzzyMatch.new [ 'Boeing 747', 'Boeing 747SR', 'Boeing ER6' ], :groupings => [ /(boeing \d{3})/i, /boeing/i ], :first_grouping_decides => true
+      d.find_all('Boeing ER6').must_equal ["Boeing ER6", "Boeing 747", "Boeing 747SR"]
 
-    d = FuzzyMatch.new [ 'Boeing 747', 'Boeing 747SR', 'Boeing ER6' ], :blockings => [ /(boeing \d{3})/i, /boeing (7|E)/i, /boeing/i ], :first_blocking_decides => true
-    d.find_all('Boeing ER6').must_equal [ 'Boeing ER6' ]
+      d = FuzzyMatch.new [ 'Boeing 747', 'Boeing 747SR', 'Boeing ER6' ], :groupings => [ /(boeing \d{3})/i, /boeing (7|E)/i, /boeing/i ], :first_grouping_decides => true
+      d.find_all('Boeing ER6').must_equal [ 'Boeing ER6' ]
 
-    # or equivalently with an identity
-    d = FuzzyMatch.new [ 'Boeing 747', 'Boeing 747SR', 'Boeing ER6' ], :blockings => [ /(boeing \d{3})/i, /boeing/i ], :first_blocking_decides => true, :identities => [ /boeing (7|E)/i ]
-    d.find_all('Boeing ER6').must_equal [ 'Boeing ER6' ]
+      # or equivalently with an identity
+      d = FuzzyMatch.new [ 'Boeing 747', 'Boeing 747SR', 'Boeing ER6' ], :groupings => [ /(boeing \d{3})/i, /boeing/i ], :first_grouping_decides => true, :identities => [ /boeing (7|E)/i ]
+      d.find_all('Boeing ER6').must_equal [ 'Boeing ER6' ]
+    end
   end
 
   describe "the :read option" do
@@ -135,70 +163,106 @@ class TestFuzzyMatch < MiniTest::Spec
       by_first.find('b').must_equal ba
       by_last.find('a').must_equal ba
     end
-
-    it %{treat the deprecrated :haystack_reader option as an alias} do
-      ab = ['a', 'b']
-      ba = ['b', 'a']
-      haystack = [ab, ba]
-      by_first = FuzzyMatch.new haystack, :haystack_reader => 0
-      by_first.find('a').must_equal ab
-      by_first.find('b').must_equal ba
-    end
   end
+  
+  describe 'the :must_match_at_least_one_word option' do
+    it %{optionally require that the matching record share at least one word with the needle} do
+      d = FuzzyMatch.new %w{ RATZ CATZ }, :must_match_at_least_one_word => true
+      d.find('RITZ').must_be_nil
+
+      d = FuzzyMatch.new ["Foo's Bar"], :must_match_at_least_one_word => true
+      d.find("Foo's").must_equal "Foo's Bar"
+      d.find("'s").must_be_nil
+      d.find("Foo").must_be_nil
+
+      d = FuzzyMatch.new ["Bolivia, Plurinational State of"], :must_match_at_least_one_word => true
+      d.find("Bolivia").must_equal "Bolivia, Plurinational State of"
+    end
 
-  it %{not return any result if the maximum score is zero} do
-    FuzzyMatch.new(['a']).find('b').must_be_nil
-  end
+    it %{use STOP WORDS} do
+      d = FuzzyMatch.new [ 'A HOTEL', 'B HTL' ]
+      d.find('A HTL', :must_match_at_least_one_word => true).must_equal 'B HTL'
 
-  it %{optionally require that the matching record share at least one word with the needle} do
-    d = FuzzyMatch.new %w{ RATZ CATZ }, :must_match_at_least_one_word => true
-    d.find('RITZ').must_be_nil
+      d = FuzzyMatch.new [ 'A HOTEL', 'B HTL' ], :must_match_at_least_one_word => true
+      d.find('A HTL').must_equal 'B HTL'
 
-    d = FuzzyMatch.new ["Foo's Bar"], :must_match_at_least_one_word => true
-    d.find("Foo's").must_equal "Foo's Bar"
-    d.find("'s").must_be_nil
-    d.find("Foo").must_be_nil
+      d = FuzzyMatch.new [ 'A HOTEL', 'B HTL' ], :must_match_at_least_one_word => true, :stop_words => [ %r{HO?TE?L} ]
+      d.find('A HTL').must_equal 'A HOTEL'
+    end
     
-    d = FuzzyMatch.new ["Bolivia, Plurinational State of"], :must_match_at_least_one_word => true
-    d.find("Bolivia").must_equal "Bolivia, Plurinational State of"
+    it %{not be fooled by substrings (but rather compare whole words to whole words)} do
+      d = FuzzyMatch.new [ 'PENINSULA HOTELS' ], :must_match_at_least_one_word => true
+      d.find('DOLCE LA HULPE BXL FI').must_be_nil
+    end
+
+    it %{not be case-sensitive when checking for sharing of words} do
+      d = FuzzyMatch.new [ 'A', 'B' ]
+      d.find('a', :must_match_at_least_one_word => true).must_equal 'A'
+    end
+  end
+  
+  describe "the :gather_last_result option" do
+    it %{not gather metadata about the last result by default} do
+      d = FuzzyMatch.new %w{ NISSAN HONDA }
+      d.find('MISSAM')
+      lambda do
+        d.last_result
+      end.must_raise ::RuntimeError, /gather_last_result/
+    end
+
+    it %{optionally gather metadata about the last result} do
+      d = FuzzyMatch.new %w{ NISSAN HONDA }
+      d.find 'MISSAM', :gather_last_result => true
+      d.last_result.score.must_equal 0.6
+      d.last_result.winner.must_equal 'NISSAN'
+    end
+  end
+  
+  describe 'quirks' do
+    it %{should not return false negatives because of one-letter similarities} do
+      # dices coefficient doesn't think these two are similar at all because it looks at pairs
+      FuzzyMatch.score_class.new('X foo', 'X bar').dices_coefficient_similar.must_equal 0
+      # so we must compensate for that somewhere
+      d = FuzzyMatch.new ['X foo', 'randomness']
+      d.find('X bar').must_equal 'X foo'
+      # without making false positives
+      d.find('Y bar').must_be_nil
+    end
   end
 
-  it %{use STOP WORDS} do
-    d = FuzzyMatch.new [ 'A HOTEL', 'B HTL' ]
-    d.find('A HTL', :must_match_at_least_one_word => true).must_equal 'B HTL'
+  describe 'deprecations' do
+    it %{takes :must_match_blocking as :must_match_grouping} do
+      d = FuzzyMatch.new [], :must_match_blocking => :a
+      d.default_options[:must_match_grouping].must_equal :a
+    end
 
-    d = FuzzyMatch.new [ 'A HOTEL', 'B HTL' ], :must_match_at_least_one_word => true
-    d.find('A HTL').must_equal 'B HTL'
+    it %{takes :first_blocking_decides as :first_grouping_decides} do
+      d = FuzzyMatch.new [], :first_blocking_decides => :b
+      d.default_options[:first_grouping_decides].must_equal :b
+    end
 
-    d = FuzzyMatch.new [ 'A HOTEL', 'B HTL' ], :must_match_at_least_one_word => true, :stop_words => [ %r{HO?TE?L} ]
-    d.find('A HTL').must_equal 'A HOTEL'
-  end
+    it %{takes :haystack_reader as :read} do
+      d = FuzzyMatch.new [], :haystack_reader => :c
+      d.read.must_equal :c
+    end
 
-  it %{print a basic explanation to stdout} do
-    require 'stringio'
-    capture = StringIO.new
-    begin
-      old_stdout = $stdout
-      $stdout = capture
-      d = FuzzyMatch.new %w{ RATZ CATZ }
-      d.explain('RITZ')
-    ensure
-      $stdout = old_stdout
+    it %{takes :blockings as :groupings} do
+      d = FuzzyMatch.new [], :blockings => [ /X/, /Y/ ]
+      d.groupings.must_equal [ FuzzyMatch::Rule::Grouping.new(/X/), FuzzyMatch::Rule::Grouping.new(/Y/) ]
     end
-    capture.rewind
-    capture.read.must_include 'CATZ'
-  end
 
-  it %{not be fooled by substrings (but rather compare whole words to whole words)} do
-    d = FuzzyMatch.new [ 'PENINSULA HOTELS' ], :must_match_at_least_one_word => true
-    d.find('DOLCE LA HULPE BXL FI').must_be_nil
-  end
+    it %{takes :tighteners as :normalizers} do
+      d = FuzzyMatch.new [], :tighteners => [ /X/, /Y/ ]
+      d.normalizers.must_equal [ FuzzyMatch::Rule::Normalizer.new(/X/), FuzzyMatch::Rule::Normalizer.new(/Y/) ]
+    end
 
-  it %{not be case-sensitive when checking for sharing of words} do
-    d = FuzzyMatch.new [ 'A', 'B' ]
-    d.find('a', :must_match_at_least_one_word => true).must_equal 'A'
+    it %{receives #free method, but doesn't do anything} do
+      d = FuzzyMatch.new %w{ A B }
+      d.free
+      d.find('A').wont_be_nil
+    end
   end
-
+  
   it %{defaults to a pure-ruby engine, but also has amatch} do
     if defined?($testing_amatch) and $testing_amatch
       FuzzyMatch.engine.must_equal :amatch