fuzzy_match 1.5.0 → 2.0.0

data/lib/fuzzy_match/similarity.rb

@@ -1,11 +1,11 @@
 class FuzzyMatch
   class Similarity
-    attr_reader :wrapper1
-    attr_reader :wrapper2
+    attr_reader :record1
+    attr_reader :record2
     
-    def initialize(wrapper1, wrapper2)
-      @wrapper1 = wrapper1
-      @wrapper2 = wrapper2
+    def initialize(record1, record2)
+      @record1 = record1
+      @record2 = record2
     end
     
     def <=>(other)
@@ -18,44 +18,22 @@ class FuzzyMatch
     end
     
     def best_score
-      @best_score ||= FuzzyMatch.score_class.new(best_wrapper1_variant, best_wrapper2_variant)
+      @best_score ||= FuzzyMatch.score_class.new(record1.clean, record2.clean)
     end
 
     def satisfy?(needle, threshold)
       best_score.dices_coefficient_similar > threshold or
-        ((wrapper2.render.length < 3 or needle.render.length < 3) and best_score.levenshtein_similar > 0) or
-        (needle.words & wrapper2.words).any?
+        ((record2.clean.length < 3 or needle.clean.length < 3) and best_score.levenshtein_similar > 0) or
+        (needle.words & record2.words).any?
     end
 
     def inspect
-      %{#<FuzzyMatch::Similarity #{wrapper2.render.inspect}=>#{best_wrapper2_variant.inspect} versus #{wrapper1.render.inspect}=>#{best_wrapper1_variant.inspect} original_weight=#{"%0.5f" % original_weight} best_score=#{best_score.inspect}>}
+      %{#{record2.clean.inspect} ~ #{record1.clean.inspect} => #{best_score.inspect}}
     end
 
     # Weight things towards short original strings
     def original_weight
-      @original_weight ||= (1.0 / (wrapper1.render.length * wrapper2.render.length))
-    end
-
-    private
-        
-    def best_wrapper1_variant
-      best_variants[0]
-    end
-    
-    def best_wrapper2_variant
-      best_variants[1]
-    end
-    
-    def best_variants
-      @best_variants ||= begin
-        wrapper1.variants.product(wrapper2.variants).sort do |tuple1, tuple2|
-          wrapper1_variant1, wrapper2_variant1 = tuple1
-          wrapper1_variant2, wrapper2_variant2 = tuple2
-          score1 = FuzzyMatch.score_class.new wrapper1_variant1, wrapper2_variant1
-          score2 = FuzzyMatch.score_class.new wrapper1_variant2, wrapper2_variant2
-          score1 <=> score2
-        end.last
-      end
+      @original_weight ||= (1.0 / (record1.clean.length * record2.clean.length))
     end
   end
 end

data/lib/fuzzy_match/version.rb

@@ -1,3 +1,3 @@
 class FuzzyMatch
-  VERSION = '1.5.0'
+  VERSION = '2.0.0'
 end

data/lib/fuzzy_match.rb

@@ -1,10 +1,8 @@
 require 'fuzzy_match/rule'
-require 'fuzzy_match/rule/normalizer'
-require 'fuzzy_match/rule/stop_word'
 require 'fuzzy_match/rule/grouping'
 require 'fuzzy_match/rule/identity'
 require 'fuzzy_match/result'
-require 'fuzzy_match/wrapper'
+require 'fuzzy_match/record'
 require 'fuzzy_match/similarity'
 require 'fuzzy_match/score'
 
@@ -33,8 +31,8 @@ class FuzzyMatch
   
   DEFAULT_ENGINE = :pure_ruby
   
+  #TODO refactor at least all the :find_X things
   DEFAULT_OPTIONS = {
-    :first_grouping_decides => false,
     :must_match_grouping => false,
     :must_match_at_least_one_word => false,
     :gather_last_result => false,
@@ -42,6 +40,7 @@ class FuzzyMatch
     :find_all_with_score => false,
     :threshold => 0,
     :find_best => false,
+    :find_with_score => false,
   }
 
   self.engine = DEFAULT_ENGINE
@@ -49,7 +48,6 @@ class FuzzyMatch
   attr_reader :haystack
   attr_reader :groupings
   attr_reader :identities
-  attr_reader :normalizers
   attr_reader :stop_words
   attr_accessor :read
   attr_reader :default_options
@@ -57,7 +55,6 @@ class FuzzyMatch
   # haystack - a bunch of records that will compete to see who best matches the needle
   #
   # Rules (can only be specified at initialization or by using a setter)
-  # * :<tt>normalizers</tt> - regexps (see README)
   # * :<tt>identities</tt> - regexps
   # * :<tt>groupings</tt> - regexps
   # * :<tt>stop_words</tt> - regexps
@@ -66,70 +63,53 @@ class FuzzyMatch
   # Options (can be specified at initialization or when calling #find)
   # * :<tt>must_match_grouping</tt> - don't return a match unless the needle fits into one of the groupings you specified
   # * :<tt>must_match_at_least_one_word</tt> - don't return a match unless the needle shares at least one word with the match
-  # * :<tt>first_grouping_decides</tt> - force records into the first grouping they match, rather than choosing a grouping that will give them a higher score
   # * :<tt>gather_last_result</tt> - enable <tt>last_result</tt>
   # * :<tt>threshold</tt> - set a score threshold below which not to return results (not generally recommended - please test the results of setting a threshold thoroughly - one set of results and their scores probably won't be enough to determine the appropriate number). Only checked against the Pair Distance score and ignored when one string or the other is of length 1.
-  def initialize(competitors, options_and_rules = {})
-    options_and_rules = options_and_rules.dup
+  def initialize(haystack, options_and_rules = {})
+    o = options_and_rules.dup
 
     # rules
-    self.groupings = options_and_rules.delete(:groupings) || options_and_rules.delete(:blockings) || []
-    self.identities = options_and_rules.delete(:identities) || []
-    self.normalizers = options_and_rules.delete(:normalizers) || options_and_rules.delete(:tighteners) || []
-    self.stop_words = options_and_rules.delete(:stop_words) || []
-    @read = options_and_rules.delete(:read) || options_and_rules.delete(:haystack_reader)
-
+    @read = o.delete(:read) || o.delete(:haystack_reader)
+    @groupings = (o.delete(:groupings) || o.delete(:blockings) || []).map { |regexp| Rule::Grouping.make(regexp) }.flatten
+    @identities = (o.delete(:identities) || []).map { |regexp| Rule::Identity.new(regexp) }
+    @stop_words = o.delete(:stop_words) || []
+    
     # options
-    if deprecated = options_and_rules.delete(:first_blocking_decides)
-      options_and_rules[:first_grouping_decides] = deprecated
-    end
-    if deprecated = options_and_rules.delete(:must_match_blocking)
-      options_and_rules[:must_match_grouping] = deprecated
+    if deprecated = o.delete(:must_match_blocking)
+      o[:must_match_grouping] = deprecated
     end
-    @default_options = DEFAULT_OPTIONS.merge(options_and_rules).freeze
+    @default_options = DEFAULT_OPTIONS.merge(o).freeze
 
-    # do this last
-    self.haystack = competitors
-  end
-  
-  def groupings=(ary)
-    @groupings = ary.map { |regexp| Rule::Grouping.new regexp }
-  end
-  
-  def identities=(ary)
-    @identities = ary.map { |regexp| Rule::Identity.new regexp }
-  end
-  
-  def normalizers=(ary)
-    @normalizers = ary.map { |regexp| Rule::Normalizer.new regexp }
-  end
-  
-  def stop_words=(ary)
-    @stop_words = ary.map { |regexp| Rule::StopWord.new regexp }
-  end
-  
-  def haystack=(ary)
-    @haystack = ary.map { |competitor| Wrapper.new self, competitor }
+    @haystack = haystack.map { |original| Record.new original, stop_words: @stop_words, read: @read }
   end
-  
+    
   def last_result
-    @last_result || raise(::RuntimeError, "[fuzzy_match] You can't access the last result until you've run a find with :gather_last_result => true")
+    @last_result or raise("You can't access the last result until you've run a find with :gather_last_result => true")
   end
   
+  # Return everything in sorted order
   def find_all(needle, options = {})
     options = options.merge(:find_all => true)
     find needle, options
   end
 
+  # Return the top results with the same score
   def find_best(needle, options = {})
     options = options.merge(:find_best => true)
     find needle, options
   end
 
+  # Return everything in sorted order with score
   def find_all_with_score(needle, options = {})
     options = options.merge(:find_all_with_score => true)
     find needle, options
   end
+
+  # Return one with score
+  def find_with_score(needle, options = {})
+    options = options.merge(:find_with_score => true)
+    find needle, options
+  end
   
   def find(needle, options = {})
     options = default_options.merge options
@@ -137,9 +117,9 @@ class FuzzyMatch
     threshold = options[:threshold]
     gather_last_result = options[:gather_last_result]
     is_find_all_with_score = options[:find_all_with_score]
+    is_find_with_score = options[:find_with_score]
     is_find_best = options[:find_best]
     is_find_all = options[:find_all] || is_find_all_with_score || is_find_best
-    first_grouping_decides = options[:first_grouping_decides]
     must_match_grouping = options[:must_match_grouping]
     must_match_at_least_one_word = options[:must_match_at_least_one_word]
     
@@ -148,37 +128,38 @@ class FuzzyMatch
       last_result.read = read
       last_result.haystack = haystack
       last_result.options = options
-      last_result.timeline << <<-EOS
-Options were set, either by you or by falling back to defaults.
-\tOptions: #{options.inspect}
-EOS
     end
     
     if gather_last_result
-      last_result.normalizers = normalizers
       last_result.identities = identities
       last_result.groupings = groupings
       last_result.stop_words = stop_words
     end
     
-    needle = Wrapper.new self, needle, true
+    needle = Record.new needle
     
     if gather_last_result
       last_result.needle = needle
-      last_result.timeline << <<-EOS
-The needle's #{needle.variants.length} variants were enumerated.
-\tVariants: #{needle.variants.map(&:inspect).join(', ')}
-EOS
     end
-    
-    if must_match_grouping and groupings.any? and groupings.none? { |grouping| grouping.match? needle }
+
+    if groupings.any?
+      first_grouping = groupings.detect { |grouping| grouping.xmatch? needle }
+      if gather_last_result
+        if first_grouping
+          last_result.timeline << "Grouping: #{first_grouping.inspect}"
+        else
+          last_result.timeline << "No grouping."
+        end
+      end
+    end
+
+    if must_match_grouping and not first_grouping
       if gather_last_result
         last_result.timeline << <<-EOS
-The needle didn't match any of the #{groupings.length} grouping, which was a requirement.
-\tGroupings (first 3): #{groupings[0,3].map(&:inspect).join(', ')}
+The needle didn't match any of the #{groupings.length} groupings, which was a requirement.
+\t#{groupings.map(&:inspect).join("\n\t")}
 EOS
       end
-      
       if is_find_all
         return []
       else
@@ -186,38 +167,39 @@ EOS
       end
     end
 
+    if groupings.any? and not first_grouping
+      passed_grouping_requirement = haystack.reject do |straw|
+        groupings.any? { |grouping| grouping.xmatch? straw }
+      end
+    else
+      passed_grouping_requirement = haystack
+    end
+
     if must_match_at_least_one_word
-      passed_word_requirement = haystack.select do |straw|
+      passed_word_requirement = passed_grouping_requirement.select do |straw|
         (needle.words & straw.words).any?
       end
       if gather_last_result
         last_result.timeline << <<-EOS
 Since :must_match_at_least_one_word => true, the competition was reduced to records sharing at least one word with the needle.
 \tNeedle words: #{needle.words.map(&:inspect).join(', ')}
-\tPassed (first 3): #{passed_word_requirement[0,3].map(&:render).map(&:inspect).join(', ')}
-\tFailed (first 3): #{(haystack-passed_word_requirement)[0,3].map(&:render).map(&:inspect).join(', ')}
+\tPassed (first 3): #{passed_word_requirement[0,3].map(&:inspect).join(', ')}
+\tFailed (first 3): #{(passed_grouping_requirement-passed_word_requirement)[0,3].map(&:inspect).join(', ')}
 EOS
       end
     else
-      passed_word_requirement = haystack
+      passed_word_requirement = passed_grouping_requirement
     end
-    
-    if groupings.any?
+        
+    if first_grouping
       joint = passed_word_requirement.select do |straw|
-        if first_grouping_decides
-          if first_grouping = groupings.detect { |grouping| grouping.match? needle }
-            first_grouping.join? needle, straw
-          end
-        else
-          groupings.any? { |grouping| grouping.join? needle, straw }
-        end
+        first_grouping.xjoin? needle, straw
       end
+      # binding.pry      
       if gather_last_result
         last_result.timeline << <<-EOS
-Since there were groupings, the competition was reduced to records in the same group as the needle.
-\tGroupings (first 3): #{groupings[0,3].map(&:inspect).join(', ')}
-\tPassed (first 3): #{joint[0,3].map(&:render).map(&:inspect).join(', ')}
-\tFailed (first 3): #{(passed_word_requirement-joint)[0,3].map(&:render).map(&:inspect).join(', ')}
+Since there were groupings, the competition was reduced to #{joint.length} records in the same group as the needle.
+\t#{joint.map(&:inspect).join("\n\t")}
 EOS
       end
     else
@@ -252,8 +234,8 @@ EOS
         last_result.timeline << <<-EOS
 Since there were identities, the competition was reduced to records that might be identical to the needle (in other words, are not certainly different)
 \tIdentities (first 10 of #{identities.length}): #{identities[0,9].map(&:inspect).join(', ')}
-\tPassed (first 10 of #{possibly_identical.length}): #{possibly_identical[0,9].map(&:render).map(&:inspect).join(', ')}
-\tFailed (first 10 of #{(joint-possibly_identical).length}): #{(joint-possibly_identical)[0,9].map(&:render).map(&:inspect).join(', ')}
+\tPassed (first 10 of #{possibly_identical.length}): #{possibly_identical[0,9].map(&:inspect).join(', ')}
+\tFailed (first 10 of #{(joint-possibly_identical).length}): #{(joint-possibly_identical)[0,9].map(&:inspect).join(', ')}
 EOS
       end
     else
@@ -265,7 +247,7 @@ EOS
     if gather_last_result
       last_result.timeline << <<-EOS
 The competition was sorted in order of similarity to the needle.
-\tSimilar (first 10 of #{similarities.length}): #{similarities[0,9].map { |s| "#{s.wrapper2.render.inspect} (#{[s.best_score.dices_coefficient_similar, s.best_score.levenshtein_similar].map { |v| '%0.5f' % v }.join('/')})" }.join(', ')}
+\t#{similarities[0,9].map { |s| "#{s.record2.similarity(needle).inspect}" }.join("\n\t")}
 EOS
     end
     
@@ -274,7 +256,7 @@ EOS
       similarities.each do |similarity|
         if similarity.satisfy?(needle, threshold)
           bs = similarity.best_score
-          memo << [similarity.wrapper2.record, bs.dices_coefficient_similar, bs.levenshtein_similar]
+          memo << [similarity.record2.original, bs.dices_coefficient_similar, bs.levenshtein_similar]
         end
       end
       return memo
@@ -288,7 +270,7 @@ EOS
           bs = similarity.best_score
           best_bs ||= bs
           if bs >= best_bs
-            memo << similarity.wrapper2.record
+            memo << similarity.record2.original
           else
             break
           end
@@ -301,7 +283,7 @@ EOS
       memo = []
       similarities.each do |similarity|
         if similarity.satisfy?(needle, threshold)
-          memo << similarity.wrapper2.record
+          memo << similarity.record2.original
         end
       end
       return memo
@@ -311,24 +293,30 @@ EOS
     winner = nil
 
     if best_similarity and best_similarity.satisfy?(needle, threshold)
-      winner = best_similarity.wrapper2.record
+      winner = best_similarity.record2.original
       if gather_last_result
         last_result.winner = winner
         last_result.score = best_similarity.best_score.dices_coefficient_similar
         last_result.timeline << <<-EOS
-A winner was determined because the Dice's Coefficient similarity (#{best_similarity.best_score.dices_coefficient_similar}) is greater than zero or because it shared a word with the needle.
+A winner was determined because the Dice's Coefficient similarity (#{'%0.5f' % best_similarity.best_score.dices_coefficient_similar}) is greater than zero or because it shared a word with the needle.
 EOS
       end
+      if is_find_with_score
+        bs = best_similarity.best_score
+        return [winner, bs.dices_coefficient_similar, bs.levenshtein_similar]
+      else
+        return winner
+      end
     elsif gather_last_result
-      best_similarity_record = if best_similarity and best_similarity.wrapper2
-        best_similarity.wrapper2.record
+      best_similarity_record = if best_similarity and best_similarity.record2
+        best_similarity.record2.original
       end
       last_result.timeline << <<-EOS
 No winner assigned because the score of the best similarity (#{best_similarity_record.inspect}) was zero and it didn't match any words with the needle (#{needle.inspect}).
 EOS
     end
-    
-    winner
+
+    nil # ugly
   end
   
   # Explain is like mysql's EXPLAIN command. You give it a needle and it tells you about how it was located (successfully or not) in the haystack.
@@ -339,8 +327,4 @@ EOS
     find needle, options.merge(:gather_last_result => true)
     last_result.explain
   end
-
-  # DEPRECATED
-  def free
-  end
 end

data/{test/test_amatch.rb → spec/amatch_spec.rb}

@@ -1,6 +1,5 @@
 unless RUBY_PLATFORM == 'java'
-  require 'helper'
-  require 'test_fuzzy_match'
+  require 'spec_helper'
   require 'amatch'
   
   describe FuzzyMatch do

data/{test/test_cache.rb → spec/cache_spec.rb}

@@ -1,4 +1,4 @@
-require 'helper'
+require 'spec_helper'
 
 require 'active_record'
 require 'cohort_analysis'
@@ -102,31 +102,31 @@ end
 describe FuzzyMatch::CachedResult do
   it %{joins aircraft to flight segments} do
     aircraft = Aircraft.find('B742')
-    aircraft.flight_segments.count.must_equal 2
+    aircraft.flight_segments.count.should == 2
   end
   
   it %{allow simple SQL operations} do
     aircraft = Aircraft.find('B742')
-    aircraft.flight_segments.sum(:passengers).must_equal 110
+    aircraft.flight_segments.sum(:passengers).should == 110
   end
   
   it %{works with weighted_average} do
     aircraft = Aircraft.find('B742')
-    aircraft.flight_segments.weighted_average(:seats, :weighted_by => :passengers).must_equal 5.45455
+    aircraft.flight_segments.weighted_average(:seats, :weighted_by => :passengers).should == 5.45455
   end
   
   it %{works with cohort_scope (albeit rather clumsily)} do
     aircraft = Aircraft.find('B742')
     cohort = FlightSegment.cohort({:aircraft_description => aircraft.flight_segments_foreign_keys}, :minimum_size => 2)
-    FlightSegment.connection.select_value(cohort.project('COUNT(*)').to_sql).must_equal 2
-    # FlightSegment.cohort(:aircraft_description => aircraft.flight_segments_foreign_keys).must_equal []
+    FlightSegment.connection.select_value(cohort.project('COUNT(*)').to_sql).should == 2
+    # FlightSegment.cohort(:aircraft_description => aircraft.flight_segments_foreign_keys).should == []
   end
   
   # def test_006_you_can_get_aircraft_from_flight_segments
   #   fs = FlightSegment.first
   #   # you need to add an aircraft_description column
   #   lambda do
-  #     fs.aircraft.count.must_equal 2
+  #     fs.aircraft.count.should == 2
   #   end.must_raise ActiveRecord::StatementInvalid
   # end
 end

data/spec/foo.rb

@@ -0,0 +1,9 @@
+require 'fileutils'
+Dir['test*.rb'].each do |f|
+  n = File.basename(f, '.rb')
+  n.sub! 'test_', ''
+  n += '_spec.rb'
+  puts f
+  puts n
+  FileUtils.cp f, n
+end