fuzzy_match 1.0.5

data/lib/fuzzy_match/identity.rb

@@ -0,0 +1,23 @@
+class FuzzyMatch
+  # Identities take effect when needle and haystack both match a regexp
+  # Then the captured part of the regexp has to match exactly
+  class Identity
+    attr_reader :regexp
+    
+    def initialize(regexp_or_str)
+      @regexp = regexp_or_str.to_regexp
+    end
+    
+    # Two strings are "identical" if they both match this identity and the captures are equal.
+    #
+    # Only returns true/false if both strings match the regexp.
+    # Otherwise returns nil.
+    def identical?(str1, str2)
+      if str1_match_data = regexp.match(str1) and match_data = regexp.match(str2)
+        str1_match_data.captures == match_data.captures
+      else
+        nil
+      end
+    end
+  end
+end

data/lib/fuzzy_match/result.rb

@@ -0,0 +1,17 @@
+class FuzzyMatch
+  class Result #:nodoc: all
+    attr_accessor :needle
+    attr_accessor :tighteners
+    attr_accessor :blockings
+    attr_accessor :identities
+    attr_accessor :stop_words
+    attr_accessor :candidates
+    attr_accessor :joint
+    attr_accessor :disjoint
+    attr_accessor :possibly_identical
+    attr_accessor :certainly_different
+    attr_accessor :similarities
+    attr_accessor :record
+    attr_accessor :score
+  end
+end

data/lib/fuzzy_match/score.rb

@@ -0,0 +1,125 @@
+begin
+  require 'amatch'
+rescue ::LoadError
+  # using native ruby similarity scoring
+end
+
+class FuzzyMatch
+  class Score
+    attr_reader :str1, :str2
+
+    def initialize(str1, str2)
+      @str1 = str1.downcase
+      @str2 = str2.downcase
+    end
+    
+    def inspect
+      %{#<Score: dices_coefficient=#{dices_coefficient} levenshtein=#{levenshtein}>}
+    end
+    
+    def <=>(other)
+      by_dices_coefficient = (dices_coefficient <=> other.dices_coefficient)
+      if by_dices_coefficient == 0
+        levenshtein <=> other.levenshtein
+      else
+        by_dices_coefficient
+      end
+    end
+        
+    def utf8?
+      (defined?(::Encoding) ? str1.encoding.to_s : $KCODE).downcase.start_with?('u')
+    end
+    
+    if defined?(::Amatch)
+      
+      def dices_coefficient
+        str1.pair_distance_similar str2
+      end
+      
+      def levenshtein
+        str1.levenshtein_similar str2
+      end
+      
+    else
+      
+      SPACE = ' '
+      # http://stackoverflow.com/questions/653157/a-better-similarity-ranking-algorithm-for-variable-length-strings
+      def dices_coefficient
+        if str1 == str2
+          return 1.0
+        elsif str1.length == 1 and str2.length == 1
+          return 0.0
+        end
+        pairs1 = (0..str1.length-2).map do |i|
+          str1[i,2]
+        end.reject do |pair|
+          pair.include? SPACE
+        end
+        pairs2 = (0..str2.length-2).map do |i|
+          str2[i,2]
+        end.reject do |pair|
+          pair.include? SPACE
+        end
+        union = pairs1.size + pairs2.size 
+        intersection = 0 
+        pairs1.each do |p1| 
+          0.upto(pairs2.size-1) do |i| 
+            if p1 == pairs2[i] 
+              intersection += 1 
+              pairs2.slice!(i) 
+              break 
+            end 
+          end 
+        end 
+        (2.0 * intersection) / union
+      end
+
+      # extracted/adapted from the text gem version 1.0.2
+      # normalization added for utf-8 strings
+      # lib/text/levenshtein.rb
+      def levenshtein
+        if utf8?
+          unpack_rule = 'U*'
+        else
+          unpack_rule = 'C*'
+        end
+        s = str1.unpack(unpack_rule)
+        t = str2.unpack(unpack_rule)
+        n = s.length
+        m = t.length
+        if n == 0 or m == 0
+          return 0.0
+        end
+        d = (0..m).to_a
+        x = nil
+        (0...n).each do |i|
+          e = i+1
+          (0...m).each do |j|
+            cost = (s[i] == t[j]) ? 0 : 1
+            x = [
+              d[j+1] + 1, # insertion
+              e + 1,      # deletion
+              d[j] + cost # substitution
+            ].min
+            d[j] = e
+            e = x
+          end
+          d[m] = x
+        end
+        # normalization logic from https://github.com/flori/amatch/blob/master/ext/amatch_ext.c#L301
+        # if (b_len > a_len) {
+        #     result = rb_float_new(1.0 - ((double) v[p][b_len]) / b_len);
+        # } else {
+        #     result = rb_float_new(1.0 - ((double) v[p][b_len]) / a_len);
+        # }
+        1.0 - x.to_f / [n, m].max
+      end
+      
+    end
+    
+    extend ::ActiveSupport::Memoizable
+    memoize :dices_coefficient
+    memoize :levenshtein
+    memoize :utf8?
+  end
+end

data/lib/fuzzy_match/similarity.rb

@@ -0,0 +1,53 @@
+class FuzzyMatch
+  class Similarity
+    attr_reader :wrapper1
+    attr_reader :wrapper2
+    
+    def initialize(wrapper1, wrapper2)
+      @wrapper1 = wrapper1
+      @wrapper2 = wrapper2
+    end
+    
+    def <=>(other)
+      by_score = best_score <=> other.best_score
+      if by_score == 0
+        original_weight <=> other.original_weight
+      else
+        by_score
+      end
+    end
+    
+    # Weight things towards short original strings
+    def original_weight
+      @original_weight ||= (1.0 / (wrapper1.render.length * wrapper2.render.length))
+    end
+    
+    def best_score
+      @best_score ||= Score.new best_wrapper1_variant, best_wrapper2_variant
+    end
+    
+    def best_wrapper1_variant
+      best_variants[0]
+    end
+    
+    def best_wrapper2_variant
+      best_variants[1]
+    end
+    
+    def best_variants
+      @best_variants ||= wrapper1.variants.product(wrapper2.variants).sort do |tuple1, tuple2|
+        wrapper1_variant1, wrapper2_variant1 = tuple1
+        wrapper1_variant2, wrapper2_variant2 = tuple2
+
+        score1 = Score.new wrapper1_variant1, wrapper2_variant1
+        score2 = Score.new wrapper1_variant2, wrapper2_variant2
+        
+        score1 <=> score2
+      end[-1]
+    end
+    
+    def inspect
+      %{#<Similarity "#{wrapper2.render}"=>"#{best_wrapper2_variant}" versus "#{wrapper1.render}"=>"#{best_wrapper1_variant}" original_weight=#{"%0.5f" % original_weight} best_score=#{best_score.inspect}>}
+    end
+  end
+end

data/lib/fuzzy_match/stop_word.rb

@@ -0,0 +1,19 @@
+class FuzzyMatch
+  # A stop word is ignored
+  class StopWord
+    attr_reader :regexp
+    
+    def initialize(regexp_or_str)
+      @regexp = regexp_or_str.to_regexp
+    end
+    
+    # Destructively remove stop words from the string
+    def apply!(str)
+      str.gsub! regexp, ''
+    end
+    
+    def inspect
+      "#<StopWord regexp=#{regexp.inspect}>"
+    end
+  end
+end

data/lib/fuzzy_match/tightener.rb

@@ -0,0 +1,28 @@
+class FuzzyMatch
+  # A tightener just strips a string down to its core
+  class Tightener
+    attr_reader :regexp
+    
+    def initialize(regexp_or_str)
+      @regexp = regexp_or_str.to_regexp
+    end
+    
+    # A tightener applies when its regexp matches and captures a new (shorter) string
+    def apply?(str)
+      !!(regexp.match(str))
+    end
+    
+    # The result of applying a tightener is just all the captures put together.
+    def apply(str)
+      if match_data = regexp.match(str)
+        match_data.captures.join
+      else
+        str
+      end
+    end
+    
+    def inspect
+      "#<Tightener regexp=#{regexp.inspect}>"
+    end
+  end
+end

data/lib/fuzzy_match/version.rb

@@ -0,0 +1,3 @@
+class FuzzyMatch
+  VERSION = '1.0.5'
+end

data/lib/fuzzy_match/wrapper.rb

@@ -0,0 +1,67 @@
+class FuzzyMatch
+  # Wrappers are the tokens that are passed around when doing scoring and optimizing.
+  class Wrapper #:nodoc: all
+    attr_reader :fuzzy_match
+    attr_reader :record
+    attr_reader :read
+
+    def initialize(fuzzy_match, record, read = nil)
+      @fuzzy_match = fuzzy_match
+      @record = record
+      @read = read
+    end
+
+    def inspect
+      "#<Wrapper render=#{render} variants=#{variants.length}>"
+    end
+
+    def render
+      return @render if rendered?
+      str = case read
+      when ::Proc
+        read.call record
+      when ::Symbol
+        if record.respond_to?(read)
+          record.send read
+        else
+          record[read]
+        end
+      when ::NilClass
+        record
+      else
+        record[read]
+      end.to_s.dup
+      fuzzy_match.stop_words.each do |stop_word|
+        stop_word.apply! str
+      end
+      str.strip!
+      @render = str.freeze
+      @rendered = true
+      @render
+    end
+
+    alias :to_str :render
+
+    WORD_BOUNDARY = %r{\s*\b\s*}
+    def words
+      @words ||= render.split(WORD_BOUNDARY)
+    end
+
+    def similarity(other)
+      Similarity.new self, other
+    end
+
+    def variants
+      @variants ||= fuzzy_match.tighteners.inject([ render ]) do |memo, tightener|
+        if tightener.apply? render
+          memo.push tightener.apply(render)
+        end
+        memo
+      end.uniq
+    end
+    
+    def rendered?
+      @rendered == true
+    end
+  end
+end

data/lib/fuzzy_match.rb

@@ -0,0 +1,252 @@
+require 'active_support'
+require 'active_support/version'
+if ::ActiveSupport::VERSION::MAJOR >= 3
+  require 'active_support/core_ext'
+end
+require 'to_regexp'
+
+# See the README for more information.
+class FuzzyMatch
+  autoload :Tightener, 'fuzzy_match/tightener'
+  autoload :StopWord, 'fuzzy_match/stop_word'
+  autoload :Blocking, 'fuzzy_match/blocking'
+  autoload :Identity, 'fuzzy_match/identity'
+  autoload :Result, 'fuzzy_match/result'
+  autoload :Wrapper, 'fuzzy_match/wrapper'
+  autoload :Similarity, 'fuzzy_match/similarity'
+  autoload :Score, 'fuzzy_match/score'
+  autoload :CachedResult, 'fuzzy_match/cached_result'
+  
+  attr_reader :haystack
+  attr_reader :blockings
+  attr_reader :identities
+  attr_reader :tighteners
+  attr_reader :stop_words
+  attr_reader :default_first_blocking_decides
+  attr_reader :default_must_match_blocking
+  attr_reader :default_must_match_at_least_one_word
+
+  # haystack - a bunch of records
+  # options
+  # * tighteners: regexps (see readme)
+  # * identities: regexps
+  # * blockings: regexps
+  # * stop_words: regexps
+  # * read: how to interpret each entry in the 'haystack', either a Proc or a symbol
+  def initialize(records, options = {})
+    options = options.symbolize_keys
+    @default_first_blocking_decides = options[:first_blocking_decides]
+    @default_must_match_blocking = options[:must_match_blocking]
+    @default_must_match_at_least_one_word = options[:must_match_at_least_one_word]
+    @blockings = options.fetch(:blockings, []).map { |regexp_or_str| Blocking.new regexp_or_str }
+    @identities = options.fetch(:identities, []).map { |regexp_or_str| Identity.new regexp_or_str }
+    @tighteners = options.fetch(:tighteners, []).map { |regexp_or_str| Tightener.new regexp_or_str }
+    @stop_words = options.fetch(:stop_words, []).map { |regexp_or_str| StopWord.new regexp_or_str }
+    read = options[:read] || options[:haystack_reader]
+    @haystack = records.map { |record| Wrapper.new self, record, read }
+  end
+  
+  def last_result
+    @last_result || raise(::RuntimeError, "[fuzzy_match] You can't access the last result until you've run a find with :gather_last_result => true")
+  end
+  
+  def find_all(needle, options = {})
+    options = options.symbolize_keys.merge(:find_all => true)
+    find needle, options
+  end
+  
+  def find(needle, options = {})
+    raise ::RuntimeError, "[fuzzy_match] Dictionary has already been freed, can't perform more finds" if freed?
+    
+    options = options.symbolize_keys
+    gather_last_result = options.fetch(:gather_last_result, false)
+    is_find_all = options.fetch(:find_all, false)
+    first_blocking_decides = options.fetch(:first_blocking_decides, default_first_blocking_decides)
+    must_match_blocking = options.fetch(:must_match_blocking, default_must_match_blocking)
+    must_match_at_least_one_word = options.fetch(:must_match_at_least_one_word, default_must_match_at_least_one_word)
+    
+    if gather_last_result
+      free_last_result
+      @last_result = Result.new
+    end
+    
+    if gather_last_result
+      last_result.tighteners = tighteners
+      last_result.identities = identities
+      last_result.blockings = blockings
+      last_result.stop_words = stop_words
+    end
+    
+    needle = Wrapper.new self, needle
+    
+    if gather_last_result
+      last_result.needle = needle
+    end
+    
+    if must_match_blocking and blockings.any? and blockings.none? { |blocking| blocking.match? needle }
+      if is_find_all
+        return []
+      else
+        return nil
+      end
+    end
+
+    candidates = if must_match_at_least_one_word
+      haystack.select do |straw|
+        (needle.words & straw.words).any?
+      end
+    else
+      haystack
+    end
+    
+    if gather_last_result
+      last_result.candidates = candidates
+    end
+    
+    joint, disjoint = if blockings.any?
+      candidates.partition do |straw|
+        if first_blocking_decides
+          blockings.detect { |blocking| blocking.match? needle }.try :join?, needle, straw
+        else
+          blockings.any? { |blocking| blocking.join? needle, straw }
+        end
+      end
+    else
+      [ candidates.dup, [] ]
+    end
+    
+    if joint.none?
+      if must_match_blocking
+        if is_find_all
+          return []
+        else
+          return nil
+        end
+      else
+        # special case: the needle didn't fit anywhere, but must_match_blocking is false, so we'll try it against everything
+        joint = disjoint
+        disjoint = []
+      end
+    end
+    
+    if gather_last_result
+      last_result.joint = joint
+      last_result.disjoint = disjoint
+    end
+    
+    possibly_identical, certainly_different = if identities.any?
+      joint.partition do |straw|
+        identities.all? do |identity|
+          answer = identity.identical? needle, straw
+          answer.nil? or answer == true
+        end
+      end
+    else
+      [ joint.dup, [] ]
+    end
+    
+    if gather_last_result
+      last_result.possibly_identical = possibly_identical
+      last_result.certainly_different = certainly_different
+    end
+    
+    if is_find_all
+      return possibly_identical.map { |straw| straw.record }
+    end
+    
+    similarities = possibly_identical.map { |straw| needle.similarity straw }.sort
+    
+    if gather_last_result
+      last_result.similarities = similarities
+    end
+    
+    if best_similarity = similarities[-1] and best_similarity.best_score.dices_coefficient > 0
+      record = best_similarity.wrapper2.record
+      if gather_last_result
+        last_result.record = record
+        last_result.score = best_similarity.best_score.dices_coefficient
+      end
+      record
+    end
+  end
+  
+  # Explain is like mysql's EXPLAIN command. You give it a needle and it tells you about how it was located (successfully or not) in the haystack.
+  #
+  #     d = FuzzyMatch.new ['737', '747', '757' ]
+  #     d.explain 'boeing 737-100'
+  def explain(needle, options = {})
+    record = find needle, options.merge(:gather_last_result => true)
+    log "#" * 150
+    log "# Match #{needle.inspect} => #{record.inspect}"
+    log "#" * 150
+    log
+    log "Needle"
+    log "-" * 150
+    log last_result.needle.render
+    log
+    log "Stop words"
+    log last_result.stop_words.blank? ? '(none)' : last_result.stop_words.map { |stop_word| stop_word.inspect }.join("\n")
+    log
+    log "Candidates"
+    log "-" * 150
+    log last_result.candidates.map { |record| record.render }.join("\n")
+    log
+    log "Tighteners"
+    log "-" * 150
+    log last_result.tighteners.blank? ? '(none)' : last_result.tighteners.map { |tightener| tightener.inspect }.join("\n")
+    log
+    log "Blockings"
+    log "-" * 150
+    log last_result.blockings.blank? ? '(none)' : last_result.blockings.map { |blocking| blocking.inspect }.join("\n")
+    log
+    log "Identities"
+    log "-" * 150
+    log last_result.identities.blank? ? '(none)' : last_result.identities.map { |blocking| blocking.inspect }.join("\n")
+    log
+    log "Joint"
+    log "-" * 150
+    log last_result.joint.blank? ? '(none)' : last_result.joint.map { |joint| joint.render }.join("\n")
+    log
+    log "Disjoint"
+    log "-" * 150
+    log last_result.disjoint.blank? ? '(none)' : last_result.disjoint.map { |disjoint| disjoint.render }.join("\n")
+    log
+    log "Possibly identical"
+    log "-" * 150
+    log last_result.possibly_identical.blank? ? '(none)' : last_result.possibly_identical.map { |possibly_identical| possibly_identical.render }.join("\n")
+    log
+    log "Certainly different"
+    log "-" * 150
+    log last_result.certainly_different.blank? ? '(none)' : last_result.certainly_different.map { |certainly_different| certainly_different.render }.join("\n")
+    log
+    log "Similarities"
+    log "-" * 150
+    log last_result.similarities.blank? ? '(none)' : last_result.similarities.reverse[0..9].map { |similarity| similarity.inspect }.join("\n")
+    log
+    log "Match"
+    log "-" * 150
+    log record.inspect
+  end
+  
+  def log(str = '') #:nodoc:
+    $stderr.puts str
+  end
+    
+  def freed?
+    @freed == true
+  end
+  
+  def free
+    free_last_result
+    @haystack.try :clear
+    @haystack = nil
+  ensure
+    @freed = true
+  end
+  
+  private
+  
+  def free_last_result
+    @last_result = nil    
+  end
+end

data/test/helper.rb

@@ -0,0 +1,12 @@
+require 'rubygems'
+require 'bundler'
+Bundler.setup
+require 'test/unit'
+require 'stringio'
+require 'remote_table'
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'fuzzy_match'
+
+class Test::Unit::TestCase
+end

data/test/test_blocking.rb

@@ -0,0 +1,23 @@
+require 'helper'
+
+class TestBlocking < Test::Unit::TestCase
+  def test_001_match_one
+    b = FuzzyMatch::Blocking.new %r{apple}
+    assert_equal true, b.match?('2 apples')
+  end
+  
+  def test_002_join_both
+    b = FuzzyMatch::Blocking.new %r{apple}
+    assert_equal true, b.join?('apple', '2 apples')    
+  end
+  
+  def test_002_doesnt_join_both
+    b = FuzzyMatch::Blocking.new %r{apple}
+    assert_equal false, b.join?('orange', '2 apples')
+  end
+  
+  def test_003_no_information
+    b = FuzzyMatch::Blocking.new %r{apple}
+    assert_equal nil, b.join?('orange', 'orange')
+  end
+end