loose_tight_dictionary 0.0.10 → 0.1.0

data/examples/bts_aircraft/tighteners.csv

@@ -0,0 +1 @@
+regexp,notes

data/examples/first_name_matching.rb

@@ -1,23 +1,15 @@
 #!/usr/bin/env ruby
-require 'rubygems'
-# require 'loose_tight_dictionary'
-require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'loose_tight_dictionary.rb'))
-right_side = [ 'seamus', 'andy', 'ben' ]
-left_side = [ 'Mr. Seamus', 'Sr. Andy', 'Master BenT' ]
-
-puts "Left side (input)"
-puts "=" * 20
-puts left_side
-puts
-
-puts "Right side (output)"
-puts "=" * 20
-puts right_side
-puts
-
-puts "Results"
-puts "=" * 20
-d = LooseTightDictionary.new right_side, :tee => STDOUT, :tee_format => :fixed_width
-d.check left_side
-
-puts d.left_to_right 'Shamus Heaney'
+unless RUBY_VERSION >= '1.9'
+  require 'rubygems'
+end
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'loose_tight_dictionary'
+
+haystack = [ 'seamus', 'andy', 'ben' ]
+needles = [ 'Mr. Seamus', 'Sr. Andy', 'Master BenT', 'Shamus Heaney' ]
+
+d = LooseTightDictionary.new haystack, :log => $stdout
+needles.each do |needle|
+  d.explain needle
+  puts
+end

data/lib/loose_tight_dictionary/blocking.rb

@@ -0,0 +1,36 @@
+class LooseTightDictionary
+  # "Record linkage typically involves two main steps: blocking and scoring..."
+  # http://en.wikipedia.org/wiki/Record_linkage
+  #
+  # Blockings effectively divide up the haystack into groups that match a pattern
+  #
+  # A blocking (as in a grouping) comes into effect when a str matches.
+  # Then the needle must also match the blocking's regexp.
+  class Blocking
+    include ExtractRegexp
+    
+    attr_reader :regexp
+    
+    def initialize(regexp_or_str)
+      @regexp = extract_regexp regexp_or_str
+    end
+
+    # If a blocking "encompasses" two strings, that means they both fit into it.
+    #
+    # Returns false if they certainly don't fit this blocking.
+    # Returns nil if the blocking doesn't apply, i.e. str2 doesn't fit the blocking.
+    def encompass?(str1, str2 = nil)
+      if str2.nil?
+        !!(regexp.match(str1))
+      elsif str2_match_data = regexp.match(str2)
+        if str1_match_data = regexp.match(str1)
+          str2_match_data.captures == str1_match_data.captures
+        else
+          false
+        end
+      else
+        nil
+      end
+    end
+  end
+end

data/lib/loose_tight_dictionary/extract_regexp.rb

@@ -0,0 +1,30 @@
+class LooseTightDictionary
+  module ExtractRegexp #:nodoc: all
+    def extract_regexp(regexp_or_str)
+      case regexp_or_str
+      when ::Regexp
+        regexp_or_str
+      when ::String
+        regexp_from_string regexp_or_str
+      else
+        raise ::ArgumentError, "Expected regexp or string"
+      end
+    end
+    
+    REGEXP_DELIMITERS = {
+      '%r{' => '}',
+      '/' => '/'
+    }
+    def regexp_from_string(str)
+      delim_start, delim_end = REGEXP_DELIMITERS.detect { |k, v| str.start_with? k }.map { |delim| ::Regexp.escape delim }
+      %r{\A#{delim_start}(.*)#{delim_end}([^#{delim_end}]*)\z} =~ str.strip
+      content = $1
+      options = $2
+      content.gsub! '\\/', '/'
+      ignore_case = options.include?('i') ? ::Regexp::IGNORECASE : nil
+      multiline = options.include?('m') ? ::Regexp::MULTILINE : nil
+      extended = options.include?('x') ? ::Regexp::EXTENDED : nil
+      ::Regexp.new content, (ignore_case||multiline||extended)
+    end
+  end
+end

data/lib/loose_tight_dictionary/identity.rb

@@ -0,0 +1,25 @@
+class LooseTightDictionary
+  # Identities take effect when needle and haystack both match a regexp
+  # Then the captured part of the regexp has to match exactly
+  class Identity
+    include ExtractRegexp
+    
+    attr_reader :regexp
+    
+    def initialize(regexp_or_str)
+      @regexp = extract_regexp regexp_or_str
+    end
+    
+    # Two strings are "identical" if they both match this identity and the captures are equal.
+    #
+    # Only returns true/false if both strings match the regexp.
+    # Otherwise returns nil.
+    def identical?(str1, str2)
+      if str1_match_data = regexp.match(str1) and match_data = regexp.match(str2)
+        str1_match_data.captures == match_data.captures
+      else
+        nil
+      end
+    end
+  end
+end

data/lib/loose_tight_dictionary/result.rb

@@ -0,0 +1,23 @@
+class LooseTightDictionary
+  class Result #:nodoc: all
+    attr_accessor :needle
+    attr_accessor :tighteners
+    attr_accessor :blockings
+    attr_accessor :identities
+    attr_accessor :encompassed
+    attr_accessor :unencompassed
+    attr_accessor :possibly_identical
+    attr_accessor :certainly_different
+    attr_accessor :similarities
+    attr_accessor :record
+    attr_accessor :score
+    
+    def haystack
+      encompassed + unencompassed
+    end
+    
+    def free
+      # nothing to see here
+    end
+  end
+end

data/lib/loose_tight_dictionary/score.rb

@@ -0,0 +1,28 @@
+require 'amatch'
+
+class LooseTightDictionary
+  class Score
+    attr_reader :str1, :str2
+
+    def initialize(str1, str2)
+      @str1 = str1
+      @str2 = str2
+    end
+    
+    def to_f
+      @to_f ||= str1.pair_distance_similar str2
+    end
+    
+    def inspect
+      %{#<Score: to_f=#{to_f}>}
+    end
+    
+    def <=>(other)
+      to_f <=> other.to_f
+    end
+    
+    def ==(other)
+      to_f == other.to_f
+    end
+  end
+end

data/lib/loose_tight_dictionary/similarity.rb

@@ -0,0 +1,62 @@
+class LooseTightDictionary
+  class Similarity
+    attr_reader :wrapper1
+    attr_reader :wrapper2
+    
+    def initialize(wrapper1, wrapper2)
+      @wrapper1 = wrapper1
+      @wrapper2 = wrapper2
+    end
+    
+    def <=>(other)
+      if best_score != other.best_score
+        best_score <=> other.best_score
+      else
+        weight <=> other.weight
+      end
+    end
+    
+    # Weight things towards short original strings
+    def weight
+      @weight ||= (1.0 / (wrapper1.to_str.length * wrapper2.to_str.length))
+    end
+    
+    def best_score
+      @best_score ||= Score.new best_wrapper1_variant, best_wrapper2_variant
+    end
+    
+    def best_wrapper1_variant
+      best_variants[0]
+    end
+    
+    def best_wrapper2_variant
+      best_variants[1]
+    end
+    
+    def best_variants
+      @best_variants ||= cart_prod(wrapper1.variants, wrapper2.variants).sort do |tuple1, tuple2|
+        wrapper1_variant1, wrapper2_variant1 = tuple1
+        wrapper1_variant2, wrapper2_variant2 = tuple2
+
+        score1 = Score.new wrapper1_variant1, wrapper2_variant1
+        score2 = Score.new wrapper1_variant2, wrapper2_variant2
+        
+        score1 <=> score2
+      end[-1]
+    end
+    
+    def inspect
+      %{#<Similarity "#{wrapper2.to_str}"=>"#{best_wrapper2_variant}" versus "#{wrapper1.to_str}"=>"#{best_wrapper1_variant}" weight=#{"%0.5f" % weight} best_score=#{"%0.5f" % best_score.to_f}>}
+    end
+    
+    # Thanks William James!
+    # http://www.ruby-forum.com/topic/95519#200484
+    def cart_prod(*args)
+      args.inject([[]]){|old,lst|
+        new = []
+        lst.each{|e| new += old.map{|c| c.dup << e }}
+        new
+      }
+    end
+  end
+end

data/lib/loose_tight_dictionary/tightener.rb

@@ -0,0 +1,30 @@
+class LooseTightDictionary
+  # A tightener just strips a string down to its core
+  class Tightener
+    include ExtractRegexp
+    
+    attr_reader :regexp
+    
+    def initialize(regexp_or_str)
+      @regexp = extract_regexp regexp_or_str
+    end
+    
+    # A tightener applies when its regexp matches and captures a new (shorter) string
+    def apply?(str)
+      !!(regexp.match(str))
+    end
+    
+    # The result of applying a tightener is just all the captures put together.
+    def apply(str)
+      if match_data = regexp.match(str)
+        match_data.captures.join
+      else
+        str
+      end
+    end
+    
+    def inspect
+      "#<Tightener regexp=#{regexp.inspect}>"
+    end
+  end
+end

data/lib/loose_tight_dictionary/version.rb

@@ -0,0 +1,3 @@
+class LooseTightDictionary
+  VERSION = '0.1.0'
+end

data/lib/loose_tight_dictionary/wrapper.rb

@@ -0,0 +1,37 @@
+class LooseTightDictionary
+  # Wrappers are the tokens that are passed around when doing scoring and optimizing.
+  class Wrapper #:nodoc: all
+    attr_reader :parent
+    attr_reader :record
+    attr_reader :reader
+
+    def initialize(attrs = {})
+      attrs.each do |k, v|
+        instance_variable_set "@#{k}", v
+      end
+    end
+
+    def inspect
+      "#<Wrapper to_str=#{to_str} variants=#{variants.length}>"
+    end
+
+    def to_str
+      @to_str ||= reader ? reader.call(record) : record.to_s
+    end
+
+    alias :to_s :to_str
+
+    def similarity(other)
+      Similarity.new self, other
+    end
+
+    def variants
+      @variants ||= parent.tighteners.inject([ to_str ]) do |memo, tightener|
+        if tightener.apply? to_str
+          memo.push tightener.apply(to_str)
+        end
+        memo
+      end.uniq
+    end
+  end
+end

data/lib/loose_tight_dictionary.rb

@@ -2,351 +2,224 @@ require 'active_support'
 require 'active_support/version'
 %w{
   active_support/core_ext/string
+  active_support/core_ext/hash
+  active_support/core_ext/object
 }.each do |active_support_3_requirement|
   require active_support_3_requirement
-end if ActiveSupport::VERSION::MAJOR == 3
-require 'amatch'
-require 'andand'
-if RUBY_VERSION >= '1.9'
-  require 'csv'
-else
-  begin
-    require 'fastercsv'
-  rescue LoadError
-    $stderr.puts "[loose_tight_dictionary gem] You probably need to manually install the fastercsv gem."
-    raise $!
-  end
-end
+end if ::ActiveSupport::VERSION::MAJOR == 3
 
+# See the README for more information.
 class LooseTightDictionary
-  class MissedChecks < RuntimeError; end
-  class Mismatch < RuntimeError; end
-  class FalsePositive < RuntimeError; end
+  autoload :ExtractRegexp, 'loose_tight_dictionary/extract_regexp'
+  autoload :Tightener, 'loose_tight_dictionary/tightener'
+  autoload :Blocking, 'loose_tight_dictionary/blocking'
+  autoload :Identity, 'loose_tight_dictionary/identity'
+  autoload :Result, 'loose_tight_dictionary/result'
+  autoload :Wrapper, 'loose_tight_dictionary/wrapper'
+  autoload :Similarity, 'loose_tight_dictionary/similarity'
+  autoload :Score, 'loose_tight_dictionary/score'
   
-  class T
-    attr_reader :str, :tightened_str
-    def initialize(str, tightened_str)
-      @str = str
-      @tightened_str = tightened_str
-    end
-    
-    def tightened?
-      str != tightened_str
-    end
-    
-    def prefix_and_score(other)
-      prefix = [ tightened_str.length, other.tightened_str.length ].min if tightened? and other.tightened?
-      score = if prefix
-        tightened_str.first(prefix).pair_distance_similar other.tightened_str.first(prefix)
-      else
-        tightened_str.pair_distance_similar other.tightened_str
-      end
-      [ prefix, score ]
-    end
-  end
-
-  include Amatch
-
-  attr_reader :right_records
-  attr_reader :case_sensitive
-
-  attr_accessor :logger
-  attr_accessor :tee
-  attr_accessor :tee_format  
-  attr_accessor :positives
-  attr_accessor :negatives
-  attr_accessor :left_reader
-  attr_accessor :right_reader
-  attr_accessor :blocking_only
+  class Freed < RuntimeError; end
+  
+  attr_reader :options
+  attr_reader :haystack
+  attr_reader :records
 
-  def initialize(right_records, options = {})
-    @right_records = right_records
-    @_raw_tightenings = options[:tightenings] || Array.new
-    @_raw_identities = options[:identities] || Array.new
-    @_raw_blockings = options[:blockings] || Array.new
-    @left_reader = options[:left_reader]
-    @right_reader = options[:right_reader]
-    @positives = options[:positives]
-    @negatives = options[:negatives]
-    @logger = options[:logger]
-    @tee = options[:tee]
-    @tee_format = options[:tee_format] || :fixed_width
-    @case_sensitive = options[:case_sensitive] || false
-    @blocking_only = options[:blocking_only] || false
+  # haystack - a bunch of records
+  # options
+  # * tighteners: regexps that essentialize strings down
+  # * identities: regexps that rule out similarities, for example a 737 cannot be identical to a 747
+  def initialize(records, options = {})
+    @options = options.symbolize_keys
+    @records = records
+    @haystack = records.map { |record| Wrapper.new :parent => self, :record => record, :reader => haystack_reader }
   end
   
-  # def tightenings
-  # def identities
-  # def blockings
-  %w{ tightenings identities blockings }.each do |name|
-    module_eval %{
-      def #{name}
-        @#{name} ||= @_raw_#{name}.map do |i|
-          next if i[0].blank?
-          literal_regexp i[0]
-        end
-      end
-    }
+  def last_result
+    @last_result ||= Result.new
   end
   
-  def blocking_only?
-    !!blocking_only
+  def log(str = '') #:nodoc:
+    (options[:log] || $stderr).puts str unless options[:log] == false
   end
-
-  def inline_check(left_record, right_record)
-    return unless positives.present? or negatives.present?
-    
-    left = read_left left_record
-    right = read_right right_record
+  
+  def find_with_score(needle)
+    record = find needle
+    [ record, last_result.score ]
+  end
+  
+  # todo fix record.record confusion (should be wrapper.record or smth)
+  def find(needle, gather_last_result = true)
+    raise Freed if freed?
+    free_last_result
     
-    if positive_record = positives.andand.detect { |record| record[0] == left }
-      correct_right = positive_record[1]
-      if correct_right.present? and right.blank?
-        logger.andand.debug "  Mismatch! (should match SOMETHING)"
-        raise Mismatch
-      elsif right != correct_right
-        logger.andand.debug "  Mismatch! (#{right} should be #{correct_right})"
-        raise Mismatch
-      end
+    if gather_last_result
+      last_result.tighteners = tighteners
+      last_result.identities = identities
+      last_result.blockings = blockings
     end
     
-    if negative_record = negatives.andand.detect { |record| record[0] == left }
-      incorrect_right = negative_record[1]
-      if incorrect_right.blank? and right.present?
-        logger.andand.debug "  False positive! (should NOT match ANYTHING)"
-        raise FalsePositive
-      elsif right == incorrect_right
-        logger.andand.debug "  False positive! (#{right} should NOT be #{incorrect_right})"
-        raise FalsePositive
-      end
-    end
-  end
-
-  def check(left_records)
-    header = [ 'Left record (input)', 'Right record (output)', 'Prefix used (if any)', 'Score' ]
-    case tee_format
-    when :csv
-      tee.andand.puts header.flatten.to_csv
-    when :fixed_width
-      tee.andand.puts header.map { |i| i.to_s.ljust(30) }.join
+    needle = Wrapper.new :parent => self, :record => needle
+    
+    if gather_last_result
+      last_result.needle = needle
     end
+    
+    return if strict_blocking and blockings.none? { |blocking| blocking.encompass? needle }
 
-    left_records.each do |left_record|
-      begin
-        right_record = left_to_right left_record
-      ensure
-        case tee_format
-        when :csv
-          tee.andand.puts $ltd_1.flatten.to_csv
-        when :fixed_width
-          tee.andand.puts $ltd_1.map { |i| i.to_s.ljust(30) }.join if $ltd_1
+    encompassed, unencompassed = if strict_blocking and blockings.any?
+      haystack.partition do |record|
+        blockings.any? do |blocking|
+          blocking.encompass?(needle, record) == true
         end
       end
+    else
+      [ haystack.dup, [] ]
     end
-  end
-  
-  def left_to_right(left_record)
-    left = read_left left_record
-    blocking_left = blocking left
-    return if blocking_only? and blocking_left.nil?
-    i_options_left = i_options left
-    t_options_left = t_options left
-    history = Hash.new
-    right_record = right_records.select do |right_record|
-      right = read_right right_record
-      blocking_right = blocking right
-      (not blocking_left and not blocking_right) or
-        (blocking_right and blocking_right.match(left)) or
-        (blocking_left and blocking_left.match(right))
-    end.max do |a_record, b_record|
-      a = read_right a_record
-      b = read_right b_record
-      i_options_a = i_options a
-      i_options_b = i_options b
-      collision_a = collision? i_options_left, i_options_a
-      collision_b = collision? i_options_left, i_options_b
-      if collision_a and collision_b
-        # neither would ever work, so randomly rank one over the other
-        rand(2) == 1 ? -1 : 1
-      elsif collision_a
-        -1
-      elsif collision_b
-        1
-      else
-        t_left_a, t_right_a = optimize t_options_left, t_options(a)
-        t_left_b, t_right_b = optimize t_options_left, t_options(b)
-        a_prefix, a_score = t_left_a.prefix_and_score t_right_a
-        b_prefix, b_score = t_left_b.prefix_and_score t_right_b
-        history[a_record] = [t_left_a.tightened_str, t_right_a.tightened_str, a_prefix ? a_prefix : 'NULL', a_score]
-        history[b_record] = [t_left_b.tightened_str, t_right_b.tightened_str, b_prefix ? b_prefix : 'NULL', b_score]
-        
-        yep_dd = ($ltd_dd_right and $ltd_dd_left and [t_left_a, t_left_b].any? { |f| f.str =~ $ltd_dd_left } and [t_right_a, t_right_b].any? { |f| f.str =~ $ltd_dd_right } and (!$ltd_dd_left_not or [t_left_a, t_left_b].none? { |f| f.str =~ $ltd_dd_left_not }))
-        
-        if $ltd_dd_print and yep_dd
-          logger.andand.debug t_left_a.inspect
-          logger.andand.debug t_right_a.inspect
-          logger.andand.debug t_left_b.inspect
-          logger.andand.debug t_right_b.inspect
-          logger.andand.debug
-        end
-
-        z = 1
-        debugger if yep_dd
-        z = 1
-        
-        if a_score != b_score
-          a_score <=> b_score
-        elsif a_prefix and b_prefix and a_prefix != b_prefix
-          a_prefix <=> b_prefix
-        else
-          b.length <=> a.length
+    
+    if gather_last_result
+      last_result.encompassed = encompassed
+      last_result.unencompassed = unencompassed
+    end
+    
+    possibly_identical, certainly_different = if identities.any?
+      encompassed.partition do |record|
+        identities.all? do |identity|
+          answer = identity.identical? needle, record
+          answer.nil? or answer == true
         end
       end
-    end
-    $ltd_1 = history[right_record]
-    right = read_right right_record
-    i_options_right = i_options right
-    z = 1
-    debugger if $ltd_left.andand.match(left) or $ltd_right.andand.match(right)
-    z = 1
-    if collision? i_options_left, i_options_right
-      $ltd_0 = nil
-      return
     else
-      $ltd_0 = right_record
+      [ encompassed.dup, [] ]
     end
-    inline_check left_record, right_record
-    right_record
-  end
-  alias_method :find, :left_to_right
-  
-  def optimize(t_options_left, t_options_right)
-    cart_prod(t_options_left, t_options_right).max do |a, b|
-      t_left_a, t_right_a = a
-      t_left_b, t_right_b = b
     
-      a_prefix, a_score = t_left_a.prefix_and_score t_right_a
-      b_prefix, b_score = t_left_b.prefix_and_score t_right_b
-      
-      yep_ddd = ($ltd_ddd_right and $ltd_ddd_left and [t_left_a, t_left_b].any? { |f| f.str =~ $ltd_ddd_left } and [t_right_a, t_right_b].any? { |f| f.str =~ $ltd_ddd_right } and (!$ltd_ddd_left_not or [t_left_a, t_left_b].none? { |f| f.str =~ $ltd_ddd_left_not }))
-      
-      if $ltd_ddd_print and yep_ddd
-        logger.andand.debug t_left_a.inspect
-        logger.andand.debug t_right_a.inspect
-        logger.andand.debug t_left_b.inspect
-        logger.andand.debug t_right_b.inspect
-        logger.andand.debug
-      end
-      
-      z = 1
-      debugger if yep_ddd
-      z = 1
-      
-      if a_score != b_score
-        a_score <=> b_score
-      elsif a_prefix and b_prefix and a_prefix != b_prefix
-        a_prefix <=> b_prefix
-      else
-        # randomly choose
-        # maybe later i can figure out how big the inputs are and apply occam's razor
-        rand(2) == 1 ? -1 : 1
-      end
+    if gather_last_result
+      last_result.possibly_identical = possibly_identical
+      last_result.certainly_different = certainly_different
     end
-  end
-  
-  def t_options(str)
-    return @_t_options[str] if @_t_options.andand.has_key?(str)
-    @_t_options ||= Hash.new
-    ary = Array.new
-    ary.push T.new(str, str)
-    tightenings.each do |regexp|
-      if match_data = regexp.match(str)
-        ary.push T.new(str, match_data.captures.compact.join)
-      end
+    
+    similarities = possibly_identical.map do |record|
+      needle.similarity record
+    end.sort
+    
+    best_similarity = similarities[-1]
+    record = best_similarity.wrapper2
+    score = best_similarity.best_score.to_f
+    
+    if gather_last_result
+      last_result.similarities = similarities
+      last_result.record = record.record
+      last_result.score = score
     end
-    @_t_options[str] = ary
+    
+    record.record
   end
   
-  class I
-    attr_reader :regexp, :str, :case_sensitive, :identity
-    def initialize(regexp, str, case_sensitive)
-      @regexp = regexp
-      @str = str
-      @identity = regexp.match(str).captures.compact.join
-      @identity = @identity.downcase if case_sensitive
-    end
+  # Explain is like mysql's EXPLAIN command. You give it a needle and it tells you about how it was located (successfully or not) in the haystack.
+  #
+  #     d = LooseTightDictionary.new ['737', '747', '757' ]
+  #     d.explain 'boeing 737-100'
+  def explain(needle)
+    record = find needle
+    log "#" * 150
+    log "# Match #{needle.inspect} => #{record.inspect}"
+    log "#" * 150
+    log
+    log "Needle"
+    log "-" * 150
+    log last_result.needle.to_str
+    log
+    log "Haystack"
+    log "-" * 150
+    log last_result.haystack.map { |record| record.to_str }.join("\n")
+    log
+    log "Tighteners"
+    log "-" * 150
+    log last_result.tighteners.blank? ? '(none)' : last_result.tighteners.map { |tightener| tightener.inspect }.join("\n")
+    log
+    log "Blockings"
+    log "-" * 150
+    log last_result.blockings.blank? ? '(none)' : last_result.blockings.map { |blocking| blocking.inspect }.join("\n")
+    log
+    log "Identities"
+    log "-" * 150
+    log last_result.identities.blank? ? '(none)' : last_result.identities.map { |blocking| blocking.inspect }.join("\n")
+    log
+    log "Included"
+    log "-" * 150
+    log last_result.encompassed.blank? ? '(none)' : last_result.encompassed.map { |encompassed| encompassed.to_str }.join("\n")
+    log
+    log "Ignored"
+    log "-" * 150
+    log last_result.unencompassed.blank? ? '(none)' : last_result.unencompassed.map { |unencompassed| unencompassed.to_str }.join("\n")
+    log
+    log "Possibly identical"
+    log "-" * 150
+    log last_result.possibly_identical.blank? ? '(none)' : last_result.possibly_identical.map { |possibly_identical| possibly_identical.to_str }.join("\n")
+    log
+    log "Certainly different"
+    log "-" * 150
+    log last_result.certainly_different.blank? ? '(none)' : last_result.certainly_different.map { |certainly_different| certainly_different.to_str }.join("\n")
+    log
+    log "Similarities"
+    log "-" * 150
+    log last_result.similarities.blank? ? '(none)' : last_result.similarities.reverse[0..9].map { |similarity| similarity.inspect }.join("\n")
+    log
+    log "Match"
+    log "-" * 150
+    log record.inspect
   end
-  
-  def collision?(i_options_left, i_options_right)
-    i_options_left.any? do |r_left|
-      i_options_right.any? do |r_right|
-        r_left.regexp == r_right.regexp and r_left.identity != r_right.identity
-      end
+    
+  def haystack_reader
+    options[:haystack_reader]
+  end
+        
+  def strict_blocking
+    options[:strict_blocking] || false
+  end
+
+  def tighteners
+    @tighteners ||= (options[:tighteners] || []).map do |regexp_or_str|
+      Tightener.new regexp_or_str
     end
   end
-  
-  def i_options(str)
-    return @_i_options[str] if @_i_options.andand.has_key?(str)
-    @_i_options ||= Hash.new
-    ary = Array.new
-    identities.each do |regexp|
-      if regexp.match str
-        ary.push I.new(regexp, str, case_sensitive)
-      end
+
+  def identities
+    @identities ||= (options[:identities] || []).map do |regexp_or_str|
+      Identity.new regexp_or_str
     end
-    @_i_options[str] = ary
   end
-  
-  def blocking(str)
-    return @_blocking[str] if @_blocking.andand.has_key?(str)
-    @_blocking ||= Hash.new
-    blockings.each do |regexp|
-      if regexp.match str
-        return @_blocking[str] = regexp
-      end
+
+  def blockings
+    @blockings ||= (options[:blockings] || []).map do |regexp_or_str|
+      Blocking.new regexp_or_str
     end
-    @_blocking[str] = nil
   end
   
-  def literal_regexp(str)
-    return @_literal_regexp[str] if @_literal_regexp.andand.has_key? str
-    @_literal_regexp ||= Hash.new
-    raw_regexp_options = str.split('/').last
-    ignore_case = (!case_sensitive or raw_regexp_options.include?('i')) ? Regexp::IGNORECASE : nil
-    multiline = raw_regexp_options.include?('m') ? Regexp::MULTILINE : nil
-    extended = raw_regexp_options.include?('x') ? Regexp::EXTENDED : nil
-    @_literal_regexp[str] = Regexp.new str.gsub(/\A\/|\/([ixm]*)\z/, ''), (ignore_case||multiline||extended)
+  def freed?
+    @freed == true
   end
   
-  def read_left(left_record)
-    return if left_record.nil?
-    if left_reader
-      left_reader.call(left_record)
-    elsif left_record.is_a?(String)
-      left_record
-    else
-      left_record[0]
-    end
+  def free
+    free_last_result
+    @options.try :clear
+    @options = nil
+    @haystack.try :clear
+    @haystack = nil
+    @tighteners.try :clear
+    @tighteners = nil
+    @identities.try :clear
+    @identities = nil
+    @blockings.try :clear
+    @blockings = nil
+  ensure
+    @freed = true
   end
   
-  def read_right(right_record)
-    return if right_record.nil?
-    if right_reader
-      right_reader.call(right_record)
-    elsif right_record.is_a?(String)
-      right_record
-    else
-      right_record[0]
-    end
-  end
+  private
   
-  # Thanks William James!
-  # http://www.ruby-forum.com/topic/95519#200484
-  def cart_prod(*args)
-    args.inject([[]]){|old,lst|
-      new = []
-      lst.each{|e| new += old.map{|c| c.dup << e }}
-      new
-    }
+  def free_last_result
+    @last_result.try :free
+    @last_result = nil
   end
 end