matching 0.14.1

data/VERSION

@@ -0,0 +1 @@
+0.14.1

data/lib/matching.rb

@@ -0,0 +1,11 @@
+files = %w(
+  attribute_pair 
+  array_store 
+  similarity
+  hash_index 
+  match 
+  matcher
+  deduplicator
+)
+
+files.each { |f| require File.expand_path(File.dirname(__FILE__) + "/matching/#{f}.rb") }

data/lib/matching/active_relation_store.rb

@@ -0,0 +1,30 @@
+require 'active_record'
+
+module Matching
+
+  #Stores and retrieves data from ActiveRelation for Matcher
+  class ActiveRelationStore
+
+    attr_reader :model, :where_clause
+
+    def initialize(model, where_clause = nil)
+      @model = model
+      @where_clause = where_clause
+    end
+
+    #Iterates over array, also returning id
+    def each(&blk)
+      @model.where(@where_clause).find_in_batches do |group|
+        group.each do |obj|
+          blk.yield(obj, obj.id)
+        end
+      end
+    end
+
+    #Return an object by its AR id
+    def find(id)
+      @model.find(id)
+    end
+
+  end
+end

data/lib/matching/array_store.rb

@@ -0,0 +1,23 @@
+module Matching
+
+  #Stores and retrieves data from arrays for Matcher
+  class ArrayStore
+
+    attr_reader :arr
+
+    def initialize(arr)
+      @arr = arr
+    end
+
+    #Iterates over array, also returning index as a kind of ID
+    def each(&blk)
+      @arr.each_with_index(&blk)
+    end
+
+    #Return an object from the array by its index position
+    def find(idx)
+      @arr[idx]
+    end
+
+  end
+end

data/lib/matching/attribute_pair.rb

@@ -0,0 +1,17 @@
+module Matching
+
+  #Defines the comparison of two attributes, one from the "left" class
+  #and one from the "right"
+  class AttributePair
+    attr_reader :left_attr, :right_attr, :weight, :is_fuzzy
+
+    def initialize(left_attr, right_attr, weight, is_fuzzy = false)
+      @left_attr = left_attr
+      @right_attr = right_attr
+      @weight = weight
+      @is_fuzzy = is_fuzzy
+
+      raise "Weight must be > 0.0" unless weight > 0.0
+    end
+  end
+end

data/lib/matching/deduplicator.rb

@@ -0,0 +1,133 @@
+module Matching
+  class Deduplicator
+
+    attr_accessor :store, :index, :criteria
+    attr_accessor :groups   # array of arrays of duplicate records in form [[1,5],[2,3,4],[6]]
+    attr_accessor :grouped  # hash of all ids present in @groups. Eventually all ids from @store will be added.
+                            # Stored in form { id => index_of_groups_object }
+
+    def initialize(store,opts={})
+      raise 'Store parameter required' unless store
+      @store = store
+
+      @criteria = []
+     
+      # Create an index using either a hash or Redis as the backing store
+      if opts[:redis_db] && opts[:redis_db].to_i >= 1
+        @index = RedisIndex.new(opts[:redis_db])
+      else
+        @index = HashIndex.new
+      end
+    end
+
+    def match_attrs(attrs)
+      @criteria << [*attrs] #converts to array if not already, doesn't affect arrays  
+    end
+
+    def unique_attrs
+      @criteria.flatten.uniq
+    end
+
+    def define(&block)
+      instance_eval(&block)
+    end
+
+    def deduplicate
+      @groups = []      # Array of arrays containing ids of grouped objects
+      @nil_group = []   # Special array of objects whose indexed values are all nil (because index isn't tracking them)
+      @grouped = {}     # Hash of each object's id to the index of @groups in which its found
+      
+      # Index all records in the store to speed search
+      create_index
+
+      # Place each object into an array in @groups that contain all
+      # records that match the defined matching logic.
+      @store.each do |obj,store_idx|
+
+        puts "On #{store_idx}" if store_idx % 100 == 0 && store_idx > 0
+
+        # Shortcut the process if there is only one array in criteria 
+        # and this object is already present (because it can't possibly match
+        # a second time)
+        next if @criteria.size == 1 && @grouped[obj.id]
+
+        @criteria.each do |arr|
+
+          # Find matching objects
+          all_matches = nil
+          arr.each do |match_attr|
+            val = obj.send(match_attr)
+
+            if val != nil
+              matches = @index.get(match_attr, val)
+              all_matches = (all_matches ? all_matches & matches : matches)
+            end
+          end
+
+          if all_matches.nil?
+            @nil_group << obj.id
+            next
+          end
+
+          # Assign matched objects to a group.
+          # Groups may be merged in this process. 
+          current_group_indexes = all_matches.inject([]) do |arr,id| 
+            arr << @grouped[id] if @grouped[id] 
+            arr
+          end.uniq.compact
+
+          next if current_group_indexes.size == 1 # can only be [obj_id]
+
+          if current_group_indexes.size > 1
+            # Merge related groups into mega_group based on first group
+            mega_group = @groups[current_group_indexes[0]] 
+            current_group_indexes[1..-1].each do |idx| 
+              @groups[idx].each { |id| mega_group << id } 
+              @groups.delete_at(idx)
+            end
+          
+            # Re-assign @grouped for all objects to new mega-group
+            mega_group.each { |obj_id| @grouped[obj_id] = current_group_indexes[0] }
+          else
+            # Create new group
+            @groups << all_matches
+            group_idx = @groups.size - 1
+            all_matches.each { |obj_id| @grouped[obj_id] = group_idx }
+          end
+        end   
+      end
+
+      # Add the contents of nil group as a single group
+      @groups << @nil_group if @nil_group.any?
+
+      #puts "Results: #{@groups.inspect}"
+    end
+
+    def create_index
+      raise 'Deduplicator requires at least one match attribute be defined' unless @criteria.any?
+
+      @store.each do |obj, id|
+        unique_attrs.each do |ma|
+          @index.put(ma, obj.send(ma), id)
+        end
+      end
+    end
+
+    # Returns each object in store along with its group's index and index within
+    # the group. For example...
+    # group_idx | idx | name
+    #         0 |   0 | Fred Smith
+    #         0 |   1 | Fred Smith
+    #         1 |   0 | Jane Green
+    #         2 |   0 | Linda Smythe
+    #         2 |   1 | Linda Smythe
+    def each_with_groups
+      @groups.each_with_index do |arr,grp_idx|
+        arr.each_with_index do |obj_id,obj_idx|
+          yield(@store.find(obj_id), grp_idx, obj_idx) 
+        end
+      end
+    end
+
+  end # class
+end # module

data/lib/matching/hash_index.rb

@@ -0,0 +1,25 @@
+module Matching
+  class HashIndex
+
+    attr_reader :hashes
+
+    def initialize
+      #one hash for each attribute
+      @hashes = {}
+    end
+
+    # Add a value to the index for a given attribute and object id
+    def put(attr, val, id)
+      unless val.nil?
+        h = @hashes[attr] || (@hashes[attr] = {})
+        (h[val] ? h[val] << id : h[val] = [id])
+      end
+    end
+
+    # Return an array of object ids for a given attribute and value
+    def get(attr, val)
+      (@hashes[attr] ? @hashes[attr][val] : nil)
+    end
+
+  end
+end

data/lib/matching/match.rb

@@ -0,0 +1,14 @@
+module Matching
+
+  #Defines a pair of objects that have been matched by the matcher
+  class Match
+    attr_reader :left_obj, :right_obj
+    attr_accessor :score
+
+    def initialize(left_obj, right_obj, score)
+      @left_obj = left_obj
+      @right_obj = right_obj
+      @score = score
+    end
+  end
+end

data/lib/matching/matcher.rb

@@ -0,0 +1,266 @@
+module Matching
+
+  class Matcher
+    attr_accessor :min_score
+    attr_reader :left_store, :right_store
+    attr_reader :join_pairs, :compare_pairs, :custom_functions, :filter_functions
+    attr_reader :left_matches, :right_matches
+    attr_reader :right_index
+
+    def self.define(opts=nil, &block)
+      m = new(opts)
+      m.define(block)
+      m
+    end
+
+    def initialize(opts={})
+      @left_store = opts[:left_store]
+      @right_store = opts[:right_store]
+      @min_score = opts[:min_score] || 1.0
+
+      @join_pairs = []
+      @compare_pairs = []
+      @custom_functions = []
+      @filter_functions = []
+      @right_matches = {} #hash keyed on right_class records, used during main rec loop
+      @left_matches = {} #hash keyed on left_class records, created after main rec loop from reverse of @right_matches
+      @left_losers = [] #array of left objects that were matched to right records then unmatched, requiring re-match attempt
+
+      # Create @right_index using either a hash or Redis as the backing store
+      if opts[:redis_db] && opts[:redis_db].to_i >= 1
+        @right_index = RedisIndex.new(opts[:redis_db])
+      else
+        @right_index = HashIndex.new
+      end
+    end
+
+    # Compare left and right arguments and return similarity as a floating point
+    # value where 0.0 represents no similarity and 1.0 represents equality.
+    def compare_values(left,right,opts={})
+      return 0.0 unless left && right
+
+      raise ArgumentError, "Cannot compare values of dissimilar type - left = #{left}, right = #{right}" unless left.class == right.class
+
+      if opts[:fuzzy]
+        raise ArgumentError, "Cannot calculate fuzzy comparison for type #{left.class}" unless left.respond_to?(:similarity_to)
+        left.similarity_to(right,opts)
+      else
+        (left == right ? 1.0 : 0.0)
+      end
+    end
+
+    def define(&block)
+      instance_eval(&block)
+    end
+
+    # One or more join attributes are required for a match between two records
+    # to occur. Attributes must be equal.
+    def join(left_attr, right_attr, weight)
+      @join_pairs << AttributePair.new(left_attr, right_attr, weight)
+    end
+
+    # For records matched via join attributes, comparisons may be applied to
+    # adjust the score.
+    def compare(left_attr, right_attr, weight, is_fuzzy = false)
+      @compare_pairs << AttributePair.new(left_attr, right_attr, weight, is_fuzzy)
+    end
+
+    # Custom functions may adjust the score beyond the simple comparisons
+    # performed via @compare_pairs.
+    def custom(lmbda)
+      @custom_functions << lmbda
+    end
+
+    # Filter lambdas must return a boolean. Returning true will prevent a match.
+    def filter(lmbda)
+      @filter_functions << lmbda
+    end
+
+    # Given join, compare, and custom rules, return the floating point
+    # matching score of two objects.
+    def score_pair(left_obj, right_obj)
+      score = 0
+
+      @join_pairs.each do |pair|
+        score += pair.weight * compare_values(left_obj.send(pair.left_attr), right_obj.send(pair.right_attr))
+      end
+
+      @compare_pairs.each do |pair|
+        score += pair.weight * compare_values(left_obj.send(pair.left_attr), right_obj.send(pair.right_attr), pair.is_fuzzy)
+      end
+
+      @custom_functions.each do |lmbda|
+        score += lmbda.call(left_obj, right_obj)
+      end
+
+      @filter_functions.each do |lmbda|
+        score = 0 unless lmbda.call(left_obj, right_obj)
+      end
+
+      score
+    end
+
+    # Perform matching
+    def match
+      unless @left_store && @right_store
+        raise ArgumentError, "Matcher requires left_store and right_store attributes"
+      end
+
+      # Index right objects to speed search
+      index_right_objects
+
+      # Evaluate each left record for matches.
+      # If more than one match is found, the best-possible match
+      # will be awarded the match unless another object is already
+      # matched to it. Conflicts are resolved in a separate method.
+      @left_store.each do |left_obj|
+
+        yield left_obj if block_given?
+
+        # Results are pre-sorted with the best matches first
+        ranked_matches = find_matches(left_obj)
+
+        # Attempt to pair the left_object with one of the 
+        # ranked right matches
+        pair_matches(left_obj, ranked_matches)
+      end #each left_obj
+
+      # Call the recursive method evaluate_left_losers which will attempt to
+      # find new matches
+      evaluate_left_losers
+
+      # Populate left_matches as the mirror of right_matches
+      @right_matches.each { |right_obj, match| @left_matches[match.left_obj] = match }
+    end
+
+    # Indexes attribues from right object in @right_index (either hash or Redis, see
+    # initialize). For each join_pair, store the attribute's values in the form:
+    #  attr:val -> [array_of_ids]
+    def index_right_objects
+
+      # Require at least one exact_pair else would execute in quadratic time
+      raise 'Matcher requires at least one join pair to be defined' unless @join_pairs.any?
+
+      @right_store.each do |right_obj, id|
+        @join_pairs.each { |jp| @right_index.put(jp.right_attr, right_obj.send(jp.right_attr), id) }
+      end
+    end
+
+    # Return of scored matches for the left_object argument.
+    # Results are in an ordered array of form [[right_obj_a, score_a], [right_obj_b, score_b], ...]
+    def find_matches(left_obj)
+      potential_matches = find_potential_matches(left_obj)
+      ranked_pairs = []
+
+      potential_matches.each do |right_obj|
+        score = score_pair(left_obj, right_obj)
+        ranked_pairs << [right_obj, score] if score >= @min_score
+      end
+
+      ranked_pairs.sort! { |a,b| a[1] <=> b[1] }
+      ranked_pairs.reverse
+    end
+
+    # Return an array of right_objects that match the left_object by
+    # join criteria. This is equivalent to an index lookup. No scoring
+    # is done by this method.
+    def find_potential_matches(left_obj)
+      right_objects = []
+
+      @join_pairs.each do |jp|
+        left_val = left_obj.send(jp.left_attr)
+        next if left_val.nil? || left_val == ''
+
+        matches = @right_index.get(jp.right_attr, left_val)
+        right_objects = right_objects | matches if matches
+      end
+
+      # At this point right_objects contains an array of right object ID's.
+      # Retrieve the matching objects now.
+      right_objects.map! { |r_id| @right_store.find(r_id) }
+    end
+
+    # Evaluate and possibly create Match objects to join the
+    # left_object to one of the right_objects from the
+    # ranked_matches array
+    def pair_matches(left_obj, ranked_matches)
+
+      ranked_matches.each do |pair|
+        (right_obj, score) = pair
+
+        if @right_matches[right_obj]
+          # A match already exists. Determine which left_obj is the best fit.
+          if score > @right_matches[right_obj].score
+            # The current left_obj is a better fit.
+            # Record the other left_obj as a loser then switch
+            # the match for the right_obj.
+            @left_losers << @right_matches[right_obj].left_obj
+            @right_matches[right_obj] = Match.new(left_obj, right_obj, score)
+            break
+          else
+            # Continue looping to try to find a better match
+          end
+        else
+          # Assign first match for this right_obj
+          @right_matches[right_obj] = Match.new(left_obj, right_obj, score)
+          break
+        end
+      end
+    end
+
+    # Attempt to find matches while any left losers remain
+    def evaluate_left_losers
+      return unless @left_losers.any?
+
+      # Use a copy of the array because it may be filled again as
+      # find_matches is called
+      working_losers = @left_losers
+      @left_losers = []
+      working_losers.each do |left_obj| 
+        ranked_matches = find_matches(left_obj)
+        pair_matches(left_obj, ranked_matches)
+      end
+
+      # To understand recursion you first must understand recursion
+      evaluate_left_losers
+    end
+
+    # Returns array of non-matched left objects
+    def left_exceptions
+      return @left_exceptions if @left_exceptions
+      @left_exceptions = exceptions(:left)
+      @left_exceptions
+    end
+
+    # Returns array of non-matched right objects
+    def right_exceptions
+      return @right_exceptions if @right_exceptions
+      @right_exceptions = exceptions(:right)
+      @right_exceptions
+    end
+
+    def exceptions(side)
+      if side == :left 
+        store, matches = @left_store, @left_matches
+      else 
+        store, matches = @right_store, @right_matches
+      end
+
+      arr = []
+      if arr.class == ArrayStore
+        arr = store.arr - matches
+      else
+        store.each do |obj|
+          arr << obj unless matches[obj]
+        end
+      end
+      arr
+    end
+
+    def matches
+      @left_matches.map do |left_obj, match|
+        match
+      end 
+    end
+  end #class
+end #module