matching 0.14.1

data/lib/matching/redis_index.rb

@@ -0,0 +1,26 @@
+require 'redis'
+
+module Matching
+  class RedisIndex
+
+    def initialize(db_num=8)
+      @redis = Redis.new
+      @redis.select(db_num)
+      @redis.flushdb
+    end
+
+    #Add a value to the index for a given attribute and object id
+    def put(attr, val, id)
+      unless val.nil?
+        @redis.sadd("#{attr}:#{val}",id)
+      end
+    end
+
+    #Return an array of object ids for a given attribute and value
+    def get(attr, val)
+      str_ids = @redis.smembers("#{attr}:#{val}")
+      (str_ids.any? ? str_ids.map { |a| a.to_i } : nil)
+    end
+
+  end
+end

data/lib/matching/similarity.rb

@@ -0,0 +1,78 @@
+require 'text/levenshtein'
+require 'date'
+
+# Adds fuzzy methods to standard classes for
+# comparing two instances on a rules-based scale
+# between 0.0 and 1.0.
+
+class Date
+  # Calculates a score between 0.0 and 1.0 for all dates within :days_scale
+  # of each other.
+  def similarity_to(other_date, opts={})
+    days_scale = opts[:days_scale] || 30
+    raise ArgumentError, 'days_scale must be numeric' unless days_scale.class == Fixnum
+    days_scale = days_scale.to_f
+
+    delta = (self - other_date).to_f.abs
+    (delta < days_scale ? (days_scale - delta) / days_scale : 0.0)
+  end
+end
+
+class String
+
+  def similarity_to(other_string, opts={})
+    case opts[:comparison] 
+      when :name
+        name_similarity_to(other_string)
+      else 
+        ## use just levenshtein edit distance (see levenshtein.rb)
+        return raw_similarity_to(other_string)
+    end
+  end
+  
+  #Given a string, return one or more tokens parsed with the following rules:
+  # 1. Turn commas into spaces
+  # 2. Split on spaces
+  # 3. Strip periods
+  # 4. Discard any tokens with single letters
+  def tokenize
+    tokens = self.gsub(/\,/,' ').gsub(/\./,'').split(' ')
+    tokens.reject! { |p| p.size == 1 }
+    tokens
+  end
+
+  # Given two names, return a floating-point evaluation 
+  # of similarity in the range 0.0 - 1.0
+  def name_similarity_to(other_string)
+    return 0.0 if self.nil? || other_string.nil? || self.size == 0 || other_string.size == 0
+    return 1.0 if self == other_string
+
+    l_tokens = self.tokenize
+    r_tokens = other_string.tokenize
+
+    total_sim = 0.0
+    l_tokens.each do |l|
+      r_tokens.each do |r|
+        total_sim += l.raw_similarity_to(r)
+      end
+    end
+
+    avg_tokens = (l_tokens.size + r_tokens.size).to_f / 2.0
+    score = total_sim / avg_tokens
+    (score > 1.0 ? 1.0 : score)
+  end
+
+  # Returns a floating point value of the similarity
+  # between this string and other.
+  # Uses 'text' gem, http://rubyforge.org/projects/text
+  def raw_similarity_to(other)
+    delta = Text::Levenshtein::distance(self.downcase, other.downcase)
+    return 0.0 unless delta
+    return 1.0 if delta == 0
+
+    avg_len = (size + other.size).to_f / 2.0
+    return 0.0 if delta > avg_len
+    (avg_len - delta.to_f) / avg_len
+  end
+end
+

data/matching.gemspec

@@ -0,0 +1,71 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{matching}
+  s.version = "0.14.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = [%q{Barry Ezell}]
+  s.date = %q{2012-02-09}
+  s.description = %q{}
+  s.email = %q{barrye@gmail.com}
+  s.extra_rdoc_files = [
+    "README.md"
+  ]
+  s.files = [
+    ".document",
+    ".rspec",
+    "Gemfile",
+    "Gemfile.lock",
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "lib/matching.rb",
+    "lib/matching/active_relation_store.rb",
+    "lib/matching/array_store.rb",
+    "lib/matching/attribute_pair.rb",
+    "lib/matching/deduplicator.rb",
+    "lib/matching/hash_index.rb",
+    "lib/matching/match.rb",
+    "lib/matching/matcher.rb",
+    "lib/matching/redis_index.rb",
+    "lib/matching/similarity.rb",
+    "matching.gemspec",
+    "spec/db/database.yml",
+    "spec/integration/bank_rec_spec.rb",
+    "spec/lib/ar_spec.rb",
+    "spec/lib/deduplicator_spec.rb",
+    "spec/lib/matcher_spec.rb",
+    "spec/lib/redis_spec.rb",
+    "spec/lib/similarity_spec.rb",
+    "spec/samples/agent_recs.csv",
+    "spec/spec_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/btedev/matching}
+  s.licenses = [%q{MIT license}]
+  s.require_paths = [%q{lib}]
+  s.rubygems_version = %q{1.8.7}
+  s.summary = %q{Dataset matching engine}
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<text>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, [">= 0"])
+      s.add_development_dependency(%q<jeweler>, [">= 0"])
+    else
+      s.add_dependency(%q<text>, [">= 0"])
+      s.add_dependency(%q<bundler>, [">= 0"])
+      s.add_dependency(%q<jeweler>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<text>, [">= 0"])
+    s.add_dependency(%q<bundler>, [">= 0"])
+    s.add_dependency(%q<jeweler>, [">= 0"])
+  end
+end
+

data/spec/db/database.yml

@@ -0,0 +1,5 @@
+development:
+  adapter: sqlite3
+  database: spec/db/development.sqlite3
+  pool: 5
+  timeout: 5000

data/spec/integration/bank_rec_spec.rb

@@ -0,0 +1,50 @@
+require File.expand_path("../../spec_helper", __FILE__)
+include Matching
+
+# Note: do not use a Struct in place of the class because matcher.rb relies
+# on object_id for determine object inclusion in exception arrays. Two
+# instances of a Struct with the same values have the same object_id.
+class Transaction
+  attr_accessor :date, :desc, :amount
+  def initialize(date, desc, amount)
+    @date, @desc, @amount = date, desc, amount     
+  end
+end
+
+describe "Bank reconciliation" do
+
+  let(:ledger_txns) do
+    [
+      Transaction.new(Date.new(2012,1,1),'Basecamp','25.0'),
+      Transaction.new(Date.new(2012,1,1),'Basecamp','25.0'),
+      Transaction.new(Date.new(2012,1,2),'Github','25.0')
+    ]
+  end
+
+  let(:bank_txns) do
+    [
+      Transaction.new(Date.new(2012,1,1),'Basecamp (37 signals)','25.0'),
+      Transaction.new(Date.new(2012,1,3),'Github','25.0')
+    ]
+  end
+
+  let(:matcher) do
+    Matching::Matcher.new(
+      :left_store => ArrayStore.new(ledger_txns),
+      :right_store => ArrayStore.new(bank_txns),
+      :min_score => 1.0
+    )
+  end
+
+  it "should rec" do
+    matcher.define do
+      join  :amount,  :amount, 1.0
+      compare :date,  :date, 0.2, :fuzzy => true
+    end
+    matcher.match 
+    matcher.left_matches.should have(2).items
+    matcher.left_exceptions.should have(1).items
+    matcher.right_exceptions.should have(0).items
+  end
+
+end

data/spec/lib/ar_spec.rb

@@ -0,0 +1,182 @@
+# Tests ActiveRecord as the data store
+
+require File.expand_path("../../spec_helper", __FILE__)
+require File.expand_path("../../../lib/matching/active_relation_store", __FILE__)
+include Matching
+
+module ArSpecHelper
+
+  class Txn < ActiveRecord::Base
+  end
+
+  def config
+    @config ||= YAML.load_file(File.expand_path(File.dirname(__FILE__) + '/../db/database.yml'))['development']
+  end
+
+  def db_connect
+    File.delete(config['database']) if File.exists?(config['database'])
+    options = {:charset => 'utf8', :collation => 'utf8_unicode_ci'}
+    ActiveRecord::Base.establish_connection config
+    sql = "create table txns(id integer primary key, company text, esn text, mdn text, date date);"
+    ActiveRecord::Base.connection.execute(sql)
+  end
+
+  #creates arrays of Transaction and ServiceChange model objects using similar structure to
+  #create_test_data above
+  def create_ar_test_data
+    db_connect
+
+    @left_a = Txn.create(:company => 'ACME', :esn => "11111111111", :mdn => "7275551111", :date => Date.new(2010,6,1))
+    @left_b = Txn.create(:company => 'ACME', :esn => "22222222222", :mdn => "8135554444", :date => Date.new(2010,6,1))
+    @left_c = Txn.create(:company => 'ACME', :esn => "33333333333", :mdn => "7275551111", :date => Date.new(2010,6,15))
+
+    @right_a = Txn.create(:company => 'Cinco', :esn => "11111111111", :mdn => "2015559999", :date => Date.new(2010,6,1))
+    @right_b = Txn.create(:company => 'Cinco', :esn => "11111111111", :mdn => "7275551111", :date => Date.new(2010,6,1))
+    @right_c = Txn.create(:company => 'Cinco', :esn => "22222222222", :mdn => "8135554444", :date => Date.new(2010,6,2))
+    @right_d = Txn.create(:company => 'Cinco', :esn => "44444444444", :mdn => "7275551111", :date => Date.new(2010,6,14))
+  end
+
+  #matcher using ActiveRecord for the data store
+  def create_ar_matcher(use_redis = false)
+    create_ar_test_data
+
+    matcher = Matcher.new(
+                            :left_store => ActiveRelationStore.new(Txn, "company = 'ACME'"),
+                            :right_store => ActiveRelationStore.new(Txn, "company = 'Cinco'"),
+                            :redis_db => (use_redis ? 8 : nil)
+                          )
+  end
+end
+
+describe ActiveRelationStore do
+  include ArSpecHelper
+
+  before(:each) do
+    create_ar_test_data
+  end
+
+  context " unfiltered" do
+    let(:store) { ActiveRelationStore.new(Txn) }
+
+    it "should enumerate left AR objects with id" do
+      cnt = 0
+      expect { store.each { |o,idx| cnt += 1 } }.to change{cnt}.from(0).to(7)
+
+      obj, id = nil, nil
+      store.each do |_obj,_id|
+        obj, id = _obj, _id
+        break
+      end
+
+      id.should == 1
+      obj.should == @left_a
+    end
+
+    it "should retrieve objects by their id through the find method" do
+      store.find(2).should == @left_b
+    end
+  end
+
+  context " filtered" do
+    let(:store) { ActiveRelationStore.new(Txn, "company = 'Cinco'") }
+
+    it "should have a where clause" do
+      store.where_clause.should == "company = 'Cinco'"
+    end
+
+    it "should enumerate left AR objects from query with where clause" do
+      cnt = 0
+      expect { store.each { |o,idx| cnt += 1 } }.to change{cnt}.from(0).to(4)
+    end
+  end
+end
+
+describe Matcher do
+  include ArSpecHelper
+
+  context "with hash index and ActiveRecord store" do
+
+    before(:each) do
+      @matcher = create_ar_matcher
+    end
+
+    let(:esn_matcher) do
+      @matcher.define { join :esn, :esn, 1.0 }
+    end
+
+    let(:ptn_esn_matcher) do
+      @matcher.define do
+        join :mdn, :mdn, 1.0 
+        join :esn, :esn, 1.0
+      end
+    end
+
+    it "requires at least one join pair to be defined" do
+      expect { @matcher.index_right_objects }.to raise_error
+    end
+
+    context "using ptn and esn matcher" do
+
+      before(:each) do
+        ptn_esn_matcher
+        @matcher.index_right_objects
+      end
+
+      it "indexes right records on join attributes" do
+        @matcher.right_index.get(:esn, "11111111111").should_not be_nil
+        @matcher.right_index.get(:esn, "11111111111").size.should == 2
+        @matcher.right_index.get(:mdn, "8135554444").size.should == 1
+      end
+
+      it "finds potential matches for left_objects from right_objects based on join criteria" do
+        right_matches = @matcher.find_potential_matches(@left_a)
+        right_matches.should have(3).items
+        right_matches.should include(@right_a)
+      end
+
+      it "finds scored matches by applying rules after finding potential matches" do
+        right_matches = @matcher.find_matches(@left_a)
+        right_matches.should have(3).items
+
+        #raise matching threshold
+        @matcher.min_score = 2.0
+        right_matches = @matcher.find_matches(@left_a)
+        right_matches.should have(1).items
+
+        #note: return value is an array of arrays, not an array of just
+        #right_objects
+        right_matches[0].should == [@right_b, 2.0]
+      end
+    end
+
+    it "should reconcile test data based on single attribute pair" do
+      esn_matcher
+      @matcher.match
+      @matcher.right_matches.size.should == 2
+      @matcher.left_matches.size.should == 2
+
+      @matcher.left_matches.should include(@left_a)
+      @matcher.left_matches.should include(@left_b)
+      @matcher.left_matches[@left_b].right_obj.should == @right_c
+    end
+
+    it "should reconcile test data based on two attribute pairs" do
+      ptn_esn_matcher
+      @matcher.match
+      @matcher.right_matches.size.should == 3
+      @matcher.left_matches.size.should == 3
+      @matcher.left_matches[@left_c].right_obj.should == @right_d
+    end
+
+    it "should fail to match records below the min_score threshold" do
+      ptn_esn_matcher
+      @matcher.min_score = 2.0
+      @matcher.match
+      @matcher.right_matches.size.should == 2
+      @matcher.left_matches.size.should == 2
+      @matcher.left_matches[@left_c].should be_nil
+    end
+
+  end #hash index and ActiveRecord tests
+
+end

data/spec/lib/deduplicator_spec.rb

@@ -0,0 +1,221 @@
+# Tests main functionality using array data stores and hash indexing.
+# See ar_spec.rb for tests of ActiveRecord as the data store
+# See redis_spec.rb for tests of Redis for indexing.
+
+require 'rspec'
+require 'date'
+require File.expand_path(File.dirname(__FILE__) + '/../../lib/matching.rb')
+include Matching
+
+module DedupeSpecHelper
+  CellTxn = Struct.new(:id, :mid, :esn, :act_date, :nilly)
+end
+
+describe Deduplicator do
+  include DedupeSpecHelper
+
+  let (:array_store) do
+    c1 = CellTxn.new(0, "7275554444", "11111111111", Date.new(2011,1,1))
+    c2 = CellTxn.new(1, "7275554444", "22222222222", Date.new(2011,1,2))
+    c3 = CellTxn.new(2, "8135552222", "22222222222", Date.new(2011,1,3))
+    c4 = CellTxn.new(3, "8135552222", "22222222222", Date.new(2011,1,2))
+    ArrayStore.new([c1,c2,c3,c4])
+  end
+
+  before(:each) do
+    @deduper = Deduplicator.new(array_store)
+  end
+
+  subject { @deduper }
+  specify { subject.index.should_not be_nil }
+  
+  describe :store do
+    context "when not empty" do
+      specify { subject.store.should_not be_nil }
+    end
+
+    context "when empty" do
+      specify { expect { Deduplicator.new }.to raise_error }
+    end
+  end
+
+  describe "match criteria" do
+    it "adds match definitions to criteria array" do
+      subject.match_attrs([:mid])
+      subject.criteria.should == [[:mid]]
+    end
+
+    it "should convert single items into arrays when adding criteria" do
+      subject.match_attrs(:mid)
+      subject.criteria.should == [[:mid]]
+    end
+
+    it "has a flattened, unique array combining any and all criteria" do
+      subject.match_attrs([:mid, :esn])
+      subject.match_attrs([:date, :mid])
+      ua = subject.unique_attrs
+      ua.should have(3).items
+      ua.should include(:mid)
+      ua.should include(:esn)
+      ua.should include(:date)
+    end
+
+    it "calls any and all via a block" do
+      subject.define do
+        match_attrs [:mid, :esn]
+        match_attrs [:date, :esn]
+      end 
+
+      subject.criteria.should == [[:mid, :esn], [:date, :esn]]
+    end
+  end
+
+  it "indexes store values" do
+    subject.define { match_attrs [:mid] }
+    subject.create_index
+    subject.index.get(:mid, "7275554444").should have(2).items
+    subject.index.get(:mid, "8135552222").should have(2).items
+    subject.index.get(:mid, "2055558888").should be_nil
+  end
+
+  describe "deduplicate" do
+    context "single criteria arrays" do
+
+      it "should deduplicate an ArrayStore on a single match criterion (1 of 3)" do
+        subject.define do
+          match_attrs   :mid 
+        end
+
+        subject.deduplicate
+        subject.groups.count.should == 2
+        subject.groups[0].count.should == 2
+        subject.groups[1].count.should == 2
+      end
+
+      it "should deduplicate an ArrayStore on a single match criterion (2 of 3)" do
+        subject.define do
+          match_attrs   :esn
+        end
+
+        subject.deduplicate
+        subject.groups.count.should == 2
+        subject.groups[0].count.should == 1
+        subject.groups[1].count.should == 3
+      end
+
+      it "should deduplicate an ArrayStore on a single match criterion (3 of 3)" do
+        subject.define do
+          match_attrs   :act_date
+        end
+
+        subject.deduplicate
+        subject.groups.count.should == 3
+      end
+
+      it "should group with only nil values" do
+        subject.define do
+          match_attrs   :nilly
+        end
+
+        subject.deduplicate
+        subject.groups.count.should == 1
+      end
+
+      it "should group with some nil values" do
+        subject.define do
+          match_attrs [:mid, :nilly]
+        end
+
+        subject.deduplicate
+        subject.groups.count.should == 2
+      end
+
+      it "should deduplicate an ArrayStore on multiple criteria" do
+        subject.define do
+          match_attrs   [:esn, :act_date]
+        end
+
+        subject.deduplicate
+        subject.groups.count.should == 3
+      end
+    end #single criteria arrays
+
+    context "multiple criteria arrays" do
+
+      let (:larger_array_store) do
+        c1 = CellTxn.new(0, "7275554444", "11111111111", Date.new(2011,1,1))
+        c2 = CellTxn.new(1, "7275554444", "22222222222", Date.new(2011,1,2))
+        c3 = CellTxn.new(2, "8135552222", "22222222222", Date.new(2011,1,3))
+        c4 = CellTxn.new(3, "8135552222", "22222222222", Date.new(2011,1,2))
+        c5 = CellTxn.new(4, "7275554444", "11111111111", Date.new(2011,1,2)) #hybrid of c1 and c2
+        ArrayStore.new([c1,c2,c3,c4,c5])
+      end
+
+      it "should join groups that are joined by different match criteria" do
+        subject = Deduplicator.new(larger_array_store)
+        subject.define do
+          match_attrs   [:mid, :esn] #joins 0 and 4
+          match_attrs   [:mid, :act_date] #joins 1 and 4
+        end
+
+        subject.deduplicate
+        subject.groups.count.should == 2 # expect [0,1,4],[2,3]
+        two_group = subject.groups.find { |grp| grp.size == 2}
+        two_group.should include(2,3)
+        three_group = subject.groups.find { |grp| grp.size == 3}
+        three_group.should include(0,1,4)
+      end
+
+      it "should return results with objects, group index, and item index" do
+        subject = Deduplicator.new(larger_array_store)
+        subject.define do
+          match_attrs   [:mid, :esn] #joins 0 and 4
+          match_attrs   [:mid, :act_date] #joins 1 and 4
+        end
+
+        subject.deduplicate
+        group_sum, item_sum = 0, 0
+        subject.each_with_groups do |obj, grp_idx, item_idx|
+          group_sum += grp_idx
+          item_sum += item_idx
+          #puts "grp: #{grp_idx}, item: #{item_idx}"
+        end
+
+        #grp: 0, item: 0
+        #grp: 0, item: 1
+        #grp: 0, item: 2
+        #grp: 1, item: 0
+        #grp: 1, item: 1
+
+        group_sum.should == 2
+        item_sum.should == 4
+      end
+    end #multiple criteria arrays
+  end #deduplication
+
+  context "integration tests" do
+
+    it "should deduplicate on a common key" do
+      txns = []
+      i = 0
+      File.open(File.join(File.dirname(__FILE__),'/../samples/agent_recs.csv'),'r').each do |line|
+        parts = line.split ','
+        txns << CellTxn.new(parts[0],parts[1],parts[2],parts[3])
+
+        i += 1
+        break if i == 200
+      end       
+
+      subject = Deduplicator.new(ArrayStore.new(txns))
+      subject.define do
+        match_attrs :act_date
+      end
+
+      subject.deduplicate
+
+      dates = txns.map { |txn| txn.act_date } 
+
+      subject.groups.size.should == dates.uniq.count
+    end
+  end #integration
+end