linkage 0.0.0

data/test/helper.rb

@@ -0,0 +1,43 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+require 'mocha'
+require 'tmpdir'
+require 'logger'
+require 'pp'
+require 'versionomy'
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'linkage'
+
+class Test::Unit::TestCase
+  def stub_field(name, options = {}, &block)
+    f = Linkage::Field.allocate
+    f.stubs(options)
+    if block
+      f.send(:instance_eval, &block)
+    end
+    f.stubs(:is_a?).returns(false)
+    f.stubs(:is_a?).with(Linkage::Field).returns(true)
+    f
+  end
+
+  def self.current_ruby_version
+    @current_ruby_version ||= Versionomy.parse(RUBY_VERSION)
+  end
+
+  def self.ruby19
+    @ruby19 ||= Versionomy.parse("1.9")
+  end
+end
+
+module UnitTests; end
+module IntegrationTests; end

data/test/integration/test_cross_linkage.rb

@@ -0,0 +1,68 @@
+require 'helper'
+
+module IntegrationTests
+  class TestCrossLinkage < Test::Unit::TestCase
+    def setup
+      @tmpdir = Dir.mktmpdir('linkage')
+      @tmpuri = "sqlite://" + File.join(@tmpdir, "foo")
+    end
+
+    def database(&block)
+      Sequel.connect(@tmpuri, &block)
+    end
+
+    def teardown
+      FileUtils.remove_entry_secure(@tmpdir)
+    end
+
+    test "one mandatory field equality on single threaded runner" do
+      # insert the test data
+      database do |db|
+        db.create_table(:foo) { primary_key(:id); Integer(:foo); Integer(:bar) }
+        db[:foo].import([:id, :foo, :bar],
+          Array.new(100) { |i| [i, i % 10, i % 5] })
+      end
+
+      ds = Linkage::Dataset.new(@tmpuri, "foo", :single_threaded => true)
+      conf = ds.link_with(ds) do
+        lhs[:foo].must == rhs[:bar]
+      end
+      runner = Linkage::SingleThreadedRunner.new(conf, @tmpuri)
+      runner.execute
+
+      database do |db|
+        assert_equal 5, db[:groups].count
+        db[:groups].order(:foo_bar).each_with_index do |row, i|
+          assert_equal i, row[:foo_bar]
+        end
+
+        assert_equal 150, db[:groups_records].count
+        db[:groups_records].order(:group_id, :dataset, :record_id).each_with_index do |row, i|
+          expected_group_id = (row[:record_id] % 5) + 1
+          assert_equal expected_group_id, row[:group_id], "Record #{row[:record_id]} should have been in group #{expected_group_id}"
+        end
+      end
+    end
+
+    test "match same field with different filters" do
+      database do |db|
+        db.create_table(:foo) { primary_key(:id); Integer(:foo); Integer(:bar) }
+        db[:foo].import([:id, :foo, :bar],
+          Array.new(100) { |i| [i, i % 10, i % 20] })
+      end
+
+      ds = Linkage::Dataset.new(@tmpuri, "foo", :single_threaded => true)
+      conf = ds.link_with(ds) do
+        lhs[:foo].must == rhs[:foo]
+        lhs[:bar].must == 0
+        rhs[:bar].must == 10
+      end
+      runner = Linkage::SingleThreadedRunner.new(conf, @tmpuri)
+      runner.execute
+
+      database do |db|
+        assert_equal 1, db[:groups].count
+      end
+    end
+  end
+end

data/test/integration/test_dual_linkage.rb

@@ -0,0 +1,85 @@
+require 'helper'
+
+module IntegrationTests
+  class TestDualLinkage < Test::Unit::TestCase
+    def setup
+      @tmpdir = Dir.mktmpdir('linkage')
+      @tmpuri = "sqlite://" + File.join(@tmpdir, "foo")
+    end
+
+    def database(&block)
+      Sequel.connect(@tmpuri, &block)
+    end
+
+    def teardown
+      FileUtils.remove_entry_secure(@tmpdir)
+    end
+
+    test "one mandatory field equality on single threaded runner" do
+      # create the test data
+      database do |db|
+        db.create_table(:foo) { primary_key(:id); String(:ssn) }
+        db[:foo].import([:id, :ssn],
+          Array.new(100) { |i| [i, "12345678#{i%10}"] })
+
+        db.create_table(:bar) { primary_key(:id); String(:ssn) }
+        db[:bar].import([:id, :ssn],
+          Array.new(100) { |i| [i, "12345678#{i%10}"] })
+      end
+
+      ds_1 = Linkage::Dataset.new(@tmpuri, "foo", :single_threaded => true)
+      ds_2 = Linkage::Dataset.new(@tmpuri, "bar", :single_threaded => true)
+      conf = ds_1.link_with(ds_2) do
+        lhs[:ssn].must == rhs[:ssn]
+      end
+      runner = Linkage::SingleThreadedRunner.new(conf, @tmpuri)
+      runner.execute
+
+      database do |db|
+        assert_equal 10, db[:groups].count
+        db[:groups].order(:ssn).each_with_index do |row, i|
+          assert_equal "12345678#{i%10}", row[:ssn]
+        end
+
+        assert_equal 200, db[:groups_records].count
+        db[:groups_records].order(:group_id, :dataset, :record_id).each_with_index do |row, i|
+          if i % 20 >= 10
+            assert_equal 2, row[:dataset], row.inspect
+          else
+            assert_equal 1, row[:dataset], row.inspect
+          end
+          expected_group_id = i / 20 + 1
+          assert_equal expected_group_id, row[:group_id], "Record #{row.inspect} should have been in group #{expected_group_id}"
+        end
+      end
+    end
+
+    test "don't ignore 1-record groups before the combining phase" do
+      # create the test data
+      database do |db|
+        db.create_table(:foo) { primary_key(:id); String(:ssn) }
+        db[:foo].import([:id, :ssn],
+          Array.new(100) { |i| [i, "1234567%03d" % i] })
+
+        db.create_table(:bar) { primary_key(:id); String(:ssn) }
+        db[:bar].import([:id, :ssn],
+          Array.new(100) { |i| [i, "1234567%03d" % i] })
+      end
+
+      ds_1 = Linkage::Dataset.new(@tmpuri, "foo", :single_threaded => true)
+      ds_2 = Linkage::Dataset.new(@tmpuri, "bar", :single_threaded => true)
+      conf = ds_1.link_with(ds_2) do
+        lhs[:ssn].must == rhs[:ssn]
+      end
+      runner = Linkage::SingleThreadedRunner.new(conf, @tmpuri)
+      runner.execute
+
+      database do |db|
+        assert_equal 100, db[:groups].count
+        db[:groups].order(:ssn).each_with_index do |row, i|
+          assert_equal "1234567%03d" % i, row[:ssn]
+        end
+      end
+    end
+  end
+end

data/test/integration/test_self_linkage.rb

@@ -0,0 +1,209 @@
+require 'helper'
+
+module IntegrationTests
+  class TestSelfLinkage < Test::Unit::TestCase
+    def setup
+      @tmpdir = Dir.mktmpdir('linkage')
+      @tmpuri = "sqlite://" + File.join(@tmpdir, "foo")
+    end
+
+    def database(&block)
+      Sequel.connect(@tmpuri, &block)
+    end
+
+    def teardown
+      FileUtils.remove_entry_secure(@tmpdir)
+    end
+
+    test "one mandatory field equality on single threaded runner" do
+      # insert the test data
+      database do |db|
+        db.create_table(:foo) { primary_key(:id); String(:ssn) }
+        db[:foo].import([:id, :ssn],
+          Array.new(100) { |i| [i, "12345678#{i%10}"] })
+      end
+
+      ds = Linkage::Dataset.new(@tmpuri, "foo", :single_threaded => true)
+      conf = ds.link_with(ds) do
+        lhs[:ssn].must == rhs[:ssn]
+      end
+      runner = Linkage::SingleThreadedRunner.new(conf, @tmpuri)
+      runner.execute
+
+      database do |db|
+        assert_equal 10, db[:groups].count
+        db[:groups].order(:ssn).each_with_index do |row, i|
+          assert_equal "12345678#{i%10}", row[:ssn]
+        end
+
+        assert_equal 100, db[:groups_records].count
+        expected_group_id = nil
+        db[:groups_records].order(:record_id).each do |row|
+          expected_group_id = (row[:record_id] % 10) + 1
+          assert_equal expected_group_id, row[:group_id], "Record #{row[:record_id]} should have been in group #{expected_group_id}"
+        end
+      end
+    end
+
+    test "two mandatory field equalities on single threaded runner" do
+      # insert the test data
+      database do |db|
+        db.create_table(:foo) { primary_key(:id); String(:ssn); Date(:dob) }
+        db[:foo].import([:id, :ssn, :dob],
+          Array.new(100) { |i| [i, "12345678#{i%10}", Date.civil(1985, 1, (i % 20) + 1)] })
+      end
+
+      ds = Linkage::Dataset.new(@tmpuri, "foo", :single_threaded => true)
+      conf = ds.link_with(ds) do
+        lhs[:ssn].must == rhs[:ssn]
+        lhs[:dob].must == rhs[:dob]
+      end
+      runner = Linkage::SingleThreadedRunner.new(conf, @tmpuri)
+      runner.execute
+
+      database do |db|
+        assert_equal 20, db[:groups].count
+        db[:groups].order(:ssn).each_with_index do |row, i|
+          assert_equal "12345678#{i/2}", row[:ssn]
+          assert_equal Date.civil(1985, 1, i / 2 + 1 + (i % 2 == 0 ? 0 : 10)), row[:dob]
+        end
+
+        assert_equal 100, db[:groups_records].count
+        expected_group_id = nil
+        db[:groups_records].order(:record_id).each do |row|
+          v = row[:record_id] % 20
+          expected_group_id = v < 10 ? 1 + 2 * v : 2 * (v % 10 + 1)
+          assert_equal expected_group_id, row[:group_id], "Record #{row[:record_id]} should have been in group #{expected_group_id}"
+        end
+      end
+    end
+
+    test "one mandatory field equality on single threaded runner, with filter" do
+      # insert the test data
+      database do |db|
+        db.create_table(:foo) { primary_key(:id); String(:ssn); Integer(:mod_5) }
+        db[:foo].import([:id, :ssn, :mod_5],
+          Array.new(100) { |i| [i, "12345678#{i%10}", i % 5] })
+      end
+
+      ds = Linkage::Dataset.new(@tmpuri, "foo", :single_threaded => true)
+      conf = ds.link_with(ds) do
+        lhs[:ssn].must == rhs[:ssn]
+        lhs[:mod_5].must == 3
+      end
+      runner = Linkage::SingleThreadedRunner.new(conf, @tmpuri)
+      runner.execute
+
+      database do |db|
+        assert_equal 2, db[:groups].count
+        db[:groups].order(:ssn).each_with_index do |row, i|
+          assert_equal "12345678#{(i * 5) + 3}", row[:ssn]
+        end
+
+        assert_equal 20, db[:groups_records].count
+        expected_group_id = nil
+        db[:groups_records].order(:record_id).each do |row|
+          expected_group_id = (row[:record_id] / 5) % 2 + 1
+          assert_equal expected_group_id, row[:group_id], "Record #{row[:record_id]} should have been in group #{expected_group_id}"
+        end
+      end
+    end
+
+    test "one mandatory field equality on single threaded runner, with inequality filters" do
+      # insert the test data
+      database do |db|
+        db.create_table(:foo) { primary_key(:id); String(:ssn); Integer(:mod_5) }
+        db[:foo].import([:id, :ssn, :mod_5],
+          Array.new(100) { |i| [i, "12345678#{i%10}", i % 5] })
+      end
+
+      ds = Linkage::Dataset.new(@tmpuri, "foo", :single_threaded => true)
+      conf = ds.link_with(ds) do
+        lhs[:ssn].must == rhs[:ssn]
+        lhs[:mod_5].must > 2
+        lhs[:mod_5].must <= 3
+      end
+      runner = Linkage::SingleThreadedRunner.new(conf, @tmpuri)
+      runner.execute
+
+      database do |db|
+        assert_equal 2, db[:groups].count
+        db[:groups].order(:ssn).each_with_index do |row, i|
+          assert_equal "12345678#{(i * 5) + 3}", row[:ssn]
+        end
+
+        assert_equal 20, db[:groups_records].count
+        expected_group_id = nil
+        db[:groups_records].order(:record_id).each do |row|
+          expected_group_id = (row[:record_id] / 5) % 2 + 1
+          assert_equal expected_group_id, row[:group_id], "Record #{row[:record_id]} should have been in group #{expected_group_id}"
+        end
+      end
+    end
+
+    test "one mandatory field equality on single threaded runner, with field filter" do
+      # insert the test data
+      database do |db|
+        db.create_table(:foo) { primary_key(:id); String(:ssn); Integer(:mod_5); Integer(:mod_20) }
+        db[:foo].import([:id, :ssn, :mod_5, :mod_20],
+          Array.new(100) { |i| [i, "123456789#{i%10}", i % 5, i % 20] })
+      end
+
+      ds = Linkage::Dataset.new(@tmpuri, "foo", :single_threaded => true)
+      conf = ds.link_with(ds) do
+        lhs[:ssn].must == rhs[:ssn]
+        lhs[:mod_5].must == lhs[:mod_20]
+      end
+      assert_equal :self, conf.linkage_type
+      runner = Linkage::SingleThreadedRunner.new(conf, @tmpuri)
+      runner.execute
+
+      database do |db|
+        assert_equal 5, db[:groups].count
+        db[:groups].order(:ssn).each_with_index do |row, i|
+          assert_equal "123456789#{i}", row[:ssn]
+        end
+
+        assert_equal 25, db[:groups_records].count
+        expected_group_id = nil
+        db[:groups_records].order(:record_id).each do |row|
+          expected_group_id = row[:record_id] % 5 + 1
+          assert_equal expected_group_id, row[:group_id], "Record #{row[:record_id]} should have been in group #{expected_group_id}"
+        end
+      end
+    end
+
+    test "one mandatory field equality on single threaded runner, with identical filters" do
+      # insert the test data
+      database do |db|
+        db.create_table(:foo) { primary_key(:id); String(:ssn); Integer(:mod_5); Integer(:mod_20) }
+        db[:foo].import([:id, :ssn, :mod_5, :mod_20],
+          Array.new(100) { |i| [i, "123456789#{i%10}", i % 5, i % 20] })
+      end
+
+      ds = Linkage::Dataset.new(@tmpuri, "foo", :single_threaded => true)
+      conf = ds.link_with(ds) do
+        lhs[:ssn].must == rhs[:ssn]
+        lhs[:mod_5].must == lhs[:mod_20]
+        rhs[:mod_5].must == rhs[:mod_20]
+      end
+      assert_equal :self, conf.linkage_type
+      runner = Linkage::SingleThreadedRunner.new(conf, @tmpuri)
+      runner.execute
+
+      database do |db|
+        assert_equal 5, db[:groups].count
+        db[:groups].order(:ssn).each_with_index do |row, i|
+          assert_equal "123456789#{i}", row[:ssn]
+        end
+
+        assert_equal 25, db[:groups_records].count
+        expected_group_id = nil
+        db[:groups_records].order(:record_id).each do |row|
+          expected_group_id = row[:record_id] % 5 + 1
+          assert_equal expected_group_id, row[:group_id], "Record #{row[:record_id]} should have been in group #{expected_group_id}"
+        end
+      end
+    end
+  end
+end

data/test/unit/test_configuration.rb

@@ -0,0 +1,145 @@
+require 'helper'
+
+class UnitTests::TestConfiguration < Test::Unit::TestCase
+  test "linkage_type is self when the two datasets are the same" do
+    dataset = stub('dataset')
+    c = Linkage::Configuration.new(dataset, dataset)
+    assert_equal :self, c.linkage_type
+  end
+
+  test "linkage_type is dual when the two datasets are different" do
+    dataset_1 = stub('dataset')
+    dataset_2 = stub('dataset')
+    c = Linkage::Configuration.new(dataset_1, dataset_2)
+    assert_equal :dual, c.linkage_type
+  end
+
+  test "linkage_type is cross when there's a 'cross-join'" do
+    dataset = mock('dataset', :set_new_id => nil)
+    c = Linkage::Configuration.new(dataset, dataset)
+    exp = stub('expectation', :kind => :cross)
+    c.add_expectation(exp)
+    assert_equal :cross, c.linkage_type
+  end
+
+  test "linkage_type is cross when there's different filters on both sides" do
+    field = stub('field')
+    dataset = stub('dataset', :set_new_id => nil)
+    dataset.stubs(:fields).returns({:foo => field})
+    c = Linkage::Configuration.new(dataset, dataset)
+    exp_1 = stub('expectation', :kind => :filter)
+    Linkage::MustExpectation.expects(:new).with(:==, field, 123, nil).returns(exp_1)
+    exp_2 = stub('expectation', :kind => :filter)
+    Linkage::MustExpectation.expects(:new).with(:==, field, 456, nil).returns(exp_2)
+    c.send(:instance_eval) do
+      lhs[:foo].must == 123
+      rhs[:foo].must == 456
+    end
+    assert_equal :cross, c.linkage_type
+  end
+
+  test "linkage_type is self when there's identical static filters on each side" do
+    field = stub('field')
+    dataset = stub('dataset', :set_new_id => nil)
+    dataset.stubs(:fields).returns({:foo => field})
+    c = Linkage::Configuration.new(dataset, dataset)
+    exp_1 = stub('expectation', :kind => :filter)
+    Linkage::MustExpectation.expects(:new).twice.with(:==, field, 123, nil).returns(exp_1)
+    c.send(:instance_eval) do
+      lhs[:foo].must == 123
+      rhs[:foo].must == 123
+    end
+    assert_equal :self, c.linkage_type
+  end
+
+  test "linkage_type is self when there's a two-field filter on one side" do
+    field_1 = stub('field 1')
+    field_2 = stub('field 2')
+    dataset = stub('dataset', :set_new_id => nil)
+    dataset.stubs(:fields).returns({:foo => field_1, :bar => field_2})
+    c = Linkage::Configuration.new(dataset, dataset)
+    exp_1 = stub('expectation', :kind => :filter)
+    Linkage::MustExpectation.expects(:new).with(:==, field_1, field_2, :filter).returns(exp_1)
+    exp_2 = stub('expectation', :kind => :self)
+    Linkage::MustExpectation.expects(:new).with(:==, field_1, field_1, nil).returns(exp_2)
+    c.send(:instance_eval) do
+      lhs[:foo].must == lhs[:bar]
+      lhs[:foo].must == rhs[:foo]
+    end
+    assert_equal :self, c.linkage_type
+  end
+
+  test "static expectation" do
+    dataset_1 = stub('dataset')
+    field = stub('field')
+    dataset_1.stubs(:fields).returns({:foo => field})
+    dataset_2 = stub('dataset')
+    c = Linkage::Configuration.new(dataset_1, dataset_2)
+    Linkage::MustExpectation.expects(:new).with(:==, field, 123, nil)
+    c.send(:instance_eval) do
+      lhs[:foo].must == 123
+    end
+  end
+
+  ## Maybe in the future
+  #test "static expectation, flopped" do
+    #dataset_1 = stub('dataset')
+    #field = stub('field')
+    #dataset_1.stubs(:fields).returns({:foo => field})
+    #dataset_2 = stub('dataset')
+    #c = Linkage::Configuration.new(dataset_1, dataset_2)
+    #Linkage::MustExpectation.expects(:new).with(:==, 123, field)
+    #c.send(:instance_eval) do
+      #123.must == lhs[:foo]
+    #end
+  #end
+
+  test "complain if an invalid field is accessed" do
+    dataset_1 = stub('dataset')
+    field_1 = stub_field('field 1')
+    dataset_1.stubs(:fields).returns({:foo => field_1})
+
+    dataset_2 = stub('dataset')
+    field_2 = stub_field('field 2')
+    dataset_2.stubs(:fields).returns({:bar => field_2})
+
+    c = Linkage::Configuration.new(dataset_1, dataset_2)
+    assert_raises(ArgumentError) do
+      c.send(:instance_eval) do
+        lhs[:foo].must == rhs[:non_existant_field]
+      end
+    end
+  end
+
+  operators = [:>, :<, :>=, :<=]
+  operators << :'!='  if current_ruby_version >= ruby19
+  operators.each do |operator|
+    test "DSL #{operator} filter operator" do
+      dataset_1 = stub('dataset 1')
+      field_1 = stub_field('field 1')
+      dataset_1.stubs(:fields).returns({:foo => field_1})
+
+      dataset_2 = stub('dataset 2')
+      field_2 = stub_field('field 2')
+      dataset_2.stubs(:fields).returns({:bar => field_2})
+
+      c = Linkage::Configuration.new(dataset_1, dataset_2)
+      Linkage::MustExpectation.expects(:new).with(operator, field_1, field_2, nil)
+      block = eval("Proc.new { lhs[:foo].must #{operator} rhs[:bar] }")
+      c.send(:instance_eval, &block)
+    end
+  end
+
+  test "must_not expectation" do
+    dataset_1 = stub('dataset 1')
+    field_1 = stub_field('field 1')
+    dataset_1.stubs(:fields).returns({:foo => field_1})
+    dataset_2 = stub('dataset 2')
+
+    c = Linkage::Configuration.new(dataset_1, dataset_2)
+    Linkage::MustNotExpectation.expects(:new).with(:==, field_1, 123, nil)
+    c.send(:instance_eval) do
+      lhs[:foo].must_not == 123
+    end
+  end
+end