linkage 0.0.8 → 0.1.0.pre

data/test/integration/test_configuration.rb

@@ -1,268 +0,0 @@
-require 'helper'
-
-module IntegrationTests
-  class TestConfiguration < Test::Unit::TestCase
-    test "linkage_type is self when the two datasets are the same" do
-      database_for('sqlite') do |db|
-        db.create_table(:foo) { primary_key(:id); String(:foo); String(:bar) }
-      end
-
-      dataset = Linkage::Dataset.new(database_options_for('sqlite'), "foo")
-      conf = Linkage::Configuration.new(dataset, dataset)
-      assert_equal :self, conf.linkage_type
-    end
-
-    test "linkage_type is dual when the two datasets are different" do
-      database_for('sqlite') do |db|
-        db.create_table(:foo) { primary_key(:id); String(:foo); String(:bar) }
-        db.create_table(:bar) { primary_key(:id); String(:foo); String(:bar) }
-      end
-
-      dataset_1 = Linkage::Dataset.new(database_options_for('sqlite'), "foo")
-      dataset_2 = Linkage::Dataset.new(database_options_for('sqlite'), "bar")
-      conf = Linkage::Configuration.new(dataset_1, dataset_2)
-      assert_equal :dual, conf.linkage_type
-    end
-
-    test "linkage_type is cross when there's different filters on both sides" do
-      database_for('sqlite') do |db|
-        db.create_table(:foo) { primary_key(:id); String(:foo); String(:bar) }
-      end
-
-      dataset = Linkage::Dataset.new(database_options_for('sqlite'), "foo")
-      conf = Linkage::Configuration.new(dataset, dataset)
-      conf.configure do
-        lhs[:foo].must == "foo"
-        rhs[:foo].must == "bar"
-      end
-      assert_equal :cross, conf.linkage_type
-    end
-
-    test "linkage_type is self when there's identical static filters on each side" do
-      database_for('sqlite') do |db|
-        db.create_table(:foo) { primary_key(:id); String(:foo); String(:bar) }
-      end
-
-      dataset = Linkage::Dataset.new(database_options_for('sqlite'), "foo")
-      conf = Linkage::Configuration.new(dataset, dataset)
-      conf.configure do
-        lhs[:foo].must == "foo"
-        rhs[:foo].must == "foo"
-      end
-      assert_equal :self, conf.linkage_type
-    end
-
-    test "linkage_type is cross when exhaustive expectations use different fields" do
-      database_for('sqlite') do |db|
-        db.create_table(:foo) { primary_key(:id); Integer(:foo); Integer(:bar) }
-      end
-
-      dataset = Linkage::Dataset.new(database_options_for('sqlite'), "foo")
-      conf = Linkage::Configuration.new(dataset, dataset)
-      conf.configure do
-        lhs[:foo].must(be_within(5).of(rhs[:bar]))
-      end
-      assert_equal :cross, conf.linkage_type
-    end
-
-    test "static expectation" do
-      database_for('sqlite') do |db|
-        db.create_table(:foo) { primary_key(:id); String(:foo); String(:bar) }
-      end
-
-      dataset_1 = Linkage::Dataset.new(database_options_for('sqlite'), "foo")
-      conf = Linkage::Configuration.new(dataset_1, dataset_1)
-      conf.configure do
-        lhs[:foo].must == "foo"
-      end
-
-      dataset_2, _ = conf.datasets_with_applied_simple_expectations
-      assert_equal dataset_2.obj, dataset_1.filter(:foo => "foo").obj
-    end
-
-    test "complain if an invalid field is accessed" do
-      database_for('sqlite') do |db|
-        db.create_table(:foo) { primary_key(:id); String(:foo); String(:bar) }
-      end
-
-      dataset = Linkage::Dataset.new(database_options_for('sqlite'), "foo")
-      conf = Linkage::Configuration.new(dataset, dataset)
-      assert_raises(ArgumentError) do
-        conf.configure do
-          lhs[:foo].must == rhs[:non_existant_field]
-        end
-      end
-    end
-
-    operators = [:>, :<, :>=, :<=]
-    operators.each do |operator|
-      test "DSL #{operator} filter operator" do
-        database_for('sqlite') do |db|
-          db.create_table(:foo) { primary_key(:id); String(:foo); String(:bar) }
-        end
-
-        dataset_1 = Linkage::Dataset.new(database_options_for('sqlite'), "foo")
-        conf = Linkage::Configuration.new(dataset_1, dataset_1)
-        conf.configure do
-          lhs[:foo].must.send(operator, 123)
-        end
-
-        expr = Sequel::SQL::BooleanExpression.new(operator, Sequel::SQL::Identifier.new(:foo), 123)
-        dataset_2, _ = conf.datasets_with_applied_simple_expectations
-        assert_equal dataset_2.obj, dataset_1.filter(expr).obj
-      end
-
-      test "comparing two data sources with #{operator}" do
-        database_for('sqlite') do |db|
-          db.create_table(:foo) { primary_key(:id); Integer(:foo); Integer(:bar) }
-        end
-
-        dataset = Linkage::Dataset.new(database_options_for('sqlite'), "foo")
-        conf = Linkage::Configuration.new(dataset, dataset)
-        conf.configure do
-          lhs[:foo].must.send(operator, rhs[:bar])
-        end
-        assert_equal 1, conf.exhaustive_expectations.length
-
-        comp = conf.exhaustive_expectations[0].comparator
-        assert_instance_of Linkage::Comparators::Compare, comp
-      end
-    end
-
-    test "must_not expectation" do
-      database_for('sqlite') do |db|
-        db.create_table(:foo) { primary_key(:id); String(:foo); String(:bar) }
-      end
-
-      dataset_1 = Linkage::Dataset.new(database_options_for('sqlite'), "foo")
-      conf = Linkage::Configuration.new(dataset_1, dataset_1)
-      conf.configure do
-        lhs[:foo].must_not == "foo"
-      end
-
-      dataset_2, _ = conf.datasets_with_applied_simple_expectations
-      assert_equal dataset_2.obj, dataset_1.filter(~{:foo => "foo"}).obj
-    end
-
-    test "static database function" do
-      database_for('sqlite') do |db|
-        db.create_table(:foo) { primary_key(:id); String(:foo); String(:bar) }
-      end
-
-      dataset_1 = Linkage::Dataset.new(database_options_for('sqlite'), "foo")
-      conf = Linkage::Configuration.new(dataset_1, dataset_1)
-      conf.configure do
-        lhs[:foo].must == trim("foo")
-      end
-
-      dataset_2, _ = conf.datasets_with_applied_simple_expectations
-      assert_equal dataset_1.filter({:foo => :trim.sql_function("foo")}).obj, dataset_2.obj
-    end
-
-    test "save_results_in" do
-      database_for('sqlite') do |db|
-        db.create_table(:foo) { primary_key(:id); String(:foo); String(:bar) }
-      end
-
-      dataset_1 = Linkage::Dataset.new(database_options_for('sqlite'), "foo")
-      conf = Linkage::Configuration.new(dataset_1, dataset_1)
-      conf.configure do
-        save_results_in("mysql://localhost/results", {:foo => 'bar'})
-      end
-      assert_equal "mysql://localhost/results", conf.results_uri
-      assert_equal({:foo => 'bar'}, conf.results_uri_options)
-    end
-
-    test "case insensitive field names" do
-      database_for('sqlite') do |db|
-        db.create_table(:foo) { primary_key(:id); String(:foo); String(:bar) }
-      end
-
-      assert_nothing_raised do
-        dataset = Linkage::Dataset.new(database_options_for('sqlite'), "foo")
-        results_uri = database_options_for('sqlite')
-        conf = dataset.link_with(dataset) do
-          lhs[:Foo].must == rhs[:baR]
-          save_results_in(results_uri)
-        end
-      end
-    end
-
-    test "decollation_needed? is false when the datasets and results dataset all have the same database and collations" do
-      database_for('mysql') do |db|
-        db.create_table!(:foo) { primary_key(:id); String(:foo, :collate => :latin1_swedish_ci) }
-        db.create_table!(:bar) { primary_key(:id); String(:foo, :collate => :latin1_swedish_ci) }
-      end
-
-      dataset_1 = Linkage::Dataset.new(database_options_for('mysql'), 'foo')
-      dataset_2 = Linkage::Dataset.new(database_options_for('mysql'), 'bar')
-      conf = dataset_1.link_with(dataset_2) do
-        lhs[:foo].must == rhs[:foo]
-      end
-      conf.results_uri = database_options_for('mysql')
-      assert !conf.decollation_needed?
-    end
-
-    test "decollation_needed? is true when the datasets have different database types" do
-      database_for('mysql') do |db|
-        db.create_table!(:foo) { primary_key(:id); String(:foo) }
-      end
-
-      database_for('sqlite') do |db|
-        db.create_table!(:foo) { primary_key(:id); String(:foo) }
-      end
-
-      dataset_1 = Linkage::Dataset.new(database_options_for('mysql'), 'foo')
-      dataset_2 = Linkage::Dataset.new(database_options_for('sqlite'), 'foo')
-      conf = dataset_1.link_with(dataset_2) do
-        lhs[:foo].must == rhs[:foo]
-      end
-      conf.results_uri = database_options_for('mysql')
-      assert conf.decollation_needed?
-    end
-
-    test "decollation_needed? is true when the result dataset has different database type than the datasets" do
-      database_for('mysql') do |db|
-        db.create_table!(:foo) { primary_key(:id); String(:foo) }
-        db.create_table!(:bar) { primary_key(:id); String(:foo) }
-      end
-
-      dataset_1 = Linkage::Dataset.new(database_options_for('mysql'), 'foo')
-      dataset_2 = Linkage::Dataset.new(database_options_for('mysql'), 'bar')
-      conf = dataset_1.link_with(dataset_2) do
-        lhs[:foo].must == rhs[:foo]
-      end
-      conf.results_uri = database_options_for('sqlite')
-      assert conf.decollation_needed?
-    end
-
-    test "decollation_needed? is false when not comparing string columns" do
-      database_for('mysql') do |db|
-        db.create_table!(:foo) { primary_key(:id); Fixnum(:foo) }
-      end
-
-      database_for('sqlite') do |db|
-        db.create_table!(:foo) { primary_key(:id); Fixnum(:foo) }
-      end
-
-      dataset_1 = Linkage::Dataset.new(database_options_for('mysql'), 'foo')
-      dataset_2 = Linkage::Dataset.new(database_options_for('sqlite'), 'foo')
-      conf = dataset_1.link_with(dataset_2) do
-        lhs[:foo].must == rhs[:foo]
-      end
-      conf.results_uri = database_options_for('mysql')
-      assert !conf.decollation_needed?
-    end
-
-    test "creating comparator expectation for within" do
-      database_for('mysql') do |db|
-        db.create_table!(:foo) { primary_key(:id); Integer(:foo) }
-      end
-      dataset = Linkage::Dataset.new(database_options_for('mysql'), 'foo')
-
-      conf = dataset.link_with(dataset) do
-        lhs[:foo].must be_within(5).of(rhs[:foo])
-      end
-    end
-  end
-end

data/test/integration/test_dataset.rb

@@ -1,116 +0,0 @@
-require 'helper'
-
-class IntegrationTests::TestDataset < Test::Unit::TestCase
-  def setup
-    @tmpdir = Dir.mktmpdir('linkage')
-    @tmpuri = "sqlite://" + File.join(@tmpdir, "foo")
-  end
-
-  def database(&block)
-    Sequel.connect(@tmpuri, &block)
-  end
-
-  def teardown
-    FileUtils.remove_entry_secure(@tmpdir)
-  end
-
-  test "methods that clone the dataset" do
-    database do |db|
-      db.create_table(:foo) do
-        primary_key :id
-        String :bar
-      end
-    end
-    ds_1 = Linkage::Dataset.new(@tmpuri, "foo")
-    ds_2 = ds_1.filter(:foo => 'bar')
-    assert_instance_of Linkage::Dataset, ds_2
-    assert_equal ds_2.field_set, ds_1.field_set
-    assert_match /`foo` = 'bar'/, ds_2.sql
-  end
-
-  test "each_group" do
-    database do |db|
-      db.create_table(:foo) do
-        primary_key :id
-        String :bar
-      end
-      db[:foo].import([:id, :bar], [[1, 'foo'], [2, 'foo'], [3, 'bar'], [4, 'baz']])
-    end
-
-    ds = Linkage::Dataset.new(@tmpuri, "foo")
-    ds = ds.group_match(Linkage::MetaObject.new(ds.field_set[:bar]))
-    ds.each_group do |group|
-      assert_equal({:bar => "foo"}, group.values)
-      assert_equal(2, group.count)
-    end
-
-    groups = []
-    ds.each_group(1) do |group|
-      groups << group
-    end
-    assert_equal 3, groups.length
-  end
-
-  test "each_group with alias" do
-    database do |db|
-      db.create_table(:foo) do
-        primary_key :id
-        String :bar
-      end
-      db[:foo].import([:id, :bar], [[1, 'foo'], [2, 'foo'], [3, 'bar'], [4, 'baz']])
-    end
-
-    ds = Linkage::Dataset.new(@tmpuri, "foo")
-    ds = ds.group_match({
-      :meta_object => Linkage::MetaObject.new(ds.field_set[:bar]),
-      :alias => :bar_baz
-    })
-    ds.each_group do |group|
-      assert_equal({:bar_baz => "foo"}, group.values)
-      assert_equal(2, group.count)
-    end
-  end
-
-  test "each_group with filters" do
-    database do |db|
-      db.create_table(:foo) do
-        primary_key :id
-        String :bar
-        Integer :baz
-      end
-      db[:foo].import([:id, :bar, :baz], [[1, 'foo', 1], [2, 'foo', 2], [3, 'bar', 3], [4, 'baz', 4]])
-    end
-
-    ds = Linkage::Dataset.new(@tmpuri, "foo")
-    ds = ds.group_match(Linkage::MetaObject.new(ds.field_set[:bar]))
-    ds = ds.filter { baz >= 3 }
-    groups = []
-    ds.each_group(1) do |group|
-      groups << group
-    end
-    assert_equal 2, groups.length
-  end
-
-  test "each_group with collation" do
-    database_for('mysql') do |db|
-      db.create_table!(:foo) do
-        primary_key :id
-        String :bar, :collate => :latin1_swedish_ci
-      end
-      db[:foo].import([:id, :bar], [[1, 'fOo'], [2, 'foO'], [3, 'bar'], [4, 'baz']])
-    end
-
-    ds = Linkage::Dataset.new(database_options_for('mysql'), "foo")
-    ds = ds.group_match(Linkage::MetaObject.new(ds.field_set[:bar]))
-    groups = []
-    ds.each_group(1) do |group|
-      groups << group
-    end
-    expected = [
-      {:bar => 'BAR'},
-      {:bar => 'BAZ'},
-      {:bar => 'FOO'}
-    ]
-    assert_equal expected, groups.collect(&:decollated_values)
-  end
-end

data/test/integration/test_functions.rb

@@ -1,88 +0,0 @@
-require 'helper'
-
-module IntegrationTests
-  class TestFunctions < Test::Unit::TestCase
-    def setup
-      @tmpdir = Dir.mktmpdir('linkage')
-      @tmpuri = "sqlite://" + File.join(@tmpdir, "foo")
-    end
-
-    def database(options = {}, &block)
-      Sequel.connect(@tmpuri, options, &block)
-    end
-
-    def teardown
-      FileUtils.remove_entry_secure(@tmpdir)
-    end
-
-    test "match functions" do
-      # insert the test data
-      database do |db|
-        db.create_table(:foo) { primary_key(:id); String(:bar) }
-        db[:foo].import([:id, :bar],
-          Array.new(100) { |i| [i, "bar%s" % (" " * (i % 10))] })
-      end
-
-      ds = Linkage::Dataset.new(@tmpuri, "foo", :single_threaded => true)
-      tmpuri = @tmpuri
-      conf = ds.link_with(ds) do
-        trim(lhs[:bar]).must == trim(rhs[:bar])
-        save_results_in(tmpuri)
-      end
-      assert_equal :self, conf.linkage_type
-      runner = Linkage::SingleThreadedRunner.new(conf)
-      runner.execute
-
-      database do |db|
-        assert_equal 1, db[:groups].count
-      end
-    end
-
-    test "strftime in sqlite" do
-      #logger = Logger.new(STDERR)
-      #database(:logger => logger) do |db|
-      database do |db|
-        db.create_table(:foo) { primary_key(:id); Date(:foo_date) }
-        db.create_table(:bar) { primary_key(:id); String(:bar_string) }
-        db[:foo].insert({:id => 1, :foo_date => Date.today})
-        db[:bar].insert({:id => 1, :bar_string => Date.today.strftime("%Y-%m-%d")})
-      end
-
-      ds_1 = Linkage::Dataset.new(@tmpuri, "foo", :single_threaded => true)
-      ds_2 = Linkage::Dataset.new(@tmpuri, "bar", :single_threaded => true)
-      tmpuri = @tmpuri
-      conf = ds_1.link_with(ds_2) do
-        strftime(lhs[:foo_date], "%Y-%m-%d").must == rhs[:bar_string]
-        save_results_in(tmpuri)
-      end
-      runner = Linkage::SingleThreadedRunner.new(conf)
-      runner.execute
-
-      database do |db|
-        assert_equal 1, db[:groups].count
-      end
-    end
-
-    test "binary function with static argument" do
-      database do |db|
-        db.create_table(:foo) { primary_key(:id); String(:bar) }
-        db[:foo].import([:id, :bar], [[1, 'foo'], [2, 'foo']])
-      end
-
-      ds = Linkage::Dataset.new(@tmpuri, "foo")
-      tmpuri = @tmpuri
-      conf = ds.link_with(ds) do
-        lhs[:bar].must == rhs[:bar]
-        binary(lhs[:bar]).must == binary('foo')
-        binary(rhs[:bar]).must == binary('foo')
-        save_results_in(tmpuri)
-      end
-      runner = Linkage::SingleThreadedRunner.new(conf)
-      runner.execute
-
-      database do |db|
-        assert_equal 1, db[:groups].count
-      end
-    end
-  end
-end

data/test/integration/test_result_set.rb

@@ -1,85 +0,0 @@
-require 'helper'
-
-module IntegrationTests
-  class TestResultSet < Test::Unit::TestCase
-    test "#create_tables! creates original_groups table when decollation is needed" do
-      database_for('sqlite') do |db|
-        db.create_table!(:foo) { primary_key(:id); String(:foo) }
-      end
-
-      database_for('mysql') do |db|
-        db.create_table!(:foo) { primary_key(:id); String(:foo) }
-      end
-
-      dataset_1 = Linkage::Dataset.new(database_options_for('sqlite'), 'foo')
-      dataset_2 = Linkage::Dataset.new(database_options_for('mysql'), 'foo')
-      results_uri = database_options_for('sqlite')
-      conf = dataset_1.link_with(dataset_2) do
-        lhs[:foo].must == rhs[:foo]
-        save_results_in(results_uri)
-      end
-      conf.result_set.create_tables!
-      assert_include conf.result_set.database.tables, :original_groups
-    end
-
-    test "#create_tables! doesn't create original_groups table when decollation is needed" do
-      database_for('sqlite') do |db|
-        db.create_table!(:foo) { primary_key(:id); String(:foo) }
-      end
-
-      dataset = Linkage::Dataset.new(database_options_for('sqlite'), 'foo')
-      results_uri = database_options_for('sqlite')
-      conf = dataset.link_with(dataset) do
-        lhs[:foo].must == rhs[:foo]
-        save_results_in(results_uri)
-      end
-      conf.result_set.create_tables!
-      assert_not_include conf.result_set.database.tables, :original_groups
-    end
-
-    test "#create_tables! doesn't create groups table when not needed" do
-      database_for('sqlite') do |db|
-        db.create_table!(:foo) { primary_key(:id); Integer(:foo) }
-      end
-
-      dataset = Linkage::Dataset.new(database_options_for('sqlite'), 'foo')
-      results_uri = database_options_for('sqlite')
-      conf = dataset.link_with(dataset) do
-        lhs[:foo].must be_within(5).of(rhs[:foo])
-        save_results_in(results_uri)
-      end
-      conf.result_set.create_tables!
-      assert_not_include conf.result_set.database.tables, :groups
-    end
-
-    test "#create_tables! creates scores table when there are exhaustive expectations" do
-      database_for('sqlite') do |db|
-        db.create_table!(:foo) { primary_key(:id); Integer(:foo) }
-      end
-
-      dataset = Linkage::Dataset.new(database_options_for('sqlite'), 'foo')
-      results_uri = database_options_for('sqlite')
-      conf = dataset.link_with(dataset) do
-        lhs[:foo].must be_within(5).of(rhs[:foo])
-        save_results_in(results_uri)
-      end
-      conf.result_set.create_tables!
-      assert_include conf.result_set.database.tables, :scores
-    end
-
-    test "#create_tables! doesn't create scores table when not needed" do
-      database_for('sqlite') do |db|
-        db.create_table!(:foo) { primary_key(:id); Integer(:foo) }
-      end
-
-      dataset = Linkage::Dataset.new(database_options_for('sqlite'), 'foo')
-      results_uri = database_options_for('sqlite')
-      conf = dataset.link_with(dataset) do
-        lhs[:foo].must == rhs[:foo]
-        save_results_in(results_uri)
-      end
-      conf.result_set.create_tables!
-      assert_not_include conf.result_set.database.tables, :scores
-    end
-  end
-end

data/test/integration/test_scoring.rb

@@ -1,84 +0,0 @@
-require 'helper'
-
-module IntegrationTests
-  class TestScoring < Test::Unit::TestCase
-    test "stop scoring if must expectation fails" do
-      database_for('sqlite') do |db|
-        db.create_table(:foo) { primary_key(:id); Integer(:num) }
-        db.create_table(:bar) { primary_key(:id); Integer(:num) }
-        db[:foo].import([:id, :num], [[1, 1]])
-        db[:bar].import([:id, :num], [[1, 5]])
-      end
-
-      db_opts = database_options_for('sqlite')
-      dataset_1 = Linkage::Dataset.new(db_opts, "foo")
-      dataset_2 = Linkage::Dataset.new(db_opts, "bar")
-      conf = dataset_1.link_with(dataset_2) do
-        lhs[:num].must_not be_within(5).of(rhs[:num])
-        lhs[:num].must be_within(5).of(rhs[:num])
-        save_results_in(db_opts)
-      end
-
-      runner = Linkage::SingleThreadedRunner.new(conf)
-      runner.execute
-
-      database_for('sqlite') do |db|
-        assert_equal db[:scores].count, 1
-        record = db[:scores].first
-        assert_equal 1, record[:score]
-      end
-    end
-
-    test "scoring phase adds matches as needed" do
-      database_for('sqlite') do |db|
-        db.create_table(:foo) { primary_key(:id); Integer(:num) }
-        db.create_table(:bar) { primary_key(:id); Integer(:num) }
-        db[:foo].import([:id, :num], (0..15).collect { |i| [i, i] })
-        db[:bar].import([:id, :num], (0..15).collect { |i| [i, i] })
-      end
-
-      db_opts = database_options_for('sqlite')
-      dataset_1 = Linkage::Dataset.new(db_opts, "foo")
-      dataset_2 = Linkage::Dataset.new(db_opts, "bar")
-      conf = dataset_1.link_with(dataset_2) do
-        lhs[:num].must be_within(10).of(rhs[:num])
-        lhs[:num].must_not be_within(5).of(rhs[:num])
-        save_results_in(db_opts)
-      end
-
-      runner = Linkage::SingleThreadedRunner.new(conf)
-      runner.execute
-
-      database_for('sqlite') do |db|
-        assert_equal 80, db[:matches].count
-        db[:matches].order(:record_1_id, :record_2_id).each do |row|
-          assert_equal 1, row[:total_score]
-          assert_include 6..10, (row[:record_1_id] - row[:record_2_id]).abs
-        end
-      end
-    end
-
-    test "optimize scoring for self linkage" do
-      database_for('sqlite') do |db|
-        db.create_table(:foo) { primary_key(:id); Integer(:num) }
-        db[:foo].import([:id, :num], [[1, 1], [2, 5], [3, 10]])
-      end
-
-      db_opts = database_options_for('sqlite')
-      dataset = Linkage::Dataset.new(db_opts, "foo")
-      conf = dataset.link_with(dataset) do
-        lhs[:num].must be_within(5).of(rhs[:num])
-        save_results_in(db_opts)
-      end
-
-      runner = Linkage::SingleThreadedRunner.new(conf)
-      runner.execute
-
-      database_for('sqlite') do |db|
-        assert_equal db[:scores].count, 3
-        scores = db[:scores].order(:record_1_id, :record_2_id).select_map(:score)
-        assert_equal [1, 0, 1], scores
-      end
-    end
-  end
-end