linkage 0.0.6 → 0.0.8

data/lib/linkage/runner/single_threaded.rb

@@ -5,28 +5,40 @@ module Linkage
   class SingleThreadedRunner < Runner
     # @return [Linkage::ResultSet]
     def execute
-      setup_datasets
-      group_records
+      result_set.create_tables!
+
+      @pk_1 = config.dataset_1.field_set.primary_key.to_expr
+      @pk_2 = config.dataset_2.field_set.primary_key.to_expr
+      if config.has_simple_expectations?
+        setup_datasets
+        group_records
+
+        if config.has_exhaustive_expectations?
+          score_records_with_groups
+        else
+          create_matches
+        end
+      else
+        dataset_1, dataset_2 = config.datasets_with_applied_exhaustive_expectations
+        score_records_without_groups(dataset_1, dataset_2)
+      end
 
+      result_set.flush!
       return result_set
     end
 
     private
 
     def setup_datasets
-      @dataset_1, @dataset_2 = config.datasets_with_applied_expectations
+      @dataset_1, @dataset_2 = config.datasets_with_applied_simple_expectations
 
-      pk = @dataset_1.field_set.primary_key
-      @dataset_1 = @dataset_1.select(pk.to_expr)
+      @dataset_1 = @dataset_1.select(@pk_1)
       if @config.linkage_type != :self
-        pk = @dataset_2.field_set.primary_key
-        @dataset_2 = @dataset_2.select(pk.to_expr)
+        @dataset_2 = @dataset_2.select(@pk_2)
       end
     end
 
     def group_records
-      result_set.create_tables!
-
       if config.linkage_type == :self
         group_records_for(@dataset_1, 1)
       else
@@ -56,7 +68,8 @@ module Linkage
       groups_dataset.field_set.values.each do |field|
         # Sort on all fields
         if !field.primary_key?
-          groups_dataset = groups_dataset.match(field.to_expr)
+          meta_object = MetaObject.new(field)
+          groups_dataset = groups_dataset.group_match_more(meta_object)
         end
       end
 
@@ -68,5 +81,107 @@ module Linkage
       sub_dataset = groups_dataset.select(:max.sql_function(:id).as(:id)).group_by_matches
       groups_dataset.filter(:id => sub_dataset.obj).delete
     end
+
+    def score_records_with_groups
+      result_set.groups_dataset.each do |group_record|
+        group = Group.from_row(group_record)
+        dataset_1, dataset_2 = config.apply_exhaustive_expectations(
+          *result_set.groups_records_datasets(group))
+        score_records_without_groups(dataset_1, dataset_2)
+      end
+    end
+
+    def score_records_without_groups(dataset_1, dataset_2)
+      if config.linkage_type == :self
+        keys = dataset_1.select_map(@pk_1)
+        unfiltered_dataset = dataset_1.unfiltered
+        cache = Hashery::LRUHash.new(config.record_cache_size) do |h, k|
+          h[k] = unfiltered_dataset.filter(@pk_1 => k).first
+        end
+        upper_bound = keys.length - 1
+
+        forward = true
+        keys.each_with_index do |key_1, key_1_index|
+          record_1 = cache[key_1]
+
+          lower_bound = key_1_index + 1
+          enum =
+            if forward
+              lower_bound.upto(upper_bound)
+            else
+              upper_bound.downto(lower_bound)
+            end
+          enum.each do |key_2_index|
+            record_2 = cache[keys[key_2_index]]
+            score(record_1, record_2)
+          end
+          forward = !forward
+        end
+      else
+        keys_2 = dataset_2.select_map(@pk_2)
+        unfiltered_dataset_2 = dataset_2.unfiltered
+        cache_2 = Hashery::LRUHash.new(config.record_cache_size) do |h, k|
+          h[k] = unfiltered_dataset_2.filter(@pk_2 => k).first
+        end
+        keys_2_last = keys_2.length - 1
+
+        forward = true
+        dataset_1.each do |record_1|
+          enum = forward ? 0.upto(keys_2_last) : keys_2_last.downto(0)
+          enum.each do |key_2_index|
+            record_2 = cache_2[keys_2[key_2_index]]
+            score(record_1, record_2)
+          end
+          forward = !forward
+        end
+      end
+    end
+
+    def score(record_1, record_2)
+      pk_1 = record_1[@pk_1]
+      pk_2 = record_2[@pk_2]
+
+      catch(:stop) do
+        total_score = 0
+        config.exhaustive_expectations.each_with_index do |expectation, comparator_id|
+          comparator = expectation.comparator
+
+          score = comparator.score(record_1, record_2)
+          result_set.add_score(comparator_id, pk_1, pk_2, score)
+
+          throw(:stop) unless expectation.satisfied?(score)
+          total_score += score
+        end
+        result_set.add_match(pk_1, pk_2, total_score)
+      end
+    end
+
+    # Only needed for linkages without exhaustive expectations
+    def create_matches
+      result_set.groups_dataset.each do |group_record|
+        group = Group.from_row(group_record)
+        dataset_1, dataset_2 = result_set.groups_records_datasets(group)
+
+        if config.linkage_type == :self
+          keys = dataset_1.select_map(@pk_1)
+          keys_last = keys.length - 1
+          keys.each_with_index do |key_1, key_1_index|
+            (key_1_index + 1).upto(keys_last) do |key_2_index|
+              key_2 = keys[key_2_index]
+              result_set.add_match(key_1, key_2, nil)
+            end
+          end
+        else
+          keys_1 = dataset_1.select_map(@pk_1)
+          keys_2 = dataset_2.select_map(@pk_2)
+
+          keys_1.each do |key_1|
+            keys_2.each do |key_2|
+              result_set.add_match(key_1, key_2, nil)
+            end
+          end
+        end
+      end
+    end
   end
 end

data/lib/linkage/version.rb

@@ -0,0 +1,3 @@
+module Linkage
+  VERSION = "0.0.8"
+end

data/lib/linkage.rb

@@ -1,13 +1,16 @@
 require 'pathname'
 require 'delegate'
 require 'sequel'
+require 'hashery'
 
 module Linkage
 end
 
 path = Pathname.new(File.expand_path(File.dirname(__FILE__))) + 'linkage'
+require path + 'version'
 require path + 'utils'
 require path + 'warnings'
+require path + 'decollation'
 require path + 'dataset'
 require path + 'runner'
 require path + 'data'
@@ -15,6 +18,14 @@ require path + 'field'
 require path + 'function'
 require path + 'group'
 require path + 'import_buffer'
+require path + 'meta_object'
+require path + 'expectation'
 require path + 'configuration'
 require path + 'result_set'
 require path + 'field_set'
+require path + 'comparator'
+
+Sequel.extension :collation
+if Sequel::Collation.respond_to?(:suppress_warnings=)
+  Sequel::Collation.suppress_warnings = true
+end

data/linkage.gemspec

@@ -1,126 +1,21 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
+require File.expand_path('../lib/linkage/version', __FILE__)
 
-Gem::Specification.new do |s|
-  s.name = "linkage"
-  s.version = "0.0.6"
+Gem::Specification.new do |gem|
+  gem.authors       = ["Jeremy Stephens"]
+  gem.email         = ["jeremy.f.stephens@vanderbilt.edu"]
+  gem.description   = %q{Performs record linkage between one or two datasets, using Sequel on the backend}
+  gem.summary       = %q{Record linkage library}
+  gem.homepage      = "http://github.com/coupler/linkage"
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Jeremy Stephens"]
-  s.date = "2012-05-08"
-  s.description = "Performs record linkage between one or two datasets, using Sequel on the backend"
-  s.email = "jeremy.f.stephens@vanderbilt.edu"
-  s.extra_rdoc_files = [
-    "LICENSE.txt",
-    "README.markdown"
-  ]
-  s.files = [
-    ".document",
-    ".vimrc",
-    "Gemfile",
-    "Gemfile.lock",
-    "Guardfile",
-    "LICENSE.txt",
-    "README.markdown",
-    "Rakefile",
-    "VERSION",
-    "lib/linkage.rb",
-    "lib/linkage/configuration.rb",
-    "lib/linkage/data.rb",
-    "lib/linkage/dataset.rb",
-    "lib/linkage/field.rb",
-    "lib/linkage/field_set.rb",
-    "lib/linkage/function.rb",
-    "lib/linkage/functions/strftime.rb",
-    "lib/linkage/functions/trim.rb",
-    "lib/linkage/group.rb",
-    "lib/linkage/import_buffer.rb",
-    "lib/linkage/result_set.rb",
-    "lib/linkage/runner.rb",
-    "lib/linkage/runner/single_threaded.rb",
-    "lib/linkage/utils.rb",
-    "lib/linkage/warnings.rb",
-    "linkage.gemspec",
-    "test/config.yml",
-    "test/helper.rb",
-    "test/integration/test_cross_linkage.rb",
-    "test/integration/test_dataset.rb",
-    "test/integration/test_dual_linkage.rb",
-    "test/integration/test_functions.rb",
-    "test/integration/test_self_linkage.rb",
-    "test/unit/functions/test_strftime.rb",
-    "test/unit/functions/test_trim.rb",
-    "test/unit/runner/test_single_threaded.rb",
-    "test/unit/test_configuration.rb",
-    "test/unit/test_data.rb",
-    "test/unit/test_dataset.rb",
-    "test/unit/test_field.rb",
-    "test/unit/test_field_set.rb",
-    "test/unit/test_function.rb",
-    "test/unit/test_group.rb",
-    "test/unit/test_import_buffer.rb",
-    "test/unit/test_linkage.rb",
-    "test/unit/test_result_set.rb",
-    "test/unit/test_runner.rb",
-    "test/unit/test_utils.rb"
-  ]
-  s.homepage = "http://github.com/coupler/linkage"
-  s.licenses = ["MIT"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.23"
-  s.summary = "Record linkage library"
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "linkage"
+  gem.require_paths = ["lib"]
+  gem.version       = Linkage::VERSION
 
-  if s.respond_to? :specification_version then
-    s.specification_version = 3
-
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<sequel>, [">= 0"])
-      s.add_development_dependency(%q<bundler>, [">= 0"])
-      s.add_development_dependency(%q<jeweler>, [">= 0"])
-      s.add_development_dependency(%q<test-unit>, [">= 0"])
-      s.add_development_dependency(%q<mocha>, [">= 0"])
-      s.add_development_dependency(%q<sqlite3>, [">= 0"])
-      s.add_development_dependency(%q<yard>, [">= 0"])
-      s.add_development_dependency(%q<rake>, [">= 0"])
-      s.add_development_dependency(%q<versionomy>, [">= 0"])
-      s.add_development_dependency(%q<mysql2>, [">= 0"])
-      s.add_development_dependency(%q<pry>, [">= 0"])
-      s.add_development_dependency(%q<rdiscount>, [">= 0"])
-      s.add_development_dependency(%q<guard-test>, [">= 0"])
-      s.add_development_dependency(%q<guard-yard>, [">= 0"])
-    else
-      s.add_dependency(%q<sequel>, [">= 0"])
-      s.add_dependency(%q<bundler>, [">= 0"])
-      s.add_dependency(%q<jeweler>, [">= 0"])
-      s.add_dependency(%q<test-unit>, [">= 0"])
-      s.add_dependency(%q<mocha>, [">= 0"])
-      s.add_dependency(%q<sqlite3>, [">= 0"])
-      s.add_dependency(%q<yard>, [">= 0"])
-      s.add_dependency(%q<rake>, [">= 0"])
-      s.add_dependency(%q<versionomy>, [">= 0"])
-      s.add_dependency(%q<mysql2>, [">= 0"])
-      s.add_dependency(%q<pry>, [">= 0"])
-      s.add_dependency(%q<rdiscount>, [">= 0"])
-      s.add_dependency(%q<guard-test>, [">= 0"])
-      s.add_dependency(%q<guard-yard>, [">= 0"])
-    end
-  else
-    s.add_dependency(%q<sequel>, [">= 0"])
-    s.add_dependency(%q<bundler>, [">= 0"])
-    s.add_dependency(%q<jeweler>, [">= 0"])
-    s.add_dependency(%q<test-unit>, [">= 0"])
-    s.add_dependency(%q<mocha>, [">= 0"])
-    s.add_dependency(%q<sqlite3>, [">= 0"])
-    s.add_dependency(%q<yard>, [">= 0"])
-    s.add_dependency(%q<rake>, [">= 0"])
-    s.add_dependency(%q<versionomy>, [">= 0"])
-    s.add_dependency(%q<mysql2>, [">= 0"])
-    s.add_dependency(%q<pry>, [">= 0"])
-    s.add_dependency(%q<rdiscount>, [">= 0"])
-    s.add_dependency(%q<guard-test>, [">= 0"])
-    s.add_dependency(%q<guard-yard>, [">= 0"])
-  end
+  gem.add_dependency "sequel"
+  gem.add_dependency "sequel-collation"
+  gem.add_dependency "hashery"
 end
-

data/test/config.yml

@@ -1,5 +1,10 @@
 mysql:
+<% if RUBY_PLATFORM == "java" %>
+  "jdbc:mysql://localhost/test?user=test"
+<% else %>
+  adapter: mysql2
   host: localhost
   port: 3306
   database: test
   user: test
+<% end %>

data/test/helper.rb

@@ -8,18 +8,35 @@ rescue Bundler::BundlerError => e
   exit e.status_code
 end
 require 'test/unit'
-require 'mocha'
+require 'mocha/setup'
 require 'tmpdir'
 require 'logger'
 require 'pp'
 require 'versionomy'
-#require 'pry'
+require 'erb'
 
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 require 'linkage'
 
 class Test::Unit::TestCase
+  def self.current_ruby_version
+    @current_ruby_version ||= Versionomy.parse(RUBY_VERSION)
+  end
+
+  def self.ruby19
+    @ruby19 ||= Versionomy.parse("1.9")
+  end
+
+  @@database_config = nil
+  def self.database_config
+    if @@database_config.nil?
+      template = File.read(File.join(File.dirname(__FILE__), "config.yml"))
+      @@database_config = YAML.load(ERB.new(template).result(binding))
+    end
+    @@database_config
+  end
+
   def stub_field(name, options = {}, &block)
     f = Linkage::Field.allocate
     f.stubs({:static? => false}.merge(options))
@@ -38,6 +55,15 @@ class Test::Unit::TestCase
     f
   end
 
+  def stub_instance(klass, options = {}, &block)
+    f = klass.allocate
+    f.stubs(options)
+    if block
+      f.send(:instance_eval, &block)
+    end
+    f
+  end
+
   def new_function(name, ruby_type = nil, params = nil, &block)
     klass = Class.new(Linkage::Function)
     klass.send(:define_singleton_method, :function_name) { name }
@@ -50,17 +76,50 @@ class Test::Unit::TestCase
     klass
   end
 
+  def new_comparator(name, params = nil, score_range = nil, &block)
+    klass = Class.new(Linkage::Comparator)
+    klass.send(:define_singleton_method, :comparator_name) { name }
+    if params
+      klass.send(:define_singleton_method, :parameters) { params }
+    end
+    if score_range
+      klass.send(:define_singleton_method, :score_range) { score_range }
+    end
+    klass.send(:define_method, :score) { |record_1, record_2| 100 }
+    if block_given?
+      klass.class_eval(&block)
+    end
+    klass
+  end
 
-  def self.current_ruby_version
-    @current_ruby_version ||= Versionomy.parse(RUBY_VERSION)
+  def database_config
+    self.class.database_config
   end
 
-  def self.ruby19
-    @ruby19 ||= Versionomy.parse("1.9")
+  def database_options_for(adapter)
+    config =
+      if adapter == 'sqlite'
+        @tmpdir ||= Dir.mktmpdir('linkage')
+        { 'adapter' => 'sqlite', 'database' => File.join(@tmpdir, "foo") }
+      else
+        database_config[adapter]
+      end
+
+    if config
+      return config
+    else
+      omit("Couldn't find configuration for adapter '#{adapter}'")
+    end
   end
 
-  def test_config
-    @test_config ||= YAML.load_file(File.join(File.dirname(__FILE__), "config.yml"))
+  def database_for(adapter, options = {}, &block)
+    config = database_options_for(adapter)
+
+    if block
+      Sequel.connect(config, options, &block)
+    else
+      Sequel.connect(config, options)
+    end
   end
 
   def prefixed_logger(prefix)
@@ -72,6 +131,12 @@ class Test::Unit::TestCase
     }
     logger
   end
+
+  def teardown
+    if @tmpdir && File.exist?(@tmpdir)
+      FileUtils.remove_entry_secure(@tmpdir)
+    end
+  end
 end
 
 module UnitTests; end

data/test/integration/test_collation.rb

@@ -0,0 +1,45 @@
+require 'helper'
+
+module IntegrationTests
+  class TestCollation < Test::Unit::TestCase
+    def setup
+      @tmpdir = Dir.mktmpdir('linkage')
+      @tmpuri = "sqlite://" + File.join(@tmpdir, "foo")
+    end
+
+    def database(options = {}, &block)
+      Sequel.connect(@tmpuri, options, &block)
+    end
+
+    def teardown
+      FileUtils.remove_entry_secure(@tmpdir)
+    end
+
+    test "comparing strings exactly in MySQL" do
+      options = database_options_for('mysql')
+      database_for('mysql') do |db|
+        db.create_table!(:foo) do
+          primary_key :id
+          String :foo
+          String :bar
+        end
+        db[:foo].import([:foo, :bar], [
+          ["Foo", "foo"],
+          ["bar", "bar "],
+        ])
+      end
+      dataset = Linkage::Dataset.new(options, :foo)
+      tmpuri = @tmpuri
+      conf = dataset.link_with(dataset) do
+        (lhs[:foo].must == rhs[:bar]).exactly
+        save_results_in(tmpuri)
+      end
+      runner = Linkage::SingleThreadedRunner.new(conf)
+      runner.execute
+
+      database do |db|
+        assert_equal 0, db[:groups].count
+      end
+    end
+  end
+end