linkage 0.0.8 → 0.1.0.pre

data/lib/linkage/result_sets/database.rb

@@ -0,0 +1,42 @@
+module Linkage
+  module ResultSets
+    class Database < ResultSet
+      def initialize(database_or_options = nil)
+        @database = nil
+        @options = {}
+
+        if database_or_options.kind_of?(Sequel::Database)
+          @database = database_or_options
+        else
+          database_opts = nil
+          case database_or_options
+          when String
+            database_opts = database_or_options
+          when Hash
+            database_opts = {}
+            database_or_options.each_pair do |key, value|
+              if key == :scores || key == :matches
+                @options[key] = value
+              else
+                database_opts[key] = value
+              end
+            end
+          else
+            raise ArgumentError, "expected Sequel::Database, a String, or a Hash, got #{database_or_options.class}"
+          end
+          @database = Sequel.connect(database_opts)
+        end
+      end
+
+      def score_set
+        @score_set ||= ScoreSet['database'].new(@database, @options[:scores] || {})
+      end
+
+      def match_set
+        @match_set ||= MatchSet['database'].new(@database, @options[:matches] || {})
+      end
+    end
+
+    ResultSet.register('database', Database)
+  end
+end

data/lib/linkage/runner.rb

@@ -1,31 +1,72 @@
 module Linkage
   # Use this class to run a configuration created by {Dataset#link_with}.
   class Runner
-    attr_reader :config, :result_set
+    attr_reader :config
 
     # @param [Linkage::Configuration] config
-    # @param [String] uri Sequel-style database URI
-    # @param [Hash] options Sequel.connect options
     # @see Dataset#link_with
-    # @see http://sequel.rubyforge.org/rdoc/files/doc/opening_databases_rdoc.html Sequel: Connecting to a database
-    def initialize(config, uri = nil, options = {})
+    def initialize(config)
       @config = config
-      if uri
-        warn("[DEPRECATION] Please use Configuration#save_results_in with the database URI and options instead")
-        @config.save_results_in(uri, options)
-      end
     end
 
-    # @abstract
     def execute
-      raise NotImplementedError
+      score_records
+      match_records
     end
 
-    def result_set
-      @config.result_set
+    def score_records
+      score_recorder = config.score_recorder
+      score_recorder.start
+      dataset_1 = config.dataset_1
+      dataset_2 = config.dataset_2
+      simple_comparators = []
+      config.comparators.each do |comparator|
+        if comparator.type == :simple
+          simple_comparators << comparator
+        else
+          if dataset_2
+            comparator.score_datasets(dataset_1, dataset_2)
+          else
+            comparator.score_dataset(dataset_1)
+          end
+        end
+      end
+
+      # Handle simple comparators
+      unless simple_comparators.empty?
+        if dataset_2
+          # Two datasets
+          dataset_1.each do |record_1|
+            dataset_2.each do |record_2|
+              simple_comparators.each do |comparator|
+                comparator.score_and_notify(record_1, record_2)
+              end
+            end
+          end
+        else
+          # One dataset
+          # NOTE: very naive implementation
+          records = dataset_1.all
+          0.upto(records.length - 2) do |i|
+            record_1 = records[i]
+            (i + 1).upto(records.length - 1) do |j|
+              record_2 = records[j]
+              simple_comparators.each do |comparator|
+                comparator.score_and_notify(record_1, record_2)
+              end
+            end
+          end
+        end
+      end
+      score_recorder.stop
+    end
+
+    def match_records
+      matcher = config.matcher
+      match_recorder = config.match_recorder(matcher)
+      match_recorder.start
+      matcher.run
+      match_recorder.stop
     end
   end
 end
-
-path = Pathname.new(File.expand_path(File.dirname(__FILE__))) + 'runner'
-require path + 'single_threaded'

data/lib/linkage/score_recorder.rb

@@ -0,0 +1,30 @@
+module Linkage
+  class ScoreRecorder
+    def initialize(comparators, score_set, primary_keys)
+      @comparators = comparators
+      @score_set = score_set
+      @primary_keys = primary_keys
+    end
+
+    def start
+      @comparators.each do |comparator|
+        comparator.add_observer(self)
+      end
+      @score_set.open_for_writing
+    end
+
+    def update(comparator, record_1, record_2, score)
+      index = @comparators.index(comparator)
+      primary_key_1 = record_1[@primary_keys[0]]
+      primary_key_2 = record_2[@primary_keys[1]]
+      @score_set.add_score(index + 1, primary_key_1, primary_key_2, score)
+    end
+
+    def stop
+      @score_set.close
+      @comparators.each do |comparator|
+        comparator.delete_observer(self)
+      end
+    end
+  end
+end

data/lib/linkage/score_set.rb

@@ -0,0 +1,49 @@
+module Linkage
+  class ScoreSet
+    # Register a score set.
+    #
+    # @param [Class] klass
+    def self.register(name, klass)
+      methods = klass.instance_methods(false)
+      missing = []
+      unless methods.include?(:add_score)
+        missing.push("#add_score")
+      end
+      unless methods.include?(:each_pair)
+        missing.push("#each_pair")
+      end
+      unless missing.empty?
+        raise ArgumentError, "class must define #{missing.join(" and ")}"
+      end
+
+      @score_sets ||= {}
+      @score_sets[name] = klass
+    end
+
+    def self.[](name)
+      @score_sets ? @score_sets[name] : nil
+    end
+
+    def open_for_reading
+    end
+
+    def open_for_writing
+    end
+
+    # @abstract
+    def add_score(comparator_id, id_1, id_2, value)
+      raise NotImplementedError
+    end
+
+    # @abstract
+    def each_pair(&block)
+      raise NotImplementedError
+    end
+
+    def close
+    end
+  end
+end
+
+require 'linkage/score_sets/csv'
+require 'linkage/score_sets/database'

data/lib/linkage/score_sets/csv.rb

@@ -0,0 +1,64 @@
+require 'csv'
+
+module Linkage
+  module ScoreSets
+    class CSV < ScoreSet
+      def initialize(filename, options = {})
+        @filename = filename
+        @overwrite = options[:overwrite]
+      end
+
+      def open_for_reading
+        raise "already open for writing, try closing first" if @mode == :write
+        return if @mode == :read
+
+        if !File.exist?(@filename)
+          raise MissingError, "#{@filename} does not exist"
+        end
+        @csv = ::CSV.open(@filename, 'rb', :headers => true)
+        @mode = :read
+      end
+
+      def open_for_writing
+        raise "already open for reading, try closing first" if @mode == :read
+        return if @mode == :write
+
+        if !@overwrite && File.exist?(@filename)
+          raise ExistsError, "#{@filename} exists and not in overwrite mode"
+        end
+
+        @csv = ::CSV.open(@filename, 'wb')
+        @csv << %w{comparator_id id_1 id_2 score}
+        @mode = :write
+      end
+
+      def add_score(comparator_id, id_1, id_2, score)
+        raise "not in write mode" if @mode != :write
+        @csv << [comparator_id, id_1, id_2, score]
+      end
+
+      def each_pair
+        open_for_reading
+
+        pairs = Hash.new { |h, k| h[k] = {} }
+        @csv.each do |row|
+          key = [row['id_1'], row['id_2']]
+          score = row['score']
+          pairs[key][row['comparator_id'].to_i] = score.to_f
+        end
+        pairs.each_pair do |pair, scores|
+          yield pair[0], pair[1], scores
+        end
+
+        close
+      end
+
+      def close
+        @mode = nil
+        @csv.close if @csv
+      end
+    end
+
+    ScoreSet.register('csv', CSV)
+  end
+end

data/lib/linkage/score_sets/database.rb

@@ -0,0 +1,77 @@
+module Linkage
+  module ScoreSets
+    class Database < ScoreSet
+      def initialize(database, options = {})
+        @database = database
+        @table_name = options[:table_name] || :scores
+        @overwrite = options[:overwrite]
+      end
+
+      def open_for_reading
+        raise "already open for writing, try closing first" if @mode == :write
+        return if @mode == :read
+
+        if !@database.table_exists?(@table_name)
+          raise MissingError, "#{@table_name} table does not exist"
+        end
+
+        @dataset = @database[@table_name]
+        @mode = :read
+      end
+
+      def open_for_writing
+        raise "already open for reading, try closing first" if @mode == :read
+        return if @mode == :write
+
+        if @overwrite
+          @database.drop_table?(@table_name)
+        elsif @database.table_exists?(@table_name)
+          raise ExistsError, "#{@table_name} table exists and not in overwrite mode"
+        end
+
+        @database.create_table(@table_name) do
+          Integer :comparator_id
+          String :id_1
+          String :id_2
+          Float :score
+        end
+        @dataset = @database[@table_name]
+        @mode = :write
+      end
+
+      def add_score(comparator_id, id_1, id_2, score)
+        raise "not in write mode" if @mode != :write
+
+        @dataset.insert({
+          :comparator_id => comparator_id,
+          :id_1 => id_1,
+          :id_2 => id_2,
+          :score => score
+        })
+      end
+
+      def each_pair
+        open_for_reading
+
+        current_pair = nil
+        @dataset.order(:id_1, :id_2, :comparator_id).each do |row|
+          if current_pair.nil? || current_pair[0] != row[:id_1] || current_pair[1] != row[:id_2]
+            yield(*current_pair) unless current_pair.nil?
+            current_pair = [row[:id_1], row[:id_2], {}]
+          end
+          scores = current_pair[2]
+          scores[row[:comparator_id]] = row[:score]
+        end
+        yield(*current_pair) unless current_pair.nil?
+
+        close
+      end
+
+      def close
+        @mode = nil
+      end
+    end
+
+    ScoreSet.register('database', Database)
+  end
+end

data/lib/linkage/version.rb

@@ -1,3 +1,3 @@
 module Linkage
-  VERSION = "0.0.8"
+  VERSION = "0.1.0.pre"
 end

data/lib/linkage.rb

@@ -1,31 +1,28 @@
 require 'pathname'
+require 'fileutils'
 require 'delegate'
 require 'sequel'
 require 'hashery'
+require 'observer'
 
 module Linkage
 end
 
 path = Pathname.new(File.expand_path(File.dirname(__FILE__))) + 'linkage'
-require path + 'version'
-require path + 'utils'
-require path + 'warnings'
-require path + 'decollation'
+require path + 'comparator'
+require path + 'configuration'
 require path + 'dataset'
-require path + 'runner'
-require path + 'data'
+require path + 'exceptions'
 require path + 'field'
-require path + 'function'
-require path + 'group'
+require path + 'field_set'
 require path + 'import_buffer'
-require path + 'meta_object'
-require path + 'expectation'
-require path + 'configuration'
+require path + 'match_recorder'
+require path + 'match_set'
+require path + 'matcher'
 require path + 'result_set'
-require path + 'field_set'
-require path + 'comparator'
+require path + 'runner'
+require path + 'score_recorder'
+require path + 'score_set'
+require path + 'version'
 
-Sequel.extension :collation
-if Sequel::Collation.respond_to?(:suppress_warnings=)
-  Sequel::Collation.suppress_warnings = true
-end
+Sequel.extension :core_extensions

data/linkage.gemspec

@@ -2,6 +2,7 @@
 require File.expand_path('../lib/linkage/version', __FILE__)
 
 Gem::Specification.new do |gem|
+  gem.name          = "linkage"
   gem.authors       = ["Jeremy Stephens"]
   gem.email         = ["jeremy.f.stephens@vanderbilt.edu"]
   gem.description   = %q{Performs record linkage between one or two datasets, using Sequel on the backend}
@@ -16,6 +17,17 @@ Gem::Specification.new do |gem|
   gem.version       = Linkage::VERSION
 
   gem.add_dependency "sequel"
-  gem.add_dependency "sequel-collation"
   gem.add_dependency "hashery"
+
+  gem.add_development_dependency "bundler", "~> 1.3"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "test-unit"
+  gem.add_development_dependency "mocha"
+  gem.add_development_dependency "versionomy"
+  gem.add_development_dependency "sqlite3"
+  gem.add_development_dependency "mysql2"
+  gem.add_development_dependency "guard-test"
+  gem.add_development_dependency "guard-yard"
+
+  gem.required_ruby_version = '>= 1.9'
 end

data/linkage.gemspec-java

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/linkage/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name          = "linkage"
+  gem.authors       = ["Jeremy Stephens"]
+  gem.email         = ["jeremy.f.stephens@vanderbilt.edu"]
+  gem.description   = %q{Performs record linkage between one or two datasets, using Sequel on the backend}
+  gem.summary       = %q{Record linkage library}
+  gem.homepage      = "http://github.com/coupler/linkage"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "linkage"
+  gem.require_paths = ["lib"]
+  gem.version       = Linkage::VERSION
+  gem.platform      = "java"
+
+  gem.add_dependency "sequel"
+  gem.add_dependency "sequel-collation"
+  gem.add_dependency "hashery"
+
+  gem.add_development_dependency "bundler", "~> 1.3"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "test-unit"
+  gem.add_development_dependency "mocha"
+  gem.add_development_dependency "versionomy"
+  gem.add_development_dependency "jdbc-sqlite3"
+  gem.add_development_dependency "jdbc-mysql"
+  gem.add_development_dependency "guard-test"
+end

data/test/helper.rb

@@ -14,6 +14,7 @@ require 'logger'
 require 'pp'
 require 'versionomy'
 require 'erb'
+require 'tempfile'
 
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
@@ -46,8 +47,8 @@ class Test::Unit::TestCase
     f
   end
 
-  def stub_function(name, options = {}, &block)
-    f = Linkage::Function.allocate
+  def stub_instance(klass, options = {}, &block)
+    f = klass.allocate
     f.stubs(options)
     if block
       f.send(:instance_eval, &block)
@@ -55,37 +56,43 @@ class Test::Unit::TestCase
     f
   end
 
-  def stub_instance(klass, options = {}, &block)
-    f = klass.allocate
-    f.stubs(options)
-    if block
-      f.send(:instance_eval, &block)
+  def new_comparator(&block)
+    klass = Class.new(Linkage::Comparator)
+    klass.send(:define_method, :score) { |record_1, record_2| 1 }
+    if block_given?
+      klass.class_eval(&block)
     end
-    f
+    klass
   end
 
-  def new_function(name, ruby_type = nil, params = nil, &block)
-    klass = Class.new(Linkage::Function)
-    klass.send(:define_singleton_method, :function_name) { name }
-    if ruby_type
-      klass.send(:define_method, :ruby_type) { ruby_type }
+  def new_score_set(&block)
+    klass = Class.new(Linkage::ScoreSet)
+    klass.send(:define_method, :add_score) do |comparator_index, id_1, id_2, value|
     end
-    if params
-      klass.send(:define_singleton_method, :parameters) { params }
+    klass.send(:define_method, :each_pair) do
+    end
+    if block_given?
+      klass.class_eval(&block)
     end
     klass
   end
 
-  def new_comparator(name, params = nil, score_range = nil, &block)
-    klass = Class.new(Linkage::Comparator)
-    klass.send(:define_singleton_method, :comparator_name) { name }
-    if params
-      klass.send(:define_singleton_method, :parameters) { params }
+  def new_match_set(&block)
+    klass = Class.new(Linkage::MatchSet)
+    klass.send(:define_method, :add_match) do |id_1, id_2, value|
+    end
+    if block_given?
+      klass.class_eval(&block)
+    end
+    klass
+  end
+
+  def new_result_set(&block)
+    klass = Class.new(Linkage::ResultSet)
+    klass.send(:define_method, :score_set) do
     end
-    if score_range
-      klass.send(:define_singleton_method, :score_range) { score_range }
+    klass.send(:define_method, :match_set) do
     end
-    klass.send(:define_method, :score) { |record_1, record_2| 100 }
     if block_given?
       klass.class_eval(&block)
     end

data/test/integration/test_cross_linkage.rb

@@ -15,7 +15,7 @@ module IntegrationTests
       FileUtils.remove_entry_secure(@tmpdir)
     end
 
-    test "one mandatory field equality on single threaded runner" do
+    test "one field equality on single threaded runner" do
       # insert the test data
       database do |db|
         db.create_table(:foo) { primary_key(:id); Integer(:foo); Integer(:bar) }
@@ -25,25 +25,32 @@ module IntegrationTests
 
       ds = Linkage::Dataset.new(@tmpuri, "foo", :single_threaded => true)
 
-      tmpuri = @tmpuri
-      conf = ds.link_with(ds) do
-        lhs[:foo].must == rhs[:bar]
-        save_results_in(tmpuri, :single_threaded => true)
+      result_set = Linkage::ResultSet['csv'].new(@tmpdir)
+      conf = ds.link_with(ds, result_set) do |conf|
+        conf.compare([:foo], [:bar], :equal)
+        conf.algorithm = :mean
+        conf.threshold = 1
       end
-      assert_equal :cross, conf.linkage_type
-      runner = Linkage::SingleThreadedRunner.new(conf)
+
+      runner = Linkage::Runner.new(conf)
       runner.execute
 
-      database do |db|
-        assert_equal 5, db[:groups].count, PP.pp(db[:groups].all, "")
-        db[:groups].order(:foo_bar).each_with_index do |row, i|
-          assert_equal i, row[:foo_bar]
-        end
+      score_csv = CSV.read(File.join(@tmpdir, 'scores.csv'), :headers => true)
+      assert_equal 1000, score_csv.length
+      score_csv.each do |row|
+        id_1 = row['id_1'].to_i
+        id_2 = row['id_2'].to_i
+        assert (id_1 % 10) == (id_2 % 5)
+        assert_equal "1", row['score']
+      end
 
-        assert_equal 1000, db[:matches].count
-        db[:matches].order(:record_1_id, :record_2_id).each do |row|
-          assert_equal row[:record_1_id] % 10, row[:record_2_id] % 5
-        end
+      match_csv = CSV.read(File.join(@tmpdir, 'matches.csv'), :headers => true)
+      assert_equal 1000, match_csv.length
+      match_csv.each do |row|
+        id_1 = row['id_1'].to_i
+        id_2 = row['id_2'].to_i
+        assert (id_1 % 10) == (id_2 % 5)
+        assert_equal "1", row['score']
       end
     end
 
@@ -54,19 +61,33 @@ module IntegrationTests
           Array.new(100) { |i| [i, i % 10, i % 20] })
       end
 
+      result_set = Linkage::ResultSet['csv'].new(@tmpdir)
       ds = Linkage::Dataset.new(@tmpuri, "foo", :single_threaded => true)
-      tmpuri = @tmpuri
-      conf = ds.link_with(ds) do
-        lhs[:foo].must == rhs[:foo]
-        lhs[:bar].must == 0
-        rhs[:bar].must == 10
-        save_results_in(tmpuri)
+      ds_1 = ds.filter(:bar => 0)
+      ds_2 = ds.filter(:bar => 10)
+      conf = ds_1.link_with(ds_2, result_set) do |conf|
+        conf.compare([:foo], [:foo], :equal)
+        conf.algorithm = :mean
+        conf.threshold = 1
       end
-      runner = Linkage::SingleThreadedRunner.new(conf)
+
+      runner = Linkage::Runner.new(conf)
       runner.execute
 
-      database do |db|
-        assert_equal 1, db[:groups].count
+      score_csv = CSV.read(File.join(@tmpdir, 'scores.csv'), :headers => true)
+      assert_equal 25, score_csv.length
+      score_csv.each do |row|
+        id_1 = row['id_1'].to_i
+        id_2 = row['id_2'].to_i
+        assert (id_1 % 10) == (id_1 % 10)
+      end
+
+      match_csv = CSV.read(File.join(@tmpdir, 'matches.csv'), :headers => true)
+      assert_equal 25, match_csv.length
+      match_csv.each do |row|
+        id_1 = row['id_1'].to_i
+        id_2 = row['id_2'].to_i
+        assert (id_1 % 10) == (id_1 % 10)
       end
     end
   end