linkage 0.0.0

data/lib/linkage/group.rb

@@ -0,0 +1,43 @@
+module Linkage
+  # This class represents a group of records that match based on criteria
+  # described via the {Dataset#link_with} method. Group's are created by
+  # subclasses of the {Runner} class during execution.
+  #
+  # @see Dataset#link_with
+  # @see SingleThreadedRunner
+  class Group
+    # @return [Array<Object>] An array of this group's record ids
+    attr_reader :records
+
+    # @return [Hash] Hash of matching values
+    attr_reader :values
+
+    # @param [Hash] matching_values Values that define this group
+    # @example
+    #   Linkage::Group.new({:foo => 123, :bar => 'baz'})
+    def initialize(matching_values)
+      @values = matching_values
+      @records = []
+    end
+
+    # Check to see if the given set of values matches this group's values.
+    #
+    # @param [Hash] values Hash of values
+    # @return [Boolean] true if match, false if not
+    def matches?(values)
+      @values == values
+    end
+
+    # Add a record id to this group's set of records.
+    #
+    # @param [Object] record_id
+    def add_record(record_id)
+      @records << record_id
+    end
+
+    # @return [Fixnum] Number of records in this group
+    def count
+      @records.count
+    end
+  end
+end

data/lib/linkage/import_buffer.rb

@@ -0,0 +1,39 @@
+module Linkage
+  class ImportBuffer
+    # @param [String] uri Sequel-style URI
+    # @param [Symbol, String] table_name
+    # @param [Array<Symbol>] headers List of fields you want to insert
+    # @param [Hash] options Sequel.connect options
+    # @param [Fixnum] limit Number of records to insert at a time
+    def initialize(uri, table_name, headers, options = {}, limit = 1000)
+      @uri = uri
+      @table_name = table_name.to_sym
+      @headers = headers
+      @options = options
+      @limit = limit
+      @values = []
+    end
+
+    def add(values)
+      @values << values
+      if @values.length == @limit
+        flush
+      end
+    end
+
+    def flush
+      return if @values.empty?
+      database do |db|
+        ds = db[@table_name]
+        ds.import(@headers, @values)
+        @values.clear
+      end
+    end
+
+    private
+
+    def database(&block)
+      Sequel.connect(@uri, @options, &block)
+    end
+  end
+end

data/lib/linkage/runner.rb

@@ -0,0 +1,59 @@
+module Linkage
+  # Use this class to run a configuration created by {Dataset#link_with}.
+  class Runner
+    attr_reader :config
+
+    # @param [Linkage::Configuration] config
+    # @param [String] uri Sequel-style database URI
+    # @param [Hash] options Sequel.connect options
+    # @see Dataset#link_with
+    # @see http://sequel.rubyforge.org/rdoc/files/doc/opening_databases_rdoc.html Sequel: Connecting to a database
+    def initialize(config, uri, options = {})
+      @config = config
+      @uri = uri
+      @options = options
+      @next_group_id = 1
+      @next_group_mutex = Mutex.new
+    end
+
+    # @abstract
+    def execute
+      raise NotImplementedError
+    end
+
+    protected
+
+    def database(&block)
+      Sequel.connect(@uri, @options, &block)
+    end
+
+    def create_tables
+      database do |db|
+        schema = config.groups_table_schema
+        db.create_table(:groups) do
+          schema.each { |col| column(*col) }
+        end
+
+        pk_type = config.dataset_1.primary_key.merge(config.dataset_2.primary_key).ruby_type
+        db.create_table(:groups_records) do
+          column(:record_id, pk_type[:type], pk_type[:opts] || {})
+          Integer :group_id
+          Integer :dataset
+          index :group_id
+        end
+      end
+    end
+
+    def next_group_id
+      result = nil
+      @next_group_mutex.synchronize do
+        result = @next_group_id
+        @next_group_id += 1
+      end
+      result
+    end
+  end
+end
+
+path = Pathname.new(File.expand_path(File.dirname(__FILE__))) + 'runner'
+require path + 'single_threaded'

data/lib/linkage/runner/single_threaded.rb

@@ -0,0 +1,114 @@
+module Linkage
+  # A runner class that only uses a single thread to execute a linkage.
+  #
+  # @see Runner
+  class SingleThreadedRunner < Runner
+    def execute
+      create_tables
+      setup_datasets
+      apply_expectations
+      group_records
+      nil
+    end
+
+    private
+
+    def setup_datasets
+      @dataset_1 = config.dataset_1.clone
+      @dataset_2 = config.dataset_2.clone if @config.linkage_type != :self
+    end
+
+    def apply_expectations
+      config.expectations.each do |exp|
+        exp.apply_to(@dataset_1)
+        exp.apply_to(@dataset_2) if config.linkage_type != :self
+      end
+    end
+
+    def group_records
+      if config.linkage_type == :self
+        add_groups(group_records_for(@dataset_1), 1)
+      else
+        add_groups(group_records_for(@dataset_1, false), 1)
+        add_groups(group_records_for(@dataset_2, false), 2)
+        combine_groups
+      end
+    end
+
+    def group_records_for(dataset, ignore_empty_groups = true)
+      groups = []
+      current_group = nil
+      dataset.each do |row|
+        if current_group.nil? || !current_group.matches?(row[:values])
+          if current_group && (!ignore_empty_groups || current_group.count > 1)
+            groups << current_group
+          end
+          new_group = Group.new(row[:values])
+          current_group = new_group
+        end
+        current_group.add_record(row[:pk])
+      end
+      if current_group && (!ignore_empty_groups || current_group.count > 1)
+        groups << current_group
+      end
+      groups
+    end
+
+    def add_groups(groups, dataset_id = nil)
+      return if groups.empty?
+
+      groups_headers = [:id] + groups[0].values.keys
+      groups_buffer = ImportBuffer.new(@uri, :groups, groups_headers, @options)
+
+      groups_records_buffer = ImportBuffer.new(@uri, :groups_records, [:group_id, :dataset, :record_id], @options)
+
+      groups.each_with_index do |group, i|
+        group_id = next_group_id
+        groups_buffer.add([group_id] + group.values.values)
+        group.records.each do |record_id|
+          groups_records_buffer.add([group_id, dataset_id, record_id])
+        end
+      end
+      groups_buffer.flush
+      groups_records_buffer.flush
+    end
+
+    def combine_groups
+      # Create a new dataset for the groups table
+      ds = Dataset.new(@uri, :groups, :single_threaded => true)
+      ds.fields.each_value do |field|
+        # Sort on all fields
+        next if field.primary_key?
+        ds.add_order(field)
+        ds.add_select(field)
+      end
+      ds.add_order(ds.primary_key) # ensure matching groups are sorted by id
+      combined_groups = group_records_for(ds, false)
+      database do |db|
+        groups_to_delete = []
+        db.transaction do  # for speed reasons
+          combined_groups.each do |group|
+            if group.count == 1
+              # Delete the empty group
+              groups_to_delete << group.records[0]
+            else
+              # Change group_id in the groups_records table to the first group
+              # id, delete other groups.
+              new_group_id = group.records[0]
+              group.records[1..-1].each do |old_group_id|
+                # There can only be a group with max size of 2, but this
+                # adds in future support for matching more than 2 datasets
+                # at once. Code smell?
+                db[:groups_records].filter(:group_id => old_group_id).
+                  update(:group_id => new_group_id)
+                groups_to_delete << old_group_id
+              end
+            end
+          end
+        end
+        db[:groups_records].filter(:group_id => groups_to_delete).delete
+        db[:groups].filter(:id => groups_to_delete).delete
+      end
+    end
+  end
+end

data/lib/linkage/utils.rb

@@ -0,0 +1,164 @@
+module Linkage
+  module Utils
+    # A "tree" used to find compatible types.
+    TYPE_CONVERSION_TREE = {
+      TrueClass => [Integer],
+      Integer => [Bignum, Float],
+      Bignum => [BigDecimal],
+      Float => [BigDecimal],
+      BigDecimal => [String],
+      String => nil,
+      DateTime => nil,
+      Date => nil,
+      Time => nil,
+      File => nil
+    }
+
+    # Create field information for a field that can hold data from two other
+    # fields. If the fields have different types, the resulting type is
+    # determined via a type-conversion tree.
+    #
+    # @param [Array] field_1 Schema information for the first field
+    # @param [Array] field_2 Schema information for the second field
+    # @return [Array] Schema information for the new field
+    def merge_fields(field_1, field_2)
+      schema_1 = column_schema_to_ruby_type(field_1)
+      schema_1.delete_if { |k, v| v.nil? }
+      schema_2 = column_schema_to_ruby_type(field_2)
+      schema_2.delete_if { |k, v| v.nil? }
+      if schema_1 == schema_2
+        result = schema_1
+      else
+        result = schema_1.dup
+
+        # type
+        if schema_1[:type] != schema_2[:type]
+          result[:type] = first_common_type(schema_1[:type], schema_2[:type])
+        end
+
+        # text
+        if schema_1[:text] != schema_2[:text]
+          # This can only be of type String.
+          result[:text] = true
+          result.delete(:size)
+        end
+
+        # size
+        if !result[:text] && schema_1[:size] != schema_2[:size]
+          types = [schema_1[:type], schema_2[:type]].uniq
+          if types.length == 1 && types[0] == BigDecimal
+            # Two decimals
+            if schema_1.has_key?(:size) && schema_2.has_key?(:size)
+              s_1 = schema_1[:size]
+              s_2 = schema_2[:size]
+              result[:size] = [ s_1[0] > s_2[0] ? s_1[0] : s_2[0] ]
+
+              if s_1[1] && s_2[1]
+                result[:size][1] = s_1[1] > s_2[1] ? s_1[1] : s_2[1]
+              else
+                result[:size][1] = s_1[1] ? s_1[1] : s_2[1]
+              end
+            else
+              result[:size] = schema_1.has_key?(:size) ? schema_1[:size] : schema_2[:size]
+            end
+          elsif types.include?(String) && types.include?(BigDecimal)
+            # Add one to the precision of the BigDecimal (for the dot)
+            if schema_1.has_key?(:size) && schema_2.has_key?(:size)
+              s_1 = schema_1[:size].is_a?(Array) ? schema_1[:size][0] + 1 : schema_1[:size]
+              s_2 = schema_2[:size].is_a?(Array) ? schema_2[:size][0] + 1 : schema_2[:size]
+              result[:size] = s_1 > s_2 ? s_1 : s_2
+            elsif schema_1.has_key?(:size)
+              result[:size] = schema_1[:size].is_a?(Array) ? schema_1[:size][0] + 1 : schema_1[:size]
+            elsif schema_2.has_key?(:size)
+              result[:size] = schema_2[:size].is_a?(Array) ? schema_2[:size][0] + 1 : schema_2[:size]
+            end
+          else
+            # Treat as two strings
+            if schema_1.has_key?(:size) && schema_2.has_key?(:size)
+              result[:size] = schema_1[:size] > schema_2[:size] ? schema_1[:size] : schema_2[:size]
+            elsif schema_1.has_key?(:size)
+              result[:size] = schema_1[:size]
+            else
+              result[:size] = schema_2[:size]
+            end
+          end
+        end
+
+        # fixed
+        if schema_1[:fixed] != schema_2[:fixed]
+          # This can only be of type String.
+          result[:fixed] = true
+        end
+      end
+
+      {:type => result.delete(:type), :opts => result}
+    end
+
+    private
+
+    # Convert the column schema information to a hash of column options, one of which must
+    # be :type.  The other options added should modify that type (e.g. :size).  If a
+    # database type is not recognized, return it as a String type.
+    #
+    # @note This method comes straight from Sequel (lib/sequel/extensions/schema_dumper.rb).
+    def column_schema_to_ruby_type(schema)
+      case t = schema[:db_type].downcase
+      when /\A(?:medium|small)?int(?:eger)?(?:\((?:\d+)\))?(?: unsigned)?\z/o
+        {:type=>Integer}
+      when /\Atinyint(?:\((\d+)\))?\z/o
+        {:type =>schema[:type] == :boolean ? TrueClass : Integer}
+      when /\Abigint(?:\((?:\d+)\))?(?: unsigned)?\z/o
+        {:type=>Bignum}
+      when /\A(?:real|float|double(?: precision)?)\z/o
+        {:type=>Float}
+      when 'boolean'
+        {:type=>TrueClass}
+      when /\A(?:(?:tiny|medium|long|n)?text|clob)\z/o
+        {:type=>String, :text=>true}
+      when 'date'
+        {:type=>Date}
+      when /\A(?:small)?datetime\z/o
+        {:type=>DateTime}
+      when /\Atimestamp(?:\((\d+)\))?(?: with(?:out)? time zone)?\z/o
+        {:type=>DateTime, :size=>($1.to_i if $1)}
+      when /\Atime(?: with(?:out)? time zone)?\z/o
+        {:type=>Time, :only_time=>true}
+      when /\An?char(?:acter)?(?:\((\d+)\))?\z/o
+        {:type=>String, :size=>($1.to_i if $1), :fixed=>true}
+      when /\A(?:n?varchar|character varying|bpchar|string)(?:\((\d+)\))?\z/o
+        {:type=>String, :size=>($1.to_i if $1)}
+      when /\A(?:small)?money\z/o
+        {:type=>BigDecimal, :size=>[19,2]}
+      when /\A(?:decimal|numeric|number)(?:\((\d+)(?:,\s*(\d+))?\))?\z/o
+        s = [($1.to_i if $1), ($2.to_i if $2)].compact
+        {:type=>BigDecimal, :size=>(s.empty? ? nil : s)}
+      when /\A(?:bytea|(?:tiny|medium|long)?blob|(?:var)?binary)(?:\((\d+)\))?\z/o
+        {:type=>File, :size=>($1.to_i if $1)}
+      when 'year'
+        {:type=>Integer}
+      else
+        {:type=>String}
+      end
+    end
+
+    def first_common_type(type_1, type_2)
+      types_1 = [type_1] + get_types(type_1)
+      types_2 = [type_2] + get_types(type_2)
+      (types_1 & types_2).first
+    end
+
+    # Get all types that the specified type can be converted to. Order
+    # matters.
+    def get_types(type)
+      result = []
+      types = TYPE_CONVERSION_TREE[type]
+      if types
+        result += types
+        types.each do |t|
+          result |= get_types(t)
+        end
+      end
+      result
+    end
+  end
+end

data/linkage.gemspec

@@ -0,0 +1,106 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "linkage"
+  s.version = "0.0.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Jeremy Stephens"]
+  s.date = "2011-09-30"
+  s.description = "Wraps Sequel to perform record linkage between one or two datasets"
+  s.email = "jeremy.f.stephens@vanderbilt.edu"
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.markdown"
+  ]
+  s.files = [
+    ".document",
+    ".vimrc",
+    "Gemfile",
+    "Gemfile.lock",
+    "Guardfile",
+    "LICENSE.txt",
+    "README.markdown",
+    "Rakefile",
+    "VERSION",
+    "lib/linkage.rb",
+    "lib/linkage/configuration.rb",
+    "lib/linkage/dataset.rb",
+    "lib/linkage/expectation.rb",
+    "lib/linkage/field.rb",
+    "lib/linkage/group.rb",
+    "lib/linkage/import_buffer.rb",
+    "lib/linkage/runner.rb",
+    "lib/linkage/runner/single_threaded.rb",
+    "lib/linkage/utils.rb",
+    "linkage.gemspec",
+    "test/helper.rb",
+    "test/integration/test_cross_linkage.rb",
+    "test/integration/test_dual_linkage.rb",
+    "test/integration/test_self_linkage.rb",
+    "test/unit/test_configuration.rb",
+    "test/unit/test_dataset.rb",
+    "test/unit/test_expectation.rb",
+    "test/unit/test_field.rb",
+    "test/unit/test_group.rb",
+    "test/unit/test_import_buffer.rb",
+    "test/unit/test_linkage.rb",
+    "test/unit/test_runner.rb",
+    "test/unit/test_single_threaded_runner.rb",
+    "test/unit/test_utils.rb"
+  ]
+  s.homepage = "http://github.com/coupler/linkage"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.10"
+  s.summary = "Sequel-based record linkage"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<sequel>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+      s.add_development_dependency(%q<guard-test>, [">= 0"])
+      s.add_development_dependency(%q<test-unit>, ["= 2.3.2"])
+      s.add_development_dependency(%q<mocha>, [">= 0"])
+      s.add_development_dependency(%q<sqlite3>, [">= 0"])
+      s.add_development_dependency(%q<yard>, [">= 0"])
+      s.add_development_dependency(%q<rake>, [">= 0"])
+      s.add_development_dependency(%q<versionomy>, [">= 0"])
+      s.add_development_dependency(%q<guard-yard>, [">= 0"])
+    else
+      s.add_dependency(%q<sequel>, [">= 0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+      s.add_dependency(%q<guard-test>, [">= 0"])
+      s.add_dependency(%q<test-unit>, ["= 2.3.2"])
+      s.add_dependency(%q<mocha>, [">= 0"])
+      s.add_dependency(%q<sqlite3>, [">= 0"])
+      s.add_dependency(%q<yard>, [">= 0"])
+      s.add_dependency(%q<rake>, [">= 0"])
+      s.add_dependency(%q<versionomy>, [">= 0"])
+      s.add_dependency(%q<guard-yard>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<sequel>, [">= 0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+    s.add_dependency(%q<guard-test>, [">= 0"])
+    s.add_dependency(%q<test-unit>, ["= 2.3.2"])
+    s.add_dependency(%q<mocha>, [">= 0"])
+    s.add_dependency(%q<sqlite3>, [">= 0"])
+    s.add_dependency(%q<yard>, [">= 0"])
+    s.add_dependency(%q<rake>, [">= 0"])
+    s.add_dependency(%q<versionomy>, [">= 0"])
+    s.add_dependency(%q<guard-yard>, [">= 0"])
+  end
+end
+