linkage 0.0.8 → 0.1.0.pre

data/lib/linkage/group.rb

@@ -1,55 +0,0 @@
-module Linkage
-  class Group
-    include Linkage::Decollation
-
-    # @return [Hash] Hash of matching values
-    attr_reader :values
-
-    # @return [Integer] Number of records in this group
-    attr_reader :count
-
-    # @return [Integer] This group's ID (if it exists)
-    attr_reader :id
-
-    def self.from_row(row)
-      values = {}
-      options = {}
-      row.each_pair do |key, value|
-        if key == :id || key == :count
-          options[key] = value
-        else
-          values[key] = value
-        end
-      end
-      new(values, options)
-    end
-
-    # @param [Hash] values Values that define this group
-    # @param [Hash] options
-    # @option options [Fixnum] :id The group ID
-    # @option options [Fixnum] :count How many records are in the group
-    # @option options [Hash] :ruby_types Hash of ruby types for each value
-    # @option options [Symbol] :database_type
-    # @example
-    #   Linkage::Group.new({:foo => 123, :bar => 'baz'}, {:count => 5, :id => 456})
-    def initialize(values, options)
-      @count = options[:count]
-      @id = options[:id]
-      @ruby_types = options[:ruby_types]
-      @database_type = options[:database_type]
-      @values = values
-    end
-
-    def decollated_values
-      @values.inject({}) do |hsh, (key, value)|
-        ruby_type = @ruby_types[key]
-        if ruby_type && ruby_type.has_key?(:opts) && ruby_type[:opts].has_key?(:collate)
-          hsh[key] = decollate(value, @database_type, ruby_type[:opts][:collate])
-        else
-          hsh[key] = value
-        end
-        hsh
-      end
-    end
-  end
-end

data/lib/linkage/meta_object.rb

@@ -1,139 +0,0 @@
-module Linkage
-  class MetaObject
-    attr_reader :object
-    attr_writer :side
-
-    # Creates a new MetaObject.
-    #
-    # @param [Object] object This can be a {Field}, {Function} or a regular
-    #   Ruby object (Fixnum, String, etc). If `object` is not static (a {Field}
-    #   or a {Function} that contains one or more {Field} objects), you should
-    #   specify which "side" of the linkage the object belongs to (left-hand
-    #   side or right-hand side) in the `side` argument.
-    # @param [Symbol] side `:lhs` for left-hand side or `:rhs` for right-hand
-    #   side
-    def initialize(object, side = nil)
-      @object = object
-      @static = object.kind_of?(Linkage::Data) ? object.static? : true
-      if !side.nil? && side != :lhs && side != :rhs
-        raise ArgumentError, "invalid `side` argument, must be :lhs or :rhs"
-      end
-      @side = side
-    end
-
-    def side
-      if !@static && @side.nil?
-        raise RuntimeError, "Object is dynamic and side is not set"
-      end
-      @side
-    end
-
-    def dataset
-      @object.kind_of?(Linkage::Data) ? @object.dataset : nil
-    end
-
-    def dataset=(dataset)
-      if @object.kind_of?(Linkage::Data)
-        @object.dataset = dataset
-      else
-        raise RuntimeError, "You can't set the dataset of a non-data object."
-      end
-    end
-
-    def database_type
-      ds = dataset
-      ds ? ds.database_type : nil
-    end
-
-    def static?
-      @static
-    end
-
-    # Returns true if the argument has the same object as the instance.
-    #
-    # @param [Linkage::MetaObject] other
-    # @return [Boolean]
-    def objects_equal?(other)
-      other.is_a?(Linkage::MetaObject) && other.object == self.object
-    end
-
-    # Returns true if the argument has the same dataset as the instance.
-    #
-    # @param [Linkage::MetaObject] other
-    # @return [Boolean]
-    def datasets_equal?(other)
-      other.is_a?(Linkage::MetaObject) && other.dataset == self.dataset
-    end
-
-    # Returns an expression suitable for use in Sequel queries.
-    # @return [Object]
-    def to_expr
-      if @object.kind_of?(Linkage::Data)
-        @object.to_expr
-      else
-        @object
-      end
-    end
-
-    # Returns a Sequel identifier for {Data} objects, or the object itself.
-    # @return [Sequel::SQL::Identifier, Object]
-    def to_identifier
-      if @object.kind_of?(Linkage::Data)
-        Sequel::SQL::Identifier.new(@object.to_expr)
-      else
-        @object
-      end
-    end
-
-    # Return the name of the object for {Data} objects, nil for others.
-    # @return [Symbol, nil]
-    def name
-      if @object.kind_of?(Linkage::Data)
-        @object.name
-      else
-        nil
-      end
-    end
-
-    # Returns a {MergeField} if both objects are {Data} objects, otherwise,
-    # raises an exception.
-    #
-    # @return [Linkage::MergeField]
-    def merge(other)
-      if @object.kind_of?(Linkage::Data) && other.object.kind_of?(Linkage::Data)
-        @object.merge(other.object)
-      else
-        raise ArgumentError, "Cannot merge a non-data object"
-      end
-    end
-
-    # Returns the Ruby type of the underlying object.
-    #
-    # @return [Hash]
-    # @see Linkage::Field#ruby_type
-    # @see Linkage::Function#ruby_type
-    def ruby_type
-      if @object.kind_of?(Linkage::Data)
-        @object.ruby_type
-      else
-        {:type => @object.class}
-      end
-    end
-
-    # Returns the collation of the underlying object.
-    #
-    # @return [Symbol]
-    def collation
-      if @object.kind_of?(Linkage::Data)
-        @object.collation
-      else
-        nil
-      end
-    end
-
-    # Returns true if underlying object is not a subclass of {Linkage::Data}.
-    def raw?
-      !@object.kind_of?(Linkage::Data)
-    end
-  end
-end

data/lib/linkage/runner/single_threaded.rb

@@ -1,187 +0,0 @@
-module Linkage
-  # A runner class that only uses a single thread to execute a linkage.
-  #
-  # @see Runner
-  class SingleThreadedRunner < Runner
-    # @return [Linkage::ResultSet]
-    def execute
-      result_set.create_tables!
-
-      @pk_1 = config.dataset_1.field_set.primary_key.to_expr
-      @pk_2 = config.dataset_2.field_set.primary_key.to_expr
-      if config.has_simple_expectations?
-        setup_datasets
-        group_records
-
-        if config.has_exhaustive_expectations?
-          score_records_with_groups
-        else
-          create_matches
-        end
-      else
-        dataset_1, dataset_2 = config.datasets_with_applied_exhaustive_expectations
-        score_records_without_groups(dataset_1, dataset_2)
-      end
-
-      result_set.flush!
-      return result_set
-    end
-
-    private
-
-    def setup_datasets
-      @dataset_1, @dataset_2 = config.datasets_with_applied_simple_expectations
-
-      @dataset_1 = @dataset_1.select(@pk_1)
-      if @config.linkage_type != :self
-        @dataset_2 = @dataset_2.select(@pk_2)
-      end
-    end
-
-    def group_records
-      if config.linkage_type == :self
-        group_records_for(@dataset_1, 1)
-      else
-        group_records_for(@dataset_1, 1, false)
-        group_records_for(@dataset_2, 2, false)
-        combine_groups
-      end
-    end
-
-    # @param [Linkage::Dataset] dataset
-    # @param [Fixnum, nil] dataset_id
-    # @param [Boolean] ignore_empty_groups
-    # @yield [Linkage::Group] If a block is given, yield completed groups to
-    #   the block. Otherwise, call ResultSet#add_group on the group.
-    def group_records_for(dataset, dataset_id, ignore_empty_groups = true)
-      group_minimum = ignore_empty_groups ? 2 : 1
-      dataset.each_group(group_minimum) do |group|
-        result_set.add_group(group, dataset_id)
-      end
-      result_set.flush!
-    end
-
-    def combine_groups
-      # Create a new dataset for the groups table
-      groups_dataset = result_set.groups_dataset
-
-      groups_dataset.field_set.values.each do |field|
-        # Sort on all fields
-        if !field.primary_key?
-          meta_object = MetaObject.new(field)
-          groups_dataset = groups_dataset.group_match_more(meta_object)
-        end
-      end
-
-      # Delete non-matching groups
-      sub_dataset = groups_dataset.select(:id).group_by_matches.having(:count.sql_function(:id) => 1)
-      groups_dataset.filter(:id => sub_dataset.obj).delete
-
-      # Delete duplicate groups
-      sub_dataset = groups_dataset.select(:max.sql_function(:id).as(:id)).group_by_matches
-      groups_dataset.filter(:id => sub_dataset.obj).delete
-    end
-
-    def score_records_with_groups
-      result_set.groups_dataset.each do |group_record|
-        group = Group.from_row(group_record)
-        dataset_1, dataset_2 = config.apply_exhaustive_expectations(
-          *result_set.groups_records_datasets(group))
-        score_records_without_groups(dataset_1, dataset_2)
-      end
-    end
-
-    def score_records_without_groups(dataset_1, dataset_2)
-      if config.linkage_type == :self
-        keys = dataset_1.select_map(@pk_1)
-        unfiltered_dataset = dataset_1.unfiltered
-        cache = Hashery::LRUHash.new(config.record_cache_size) do |h, k|
-          h[k] = unfiltered_dataset.filter(@pk_1 => k).first
-        end
-        upper_bound = keys.length - 1
-
-        forward = true
-        keys.each_with_index do |key_1, key_1_index|
-          record_1 = cache[key_1]
-
-          lower_bound = key_1_index + 1
-          enum =
-            if forward
-              lower_bound.upto(upper_bound)
-            else
-              upper_bound.downto(lower_bound)
-            end
-          enum.each do |key_2_index|
-            record_2 = cache[keys[key_2_index]]
-            score(record_1, record_2)
-          end
-          forward = !forward
-        end
-      else
-        keys_2 = dataset_2.select_map(@pk_2)
-        unfiltered_dataset_2 = dataset_2.unfiltered
-        cache_2 = Hashery::LRUHash.new(config.record_cache_size) do |h, k|
-          h[k] = unfiltered_dataset_2.filter(@pk_2 => k).first
-        end
-        keys_2_last = keys_2.length - 1
-
-        forward = true
-        dataset_1.each do |record_1|
-          enum = forward ? 0.upto(keys_2_last) : keys_2_last.downto(0)
-          enum.each do |key_2_index|
-            record_2 = cache_2[keys_2[key_2_index]]
-            score(record_1, record_2)
-          end
-          forward = !forward
-        end
-      end
-    end
-
-    def score(record_1, record_2)
-      pk_1 = record_1[@pk_1]
-      pk_2 = record_2[@pk_2]
-
-      catch(:stop) do
-        total_score = 0
-        config.exhaustive_expectations.each_with_index do |expectation, comparator_id|
-          comparator = expectation.comparator
-
-          score = comparator.score(record_1, record_2)
-          result_set.add_score(comparator_id, pk_1, pk_2, score)
-
-          throw(:stop) unless expectation.satisfied?(score)
-          total_score += score
-        end
-        result_set.add_match(pk_1, pk_2, total_score)
-      end
-    end
-
-    # Only needed for linkages without exhaustive expectations
-    def create_matches
-      result_set.groups_dataset.each do |group_record|
-        group = Group.from_row(group_record)
-        dataset_1, dataset_2 = result_set.groups_records_datasets(group)
-
-        if config.linkage_type == :self
-          keys = dataset_1.select_map(@pk_1)
-          keys_last = keys.length - 1
-          keys.each_with_index do |key_1, key_1_index|
-            (key_1_index + 1).upto(keys_last) do |key_2_index|
-              key_2 = keys[key_2_index]
-              result_set.add_match(key_1, key_2, nil)
-            end
-          end
-        else
-          keys_1 = dataset_1.select_map(@pk_1)
-          keys_2 = dataset_2.select_map(@pk_2)
-
-          keys_1.each do |key_1|
-            keys_2.each do |key_2|
-              result_set.add_match(key_1, key_2, nil)
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/linkage/utils.rb

@@ -1,164 +0,0 @@
-module Linkage
-  module Utils
-    # A "tree" used to find compatible types.
-    TYPE_CONVERSION_TREE = {
-      TrueClass => [Integer],
-      Integer => [Bignum, Float],
-      Bignum => [BigDecimal],
-      Float => [BigDecimal],
-      BigDecimal => [String],
-      String => nil,
-      DateTime => nil,
-      Date => nil,
-      Time => nil,
-      File => nil
-    }
-
-    # Create field information for a field that can hold data from two other
-    # fields. If the fields have different types, the resulting type is
-    # determined via a type-conversion tree.
-    #
-    # @param [Array] field_1 Schema information for the first field
-    # @param [Array] field_2 Schema information for the second field
-    # @return [Array] Schema information for the new field
-    def merge_fields(field_1, field_2)
-      schema_1 = column_schema_to_ruby_type(field_1)
-      schema_1.delete_if { |k, v| v.nil? }
-      schema_2 = column_schema_to_ruby_type(field_2)
-      schema_2.delete_if { |k, v| v.nil? }
-      if schema_1 == schema_2
-        result = schema_1
-      else
-        result = schema_1.dup
-
-        # type
-        if schema_1[:type] != schema_2[:type]
-          result[:type] = first_common_type(schema_1[:type], schema_2[:type])
-        end
-
-        # text
-        if schema_1[:text] != schema_2[:text]
-          # This can only be of type String.
-          result[:text] = true
-          result.delete(:size)
-        end
-
-        # size
-        if !result[:text] && schema_1[:size] != schema_2[:size]
-          types = [schema_1[:type], schema_2[:type]].uniq
-          if types.length == 1 && types[0] == BigDecimal
-            # Two decimals
-            if schema_1.has_key?(:size) && schema_2.has_key?(:size)
-              s_1 = schema_1[:size]
-              s_2 = schema_2[:size]
-              result[:size] = [ s_1[0] > s_2[0] ? s_1[0] : s_2[0] ]
-
-              if s_1[1] && s_2[1]
-                result[:size][1] = s_1[1] > s_2[1] ? s_1[1] : s_2[1]
-              else
-                result[:size][1] = s_1[1] ? s_1[1] : s_2[1]
-              end
-            else
-              result[:size] = schema_1.has_key?(:size) ? schema_1[:size] : schema_2[:size]
-            end
-          elsif types.include?(String) && types.include?(BigDecimal)
-            # Add one to the precision of the BigDecimal (for the dot)
-            if schema_1.has_key?(:size) && schema_2.has_key?(:size)
-              s_1 = schema_1[:size].is_a?(Array) ? schema_1[:size][0] + 1 : schema_1[:size]
-              s_2 = schema_2[:size].is_a?(Array) ? schema_2[:size][0] + 1 : schema_2[:size]
-              result[:size] = s_1 > s_2 ? s_1 : s_2
-            elsif schema_1.has_key?(:size)
-              result[:size] = schema_1[:size].is_a?(Array) ? schema_1[:size][0] + 1 : schema_1[:size]
-            elsif schema_2.has_key?(:size)
-              result[:size] = schema_2[:size].is_a?(Array) ? schema_2[:size][0] + 1 : schema_2[:size]
-            end
-          else
-            # Treat as two strings
-            if schema_1.has_key?(:size) && schema_2.has_key?(:size)
-              result[:size] = schema_1[:size] > schema_2[:size] ? schema_1[:size] : schema_2[:size]
-            elsif schema_1.has_key?(:size)
-              result[:size] = schema_1[:size]
-            else
-              result[:size] = schema_2[:size]
-            end
-          end
-        end
-
-        # fixed
-        if schema_1[:fixed] != schema_2[:fixed]
-          # This can only be of type String.
-          result[:fixed] = true
-        end
-      end
-
-      {:type => result.delete(:type), :opts => result}
-    end
-
-    private
-
-    # Convert the column schema information to a hash of column options, one of which must
-    # be :type.  The other options added should modify that type (e.g. :size).  If a
-    # database type is not recognized, return it as a String type.
-    #
-    # @note This method comes straight from Sequel (lib/sequel/extensions/schema_dumper.rb).
-    def column_schema_to_ruby_type(schema)
-      case t = schema[:db_type].downcase
-      when /\A(?:medium|small)?int(?:eger)?(?:\((?:\d+)\))?(?: unsigned)?\z/o
-        {:type=>Integer}
-      when /\Atinyint(?:\((\d+)\))?\z/o
-        {:type =>schema[:type] == :boolean ? TrueClass : Integer}
-      when /\Abigint(?:\((?:\d+)\))?(?: unsigned)?\z/o
-        {:type=>Bignum}
-      when /\A(?:real|float|double(?: precision)?)\z/o
-        {:type=>Float}
-      when 'boolean'
-        {:type=>TrueClass}
-      when /\A(?:(?:tiny|medium|long|n)?text|clob)\z/o
-        {:type=>String, :text=>true}
-      when 'date'
-        {:type=>Date}
-      when /\A(?:small)?datetime\z/o
-        {:type=>DateTime}
-      when /\Atimestamp(?:\((\d+)\))?(?: with(?:out)? time zone)?\z/o
-        {:type=>DateTime, :size=>($1.to_i if $1)}
-      when /\Atime(?: with(?:out)? time zone)?\z/o
-        {:type=>Time, :only_time=>true}
-      when /\An?char(?:acter)?(?:\((\d+)\))?\z/o
-        {:type=>String, :size=>($1.to_i if $1), :fixed=>true}
-      when /\A(?:n?varchar|character varying|bpchar|string)(?:\((\d+)\))?\z/o
-        {:type=>String, :size=>($1.to_i if $1)}
-      when /\A(?:small)?money\z/o
-        {:type=>BigDecimal, :size=>[19,2]}
-      when /\A(?:decimal|numeric|number)(?:\((\d+)(?:,\s*(\d+))?\))?\z/o
-        s = [($1.to_i if $1), ($2.to_i if $2)].compact
-        {:type=>BigDecimal, :size=>(s.empty? ? nil : s)}
-      when /\A(?:bytea|(?:tiny|medium|long)?blob|(?:var)?binary)(?:\((\d+)\))?\z/o
-        {:type=>File, :size=>($1.to_i if $1)}
-      when 'year'
-        {:type=>Integer}
-      else
-        {:type=>String}
-      end
-    end
-
-    def first_common_type(type_1, type_2)
-      types_1 = [type_1] + get_types(type_1)
-      types_2 = [type_2] + get_types(type_2)
-      (types_1 & types_2).first
-    end
-
-    # Get all types that the specified type can be converted to. Order
-    # matters.
-    def get_types(type)
-      result = []
-      types = TYPE_CONVERSION_TREE[type]
-      if types
-        result += types
-        types.each do |t|
-          result |= get_types(t)
-        end
-      end
-      result
-    end
-  end
-end

data/lib/linkage/warnings.rb

@@ -1,5 +0,0 @@
-module Linkage
-  module Warnings
-    # TODO :)
-  end
-end

data/test/integration/test_collation.rb

@@ -1,45 +0,0 @@
-require 'helper'
-
-module IntegrationTests
-  class TestCollation < Test::Unit::TestCase
-    def setup
-      @tmpdir = Dir.mktmpdir('linkage')
-      @tmpuri = "sqlite://" + File.join(@tmpdir, "foo")
-    end
-
-    def database(options = {}, &block)
-      Sequel.connect(@tmpuri, options, &block)
-    end
-
-    def teardown
-      FileUtils.remove_entry_secure(@tmpdir)
-    end
-
-    test "comparing strings exactly in MySQL" do
-      options = database_options_for('mysql')
-      database_for('mysql') do |db|
-        db.create_table!(:foo) do
-          primary_key :id
-          String :foo
-          String :bar
-        end
-        db[:foo].import([:foo, :bar], [
-          ["Foo", "foo"],
-          ["bar", "bar "],
-        ])
-      end
-      dataset = Linkage::Dataset.new(options, :foo)
-      tmpuri = @tmpuri
-      conf = dataset.link_with(dataset) do
-        (lhs[:foo].must == rhs[:bar]).exactly
-        save_results_in(tmpuri)
-      end
-      runner = Linkage::SingleThreadedRunner.new(conf)
-      runner.execute
-
-      database do |db|
-        assert_equal 0, db[:groups].count
-      end
-    end
-  end
-end