linkage 0.0.6 → 0.0.8

data/lib/linkage/configuration.rb

@@ -1,7 +1,6 @@
 module Linkage
   class Configuration
     class DSL
-
       # Class for visually comparing matched records
       class VisualComparisonWrapper
         attr_reader :dsl, :lhs, :rhs
@@ -33,119 +32,97 @@ module Linkage
           :>=   => :<
         }
 
-        attr_reader :kind, :side, :lhs, :rhs
-
-        def initialize(dsl, type, lhs)
+        def initialize(dsl, type, lhs, *args)
           @dsl = dsl
           @type = type
           @lhs = lhs
-          @rhs = nil
-          @side = nil
-          @kind = nil
         end
 
-        VALID_OPERATORS.each do |operator|
-          define_method(operator) do |rhs|
-            # NOTE: lhs is always a DataWrapper
-
-            @rhs = rhs
-            if !@rhs.is_a?(DataWrapper) || @lhs.static? || @rhs.static? || @lhs.side == @rhs.side
-              @side = @lhs.side
-              @side = @rhs.side if @side.nil? && @rhs.is_a?(DataWrapper)
-              @kind = :filter
-            elsif @lhs.same_except_side?(@rhs)
-              @kind = :self
-            elsif @lhs.dataset == @rhs.dataset
-              @kind = :cross
-            else
-              @kind = :dual
-            end
-            @operator = @type == :must_not ? OPERATOR_OPPOSITES[operator] : operator
-            @dsl.add_expectation(self)
-          end
-        end
+        def compare_with(operator, rhs)
+          # NOTE: lhs is always a DataWrapper
 
-        def merged_field
-          @merged_field ||= @lhs.data.merge(@rhs.data)
-        end
+          if !rhs.is_a?(DataWrapper) || @lhs.static? || rhs.static? || @lhs.side == rhs.side
+            @side = !@lhs.static? ? @lhs.side : rhs.side
 
-        def filter_expr
-          if @filter_expr.nil? && @kind == :filter
-            if @lhs.is_a?(DataWrapper) && !@lhs.static?
-              target = @lhs
-              other = @rhs
-            elsif @rhs.is_a?(DataWrapper) && !@rhs.static?
-              target = @rhs
-              other = @lhs
-            else
-              raise "Wonky filter"
+            # If one of the objects in this comparison is a static function, we need to set the side
+            # and the dataset based on the other object
+            if rhs.is_a?(DataWrapper) && !rhs.static? && @lhs.is_a?(FunctionWrapper) && @lhs.static?
+              @lhs.dataset = rhs.dataset
+              @lhs.side = @side
+            elsif @lhs.is_a?(DataWrapper) && !@lhs.static? && rhs.is_a?(FunctionWrapper) && rhs.static?
+              rhs.dataset = @lhs.dataset
+              rhs.side = @side
             end
+          elsif rhs.is_a?(DataWrapper) && operator != :==
+            # create an exhaustive expectation with the Compare comparator instead
+            comparator = Comparators::Compare.new(@lhs.meta_object,
+              MetaObject.new(operator.to_s), rhs.meta_object)
 
-            arg1 = target.to_expr(@side)
-            arg2 = other.is_a?(DataWrapper) ? other.to_expr(@side) : other
-            @filter_expr =
-              case @operator
-              when :==
-                { arg1 => arg2 }
-              when :'!='
-                ~{ arg1 => arg2 }
-              else
-                arg1 = Sequel::SQL::Identifier.new(arg1)
-                arg2 = arg2.is_a?(Symbol) ? Sequel::SQL::Identifier.new(arg2) : arg2
-                Sequel::SQL::BooleanExpression.new(@operator, arg1, arg2)
-              end
-          end
-          @filter_expr
-        end
+            score_range = Comparators::Compare.score_range
+            threshold = @type == :must ? score_range.last : score_range.first
 
-        def apply_to(dataset, side)
-          if @kind == :filter
-            if @side == side
-              return dataset.filter(filter_expr)
-            else
-              # Doesn't apply
-              return dataset
-            end
+            expectation = Expectations::Exhaustive.new(comparator, threshold, :equal)
+            @dsl.add_exhaustive_expectation(expectation)
+            return self
           end
 
-          if @lhs.is_a?(DataWrapper) && @lhs.side == side
-            target = @lhs
-          elsif @rhs.is_a?(DataWrapper) && @rhs.side == side
-            target = @rhs
-          else
-            raise "Wonky expectation"
-          end
+          exp_operator = @type == :must_not ? OPERATOR_OPPOSITES[operator] : operator
 
-          expr = target.to_expr(side)
-          aliaz = nil
-          if expr != merged_field.name
-            aliaz = merged_field.name
-          end
+          rhs_meta_object = rhs.is_a?(DataWrapper) ? rhs.meta_object : MetaObject.new(rhs)
+          @expectation = Expectations::Simple.create(@lhs.meta_object,
+            rhs_meta_object, exp_operator)
+          @dsl.add_simple_expectation(@expectation)
+          self
+        end
 
-          dataset.match(expr, aliaz)
+        VALID_OPERATORS.each do |operator|
+          define_method(operator) do |rhs|
+            compare_with(operator, rhs)
+          end
         end
 
-        def same_filter?(other)
-          kind == :filter && other.kind == :filter && filter_expr == other.filter_expr
+        def exactly
+          if !@exact_match
+            @expectation.exactly!
+          end
         end
       end
 
       class DataWrapper
-        attr_reader :side, :dataset
+        attr_reader :meta_object
 
         def initialize
           raise NotImplementedError
         end
 
         [:must, :must_not].each do |type|
-          define_method(type) do
-            ExpectationWrapper.new(@dsl, type, self)
+          define_method(type) do |*args|
+            if args.length > 0
+              wrapper = args[0]
+              comparator = wrapper.to_comparator(self)
+
+              score_range = wrapper.klass.score_range
+              threshold = type == :must ? score_range.last : score_range.first
+
+              expectation = Expectations::Exhaustive.new(comparator, threshold, :equal)
+              @dsl.add_exhaustive_expectation(expectation)
+            else
+              ExpectationWrapper.new(@dsl, type, self)
+            end
           end
         end
 
         def compare_with(other)
           VisualComparisonWrapper.new(@dsl, self, other)
         end
+
+        def method_missing(m, *args, &block)
+          if meta_object.respond_to?(m)
+            meta_object.send(m, *args, &block)
+          else
+            super(m, *args, &block)
+          end
+        end
       end
 
       class FieldWrapper < DataWrapper
@@ -153,80 +130,51 @@ module Linkage
 
         def initialize(dsl, side, dataset, name)
           @dsl = dsl
-          @side = side
-          @dataset = dataset
-          @name = name
-        end
-
-        def static?
-          false
-        end
-
-        def same_except_side?(other)
-          other.is_a?(FieldWrapper) && name == other.name
-        end
-
-        def data
-          @dataset.field_set[@name]
-        end
-
-        def to_expr(side = nil)
-          data.to_expr
+          @meta_object = MetaObject.new(dataset.field_set[name], side)
         end
       end
 
       class FunctionWrapper < DataWrapper
-        attr_reader :klass, :args
-
         def initialize(dsl, klass, args)
           @dsl = dsl
-          @klass = klass
-          @args = args
-          @side = nil
-          @static = true
+
+          side = dataset = nil
+          static = true
+          function_args = []
           args.each do |arg|
             if arg.kind_of?(DataWrapper)
-              raise "conflicting sides" if @side && @side != arg.side
-              @side = arg.side
-              @static &&= arg.static?
+              raise "conflicting sides" if side && side != arg.side
+              side = arg.side
+              static &&= arg.static?
+              dataset = arg.dataset
+              function_args << arg.object
+            else
+              function_args << arg
             end
           end
+          @meta_object = MetaObject.new(klass.new(*function_args), side)
         end
+      end
 
-        def data
-          @data ||= @klass.new(*@args.collect { |arg| arg.kind_of?(DataWrapper) ? arg.data : arg })
-        end
-
-        def to_expr(side)
-          dataset = side == :lhs ? @dsl.lhs : @dsl.rhs
-          data.to_expr(dataset.dataset.adapter_scheme)
-        end
+      class ComparatorWrapper
+        attr_reader :klass, :args
 
-        def name
-          data.name
+        def initialize(dsl, klass, args)
+          @dsl = dsl
+          @klass = klass
+          @args = args
         end
 
-        def static?
-          @static
+        def of(*args)
+          @args.push(*args)
+          self
         end
 
-        def same_except_side?(other)
-          if other.is_a?(FunctionWrapper) && klass == other.klass
-            args.each_with_index do |arg, i|
-              other_arg = other.args[i]
-              if arg.is_a?(DataWrapper) && other_arg.is_a?(DataWrapper)
-                if !arg.same_except_side?(other_arg)
-                  return false
-                end
-              else
-                if arg != other_arg
-                  return false
-                end
-              end
-            end
-            return true
+        def to_comparator(receiver)
+          comparator_args = ([receiver] + @args).collect do |arg|
+            arg.is_a?(DataWrapper) ? arg.meta_object : MetaObject.new(arg)
           end
-          false
+          comparator = klass.new(*comparator_args)
         end
       end
 
@@ -268,8 +216,12 @@ module Linkage
         @config.results_uri_options = options
       end
 
-      def add_expectation(expectation)
-        @config.expectations << expectation
+      def set_record_cache_size(num)
+        @config.record_cache_size = num
+      end
+
+      def add_simple_expectation(expectation)
+        @config.add_simple_expectation(expectation)
 
         if @config.linkage_type == :self
           case expectation.kind
@@ -288,7 +240,7 @@ module Linkage
 
             these_filters << expectation
             other_filters.each do |other|
-              if !expectation.same_filter?(other)
+              if !expectation.same_except_side?(other)
                 @config.linkage_type = :cross
                 break
               end
@@ -297,36 +249,98 @@ module Linkage
         end
       end
 
+      def add_exhaustive_expectation(expectation)
+        @config.add_exhaustive_expectation(expectation)
+        if @config.linkage_type == :self
+          @config.linkage_type = expectation.kind
+        end
+      end
+
       def add_visual_comparison(visual_comparison)
         @config.visual_comparisons << visual_comparison
       end
 
-      # For handling functions
+      def groups_table_name(new_name)
+        @config.groups_table_name = new_name
+      end
+
+      def original_groups_table_name(new_name)
+        @config.original_groups_table_name = new_name
+      end
+
+      def scores_table_name(new_name)
+        @config.scores_table_name = new_name
+      end
+
+      def matches_table_name(new_name)
+        @config.matches_table_name = new_name
+      end
+
       def method_missing(name, *args, &block)
-        klass = Function[name.to_s]
-        if klass
-          FunctionWrapper.new(self, klass, args)
+        # check for comparators
+        md = name.to_s.match(/^be_(.+)$/)
+        if md
+          klass = Comparator[md[1]]
+          if klass
+            ComparatorWrapper.new(self, klass, args)
+          else
+            super
+          end
         else
-          super
+          # check for functions
+          klass = Function[name.to_s]
+          if klass
+            FunctionWrapper.new(self, klass, args)
+          else
+            super
+          end
         end
       end
     end
 
-    attr_reader :dataset_1, :dataset_2, :expectations, :visual_comparisons
-    attr_accessor :linkage_type, :results_uri, :results_uri_options
+    attr_reader :dataset_1, :dataset_2, :simple_expectations,
+      :exhaustive_expectations, :visual_comparisons
+    attr_accessor :linkage_type, :results_uri, :results_uri_options,
+      :record_cache_size, :groups_table_name, :original_groups_table_name,
+      :scores_table_name, :matches_table_name
 
     def initialize(dataset_1, dataset_2)
       @dataset_1 = dataset_1
       @dataset_2 = dataset_2
       @linkage_type = dataset_1 == dataset_2 ? :self : :dual
-      @expectations = []
+      @simple_expectations = []
+      @exhaustive_expectations = []
       @visual_comparisons = []
+      @results_uri_options = {}
+      @decollation_needed = false
+      @record_cache_size = 10_000
+      @groups_table_name = :groups
+      @original_groups_table_name = :original_groups
+      @scores_table_name = :scores
+      @matches_table_name = :matches
     end
 
     def configure(&block)
       DSL.new(self, &block)
     end
 
+    def results_uri=(uri)
+      @results_uri = uri
+      if !@decollation_needed
+        @simple_expectations.each do |expectation|
+          if decollation_needed_for_simple_expectation?(expectation)
+            @decollation_needed = true
+            break
+          end
+        end
+      end
+      uri
+    end
+
+    def decollation_needed?
+      @decollation_needed
+    end
+
     def groups_table_schema
       schema = []
 
@@ -334,29 +348,147 @@ module Linkage
       schema << [:id, Integer, {:primary_key => true}]
 
       # add values
-      @expectations.each do |exp|
+      @simple_expectations.each do |exp|
         next  if exp.kind == :filter
 
         merged_field = exp.merged_field
         merged_type = merged_field.ruby_type
-        schema << [merged_field.name, merged_type[:type], merged_type[:opts] || {}]
+
+        # if the merged field's database type is different than the result
+        # database, strip collation information
+        result_db_type = nil
+        result_set.database do |db|
+          result_db_type = db.database_type
+        end
+        if merged_field.database_type != result_db_type && merged_type.has_key?(:opts)
+          new_opts = merged_type[:opts].reject { |k, v| k == :collate }
+          merged_type = merged_type.merge(:opts => new_opts)
+        end
+
+        col = [merged_field.name, merged_type[:type], merged_type[:opts] || {}]
+        schema << col
       end
 
       schema
     end
 
+    def scores_table_schema
+      schema = []
+
+      # add id
+      schema << [:id, Integer, {:primary_key => true}]
+
+      # add comparator id
+      schema << [:comparator_id, Integer, {}]
+
+      # add record ids
+      pk = dataset_1.field_set.primary_key
+      ruby_type = pk.ruby_type
+      schema << [:record_1_id, ruby_type[:type], ruby_type[:opts] || {}]
+
+      pk = dataset_2.field_set.primary_key
+      ruby_type = pk.ruby_type
+      schema << [:record_2_id, ruby_type[:type], ruby_type[:opts] || {}]
+
+      # add score
+      schema << [:score, Integer, {}]
+
+      schema
+    end
+
+    def matches_table_schema
+      schema = []
+
+      # add id
+      schema << [:id, Integer, {:primary_key => true}]
+
+      # add record ids
+      pk = dataset_1.field_set.primary_key
+      ruby_type = pk.ruby_type
+      schema << [:record_1_id, ruby_type[:type], ruby_type[:opts] || {}]
+
+      pk = dataset_2.field_set.primary_key
+      ruby_type = pk.ruby_type
+      schema << [:record_2_id, ruby_type[:type], ruby_type[:opts] || {}]
+
+      # add score
+      schema << [:total_score, Integer, {}]
+
+      schema
+    end
+
+    def add_simple_expectation(expectation)
+      @simple_expectations << expectation
+      @decollation_needed ||= decollation_needed_for_simple_expectation?(expectation)
+      expectation
+    end
+
+    def add_exhaustive_expectation(expectation)
+      @exhaustive_expectations << expectation
+      expectation
+    end
+
     def result_set
       @result_set ||= ResultSet.new(self)
     end
 
-    def datasets_with_applied_expectations
+    def datasets_with_applied_simple_expectations
       dataset_1 = @dataset_1
       dataset_2 = @dataset_2
-      @expectations.each do |exp|
+      @simple_expectations.each do |exp|
         dataset_1 = exp.apply_to(dataset_1, :lhs)
         dataset_2 = exp.apply_to(dataset_2, :rhs) if @linkage_type != :self
       end
       @linkage_type == :self ? [dataset_1, dataset_1] : [dataset_1, dataset_2]
     end
+
+    def datasets_with_applied_exhaustive_expectations
+      apply_exhaustive_expectations(@dataset_1, @dataset_2)
+    end
+
+    def apply_exhaustive_expectations(dataset_1, dataset_2)
+      dataset_1 = dataset_1.select(dataset_1.field_set.primary_key.to_expr)
+      dataset_2 = dataset_2.select(dataset_2.field_set.primary_key.to_expr)
+      @exhaustive_expectations.each do |exp|
+        dataset_1 = exp.apply_to(dataset_1, :lhs)
+        dataset_2 = exp.apply_to(dataset_2, :rhs)
+      end
+      [dataset_1, dataset_2]
+    end
+
+    def groups_table_needed?
+      has_simple_expectations?
+    end
+
+    def scores_table_needed?
+      has_exhaustive_expectations?
+    end
+
+    def has_simple_expectations?
+      !@simple_expectations.empty?
+    end
+
+    def has_exhaustive_expectations?
+      !@exhaustive_expectations.empty?
+    end
+
+    private
+
+    def decollation_needed_for_simple_expectation?(expectation)
+      if expectation.decollation_needed?
+        true
+      elsif results_uri && expectation.kind != :filter
+        result_set_database_type = ResultSet.new(self).database.database_type
+        database_types_differ =
+          result_set_database_type != dataset_1.database_type ||
+          result_set_database_type != dataset_2.database_type
+
+        merged_field = expectation.merged_field
+        merged_field.ruby_type[:type] == String &&
+          !merged_field.collation.nil? && database_types_differ
+      else
+        false
+      end
+    end
   end
 end

data/lib/linkage/data.rb

@@ -17,9 +17,14 @@ module Linkage
       File => nil
     }
 
-    # @return [Symbol] This object's name
+    # @!attribute [r] name
+    #   @return [Symbol] This object's name
     attr_reader :name
 
+    # @!attribute [r] dataset
+    #   @return [Linkage::Dataset, nil] This object's dataset, or nil
+    attr_reader :dataset
+
     def initialize(name)
       @name = name
     end
@@ -28,20 +33,35 @@ module Linkage
       raise NotImplementedError
     end
 
-    def to_expr(adapter = nil)
+    def to_expr
+      raise NotImplementedError
+    end
+
+    def collation
+      nil
+    end
+
+    def database_type
+      ds = dataset
+      ds ? ds.database_type : nil
+    end
+
+    def static?
       raise NotImplementedError
     end
 
-    # Create a data object that can hold data from two other fields. If the fields
-    # have different types, the resulting type is determined via a
+    # Create a merge field that can hold data from two data sources. If the
+    # fields have different types, the resulting type is determined via a
     # type-conversion tree.
     #
     # @param [Linkage::Data] other
-    # @return [Linkage::Field]
+    # @return [Linkage::MergeField]
     def merge(other, new_name = nil)
       schema_1 = self.ruby_type
+      db_type_1 = self.database_type
       schema_2 = other.ruby_type
-      if schema_1 == schema_2
+      db_type_2 = other.database_type
+      if schema_1 == schema_2 && db_type_1 == db_type_2
         result = schema_1
       else
         type_1 = schema_1[:type]
@@ -113,6 +133,11 @@ module Linkage
           result_opts[:fixed] = true
         end
 
+        # collation
+        if opts_1[:collate] != opts_2[:collate] || db_type_1 != db_type_2
+          result_opts.delete(:collate)
+        end
+
         result = {:type => result_type}
         result[:opts] = result_opts  unless result_opts.empty?
       end
@@ -122,7 +147,7 @@ module Linkage
       else
         name = self.name == other.name ? self.name : :"#{self.name}_#{other.name}"
       end
-      Field.new(name, nil, result)
+      MergeField.new(name, result, db_type_1 == db_type_2 ? db_type_1 : nil)
     end
 
     private