linkage 0.0.6 → 0.0.8

data/lib/linkage/dataset.rb

@@ -2,13 +2,28 @@ module Linkage
   # Delegator around Sequel::Dataset with some extra functionality.
   class Dataset
     attr_reader :field_set, :table_name
+    attr_accessor :linkage_options
 
-    def initialize(uri, table, options = {})
-      @table_name = table.to_sym
-      db = Sequel.connect(uri, options)
-      @dataset = db[@table_name]
-      @field_set = FieldSet.new(db.schema(@table_name))
-      @_match = []
+    def initialize(*args)
+      if args.length == 1
+        @dataset = args[0]
+        @db = @dataset.db
+        @table_name = @dataset.first_source_table
+
+        if !@db.kind_of?(Sequel::Collation)
+          @db.extend(Sequel::Collation)
+        end
+      else
+        uri, table, options = args
+        options ||= {}
+
+        @table_name = table.to_sym
+        @db = Sequel.connect(uri, options)
+        @db.extend(Sequel::Collation)
+        @dataset = @db[@table_name]
+      end
+      @field_set = FieldSet.new(self)
+      @linkage_options = {}
     end
 
     def obj
@@ -28,67 +43,127 @@ module Linkage
       conf
     end
 
-    def adapter_scheme
-      @dataset.db.adapter_scheme
+    def database_type
+      @db.database_type
     end
 
-    def match(expr, aliaz = nil)
-      clone(:match => {:expr => expr, :alias => aliaz})
+    # Set objects to use for group matching. Accepts either {Linkage::MetaObject} or a
+    # hash with options (valid options are :meta_object, :alias, and :cast).
+    #
+    # @example
+    #   dataset.group_match(meta_object_1,
+    #     {:meta_object => meta_object_2, :alias => :foo})
+    def group_match(*args)
+      args.collect! do |arg|
+        case arg
+        when Linkage::MetaObject
+          { :meta_object => arg }
+        when Hash
+          if !arg.has_key?(:meta_object)
+            raise ArgumentError, "Invalid option hash, missing :meta_object key"
+          end
+          (arg.keys - [:meta_object, :alias, :cast]).each do |invalid_key|
+            warn "Invalid key in option hash: #{invalid_key}"
+          end
+          arg
+        else
+          raise ArgumentError, "expected Hash or MetaObject, got #{arg.class}"
+        end
+      end
+      clone(:group_match => args)
+    end
+
+    # Add additional objects to use for group matching.
+    def group_match_more(*args)
+      args = @linkage_options[:group_match] + args  if @linkage_options[:group_match]
+      group_match(*args)
     end
 
-    def clone(new_opts={})
-      new_opts = new_opts.dup
-      new_obj = new_opts.delete(:new_obj)
+    def clone(new_options = {})
+      new_linkage_options = {}
+      new_obj_options = {}
+      new_options.each_pair do |k, v|
+        case k
+        when :group_match
+          new_linkage_options[k] = v
+        else
+          new_obj_options[k] = v
+        end
+      end
+      new_obj = new_options[:new_obj]
 
-      match = new_opts.delete(:match)
       result = super()
-      result.send(:_match, match)
+      result.linkage_options = @linkage_options.merge(new_linkage_options)
 
       if new_obj
         result.obj = new_obj
       else
-        result.obj = obj.clone(new_opts)
+        result.obj = obj.clone(new_options)
       end
+
       result
     end
 
     def each_group(min = 2)
-      @dataset.group_and_count(*aliased_match_expressions).having{count >= min}.each do |row|
+      group_match = @linkage_options[:group_match] || []
+      ruby_types = group_match.inject({}) do |hsh, m|
+        key = m[:alias] || m[:meta_object].to_expr
+        hsh[key] = m[:meta_object].ruby_type
+        hsh
+      end
+      options = {:database_type => database_type, :ruby_types => ruby_types }
+      @dataset.group_and_count(*match_expressions).having{count >= min}.each do |row|
         count = row.delete(:count)
-        yield Group.new(row, {:count => count})
+        group = Group.new(row, options.merge(:count => count))
+        yield group
       end
     end
 
-    def group_by_matches(aliased = false)
-      expr = aliased ? aliased_match_expressions : match_expressions
+    def group_by_matches(raw = true)
+      expr = raw ? raw_match_expressions : match_expressions
       group(*expr)
     end
 
     def dataset_for_group(group)
       filters = []
+      group_match = @linkage_options[:group_match] || []
       group.values.each_pair do |key, value|
         # find a matched expression with this alias
-        m = @_match.detect { |h| h[:alias] ? h[:alias] == key : h[:expr] == key }
-        raise "this dataset isn't compatible with the given group" if !m
-        filters << {m[:expr] => value}
+        found = false
+        group_match.each do |m|
+          expr = m[:meta_object].to_expr
+          if (m[:alias] && m[:alias] == key) || expr == key
+            found = true
+            filters << {expr => value}
+            break
+          end
+        end
+        if !found
+          raise "this dataset isn't compatible with the given group"
+        end
       end
       filter(*filters)
     end
 
+    def schema
+      @db.schema(@table_name)
+    end
+
     private
 
-    def _match(opts)
-      if opts
-        @_match += [opts]
-      end
+    def raw_match_expressions
+      group_match = @linkage_options[:group_match] || []
+      group_match.collect { |m| m[:meta_object].to_expr }
     end
 
     def match_expressions
-      @_match.collect { |m| m[:expr] }
-    end
-
-    def aliased_match_expressions
-      @_match.collect { |m| m[:alias] ? m[:expr].as(m[:alias]) : m[:expr] }
+      group_match = @linkage_options[:group_match] || []
+      group_match.collect do |m|
+        expr = m[:meta_object].to_expr
+        expr = expr.as(m[:alias]) if m[:alias]
+        expr = expr.cast(m[:cast]) if m[:cast]
+        expr
+      end
     end
 
     def method_missing(name, *args, &block)

data/lib/linkage/decollation.rb

@@ -0,0 +1,93 @@
+# encoding: utf-8
+module Linkage
+  module Decollation
+    def decollate(string, database_type, collation)
+      case database_type
+      when :mysql
+        decollate_mysql(string, collation)
+      else
+        string
+      end
+    end
+
+    def decollate_mysql(string, collation)
+      case collation
+      when "latin1_swedish_ci"
+        decollate_mysql_latin1_swedish_ci(string)
+      else
+        string
+      end
+    end
+
+    def decollate_mysql_latin1_swedish_ci(string)
+      result = string.strip
+      result.each_char.with_index do |char, i|
+        case char
+        when 'A', 'a', 'À', 'Á', 'Â', 'Ã', 'à', 'á', 'â', 'ã'
+          result[i] = 'A'
+        when 'B', 'b'
+          result[i] = 'B'
+        when 'C', 'c', 'Ç', 'ç'
+          result[i] = 'C'
+        when 'D', 'd', 'Ð', 'ð'
+          result[i] = 'D'
+        when 'E', 'e', 'È', 'É', 'Ê', 'Ë', 'è', 'é', 'ê', 'ë'
+          result[i] = 'E'
+        when 'F', 'f'
+          result[i] = 'F'
+        when 'G', 'g'
+          result[i] = 'G'
+        when 'H', 'h'
+          result[i] = 'H'
+        when 'I', 'i', 'Ì', 'Í', 'Î', 'Ï', 'ì', 'í', 'î', 'ï'
+          result[i] = 'I'
+        when 'J', 'j'
+          result[i] = 'J'
+        when 'K', 'k'
+          result[i] = 'K'
+        when 'L', 'l'
+          result[i] = 'L'
+        when 'M', 'm'
+          result[i] = 'M'
+        when 'N', 'n', 'Ñ', 'ñ'
+          result[i] = 'N'
+        when 'O', 'o', 'Ò', 'Ó', 'Ô', 'Õ', 'ò', 'ó', 'ô', 'õ'
+          result[i] = 'O'
+        when 'P', 'p'
+          result[i] = 'P'
+        when 'Q', 'q'
+          result[i] = 'Q'
+        when 'R', 'r'
+          result[i] = 'R'
+        when 'S', 's'
+          result[i] = 'S'
+        when 'T', 't'
+          result[i] = 'T'
+        when 'U', 'u', 'Ù', 'Ú', 'Û', 'ù', 'ú', 'û'
+          result[i] = 'U'
+        when 'V', 'v'
+          result[i] = 'V'
+        when 'W', 'w'
+          result[i] = 'W'
+        when 'X', 'x'
+          result[i] = 'X'
+        when 'Y', 'y', 'Ü', 'Ý', 'ü', 'ý'
+          result[i] = 'Y'
+        when 'Z', 'z'
+          result[i] = 'Z'
+        when '[', 'Å', 'å'
+          result[i] = '['
+        when '\\', 'Ä', 'Æ', 'ä', 'æ'
+          result[i] = '\\'
+        when ']', 'Ö', 'ö'
+          result[i] = ']'
+        when 'Ø', 'ø'
+          result[i] = 'Ø'
+        when 'Þ', 'þ'
+          result[i] = 'Þ'
+        end
+      end
+      result
+    end
+  end
+end

data/lib/linkage/expectation.rb

@@ -0,0 +1,21 @@
+module Linkage
+  # The Expectation class contains information about how two datasets
+  # should be linked.
+  class Expectation
+    def kind
+      raise NotImplementedError
+    end
+
+    def apply_to(*args)
+      raise NotImplementedError
+    end
+
+    def decollation_needed?
+      false
+    end
+  end
+end
+
+Dir.glob(File.expand_path(File.join(File.dirname(__FILE__), "expectations", "*.rb"))).each do |filename|
+  require filename
+end

data/lib/linkage/expectations/exhaustive.rb

@@ -0,0 +1,63 @@
+module Linkage
+  module Expectations
+    class Exhaustive < Expectation
+      attr_reader :comparator, :threshold, :mode
+
+      def initialize(comparator, threshold, mode)
+        @comparator = comparator
+        @threshold = threshold
+        @mode = mode
+      end
+
+      def kind
+        if @kind.nil?
+          if @comparator.lhs_args.length != @comparator.rhs_args.length
+            @kind = :cross
+          else
+            @kind = :self
+            @comparator.lhs_args.each_with_index do |lhs_arg, index|
+              rhs_arg = @comparator.rhs_args[index]
+              if !lhs_arg.objects_equal?(rhs_arg)
+                @kind = :cross
+                break
+              end
+            end
+          end
+
+          # Check for dual-linkage.
+          if @kind == :cross
+            # Assume that all lhs arguments have the same dataset, as well
+            # as all the rhs arguments. Only check the first argument of each
+            # side.
+            lhs_arg = @comparator.lhs_args[0]
+            rhs_arg = @comparator.rhs_args[0]
+            if !lhs_arg.datasets_equal?(rhs_arg)
+              @kind = :dual
+            end
+          end
+        end
+        @kind
+      end
+
+      def apply_to(dataset, side)
+        exprs =
+          case side
+          when :lhs
+            comparator.lhs_args.collect { |arg| arg.to_expr.as(arg.name) }
+          when :rhs
+            comparator.rhs_args.collect { |arg| arg.to_expr.as(arg.name) }
+          end
+        dataset.select_more(*exprs)
+      end
+
+      def satisfied?(score)
+        case mode
+        when :equal
+          score == threshold
+        when :min
+          score >= threshold
+        end
+      end
+    end
+  end
+end

data/lib/linkage/expectations/simple.rb

@@ -0,0 +1,168 @@
+module Linkage
+  module Expectations
+    class Simple < Expectation
+      # The dataset this expectation applies to: `:lhs` or `:rhs`. This
+      # only applies to filter expectations.
+      # @return [Symbol]
+      attr_reader :side
+
+      attr_reader :meta_object_1, :meta_object_2, :operator
+
+      VALID_OPERATORS = [:==, :'!=', :>, :<, :>=, :<=]
+
+      # Automatically create an expectation type depending on the arguments.
+      #
+      # @param [Linkage::MetaObject] meta_object_1
+      # @param [Linkage::MetaObject] meta_object_2
+      # @param [Symbol] operator Valid operators: `:==`, `:'!='`, `:>`, `:<`, `:>=`, `:<=`
+      def self.create(meta_object_1, meta_object_2, operator)
+        klass =
+          if meta_object_1.static? && meta_object_2.static?
+            raise ArgumentError, "An expectation with two static objects is invalid"
+          elsif meta_object_1.static? || meta_object_2.static?
+            Filter
+          elsif meta_object_1.side == meta_object_2.side
+            if !meta_object_1.datasets_equal?(meta_object_2)
+              raise ArgumentError, "An expectation with two dynamic objects with the same side but different datasets is invalid"
+            end
+            Filter
+          elsif meta_object_1.objects_equal?(meta_object_2)
+            Self
+          elsif meta_object_1.datasets_equal?(meta_object_2)
+            Cross
+          else
+            Dual
+          end
+
+        klass.new(meta_object_1, meta_object_2, operator)
+      end
+
+      # Creates a new Simple.
+      #
+      # @param [Linkage::MetaObject] meta_object_1
+      # @param [Linkage::MetaObject] meta_object_2
+      # @param [Symbol] operator Valid operators: `:==`, `:'!='`, `:>`, `:<`, `:>=`, `:<=`
+      def initialize(meta_object_1, meta_object_2, operator)
+        @meta_object_1 = meta_object_1
+        @meta_object_2 = meta_object_2
+        @operator = operator
+
+        if !VALID_OPERATORS.include?(operator)
+          raise ArgumentError, "Invalid operator: #{operator.inspect}"
+        end
+
+        after_initialize
+      end
+
+      def same_except_side?(other)
+        other.is_a?(Simple) &&
+          operator == other.operator &&
+          meta_object_1.objects_equal?(other.meta_object_1) &&
+          meta_object_2.objects_equal?(other.meta_object_2)
+      end
+
+      def exactly!
+        function_1 = Function['binary'].new(@meta_object_1.object, :dataset => @meta_object_1.dataset)
+        function_2 = Function['binary'].new(@meta_object_2.object, :dataset => @meta_object_2.dataset)
+        @meta_object_1 = MetaObject.new(function_1, @meta_object_1.side)
+        @meta_object_2 = MetaObject.new(function_2, @meta_object_2.side)
+      end
+
+      # Display any warnings about this expectation.
+      def display_warnings
+      end
+
+      def decollation_needed?
+        merged_field.ruby_type[:type] == String && (
+          @meta_object_1.collation != @meta_object_2.collation ||
+          @meta_object_1.database_type != @meta_object_2.database_type
+        )
+      end
+
+      protected
+
+      def after_initialize
+      end
+    end
+
+    class Filter < Simple
+      def kind; :filter; end
+
+      def to_expr
+        case @operator
+        when :==, :'!='
+          expr = { @meta_object_1.to_expr => @meta_object_2.to_expr }
+          @operator == :== ? expr : ~expr
+        else
+          Sequel::SQL::BooleanExpression.new(@operator,
+            @meta_object_1.to_identifier, @meta_object_2.to_identifier)
+        end
+      end
+
+      def apply_to(dataset, side)
+        if side != @side
+          return dataset
+        end
+
+        dataset.filter(self.to_expr)
+      end
+
+      def decollation_needed?
+        false
+      end
+
+      private
+
+      def after_initialize
+        super
+        @side = @meta_object_1.static? ? @meta_object_2.side : @meta_object_1.side
+      end
+    end
+
+    class Match < Simple
+      def apply_to(dataset, side)
+        target =
+          if @meta_object_1.side == side
+            @meta_object_1
+          elsif @meta_object_2.side == side
+            @meta_object_2
+          else
+            raise ArgumentError, "Invalid `side` argument: #{side}"
+          end
+
+        dataset.group_match_more({
+          :meta_object => target,
+          :alias => merged_field.name
+        })
+      end
+
+      def merged_field
+        @merged_field ||= @meta_object_1.merge(@meta_object_2)
+      end
+
+      def display_warnings
+        object_1 = @meta_object_1.object
+        object_2 = @meta_object_2.object
+        if object_1.ruby_type[:type] == String && object_2.ruby_type[:type] == String
+          if @meta_object_1.dataset.database_type != @meta_object_2.dataset.database_type
+            warn "NOTE: You are comparing two string fields (#{object_1.name} and #{object_2.name}) from different databases. This may result in unexpected results, as different databases compare strings differently. Consider using the =binary= function."
+          elsif object_1.respond_to?(:collation) && object_1.respond_to?(:collation) && object_1.collation != object_2.collation
+            warn "NOTE: The two string fields you are comparing (#{object_1.name} and #{object_2.name}) have different collations (#{ldata.collation} vs. #{rdata.collation}). This may result in unexpected results, as the database may compare them differently. Consider using the =exactly= method."
+          end
+        end
+      end
+    end
+
+    class Self < Match
+      def kind; :self; end
+    end
+
+    class Cross < Match
+      def kind; :cross; end
+    end
+
+    class Dual < Match
+      def kind; :dual; end
+    end
+  end
+end

data/lib/linkage/field.rb

@@ -7,13 +7,13 @@ module Linkage
 
     # Create a new instance of Field.
     #
+    # @param [Linkage::Dataset] dataset
     # @param [Symbol] name The field's name
     # @param [Hash] schema The field's schema information
-    # @param [Hash] ruby_type The field's ruby type
-    def initialize(name, schema, ruby_type = nil)
+    def initialize(dataset, name, schema)
+      @dataset = dataset
       @name = name
       @schema = schema
-      @ruby_type = ruby_type
     end
 
     # Convert the column schema information to a hash of column options, one of
@@ -63,6 +63,8 @@ module Linkage
           else
             {:type=>String}
           end
+        hsh[:collate] = collation
+
         hsh.delete_if { |k, v| v.nil? }
         @ruby_type = {:type => hsh.delete(:type)}
         @ruby_type[:opts] = hsh if !hsh.empty?
@@ -70,7 +72,7 @@ module Linkage
       @ruby_type
     end
 
-    def to_expr(adapter = nil)
+    def to_expr(options = {})
       @name
     end
 
@@ -81,5 +83,29 @@ module Linkage
     def primary_key?
       schema && schema[:primary_key]
     end
+
+    def collation
+      schema[:collation]
+    end
+  end
+
+  # A special field used for merging two {Data} objects together. It
+  # has no dataset or schema.
+  class MergeField < Field
+    attr_reader :database_type
+
+    # Create a new instance of MergeField.
+    #
+    # @param [Symbol] name The field's name
+    # @param [Hash] ruby_type The field's schema information
+    def initialize(name, ruby_type, database_type = nil)
+      @name = name
+      @ruby_type = ruby_type
+      @database_type = database_type
+    end
+
+    def collation
+      @ruby_type.has_key?(:opts) ? @ruby_type[:opts][:collate] : nil
+    end
   end
 end

data/lib/linkage/field_set.rb

@@ -2,9 +2,12 @@ module Linkage
   class FieldSet < Hash
     attr_reader :primary_key
 
-    def initialize(schema)
-      schema.each do |(name, column_schema)|
-        f = Field.new(name, column_schema)
+    # Create a new FieldSet.
+    #
+    # @param [Linkage::Dataset] dataset
+    def initialize(dataset)
+      dataset.schema.each do |(name, column_schema)|
+        f = Field.new(dataset, name, column_schema)
         self[name] = f
 
         if @primary_key.nil? && column_schema[:primary_key]