linkage 0.0.6 → 0.0.8

data/lib/linkage/function.rb

@@ -38,10 +38,23 @@ module Linkage
       nil
     end
 
-    # @param [Linkage::Field, Object] args Function arguments
+    attr_reader :args
+
+    # Creates a new Function object. If the arguments contain only
+    # static objects, you should specify the dataset that this function
+    # belongs to as the last argument like so:
+    #
+    #   Function.new(foo, bar, :dataset => dataset)
+    #
+    # Optionally, you can use the `dataset=` setter to do it later. Many
+    # functions require a dataset to work properly. If you try to use
+    # such a function without setting a  dataset, it will raise a RuntimeError.
+    #
+    # @param [Linkage::Data, Object] args Function arguments
     def initialize(*args)
       @names = [self.class.function_name]
       @args = args
+      @options = args.last.is_a?(Hash) ? args.pop : {}
       process_args
     end
 
@@ -49,10 +62,25 @@ module Linkage
       @name ||= @names.join("_").to_sym
     end
 
+    def dataset
+      if @dataset.nil?
+        raise RuntimeError, "You must specify a dataset for static functions"
+      end
+      @dataset
+    end
+
+    def dataset=(dataset)
+      @dataset = dataset
+    end
+
     def static?
       @static
     end
 
+    def ==(other)
+      equal?(other) || (other.is_a?(Function) && name == other.name && args == other.args && dataset == other.dataset)
+    end
+
     # Subclasses must define this. The return value should be a Hash with
     # the following elements:
     #   :type - column type (Ruby class) of the result
@@ -61,8 +89,14 @@ module Linkage
       raise NotImplementedError
     end
 
+    # Returns `nil` by default. Subclasses should redefine this if
+    # there is a collation.
+    def collation
+      nil
+    end
+
     # @return [Sequel::SQL::Function]
-    def to_expr(adapter = nil)
+    def to_expr(options = {})
       self.class.function_name.to_sym.sql_function(*@values)
     end
 
@@ -80,6 +114,14 @@ module Linkage
         if arg.kind_of?(Data)
           @names << arg.name
           @static &&= arg.static?
+
+          # possibly set dataset
+          if @dataset.nil?
+            @dataset = arg.dataset
+          elsif @dataset != arg.dataset
+            raise ArgumentError, "Using dynamic data sources with different datasets is not permitted"
+          end
+
           type = arg.ruby_type[:type]
           value = arg.to_expr
         else
@@ -87,11 +129,16 @@ module Linkage
           type = arg.class
           value = arg
         end
-        if parameters && !parameters[i].include?(type)
+        if parameters && parameters[i] != [:any] && !parameters[i].include?(type)
           raise TypeError, "expected type #{parameters[i].join(" or ")}, got #{type}"
         end
         @values << value
       end
+
+      if @dataset.nil? && @options[:dataset]
+        # Set dataset for static functions manually
+        @dataset = @options[:dataset]
+      end
     end
   end
 end

data/lib/linkage/functions/binary.rb

@@ -0,0 +1,30 @@
+module Linkage
+  module Functions
+    class Binary < Function
+      def self.function_name
+        "binary"
+      end
+
+      def self.parameters
+        [[String]]
+      end
+
+      def ruby_type
+        {:type => File}
+      end
+
+      def to_expr(options = {})
+        expr =
+          case dataset.database_type
+          when :sqlite
+            @values[0].cast(:blob)
+          when :postgres
+            @values[0].cast(:bytea)
+          else
+            @values[0].cast(:binary)
+          end
+      end
+    end
+    Function.register(Binary)
+  end
+end

data/lib/linkage/functions/cast.rb

@@ -0,0 +1,54 @@
+module Linkage
+  module Functions
+    class Cast < Function
+      def self.function_name
+        "cast"
+      end
+
+      def self.parameters
+        [[:any], [String]]
+      end
+
+      def ruby_type
+        type =
+          case @values[1]
+          when 'integer'
+            Fixnum
+          when 'binary'
+            File
+          else
+            raise "unknown type: #{@values[1]}"
+          end
+
+        {:type => type}
+      end
+
+      def to_expr(options = {})
+        cast =
+          case @values[1]
+          when 'integer'
+            case dataset.database_type
+            when :sqlite, :postgres, :h2
+              :integer
+            when :mysql
+              :signed
+            end
+          when 'binary'
+            case dataset.database_type
+            when :sqlite
+              :blob
+            when :postgres
+              :bytea
+            when :mysql, :h2
+              :binary
+            end
+          end
+
+        if cast
+          @values[0].cast(cast)
+        end
+      end
+    end
+    Function.register(Cast)
+  end
+end

data/lib/linkage/functions/length.rb

@@ -0,0 +1,29 @@
+module Linkage
+  module Functions
+    # Returns the number of characters in a string.
+    class Length < Function
+      def self.function_name
+        "length"
+      end
+
+      def self.parameters
+        [[String]]
+      end
+
+      def ruby_type
+        {:type => Fixnum}
+      end
+
+      def to_expr(options = {})
+        expr =
+          case dataset.database_type
+          when :mysql, :postgres
+            :char_length.sql_function(@values[0])
+          else
+            :length.sql_function(@values[0])
+          end
+      end
+    end
+    Function.register(Length)
+  end
+end

data/lib/linkage/functions/strftime.rb

@@ -14,17 +14,18 @@ module Linkage
         {:type => String}
       end
 
-      def to_expr(adapter = nil)
-        case adapter
-        when :mysql, :mysql2
-          :date_format.sql_function(*@values)
-        when :sqlite
-          :strftime.sql_function(@values[1], @values[0])
-        when :postgres
-          :to_char.sql_function(*@values)
-        else
-          :strftime.sql_function(@values[0], @values[1])
-        end
+      def to_expr(options = {})
+        expr =
+          case dataset.database_type
+          when :mysql
+            :date_format.sql_function(*@values)
+          when :sqlite
+            :strftime.sql_function(@values[1], @values[0])
+          when :postgres
+            :to_char.sql_function(*@values)
+          else
+            :strftime.sql_function(@values[0], @values[1])
+          end
       end
     end
     Function.register(Strftime)

data/lib/linkage/functions/trim.rb

@@ -16,6 +16,14 @@ module Linkage
           {:type => String}
         end
       end
+
+      def collation
+        if @args[0].kind_of?(Data)
+          @args[0].collation
+        else
+          super
+        end
+      end
     end
     Function.register(Trim)
   end

data/lib/linkage/group.rb

@@ -1,5 +1,7 @@
 module Linkage
   class Group
+    include Linkage::Decollation
+
     # @return [Hash] Hash of matching values
     attr_reader :values
 
@@ -24,12 +26,30 @@ module Linkage
 
     # @param [Hash] values Values that define this group
     # @param [Hash] options
+    # @option options [Fixnum] :id The group ID
+    # @option options [Fixnum] :count How many records are in the group
+    # @option options [Hash] :ruby_types Hash of ruby types for each value
+    # @option options [Symbol] :database_type
     # @example
     #   Linkage::Group.new({:foo => 123, :bar => 'baz'}, {:count => 5, :id => 456})
     def initialize(values, options)
       @count = options[:count]
       @id = options[:id]
+      @ruby_types = options[:ruby_types]
+      @database_type = options[:database_type]
       @values = values
     end
+
+    def decollated_values
+      @values.inject({}) do |hsh, (key, value)|
+        ruby_type = @ruby_types[key]
+        if ruby_type && ruby_type.has_key?(:opts) && ruby_type[:opts].has_key?(:collate)
+          hsh[key] = decollate(value, @database_type, ruby_type[:opts][:collate])
+        else
+          hsh[key] = value
+        end
+        hsh
+      end
+    end
   end
 end

data/lib/linkage/import_buffer.rb

@@ -1,15 +1,11 @@
 module Linkage
   class ImportBuffer
-    # @param [String] uri Sequel-style URI
-    # @param [Symbol, String] table_name
+    # @param [Sequel::Dataset] dataset
     # @param [Array<Symbol>] headers List of fields you want to insert
-    # @param [Hash] options Sequel.connect options
     # @param [Fixnum] limit Number of records to insert at a time
-    def initialize(uri, table_name, headers, options = {}, limit = 1000)
-      @uri = uri
-      @table_name = table_name.to_sym
+    def initialize(dataset, headers, limit = 1000)
+      @dataset = dataset
       @headers = headers
-      @options = options
       @limit = limit
       @values = []
     end
@@ -23,17 +19,10 @@ module Linkage
 
     def flush
       return if @values.empty?
-      database do |db|
-        ds = db[@table_name]
-        ds.import(@headers, @values)
+      @dataset.db.synchronize do
+        @dataset.import(@headers, @values)
         @values.clear
       end
     end
-
-    private
-
-    def database(&block)
-      Sequel.connect(@uri, @options, &block)
-    end
   end
 end

data/lib/linkage/meta_object.rb

@@ -0,0 +1,139 @@
+module Linkage
+  class MetaObject
+    attr_reader :object
+    attr_writer :side
+
+    # Creates a new MetaObject.
+    #
+    # @param [Object] object This can be a {Field}, {Function} or a regular
+    #   Ruby object (Fixnum, String, etc). If `object` is not static (a {Field}
+    #   or a {Function} that contains one or more {Field} objects), you should
+    #   specify which "side" of the linkage the object belongs to (left-hand
+    #   side or right-hand side) in the `side` argument.
+    # @param [Symbol] side `:lhs` for left-hand side or `:rhs` for right-hand
+    #   side
+    def initialize(object, side = nil)
+      @object = object
+      @static = object.kind_of?(Linkage::Data) ? object.static? : true
+      if !side.nil? && side != :lhs && side != :rhs
+        raise ArgumentError, "invalid `side` argument, must be :lhs or :rhs"
+      end
+      @side = side
+    end
+
+    def side
+      if !@static && @side.nil?
+        raise RuntimeError, "Object is dynamic and side is not set"
+      end
+      @side
+    end
+
+    def dataset
+      @object.kind_of?(Linkage::Data) ? @object.dataset : nil
+    end
+
+    def dataset=(dataset)
+      if @object.kind_of?(Linkage::Data)
+        @object.dataset = dataset
+      else
+        raise RuntimeError, "You can't set the dataset of a non-data object."
+      end
+    end
+
+    def database_type
+      ds = dataset
+      ds ? ds.database_type : nil
+    end
+
+    def static?
+      @static
+    end
+
+    # Returns true if the argument has the same object as the instance.
+    #
+    # @param [Linkage::MetaObject] other
+    # @return [Boolean]
+    def objects_equal?(other)
+      other.is_a?(Linkage::MetaObject) && other.object == self.object
+    end
+
+    # Returns true if the argument has the same dataset as the instance.
+    #
+    # @param [Linkage::MetaObject] other
+    # @return [Boolean]
+    def datasets_equal?(other)
+      other.is_a?(Linkage::MetaObject) && other.dataset == self.dataset
+    end
+
+    # Returns an expression suitable for use in Sequel queries.
+    # @return [Object]
+    def to_expr
+      if @object.kind_of?(Linkage::Data)
+        @object.to_expr
+      else
+        @object
+      end
+    end
+
+    # Returns a Sequel identifier for {Data} objects, or the object itself.
+    # @return [Sequel::SQL::Identifier, Object]
+    def to_identifier
+      if @object.kind_of?(Linkage::Data)
+        Sequel::SQL::Identifier.new(@object.to_expr)
+      else
+        @object
+      end
+    end
+
+    # Return the name of the object for {Data} objects, nil for others.
+    # @return [Symbol, nil]
+    def name
+      if @object.kind_of?(Linkage::Data)
+        @object.name
+      else
+        nil
+      end
+    end
+
+    # Returns a {MergeField} if both objects are {Data} objects, otherwise,
+    # raises an exception.
+    #
+    # @return [Linkage::MergeField]
+    def merge(other)
+      if @object.kind_of?(Linkage::Data) && other.object.kind_of?(Linkage::Data)
+        @object.merge(other.object)
+      else
+        raise ArgumentError, "Cannot merge a non-data object"
+      end
+    end
+
+    # Returns the Ruby type of the underlying object.
+    #
+    # @return [Hash]
+    # @see Linkage::Field#ruby_type
+    # @see Linkage::Function#ruby_type
+    def ruby_type
+      if @object.kind_of?(Linkage::Data)
+        @object.ruby_type
+      else
+        {:type => @object.class}
+      end
+    end
+
+    # Returns the collation of the underlying object.
+    #
+    # @return [Symbol]
+    def collation
+      if @object.kind_of?(Linkage::Data)
+        @object.collation
+      else
+        nil
+      end
+    end
+
+    # Returns true if underlying object is not a subclass of {Linkage::Data}.
+    def raw?
+      !@object.kind_of?(Linkage::Data)
+    end
+  end
+end

data/lib/linkage/result_set.rb

@@ -7,41 +7,98 @@ module Linkage
     end
 
     def groups_dataset
-      @groups_dataset ||= Dataset.new(@config.results_uri, :groups, @config.results_uri_options)
+      @groups_dataset ||= Dataset.new(database[:groups])
     end
 
-    def database(&block)
-      Sequel.connect(@config.results_uri, @config.results_uri_options, &block)
+    def database
+      # FIXME: If the results database is the same as one of the datasets
+      # being linked, there will be two connections to said database. This
+      # could result in unexpected locking for non-concurrent databases (like
+      # SQLite).
+      @database ||= Sequel.connect(@config.results_uri, @config.results_uri_options)
     end
 
     def create_tables!
-      database do |db|
+      if @config.groups_table_needed?
         schema = @config.groups_table_schema
-        db.create_table(:groups) do
+        if @config.decollation_needed?
+          database.create_table(@config.original_groups_table_name) do
+            schema.each { |col| column(*col) }
+          end
+        end
+
+        database.create_table(@config.groups_table_name) do
           schema.each { |col| column(*col) }
         end
+      end
 
-        pk_type = @config.dataset_1.field_set.primary_key.merge(@config.dataset_2.field_set.primary_key).ruby_type
-        db.create_table(:groups_records) do
-          column(:record_id, pk_type[:type], pk_type[:opts] || {})
-          Integer :group_id
-          Integer :dataset
-          index :group_id
+      if @config.scores_table_needed?
+        schema = @config.scores_table_schema
+        database.create_table(@config.scores_table_name) do
+          schema.each { |col| column(*col) }
         end
       end
+
+      schema = @config.matches_table_schema
+      database.create_table(@config.matches_table_name) do
+        schema.each { |col| column(*col) }
+      end
     end
 
     def add_group(group, dataset_id = nil)
-      if !@groups_buffer
-        groups_headers = [:id] + group.values.keys
-        @groups_buffer = ImportBuffer.new(@config.results_uri, :groups, groups_headers, @config.results_uri_options)
+      if @config.decollation_needed?
+        original_values = group.values
+        values = group.decollated_values
+        if !@groups_buffer
+          groups_headers = [:id] + values.keys
+          @groups_buffer = ImportBuffer.new(database[@config.groups_table_name],
+            groups_headers)
+
+          original_groups_headers = [:id] + original_values.keys
+          @original_groups_buffer = ImportBuffer.new(
+            database[@config.original_groups_table_name],
+            original_groups_headers)
+        end
+
+        group_id = next_group_id
+        @groups_buffer.add([group_id] + values.values)
+        @original_groups_buffer.add([group_id] + original_values.values)
+      else
+        # Non-DRY for minute speed improvements
+        values = group.values
+        if !@groups_buffer
+          groups_headers = [:id] + values.keys
+          @groups_buffer = ImportBuffer.new(database[@config.groups_table_name],
+            groups_headers)
+        end
+        group_id = next_group_id
+        @groups_buffer.add([group_id] + values.values)
+      end
+    end
+
+    def add_score(comparator_id, record_1_id, record_2_id, score)
+      if !@scores_buffer
+        scores_headers = [:comparator_id, :record_1_id, :record_2_id, :score]
+        @scores_buffer = ImportBuffer.new(database[@config.scores_table_name],
+          scores_headers)
+      end
+      @scores_buffer.add([comparator_id, record_1_id, record_2_id, score])
+    end
+
+    def add_match(record_1_id, record_2_id, total_score)
+      if !@matches_buffer
+        matches_headers = [:record_1_id, :record_2_id, :total_score]
+        @matches_buffer = ImportBuffer.new(database[@config.matches_table_name],
+          matches_headers)
       end
-      group_id = next_group_id
-      @groups_buffer.add([group_id] + group.values.values)
+      @matches_buffer.add([record_1_id, record_2_id, total_score])
     end
 
     def flush!
       @groups_buffer.flush if @groups_buffer
+      @original_groups_buffer.flush if @original_groups_buffer
+      @scores_buffer.flush if @scores_buffer
+      @matches_buffer.flush if @matches_buffer
     end
 
     def get_group(index)
@@ -50,7 +107,7 @@ module Linkage
     end
 
     def groups_records_datasets(group)
-      datasets = @config.datasets_with_applied_expectations
+      datasets = @config.datasets_with_applied_simple_expectations
       datasets.collect! { |ds| ds.dataset_for_group(group) }
     end