linkage 0.0.8 → 0.1.0.pre

data/lib/linkage/dataset.rb

@@ -2,28 +2,33 @@ module Linkage
   # Delegator around Sequel::Dataset with some extra functionality.
   class Dataset
     attr_reader :field_set, :table_name
-    attr_accessor :linkage_options
 
     def initialize(*args)
+      if args.length == 0 || args.length > 3
+        raise ArgumentError, "wrong number of arguments (#{args.length} for 1..3)"
+      end
+
       if args.length == 1
+        unless args[0].kind_of?(Sequel::Dataset)
+          raise ArgumentError, "expected Sequel::Dataset, got #{args[0].class}"
+        end
+
         @dataset = args[0]
         @db = @dataset.db
         @table_name = @dataset.first_source_table
-
-        if !@db.kind_of?(Sequel::Collation)
-          @db.extend(Sequel::Collation)
-        end
+      elsif args.length == 2 && args[0].kind_of?(Sequel::Database)
+        @db = args[0]
+        @table_name = args[1].to_sym
+        @dataset = @db[@table_name]
       else
-        uri, table, options = args
+        uri, table_name, options = args
         options ||= {}
 
-        @table_name = table.to_sym
         @db = Sequel.connect(uri, options)
-        @db.extend(Sequel::Collation)
+        @table_name = table_name.to_sym
         @dataset = @db[@table_name]
       end
       @field_set = FieldSet.new(self)
-      @linkage_options = {}
     end
 
     def obj
@@ -37,9 +42,12 @@ module Linkage
     # Setup a linkage with another dataset
     #
     # @return [Linkage::Configuration]
-    def link_with(dataset, &block)
-      conf = Configuration.new(self, dataset)
-      conf.configure(&block)
+    def link_with(dataset, result_set)
+      other = dataset.eql?(self) ? nil : dataset
+      conf = Configuration.new(self, other, result_set)
+      if block_given?
+        yield conf
+      end
       conf
     end
 
@@ -47,132 +55,25 @@ module Linkage
       @db.database_type
     end
 
-    # Set objects to use for group matching. Accepts either {Linkage::MetaObject} or a
-    # hash with options (valid options are :meta_object, :alias, and :cast).
-    #
-    # @example
-    #   dataset.group_match(meta_object_1,
-    #     {:meta_object => meta_object_2, :alias => :foo})
-    def group_match(*args)
-      args.collect! do |arg|
-        case arg
-        when Linkage::MetaObject
-          { :meta_object => arg }
-        when Hash
-          if !arg.has_key?(:meta_object)
-            raise ArgumentError, "Invalid option hash, missing :meta_object key"
-          end
-          (arg.keys - [:meta_object, :alias, :cast]).each do |invalid_key|
-            warn "Invalid key in option hash: #{invalid_key}"
-          end
-          arg
-        else
-          raise ArgumentError, "expected Hash or MetaObject, got #{arg.class}"
-        end
-      end
-      clone(:group_match => args)
-    end
-
-    # Add additional objects to use for group matching.
-    def group_match_more(*args)
-      args = @linkage_options[:group_match] + args  if @linkage_options[:group_match]
-      group_match(*args)
-    end
-
-    def clone(new_options = {})
-      new_linkage_options = {}
-      new_obj_options = {}
-      new_options.each_pair do |k, v|
-        case k
-        when :group_match
-          new_linkage_options[k] = v
-        else
-          new_obj_options[k] = v
-        end
-      end
-      new_obj = new_options[:new_obj]
-
-      result = super()
-      result.linkage_options = @linkage_options.merge(new_linkage_options)
-
-      if new_obj
-        result.obj = new_obj
-      else
-        result.obj = obj.clone(new_options)
-      end
-
-      result
-    end
-
-    def each_group(min = 2)
-      group_match = @linkage_options[:group_match] || []
-      ruby_types = group_match.inject({}) do |hsh, m|
-        key = m[:alias] || m[:meta_object].to_expr
-        hsh[key] = m[:meta_object].ruby_type
-        hsh
-      end
-      options = {:database_type => database_type, :ruby_types => ruby_types }
-      @dataset.group_and_count(*match_expressions).having{count >= min}.each do |row|
-        count = row.delete(:count)
-        group = Group.new(row, options.merge(:count => count))
-        yield group
-      end
-    end
-
-    def group_by_matches(raw = true)
-      expr = raw ? raw_match_expressions : match_expressions
-      group(*expr)
-    end
-
-    def dataset_for_group(group)
-      filters = []
-      group_match = @linkage_options[:group_match] || []
-      group.values.each_pair do |key, value|
-        # find a matched expression with this alias
-        found = false
-        group_match.each do |m|
-          expr = m[:meta_object].to_expr
-          if (m[:alias] && m[:alias] == key) || expr == key
-            found = true
-            filters << {expr => value}
-            break
-          end
-        end
-        if !found
-          raise "this dataset isn't compatible with the given group"
-        end
-      end
-      filter(*filters)
-    end
-
     def schema
       @db.schema(@table_name)
     end
 
-    private
-
-    def raw_match_expressions
-      group_match = @linkage_options[:group_match] || []
-      group_match.collect { |m| m[:meta_object].to_expr }
+    def primary_key
+      @field_set.primary_key
     end
 
-    def match_expressions
-      group_match = @linkage_options[:group_match] || []
-      group_match.collect do |m|
-        expr = m[:meta_object].to_expr
-        expr = expr.as(m[:alias]) if m[:alias]
-        expr = expr.cast(m[:cast]) if m[:cast]
-        expr
-      end
-    end
+    protected
 
     def method_missing(name, *args, &block)
       result = @dataset.send(name, *args, &block)
       if result.kind_of?(Sequel::Dataset)
-        new_obj = result
-        result = clone(:new_obj => result)
+        new_object = clone
+        new_object.obj = result
+        new_object
+      else
+        result
       end
-      result
     end
   end
 end

data/lib/linkage/exceptions.rb

@@ -0,0 +1,5 @@
+module Linkage
+  class Error < Exception; end
+  class ExistsError < Error; end
+  class MissingError < Error; end
+end

data/lib/linkage/field.rb

@@ -1,17 +1,19 @@
 module Linkage
   # This class is for holding information about a particular field in a
   # dataset.
-  class Field < Data
+  class Field
+    # @!attribute [r] name
+    #   @return [Symbol] This object's name
+    attr_reader :name
+
     # @return [Symbol] This field's schema information
     attr_reader :schema
 
     # Create a new instance of Field.
     #
-    # @param [Linkage::Dataset] dataset
     # @param [Symbol] name The field's name
     # @param [Hash] schema The field's schema information
-    def initialize(dataset, name, schema)
-      @dataset = dataset
+    def initialize(name, schema)
       @name = name
       @schema = schema
     end
@@ -63,7 +65,6 @@ module Linkage
           else
             {:type=>String}
           end
-        hsh[:collate] = collation
 
         hsh.delete_if { |k, v| v.nil? }
         @ruby_type = {:type => hsh.delete(:type)}
@@ -72,40 +73,8 @@ module Linkage
       @ruby_type
     end
 
-    def to_expr(options = {})
-      @name
-    end
-
-    def static?
-      false
-    end
-
     def primary_key?
       schema && schema[:primary_key]
     end
-
-    def collation
-      schema[:collation]
-    end
-  end
-
-  # A special field used for merging two {Data} objects together. It
-  # has no dataset or schema.
-  class MergeField < Field
-    attr_reader :database_type
-
-    # Create a new instance of MergeField.
-    #
-    # @param [Symbol] name The field's name
-    # @param [Hash] ruby_type The field's schema information
-    def initialize(name, ruby_type, database_type = nil)
-      @name = name
-      @ruby_type = ruby_type
-      @database_type = database_type
-    end
-
-    def collation
-      @ruby_type.has_key?(:opts) ? @ruby_type[:opts][:collate] : nil
-    end
   end
 end

data/lib/linkage/field_set.rb

@@ -7,11 +7,11 @@ module Linkage
     # @param [Linkage::Dataset] dataset
     def initialize(dataset)
       dataset.schema.each do |(name, column_schema)|
-        f = Field.new(dataset, name, column_schema)
-        self[name] = f
+        field = Field.new(name, column_schema)
+        self[name] = field
 
         if @primary_key.nil? && column_schema[:primary_key]
-          @primary_key = f
+          @primary_key = field
         end
       end
     end

data/lib/linkage/match_recorder.rb

@@ -0,0 +1,22 @@
+module Linkage
+  class MatchRecorder
+    def initialize(matcher, match_set)
+      @matcher = matcher
+      @match_set = match_set
+    end
+
+    def start
+      @matcher.add_observer(self)
+      @match_set.open_for_writing
+    end
+
+    def update(id_1, id_2, score)
+      @match_set.add_match(id_1, id_2, score)
+    end
+
+    def stop
+      @match_set.close
+      @matcher.delete_observer(self)
+    end
+  end
+end

data/lib/linkage/match_set.rb

@@ -0,0 +1,34 @@
+module Linkage
+  class MatchSet
+    # Register a match set.
+    #
+    # @param [Class] klass
+    def self.register(name, klass)
+      methods = klass.instance_methods(false)
+      unless methods.include?(:add_match)
+        raise ArgumentError, "class must define #add_match"
+      end
+
+      @match_sets ||= {}
+      @match_sets[name] = klass
+    end
+
+    def self.[](name)
+      @match_sets ? @match_sets[name] : nil
+    end
+
+    def open_for_writing
+    end
+
+    # @abstract
+    def add_match(id_1, id_2, score)
+      raise NotImplementedError
+    end
+
+    def close
+    end
+  end
+end
+
+require 'linkage/match_sets/csv'
+require 'linkage/match_sets/database'

data/lib/linkage/match_sets/csv.rb

@@ -0,0 +1,39 @@
+require 'csv'
+
+module Linkage
+  module MatchSets
+    class CSV < MatchSet
+      def initialize(filename, options = {})
+        @filename = filename
+        @overwrite = options[:overwrite]
+      end
+
+      def open_for_writing
+        return if @mode == :write
+
+        if !@overwrite && File.exist?(@filename)
+          raise ExistsError, "#{@filename} exists and not in overwrite mode"
+        end
+
+        @csv = ::CSV.open(@filename, 'wb')
+        @csv << %w{id_1 id_2 score}
+        @mode = :write
+      end
+
+      def add_match(id_1, id_2, score)
+        raise "not in write mode" if @mode != :write
+        if score.equal?(1.0) || score.equal?(0.0)
+          score = score.floor
+        end
+        @csv << [id_1, id_2, score]
+      end
+
+      def close
+        @mode = nil
+        @csv.close if @csv
+      end
+    end
+
+    MatchSet.register('csv', CSV)
+  end
+end

data/lib/linkage/match_sets/database.rb

@@ -0,0 +1,45 @@
+module Linkage
+  module MatchSets
+    class Database < MatchSet
+      def initialize(database, options = {})
+        @database = database
+        @table_name = options[:table_name] || :matches
+        @overwrite = options[:overwrite]
+      end
+
+      def open_for_writing
+        return if @mode == :write
+
+        if @overwrite
+          @database.drop_table?(@table_name)
+        elsif @database.table_exists?(@table_name)
+          raise ExistsError, "#{@table_name} table exists and not in overwrite mode"
+        end
+
+        @database.create_table(@table_name) do
+          String :id_1
+          String :id_2
+          Float :score
+        end
+        @dataset = @database[@table_name]
+        @mode = :write
+      end
+
+      def add_match(id_1, id_2, score)
+        raise "not in write mode" if @mode != :write
+
+        @dataset.insert({
+          :id_1 => id_1,
+          :id_2 => id_2,
+          :score => score
+        })
+      end
+
+      def close
+        @mode = nil
+      end
+    end
+
+    MatchSet.register('database', Database)
+  end
+end

data/lib/linkage/matcher.rb

@@ -0,0 +1,30 @@
+module Linkage
+  class Matcher
+    include Observable
+
+    attr_reader :comparators, :score_set, :algorithm, :threshold
+
+    def initialize(comparators, score_set, algorithm, threshold)
+      @comparators = comparators
+      @score_set = score_set
+      @algorithm = algorithm
+      @threshold = threshold
+    end
+
+    def run
+      send(@algorithm)
+    end
+
+    private
+
+    def mean
+      @score_set.each_pair do |id_1, id_2, scores|
+        mean = scores.values.inject(:+) / @comparators.length.to_f
+        if mean >= @threshold
+          changed
+          notify_observers(id_1, id_2, mean)
+        end
+      end
+    end
+  end
+end

data/lib/linkage/result_set.rb

@@ -1,125 +1,40 @@
 module Linkage
   class ResultSet
-    def initialize(config)
-      @config = config
-      @next_group_id = 1
-      @next_group_mutex = Mutex.new
-    end
-
-    def groups_dataset
-      @groups_dataset ||= Dataset.new(database[:groups])
-    end
-
-    def database
-      # FIXME: If the results database is the same as one of the datasets
-      # being linked, there will be two connections to said database. This
-      # could result in unexpected locking for non-concurrent databases (like
-      # SQLite).
-      @database ||= Sequel.connect(@config.results_uri, @config.results_uri_options)
-    end
-
-    def create_tables!
-      if @config.groups_table_needed?
-        schema = @config.groups_table_schema
-        if @config.decollation_needed?
-          database.create_table(@config.original_groups_table_name) do
-            schema.each { |col| column(*col) }
-          end
-        end
-
-        database.create_table(@config.groups_table_name) do
-          schema.each { |col| column(*col) }
-        end
-      end
-
-      if @config.scores_table_needed?
-        schema = @config.scores_table_schema
-        database.create_table(@config.scores_table_name) do
-          schema.each { |col| column(*col) }
-        end
-      end
-
-      schema = @config.matches_table_schema
-      database.create_table(@config.matches_table_name) do
-        schema.each { |col| column(*col) }
-      end
-    end
-
-    def add_group(group, dataset_id = nil)
-      if @config.decollation_needed?
-        original_values = group.values
-        values = group.decollated_values
-        if !@groups_buffer
-          groups_headers = [:id] + values.keys
-          @groups_buffer = ImportBuffer.new(database[@config.groups_table_name],
-            groups_headers)
-
-          original_groups_headers = [:id] + original_values.keys
-          @original_groups_buffer = ImportBuffer.new(
-            database[@config.original_groups_table_name],
-            original_groups_headers)
-        end
-
-        group_id = next_group_id
-        @groups_buffer.add([group_id] + values.values)
-        @original_groups_buffer.add([group_id] + original_values.values)
-      else
-        # Non-DRY for minute speed improvements
-        values = group.values
-        if !@groups_buffer
-          groups_headers = [:id] + values.keys
-          @groups_buffer = ImportBuffer.new(database[@config.groups_table_name],
-            groups_headers)
-        end
-        group_id = next_group_id
-        @groups_buffer.add([group_id] + values.values)
+    # Register a result set.
+    #
+    # @param [Class] klass
+    def self.register(name, klass)
+      methods = klass.instance_methods(false)
+      missing = []
+      unless methods.include?(:score_set)
+        missing.push("#score_set")
       end
-    end
-
-    def add_score(comparator_id, record_1_id, record_2_id, score)
-      if !@scores_buffer
-        scores_headers = [:comparator_id, :record_1_id, :record_2_id, :score]
-        @scores_buffer = ImportBuffer.new(database[@config.scores_table_name],
-          scores_headers)
+      unless methods.include?(:match_set)
+        missing.push("#match_set")
       end
-      @scores_buffer.add([comparator_id, record_1_id, record_2_id, score])
-    end
-
-    def add_match(record_1_id, record_2_id, total_score)
-      if !@matches_buffer
-        matches_headers = [:record_1_id, :record_2_id, :total_score]
-        @matches_buffer = ImportBuffer.new(database[@config.matches_table_name],
-          matches_headers)
+      unless missing.empty?
+        raise ArgumentError, "class must define #{missing.join(" and ")}"
       end
-      @matches_buffer.add([record_1_id, record_2_id, total_score])
-    end
 
-    def flush!
-      @groups_buffer.flush if @groups_buffer
-      @original_groups_buffer.flush if @original_groups_buffer
-      @scores_buffer.flush if @scores_buffer
-      @matches_buffer.flush if @matches_buffer
+      @result_set ||= {}
+      @result_set[name] = klass
     end
 
-    def get_group(index)
-      values = groups_dataset.order(:id).limit(1, index).first
-      Group.from_row(values)
+    def self.[](name)
+      @result_set ? @result_set[name] : nil
     end
 
-    def groups_records_datasets(group)
-      datasets = @config.datasets_with_applied_simple_expectations
-      datasets.collect! { |ds| ds.dataset_for_group(group) }
+    # @abstract
+    def score_set
+      raise NotImplementedError
     end
 
-    private
-
-    def next_group_id
-      result = nil
-      @next_group_mutex.synchronize do
-        result = @next_group_id
-        @next_group_id += 1
-      end
-      result
+    # @abstract
+    def match_set
+      raise NotImplementedError
     end
   end
 end
+
+require 'linkage/result_sets/csv'
+require 'linkage/result_sets/database'

data/lib/linkage/result_sets/csv.rb

@@ -0,0 +1,54 @@
+module Linkage
+  module ResultSets
+    class CSV < ResultSet
+      def initialize(dir_or_options = nil)
+        opts =
+          case dir_or_options
+          when nil
+            {}
+          when String
+            {:dir => dir_or_options}
+          when Hash
+            dir_or_options
+          else
+            raise ArgumentError, "expected nil, a String, or a Hash, got #{dir_or_options.class}"
+          end
+
+        if opts[:dir]
+          opts[:dir] = File.expand_path(opts[:dir])
+          FileUtils.mkdir_p(opts[:dir])
+        end
+
+        @score_set_args = extract_args_for(:scores, opts)
+        @match_set_args = extract_args_for(:matches, opts)
+      end
+
+      def score_set
+        @score_set ||= ScoreSet['csv'].new(*@score_set_args)
+      end
+
+      def match_set
+        @match_set ||= MatchSet['csv'].new(*@match_set_args)
+      end
+
+      private
+
+      def extract_args_for(name, opts)
+        dir = opts[:dir] || '.'
+        opts = opts[name]
+
+        filename =
+          case opts
+          when Hash, nil
+            opts = opts ? opts.dup : {}
+            opts.delete(:filename) || "#{name}.csv"
+          when String
+            opts
+          end
+        [File.join(dir, filename), opts]
+      end
+    end
+
+    ResultSet.register('csv', CSV)
+  end
+end