linkage 0.0.2 → 0.0.3

data/lib/linkage/dataset.rb

@@ -1,205 +1,48 @@
 module Linkage
-  # Wrapper for a Sequel dataset
-  class Dataset
-    @@next_id = 1 # Internal ID used for expectations
-    @@next_id_mutex = Mutex.new
+  class Dataset < Delegator
+    attr_reader :field_set, :table_name
 
-    # @private
-    def self.next_id
-      result = nil
-      @@next_id_mutex.synchronize do
-        result = @@next_id
-        @@next_id += 1
-      end
-      result
-    end
-
-    # @return [Array] Schema information about the dataset's primary key
-    attr_reader :primary_key
-
-    # @return [Array] Schema information for this dataset
-    attr_reader :schema
-
-    # @return [String] Database URI
-    attr_reader :uri
-
-    # @return [Symbol] Database table name
-    attr_reader :table
-
-    # @return [Array<Linkage::Field>] List of {Linkage::Field}'s
-    attr_reader :fields
-
-    # @private
-    attr_reader :id
-
-    # @param [String] uri Sequel-style database URI
-    # @param [String, Symbol] table Database table name
-    # @param [Hash] options Options to pass to Sequel.connect
-    # @see http://sequel.rubyforge.org/rdoc/files/doc/opening_databases_rdoc.html Sequel: Connecting to a database
     def initialize(uri, table, options = {})
-      @id = self.class.next_id
-      @uri = uri
-      @table = table.to_sym
-      @options = options
-      schema = nil
-      database { |db| schema = db.schema(@table) }
-      @schema = schema
-      @order = []
-      @select = []
-      @filter = []
-      create_fields
+      @table_name = table.to_sym
+      db = Sequel.connect(uri, options)
+      ds = db[@table_name]
+      super(ds)
+      @field_set = FieldSet.new(db.schema(@table_name))
     end
 
+    def __setobj__(obj); @dataset = obj; end
+    def __getobj__; @dataset; end
+
     # Setup a linkage with another dataset
     #
     # @return [Linkage::Configuration]
     def link_with(dataset, &block)
       conf = Configuration.new(self, dataset)
-      conf.instance_eval(&block)
+      conf.configure(&block)
       conf
     end
 
-    # Compare URI and database table name
-    #
-    # @return [Boolean]
-    def ==(other)
-      if !other.is_a?(Dataset)
-        super
-      else
-        uri == other.uri && table == other.table
-      end
-    end
-
-    # Create a copy of this instance of Dataset, using {Dataset#initialize}.
-    #
-    # @return [Linkage::Dataset]
-    def dup
-      self.class.new(uri, table)
-    end
-
-    # Clone the dataset and its associated {Linkage::Field}'s (without hitting
-    # the database).
-    #
-    # @return [Linkage::Dataset]
-    def clone
-      other = self.class.allocate
-      other.send(:initialize_copy, self, {
-        :order => @order.clone, :select => @select.clone,
-        :filter => @filter.clone, :options => @options.clone
-      })
-    end
-
-    # Add a data source to use for ordering the dataset.
-    #
-    # @param [Linkage::Data] data
-    # @param [nil, Symbol] desc nil or :desc (for descending order)
-    def add_order(data, desc = nil)
-      expr = desc == :desc ? data.to_expr.desc : data.to_expr
-      unless @order.include?(expr)
-        @order << expr
-      end
-    end
-
-    # Add a data source to be selected on the dataset. If you don't add any
-    # selects, all fields will be selected. The primary key is always selected
-    # in either case.
-    #
-    # @param [Linkage::Data] data
-    # @param [Symbol] as Optional field/function alias
-    def add_select(data, as = nil)
-      expr = as ? data.to_expr.as(as) : data.to_expr
-      unless @select.include?(expr)
-        @select << expr
-      end
+    def adapter_scheme
+      @dataset.db.adapter_scheme
     end
 
-    # Add a filter (SQL WHERE) condition to the dataset.
-    #
-    # @param [Linkage::Data] data
-    # @param [Symbol] operator
-    # @param [Linkage::Data, Object] other
-    def add_filter(data, operator, other)
-      arg1 = data.to_expr
-      arg2 = other.is_a?(Data) ? other.to_expr : other
-      expr =
-        case operator
-        when :==
-          { arg1 => arg2 }
-        when :'!='
-          ~{ arg1 => arg2 }
-        else
-          arg1 = Sequel::SQL::Identifier.new(arg1)
-          arg2 = arg2.is_a?(Symbol) ? Sequel::SQL::Identifier.new(arg2) : arg2
-          Sequel::SQL::BooleanExpression.new(operator, arg1, arg2)
-        end
-      @filter << expr
-    end
-
-    # Yield each row of the dataset in a block.
-    #
-    # @yield [row] A Hash of two elements, :pk and :values, where row[:pk] is
-    #   the row's primary key value, and row[:values] is an array of all
-    #   selected values (except the primary key).
-    def each
-      database do |db|
-        ds = db[@table]
-
-        pk = @primary_key.name
-        if !@select.empty?
-          ds = ds.select(pk, *@select)
-        end
-        if !@order.empty?
-          ds = ds.order(*@order)
-        end
-        if !@filter.empty?
-          ds = ds.filter(*@filter)
-        end
-        ds.each do |row|
-          yield({:pk => row.delete(pk), :values => row})
-        end
-      end
-    end
-
-    private
-
-    def initialize_copy(dataset, options = {})
-      @id = dataset.id
-      @uri = dataset.uri
-      @table = dataset.table
-      @schema = dataset.schema
-      @options = options[:options]
-      @order = options[:order]
-      @select = options[:select]
-      @filter = options[:filter]
-      @fields = dataset.fields.inject({}) do |hsh, (name, field)|
-        new_field = field.clone
-        new_field.dataset = self
-        hsh[name] = new_field
-        hsh
+    def initialize_clone(obj)
+      new_obj = obj.instance_variable_get(:@new_obj)
+      if new_obj
+        __setobj__(new_obj)
+      else
+        super
       end
-      @primary_key = @fields[dataset.primary_key.name]
-      self
     end
 
-    def database(&block)
-      Sequel.connect(uri, @options, &block)
-    end
-
-    def create_fields
-      @fields = {}
-      @schema.each do |(name, column_schema)|
-        f = Field.new(name, column_schema)
-        f.dataset = self
-        @fields[name] = f
-
-        if @primary_key.nil? && column_schema[:primary_key]
-          @primary_key = f
-        end
+    def method_missing(name, *args, &block)
+      result = super
+      if result.kind_of?(Sequel::Dataset)
+        @new_obj = result
+        result = clone
+        @new_obj = nil
       end
-    end
-
-    def set_new_id
-      @id = self.class.next_id
+      result
     end
   end
 end

data/lib/linkage/field.rb

@@ -5,9 +5,6 @@ module Linkage
     # @return [Symbol] This field's schema information
     attr_reader :schema
 
-    # @attr [Linkage::Dataset] This field's associated dataset
-    attr_accessor :dataset
-
     # Create a new instance of Field.
     #
     # @param [Symbol] name The field's name

data/lib/linkage/field_set.rb

@@ -0,0 +1,16 @@
+module Linkage
+  class FieldSet < Hash
+    attr_reader :primary_key
+
+    def initialize(schema)
+      schema.each do |(name, column_schema)|
+        f = Field.new(name, column_schema)
+        self[name] = f
+
+        if @primary_key.nil? && column_schema[:primary_key]
+          @primary_key = f
+        end
+      end
+    end
+  end
+end

data/lib/linkage/function.rb

@@ -38,13 +38,10 @@ module Linkage
       nil
     end
 
-    attr_reader :dataset
-
     # @param [Linkage::Field, Object] args Function arguments
     def initialize(*args)
       @names = [self.class.function_name]
       @args = args
-      @dataset = nil
       process_args
     end
 
@@ -83,10 +80,6 @@ module Linkage
         if arg.kind_of?(Data)
           @names << arg.name
           @static &&= arg.static?
-          if @dataset && !arg.static? && @dataset != arg.dataset
-            raise ArgumentError, "You cannot supply fields from different datasets as arguments to the same function)"
-          end
-          @dataset ||= arg.dataset
           type = arg.ruby_type[:type]
           value = arg.is_a?(Field) ? arg.name : arg.to_expr
         else

data/lib/linkage/result_set.rb

@@ -0,0 +1,68 @@
+module Linkage
+  class ResultSet
+    def initialize(config)
+      @config = config
+      @next_group_id = 1
+      @next_group_mutex = Mutex.new
+    end
+
+    def groups_dataset
+      Dataset.new(@config.results_uri, :groups, @config.results_uri_options)
+    end
+
+    def groups_records_dataset
+      Dataset.new(@config.results_uri, :groups_records, @config.results_uri_options)
+    end
+
+    def database(&block)
+      Sequel.connect(@config.results_uri, @config.results_uri_options, &block)
+    end
+
+    def create_tables!
+      database do |db|
+        schema = @config.groups_table_schema
+        db.create_table(:groups) do
+          schema.each { |col| column(*col) }
+        end
+
+        pk_type = @config.dataset_1.field_set.primary_key.merge(@config.dataset_2.field_set.primary_key).ruby_type
+        db.create_table(:groups_records) do
+          column(:record_id, pk_type[:type], pk_type[:opts] || {})
+          Integer :group_id
+          Integer :dataset
+          index :group_id
+        end
+      end
+    end
+
+    def add_group(group, dataset_id = nil)
+      if !@groups_buffer
+        groups_headers = [:id] + group.values.keys
+        @groups_buffer = ImportBuffer.new(@config.results_uri, :groups, groups_headers, @config.results_uri_options)
+      end
+      @groups_records_buffer ||= ImportBuffer.new(@config.results_uri, :groups_records, [:group_id, :dataset, :record_id], @config.results_uri_options)
+
+      group_id = next_group_id
+      @groups_buffer.add([group_id] + group.values.values)
+      group.records.each do |record_id|
+        @groups_records_buffer.add([group_id, dataset_id, record_id])
+      end
+    end
+
+    def flush!
+      @groups_buffer.flush if @groups_buffer
+      @groups_records_buffer.flush if @groups_records_buffer
+    end
+
+    private
+
+    def next_group_id
+      result = nil
+      @next_group_mutex.synchronize do
+        result = @next_group_id
+        @next_group_id += 1
+      end
+      result
+    end
+  end
+end

data/lib/linkage/runner/single_threaded.rb

@@ -3,29 +3,36 @@ module Linkage
   #
   # @see Runner
   class SingleThreadedRunner < Runner
+    # @return [Linkage::ResultSet]
     def execute
-      create_tables
       setup_datasets
       apply_expectations
       group_records
-      nil
+
+      return result_set
     end
 
     private
 
     def setup_datasets
-      @dataset_1 = config.dataset_1.clone
-      @dataset_2 = config.dataset_2.clone if @config.linkage_type != :self
+      pk = config.dataset_1.field_set.primary_key
+      @dataset_1 = config.dataset_1.select(pk.to_expr)
+      if @config.linkage_type != :self
+        pk = config.dataset_2.field_set.primary_key
+        @dataset_2 = config.dataset_2.select(pk.to_expr)
+      end
     end
 
     def apply_expectations
       config.expectations.each do |exp|
-        exp.apply_to(@dataset_1)
-        exp.apply_to(@dataset_2) if config.linkage_type != :self
+        @dataset_1 = exp.apply_to(@dataset_1, :lhs)
+        @dataset_2 = exp.apply_to(@dataset_2, :rhs) if config.linkage_type != :self
       end
     end
 
     def group_records
+      result_set.create_tables!
+
       if config.linkage_type == :self
         group_records_for(@dataset_1, 1)
       else
@@ -39,59 +46,42 @@ module Linkage
     # @param [Fixnum, nil] dataset_id
     # @param [Boolean] ignore_empty_groups
     # @yield [Linkage::Group] If a block is given, yield completed groups to
-    #   the block. Otherwise, call save_group on the group.
+    #   the block. Otherwise, call ResultSet#add_group on the group.
     def group_records_for(dataset, dataset_id = nil, ignore_empty_groups = true, &block)
       current_group = nil
-      block ||= lambda { |group| save_group(current_group, dataset_id) }
+      block ||= lambda { |group| result_set.add_group(current_group, dataset_id) }
+      primary_key = dataset.field_set.primary_key.to_expr
       dataset.each do |row|
-        if current_group.nil? || !current_group.matches?(row[:values])
+        pk = row.delete(primary_key)
+        if current_group.nil? || !current_group.matches?(row)
           if current_group && (!ignore_empty_groups || current_group.count > 1)
             block.call(current_group)
           end
-          new_group = Group.new(row[:values])
+          new_group = Group.new(row)
           current_group = new_group
         end
-        current_group.add_record(row[:pk])
+        current_group.add_record(pk)
       end
       if current_group && (!ignore_empty_groups || current_group.count > 1)
         block.call(current_group)
       end
-      flush_buffers
-    end
-
-    def save_group(group, dataset_id = nil)
-      if !@groups_buffer
-        groups_headers = [:id] + group.values.keys
-        @groups_buffer = ImportBuffer.new(@uri, :groups, groups_headers, @options)
-      end
-      @groups_records_buffer ||= ImportBuffer.new(@uri, :groups_records, [:group_id, :dataset, :record_id], @options)
-
-      group_id = next_group_id
-      @groups_buffer.add([group_id] + group.values.values)
-      group.records.each do |record_id|
-        @groups_records_buffer.add([group_id, dataset_id, record_id])
-      end
-    end
-
-    def flush_buffers
-      @groups_buffer.flush if @groups_buffer
-      @groups_records_buffer.flush if @groups_records_buffer
+      result_set.flush!
     end
 
     def combine_groups
       # Create a new dataset for the groups table
-      ds = Dataset.new(@uri, :groups, @options)
-      ds.fields.each_value do |field|
+      groups_dataset = result_set.groups_dataset
+
+      exprs = groups_dataset.field_set.values.inject([]) do |arr, field|
         # Sort on all fields
-        next if field.primary_key?
-        ds.add_order(field)
-        ds.add_select(field)
+        field.primary_key? ? arr : arr << field.to_expr
       end
-      ds.add_order(ds.primary_key) # ensure matching groups are sorted by id
-      database do |db|
+      groups_dataset = groups_dataset.select(*exprs, groups_dataset.field_set.primary_key.to_expr).order(*exprs) # ensure matching groups are sorted by id
+
+      result_set.database do |db|
         groups_to_delete = []
         db.transaction do  # for speed reasons
-          group_records_for(ds, nil, false) do |group|
+          group_records_for(groups_dataset, nil, false) do |group|
             if group.count == 1
               # Delete the empty group
               groups_to_delete << group.records[0]

data/lib/linkage/runner.rb

@@ -1,19 +1,19 @@
 module Linkage
   # Use this class to run a configuration created by {Dataset#link_with}.
   class Runner
-    attr_reader :config
+    attr_reader :config, :result_set
 
     # @param [Linkage::Configuration] config
     # @param [String] uri Sequel-style database URI
     # @param [Hash] options Sequel.connect options
     # @see Dataset#link_with
     # @see http://sequel.rubyforge.org/rdoc/files/doc/opening_databases_rdoc.html Sequel: Connecting to a database
-    def initialize(config, uri, options = {})
+    def initialize(config, uri = nil, options = {})
       @config = config
-      @uri = uri
-      @options = options
-      @next_group_id = 1
-      @next_group_mutex = Mutex.new
+      if uri
+        warn("[DEPRECATION] Please use Configuration#save_results_in with the database URI and options instead")
+        @config.save_results_in(uri, options)
+      end
     end
 
     # @abstract
@@ -21,36 +21,8 @@ module Linkage
       raise NotImplementedError
     end
 
-    protected
-
-    def database(&block)
-      Sequel.connect(@uri, @options, &block)
-    end
-
-    def create_tables
-      database do |db|
-        schema = config.groups_table_schema
-        db.create_table(:groups) do
-          schema.each { |col| column(*col) }
-        end
-
-        pk_type = config.dataset_1.primary_key.merge(config.dataset_2.primary_key).ruby_type
-        db.create_table(:groups_records) do
-          column(:record_id, pk_type[:type], pk_type[:opts] || {})
-          Integer :group_id
-          Integer :dataset
-          index :group_id
-        end
-      end
-    end
-
-    def next_group_id
-      result = nil
-      @next_group_mutex.synchronize do
-        result = @next_group_id
-        @next_group_id += 1
-      end
-      result
+    def result_set
+      @config.result_set
     end
   end
 end

data/lib/linkage.rb

@@ -1,4 +1,5 @@
 require 'pathname'
+require 'delegate'
 require 'sequel'
 
 module Linkage
@@ -9,10 +10,11 @@ require path + 'utils'
 require path + 'warnings'
 require path + 'dataset'
 require path + 'runner'
-require path + 'expectation'
 require path + 'data'
 require path + 'field'
 require path + 'function'
 require path + 'group'
 require path + 'import_buffer'
 require path + 'configuration'
+require path + 'result_set'
+require path + 'field_set'

data/linkage.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "linkage"
-  s.version = "0.0.2"
+  s.version = "0.0.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jeremy Stephens"]
-  s.date = "2011-12-06"
+  s.date = "2012-02-28"
   s.description = "Wraps Sequel to perform record linkage between one or two datasets"
   s.email = "jeremy.f.stephens@vanderbilt.edu"
   s.extra_rdoc_files = [
@@ -30,12 +30,13 @@ Gem::Specification.new do |s|
     "lib/linkage/configuration.rb",
     "lib/linkage/data.rb",
     "lib/linkage/dataset.rb",
-    "lib/linkage/expectation.rb",
     "lib/linkage/field.rb",
+    "lib/linkage/field_set.rb",
     "lib/linkage/function.rb",
     "lib/linkage/functions/trim.rb",
     "lib/linkage/group.rb",
     "lib/linkage/import_buffer.rb",
+    "lib/linkage/result_set.rb",
     "lib/linkage/runner.rb",
     "lib/linkage/runner/single_threaded.rb",
     "lib/linkage/utils.rb",
@@ -44,6 +45,7 @@ Gem::Specification.new do |s|
     "test/config.yml",
     "test/helper.rb",
     "test/integration/test_cross_linkage.rb",
+    "test/integration/test_dataset.rb",
     "test/integration/test_dual_linkage.rb",
     "test/integration/test_self_linkage.rb",
     "test/unit/functions/test_trim.rb",
@@ -51,19 +53,20 @@ Gem::Specification.new do |s|
     "test/unit/test_configuration.rb",
     "test/unit/test_data.rb",
     "test/unit/test_dataset.rb",
-    "test/unit/test_expectation.rb",
     "test/unit/test_field.rb",
+    "test/unit/test_field_set.rb",
     "test/unit/test_function.rb",
     "test/unit/test_group.rb",
     "test/unit/test_import_buffer.rb",
     "test/unit/test_linkage.rb",
+    "test/unit/test_result_set.rb",
     "test/unit/test_runner.rb",
     "test/unit/test_utils.rb"
   ]
   s.homepage = "http://github.com/coupler/linkage"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.11"
+  s.rubygems_version = "1.3.9.4"
   s.summary = "Sequel-based record linkage"
 
   if s.respond_to? :specification_version then
@@ -73,54 +76,48 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<sequel>, [">= 0"])
       s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
-      s.add_development_dependency(%q<rcov>, [">= 0"])
-      s.add_development_dependency(%q<guard-test>, [">= 0"])
       s.add_development_dependency(%q<test-unit>, ["= 2.3.2"])
       s.add_development_dependency(%q<mocha>, [">= 0"])
       s.add_development_dependency(%q<sqlite3>, [">= 0"])
       s.add_development_dependency(%q<yard>, [">= 0"])
       s.add_development_dependency(%q<rake>, [">= 0"])
       s.add_development_dependency(%q<versionomy>, [">= 0"])
-      s.add_development_dependency(%q<guard-yard>, [">= 0"])
-      s.add_development_dependency(%q<rb-inotify>, [">= 0"])
       s.add_development_dependency(%q<mysql2>, [">= 0"])
       s.add_development_dependency(%q<pry>, [">= 0"])
       s.add_development_dependency(%q<rdiscount>, [">= 0"])
+      s.add_development_dependency(%q<guard-test>, [">= 0"])
+      s.add_development_dependency(%q<guard-yard>, [">= 0"])
     else
       s.add_dependency(%q<sequel>, [">= 0"])
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
-      s.add_dependency(%q<rcov>, [">= 0"])
-      s.add_dependency(%q<guard-test>, [">= 0"])
       s.add_dependency(%q<test-unit>, ["= 2.3.2"])
       s.add_dependency(%q<mocha>, [">= 0"])
       s.add_dependency(%q<sqlite3>, [">= 0"])
       s.add_dependency(%q<yard>, [">= 0"])
       s.add_dependency(%q<rake>, [">= 0"])
       s.add_dependency(%q<versionomy>, [">= 0"])
-      s.add_dependency(%q<guard-yard>, [">= 0"])
-      s.add_dependency(%q<rb-inotify>, [">= 0"])
       s.add_dependency(%q<mysql2>, [">= 0"])
       s.add_dependency(%q<pry>, [">= 0"])
       s.add_dependency(%q<rdiscount>, [">= 0"])
+      s.add_dependency(%q<guard-test>, [">= 0"])
+      s.add_dependency(%q<guard-yard>, [">= 0"])
     end
   else
     s.add_dependency(%q<sequel>, [">= 0"])
     s.add_dependency(%q<bundler>, ["~> 1.0.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
-    s.add_dependency(%q<rcov>, [">= 0"])
-    s.add_dependency(%q<guard-test>, [">= 0"])
     s.add_dependency(%q<test-unit>, ["= 2.3.2"])
     s.add_dependency(%q<mocha>, [">= 0"])
     s.add_dependency(%q<sqlite3>, [">= 0"])
     s.add_dependency(%q<yard>, [">= 0"])
     s.add_dependency(%q<rake>, [">= 0"])
     s.add_dependency(%q<versionomy>, [">= 0"])
-    s.add_dependency(%q<guard-yard>, [">= 0"])
-    s.add_dependency(%q<rb-inotify>, [">= 0"])
     s.add_dependency(%q<mysql2>, [">= 0"])
     s.add_dependency(%q<pry>, [">= 0"])
     s.add_dependency(%q<rdiscount>, [">= 0"])
+    s.add_dependency(%q<guard-test>, [">= 0"])
+    s.add_dependency(%q<guard-yard>, [">= 0"])
   end
 end
 

data/test/helper.rb

@@ -13,7 +13,7 @@ require 'tmpdir'
 require 'logger'
 require 'pp'
 require 'versionomy'
-require 'pry'
+#require 'pry'
 
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))