linkage 0.0.5 → 0.0.6

data/Gemfile.lock

@@ -2,7 +2,7 @@ GEM
   remote: http://rubygems.org/
   specs:
     blockenspiel (0.4.3)
-    coderay (1.0.5)
+    coderay (1.0.6)
     ffi (1.0.11)
     git (1.2.5)
     guard (1.0.1)
@@ -25,7 +25,7 @@ GEM
     mocha (0.10.5)
       metaclass (~> 0.0.1)
     mysql2 (0.3.11)
-    pry (0.9.8.4)
+    pry (0.9.9)
       coderay (~> 1.0.5)
       method_source (~> 0.7.1)
       slop (>= 2.4.4, < 3)
@@ -33,12 +33,12 @@ GEM
     rdiscount (1.6.8)
     rdoc (3.12)
       json (~> 1.4)
-    sequel (3.33.0)
+    sequel (3.34.1)
     slop (2.4.4)
-    sqlite3 (1.3.5)
+    sqlite3 (1.3.6)
     test-unit (2.4.8)
     thor (0.14.6)
-    versionomy (0.4.2)
+    versionomy (0.4.3)
       blockenspiel (>= 0.4.3)
     yard (0.7.5)
 

data/Rakefile

@@ -48,3 +48,5 @@ require 'yard'
 YARD::Rake::YardocTask.new do |t|
   t.files = ['lib/**/*.rb']
 end
+
+task :build => :gemspec

data/VERSION

@@ -1 +1 @@
-0.0.5
+0.0.6

data/lib/linkage/configuration.rb

@@ -117,12 +117,12 @@ module Linkage
           end
 
           expr = target.to_expr(side)
-          aliased_expr = expr
+          aliaz = nil
           if expr != merged_field.name
-            aliased_expr = expr.as(merged_field.name)
+            aliaz = merged_field.name
           end
 
-          dataset.order_more(expr).select_more(aliased_expr)
+          dataset.match(expr, aliaz)
         end
 
         def same_filter?(other)
@@ -318,9 +318,9 @@ module Linkage
     def initialize(dataset_1, dataset_2)
       @dataset_1 = dataset_1
       @dataset_2 = dataset_2
+      @linkage_type = dataset_1 == dataset_2 ? :self : :dual
       @expectations = []
       @visual_comparisons = []
-      @linkage_type = dataset_1 == dataset_2 ? :self : :dual
     end
 
     def configure(&block)
@@ -348,5 +348,15 @@ module Linkage
     def result_set
       @result_set ||= ResultSet.new(self)
     end
+
+    def datasets_with_applied_expectations
+      dataset_1 = @dataset_1
+      dataset_2 = @dataset_2
+      @expectations.each do |exp|
+        dataset_1 = exp.apply_to(dataset_1, :lhs)
+        dataset_2 = exp.apply_to(dataset_2, :rhs) if @linkage_type != :self
+      end
+      @linkage_type == :self ? [dataset_1, dataset_1] : [dataset_1, dataset_2]
+    end
   end
 end

data/lib/linkage/dataset.rb

@@ -1,17 +1,23 @@
 module Linkage
-  class Dataset < Delegator
+  # Delegator around Sequel::Dataset with some extra functionality.
+  class Dataset
     attr_reader :field_set, :table_name
 
     def initialize(uri, table, options = {})
       @table_name = table.to_sym
       db = Sequel.connect(uri, options)
-      ds = db[@table_name]
-      super(ds)
+      @dataset = db[@table_name]
       @field_set = FieldSet.new(db.schema(@table_name))
+      @_match = []
     end
 
-    def __setobj__(obj); @dataset = obj; end
-    def __getobj__; @dataset; end
+    def obj
+      @dataset
+    end
+
+    def obj=(value)
+      @dataset = value
+    end
 
     # Setup a linkage with another dataset
     #
@@ -26,21 +32,70 @@ module Linkage
       @dataset.db.adapter_scheme
     end
 
-    def initialize_clone(obj)
-      new_obj = obj.instance_variable_get(:@new_obj)
+    def match(expr, aliaz = nil)
+      clone(:match => {:expr => expr, :alias => aliaz})
+    end
+
+    def clone(new_opts={})
+      new_opts = new_opts.dup
+      new_obj = new_opts.delete(:new_obj)
+
+      match = new_opts.delete(:match)
+      result = super()
+      result.send(:_match, match)
+
       if new_obj
-        __setobj__(new_obj)
+        result.obj = new_obj
       else
-        super
+        result.obj = obj.clone(new_opts)
+      end
+      result
+    end
+
+    def each_group(min = 2)
+      @dataset.group_and_count(*aliased_match_expressions).having{count >= min}.each do |row|
+        count = row.delete(:count)
+        yield Group.new(row, {:count => count})
+      end
+    end
+
+    def group_by_matches(aliased = false)
+      expr = aliased ? aliased_match_expressions : match_expressions
+      group(*expr)
+    end
+
+    def dataset_for_group(group)
+      filters = []
+      group.values.each_pair do |key, value|
+        # find a matched expression with this alias
+        m = @_match.detect { |h| h[:alias] ? h[:alias] == key : h[:expr] == key }
+        raise "this dataset isn't compatible with the given group" if !m
+        filters << {m[:expr] => value}
+      end
+      filter(*filters)
+    end
+
+    private
+
+    def _match(opts)
+      if opts
+        @_match += [opts]
       end
     end
 
+    def match_expressions
+      @_match.collect { |m| m[:expr] }
+    end
+
+    def aliased_match_expressions
+      @_match.collect { |m| m[:alias] ? m[:expr].as(m[:alias]) : m[:expr] }
+    end
+
     def method_missing(name, *args, &block)
-      result = super
+      result = @dataset.send(name, *args, &block)
       if result.kind_of?(Sequel::Dataset)
-        @new_obj = result
-        result = clone
-        @new_obj = nil
+        new_obj = result
+        result = clone(:new_obj => result)
       end
       result
     end

data/lib/linkage/group.rb

@@ -1,43 +1,35 @@
 module Linkage
-  # This class represents a group of records that match based on criteria
-  # described via the {Dataset#link_with} method. Group's are created by
-  # subclasses of the {Runner} class during execution.
-  #
-  # @see Dataset#link_with
-  # @see SingleThreadedRunner
   class Group
-    # @return [Array<Object>] An array of this group's record ids
-    attr_reader :records
-
     # @return [Hash] Hash of matching values
     attr_reader :values
 
-    # @param [Hash] matching_values Values that define this group
-    # @example
-    #   Linkage::Group.new({:foo => 123, :bar => 'baz'})
-    def initialize(matching_values)
-      @values = matching_values
-      @records = []
-    end
+    # @return [Integer] Number of records in this group
+    attr_reader :count
 
-    # Check to see if the given set of values matches this group's values.
-    #
-    # @param [Hash] values Hash of values
-    # @return [Boolean] true if match, false if not
-    def matches?(values)
-      @values == values
-    end
+    # @return [Integer] This group's ID (if it exists)
+    attr_reader :id
 
-    # Add a record id to this group's set of records.
-    #
-    # @param [Object] record_id
-    def add_record(record_id)
-      @records << record_id
+    def self.from_row(row)
+      values = {}
+      options = {}
+      row.each_pair do |key, value|
+        if key == :id || key == :count
+          options[key] = value
+        else
+          values[key] = value
+        end
+      end
+      new(values, options)
     end
 
-    # @return [Fixnum] Number of records in this group
-    def count
-      @records.count
+    # @param [Hash] values Values that define this group
+    # @param [Hash] options
+    # @example
+    #   Linkage::Group.new({:foo => 123, :bar => 'baz'}, {:count => 5, :id => 456})
+    def initialize(values, options)
+      @count = options[:count]
+      @id = options[:id]
+      @values = values
     end
   end
 end

data/lib/linkage/result_set.rb

@@ -7,11 +7,7 @@ module Linkage
     end
 
     def groups_dataset
-      Dataset.new(@config.results_uri, :groups, @config.results_uri_options)
-    end
-
-    def groups_records_dataset
-      Dataset.new(@config.results_uri, :groups_records, @config.results_uri_options)
+      @groups_dataset ||= Dataset.new(@config.results_uri, :groups, @config.results_uri_options)
     end
 
     def database(&block)
@@ -40,18 +36,22 @@ module Linkage
         groups_headers = [:id] + group.values.keys
         @groups_buffer = ImportBuffer.new(@config.results_uri, :groups, groups_headers, @config.results_uri_options)
       end
-      @groups_records_buffer ||= ImportBuffer.new(@config.results_uri, :groups_records, [:group_id, :dataset, :record_id], @config.results_uri_options)
-
       group_id = next_group_id
       @groups_buffer.add([group_id] + group.values.values)
-      group.records.each do |record_id|
-        @groups_records_buffer.add([group_id, dataset_id, record_id])
-      end
     end
 
     def flush!
       @groups_buffer.flush if @groups_buffer
-      @groups_records_buffer.flush if @groups_records_buffer
+    end
+
+    def get_group(index)
+      values = groups_dataset.order(:id).limit(1, index).first
+      Group.from_row(values)
+    end
+
+    def groups_records_datasets(group)
+      datasets = @config.datasets_with_applied_expectations
+      datasets.collect! { |ds| ds.dataset_for_group(group) }
     end
 
     private

data/lib/linkage/runner/single_threaded.rb

@@ -6,7 +6,6 @@ module Linkage
     # @return [Linkage::ResultSet]
     def execute
       setup_datasets
-      apply_expectations
       group_records
 
       return result_set
@@ -15,18 +14,13 @@ module Linkage
     private
 
     def setup_datasets
-      pk = config.dataset_1.field_set.primary_key
-      @dataset_1 = config.dataset_1.select(pk.to_expr)
-      if @config.linkage_type != :self
-        pk = config.dataset_2.field_set.primary_key
-        @dataset_2 = config.dataset_2.select(pk.to_expr)
-      end
-    end
+      @dataset_1, @dataset_2 = config.datasets_with_applied_expectations
 
-    def apply_expectations
-      config.expectations.each do |exp|
-        @dataset_1 = exp.apply_to(@dataset_1, :lhs)
-        @dataset_2 = exp.apply_to(@dataset_2, :rhs) if config.linkage_type != :self
+      pk = @dataset_1.field_set.primary_key
+      @dataset_1 = @dataset_1.select(pk.to_expr)
+      if @config.linkage_type != :self
+        pk = @dataset_2.field_set.primary_key
+        @dataset_2 = @dataset_2.select(pk.to_expr)
       end
     end
 
@@ -47,23 +41,10 @@ module Linkage
     # @param [Boolean] ignore_empty_groups
     # @yield [Linkage::Group] If a block is given, yield completed groups to
     #   the block. Otherwise, call ResultSet#add_group on the group.
-    def group_records_for(dataset, dataset_id = nil, ignore_empty_groups = true, &block)
-      current_group = nil
-      block ||= lambda { |group| result_set.add_group(current_group, dataset_id) }
-      primary_key = dataset.field_set.primary_key.to_expr
-      dataset.each do |row|
-        pk = row.delete(primary_key)
-        if current_group.nil? || !current_group.matches?(row)
-          if current_group && (!ignore_empty_groups || current_group.count > 1)
-            block.call(current_group)
-          end
-          new_group = Group.new(row)
-          current_group = new_group
-        end
-        current_group.add_record(pk)
-      end
-      if current_group && (!ignore_empty_groups || current_group.count > 1)
-        block.call(current_group)
+    def group_records_for(dataset, dataset_id, ignore_empty_groups = true)
+      group_minimum = ignore_empty_groups ? 2 : 1
+      dataset.each_group(group_minimum) do |group|
+        result_set.add_group(group, dataset_id)
       end
       result_set.flush!
     end
@@ -72,37 +53,20 @@ module Linkage
       # Create a new dataset for the groups table
       groups_dataset = result_set.groups_dataset
 
-      exprs = groups_dataset.field_set.values.inject([]) do |arr, field|
+      groups_dataset.field_set.values.each do |field|
         # Sort on all fields
-        field.primary_key? ? arr : arr << field.to_expr
-      end
-      groups_dataset = groups_dataset.select(*exprs, groups_dataset.field_set.primary_key.to_expr).order(*exprs) # ensure matching groups are sorted by id
-
-      result_set.database do |db|
-        groups_to_delete = []
-        db.transaction do  # for speed reasons
-          group_records_for(groups_dataset, nil, false) do |group|
-            if group.count == 1
-              # Delete the empty group
-              groups_to_delete << group.records[0]
-            else
-              # Change group_id in the groups_records table to the first group
-              # id, delete other groups.
-              new_group_id = group.records[0]
-              group.records[1..-1].each do |old_group_id|
-                # NOTE: There can only be a group with max size of 2, but
-                #       this adds in future support for matching more than
-                #       2 datasets at once.
-                db[:groups_records].filter(:group_id => old_group_id).
-                  update(:group_id => new_group_id)
-                groups_to_delete << old_group_id
-              end
-            end
-          end
+        if !field.primary_key?
+          groups_dataset = groups_dataset.match(field.to_expr)
         end
-        db[:groups_records].filter(:group_id => groups_to_delete).delete
-        db[:groups].filter(:id => groups_to_delete).delete
       end
+
+      # Delete non-matching groups
+      sub_dataset = groups_dataset.select(:id).group_by_matches.having(:count.sql_function(:id) => 1)
+      groups_dataset.filter(:id => sub_dataset.obj).delete
+
+      # Delete duplicate groups
+      sub_dataset = groups_dataset.select(:max.sql_function(:id).as(:id)).group_by_matches
+      groups_dataset.filter(:id => sub_dataset.obj).delete
     end
   end
 end

data/linkage.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "linkage"
-  s.version = "0.0.5"
+  s.version = "0.0.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jeremy Stephens"]
-  s.date = "2012-03-30"
+  s.date = "2012-05-08"
   s.description = "Performs record linkage between one or two datasets, using Sequel on the backend"
   s.email = "jeremy.f.stephens@vanderbilt.edu"
   s.extra_rdoc_files = [
@@ -69,7 +69,7 @@ Gem::Specification.new do |s|
   s.homepage = "http://github.com/coupler/linkage"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.18"
+  s.rubygems_version = "1.8.23"
   s.summary = "Record linkage library"
 
   if s.respond_to? :specification_version then

data/test/helper.rb

@@ -62,6 +62,16 @@ class Test::Unit::TestCase
   def test_config
     @test_config ||= YAML.load_file(File.join(File.dirname(__FILE__), "config.yml"))
   end
+
+  def prefixed_logger(prefix)
+    logger = Logger.new(STDERR)
+    original_formatter = Logger::Formatter.new
+    logger.formatter = proc { |severity, datetime, progname, msg|
+      result = original_formatter.call(severity, datetime, progname, msg)
+      "[#{prefix}] #{result}"
+    }
+    logger
+  end
 end
 
 module UnitTests; end

data/test/integration/test_cross_linkage.rb

@@ -7,8 +7,8 @@ module IntegrationTests
       @tmpuri = "sqlite://" + File.join(@tmpdir, "foo")
     end
 
-    def database(&block)
-      Sequel.connect(@tmpuri, &block)
+    def database(options = {}, &block)
+      Sequel.connect(@tmpuri, options, &block)
     end
 
     def teardown
@@ -16,6 +16,10 @@ module IntegrationTests
     end
 
     test "one mandatory field equality on single threaded runner" do
+      #setup_logger = Logger.new(STDERR)
+      #setup_logger.formatter = lambda { |severity, time, progname, msg|
+        #" SETUP : %s [%s]: %s\n" % [severity, time, msg]
+      #}
       # insert the test data
       database do |db|
         db.create_table(:foo) { primary_key(:id); Integer(:foo); Integer(:bar) }
@@ -23,12 +27,22 @@ module IntegrationTests
           Array.new(100) { |i| [i, i % 10, i % 5] })
       end
 
+      #ds_logger = Logger.new(STDERR)
+      #ds_logger.formatter = lambda { |severity, time, progname, msg|
+        #"DATASET: %s [%s]: %s\n" % [severity, time, msg]
+      #}
       ds = Linkage::Dataset.new(@tmpuri, "foo", :single_threaded => true)
+
+      #rs_logger = Logger.new(STDERR)
+      #rs_logger.formatter = lambda { |severity, time, progname, msg|
+        #"RESULTS: %s [%s]: %s\n" % [severity, time, msg]
+      #}
       tmpuri = @tmpuri
       conf = ds.link_with(ds) do
         lhs[:foo].must == rhs[:bar]
         save_results_in(tmpuri, :single_threaded => true)
       end
+      assert_equal :cross, conf.linkage_type
       runner = Linkage::SingleThreadedRunner.new(conf)
       runner.execute
 
@@ -38,11 +52,11 @@ module IntegrationTests
           assert_equal i, row[:foo_bar]
         end
 
-        assert_equal 150, db[:groups_records].count
-        db[:groups_records].order(:group_id, :dataset, :record_id).each_with_index do |row, i|
-          expected_group_id = (row[:record_id] % 5) + 1
-          assert_equal expected_group_id, row[:group_id], "Record #{row[:record_id]} should have been in group #{expected_group_id}"
-        end
+        #assert_equal 150, db[:groups_records].count
+        #db[:groups_records].order(:group_id, :dataset, :record_id).each_with_index do |row, i|
+          #expected_group_id = (row[:record_id] % 5) + 1
+          #assert_equal expected_group_id, row[:group_id], "Record #{row[:record_id]} should have been in group #{expected_group_id}"
+        #end
       end
     end
 

data/test/integration/test_dataset.rb

@@ -27,4 +27,47 @@ class IntegrationTests::TestDataset < Test::Unit::TestCase
     assert_equal ds_2.field_set, ds_1.field_set
     assert_match /`foo` = 'bar'/, ds_2.sql
   end
+
+  test "each_group" do
+    database do |db|
+      db.create_table(:foo) do
+        primary_key :id
+        String :bar
+      end
+      db[:foo].import([:id, :bar], [[1, 'foo'], [2, 'foo'], [3, 'bar'], [4, 'baz']])
+    end
+
+    ds = Linkage::Dataset.new(@tmpuri, "foo")
+    ds = ds.match(:bar)
+    ds.each_group do |group|
+      assert_equal({:bar => "foo"}, group.values)
+      assert_equal(2, group.count)
+    end
+
+    groups = []
+    ds.each_group(1) do |group|
+      groups << group
+    end
+    assert_equal 3, groups.length
+  end
+
+  test "each_group with filters" do
+    database do |db|
+      db.create_table(:foo) do
+        primary_key :id
+        String :bar
+        Integer :baz
+      end
+      db[:foo].import([:id, :bar, :baz], [[1, 'foo', 1], [2, 'foo', 2], [3, 'bar', 3], [4, 'baz', 4]])
+    end
+
+    ds = Linkage::Dataset.new(@tmpuri, "foo")
+    ds = ds.match(:bar)
+    ds = ds.filter { baz >= 3 }
+    groups = []
+    ds.each_group(1) do |group|
+      groups << group
+    end
+    assert_equal 2, groups.length
+  end
 end

data/test/integration/test_dual_linkage.rb

@@ -7,8 +7,8 @@ module IntegrationTests
       @tmpuri = "sqlite://" + File.join(@tmpdir, "foo")
     end
 
-    def database(&block)
-      Sequel.connect(@tmpuri, &block)
+    def database(options = {}, &block)
+      Sequel.connect(@tmpuri, options, &block)
     end
 
     def teardown
@@ -43,16 +43,16 @@ module IntegrationTests
           assert_equal "12345678#{i%10}", row[:ssn]
         end
 
-        assert_equal 200, db[:groups_records].count
-        db[:groups_records].order(:group_id, :dataset, :record_id).each_with_index do |row, i|
-          if i % 20 >= 10
-            assert_equal 2, row[:dataset], row.inspect
-          else
-            assert_equal 1, row[:dataset], row.inspect
-          end
-          expected_group_id = i / 20 + 1
-          assert_equal expected_group_id, row[:group_id], "Record #{row.inspect} should have been in group #{expected_group_id}"
-        end
+        #assert_equal 200, db[:groups_records].count
+        #db[:groups_records].order(:group_id, :dataset, :record_id).each_with_index do |row, i|
+          #if i % 20 >= 10
+            #assert_equal 2, row[:dataset], row.inspect
+          #else
+            #assert_equal 1, row[:dataset], row.inspect
+          #end
+          #expected_group_id = i / 20 + 1
+          #assert_equal expected_group_id, row[:group_id], "Record #{row.inspect} should have been in group #{expected_group_id}"
+        #end
       end
     end
 
@@ -86,8 +86,7 @@ module IntegrationTests
       end
     end
 
-    test "handles MySQL's ignorance of trailing spaces when comparing strings" do
-      pend
+    test "reacts properly when using two databases with different string equality methods" do
       if !test_config['mysql']
         omission("No MySQL test configuration found")
       end
@@ -98,24 +97,21 @@ module IntegrationTests
 
         db.create_table!(:bar) { primary_key(:id); String(:one); String(:two) }
         db[:bar].import([:id, :one, :two], [[1, "", "junk"]])
-
-        db.run("DROP TABLE IF EXISTS groups")
-        db.run("DROP TABLE IF EXISTS groups_records")
       end
 
       ds_1 = Linkage::Dataset.new(uri, "foo", :single_threaded => true)
       ds_2 = Linkage::Dataset.new(uri, "bar", :single_threaded => true)
-      logger = Logger.new(STDERR)
+      tmpuri = @tmpuri
       conf = ds_1.link_with(ds_2) do
         lhs[:one].must == rhs[:one]
         lhs[:two].must == rhs[:two]
-        save_results_in(uri, :logger => logger)
+        save_results_in(tmpuri)
       end
 
       runner = Linkage::SingleThreadedRunner.new(conf)
       runner.execute
 
-      Sequel.connect(@tmpuri) do |db|
+      database do |db|
         assert_equal 1, db[:groups].count
       end
     end

data/test/integration/test_functions.rb

@@ -39,20 +39,21 @@ module IntegrationTests
     end
 
     test "strftime in sqlite" do
-      logger = Logger.new(STDERR)
-      database(:logger => logger) do |db|
+      #logger = Logger.new(STDERR)
+      #database(:logger => logger) do |db|
+      database do |db|
         db.create_table(:foo) { primary_key(:id); Date(:foo_date) }
         db.create_table(:bar) { primary_key(:id); String(:bar_string) }
         db[:foo].insert({:id => 1, :foo_date => Date.today})
         db[:bar].insert({:id => 1, :bar_string => Date.today.strftime("%Y-%m-%d")})
       end
 
-      ds_1 = Linkage::Dataset.new(@tmpuri, "foo", :single_threaded => true, :logger => logger)
-      ds_2 = Linkage::Dataset.new(@tmpuri, "bar", :single_threaded => true, :logger => logger)
+      ds_1 = Linkage::Dataset.new(@tmpuri, "foo", :single_threaded => true)
+      ds_2 = Linkage::Dataset.new(@tmpuri, "bar", :single_threaded => true)
       tmpuri = @tmpuri
       conf = ds_1.link_with(ds_2) do
         strftime(lhs[:foo_date], "%Y-%m-%d").must == rhs[:bar_string]
-        save_results_in(tmpuri, :logger => logger)
+        save_results_in(tmpuri)
       end
       runner = Linkage::SingleThreadedRunner.new(conf)
       runner.execute

data/test/integration/test_self_linkage.rb

@@ -37,13 +37,10 @@ module IntegrationTests
         assert_equal 10, db[:groups].count
         db[:groups].order(:ssn).each_with_index do |row, i|
           assert_equal "12345678#{i%10}", row[:ssn]
-        end
 
-        assert_equal 100, db[:groups_records].count
-        expected_group_id = nil
-        db[:groups_records].order(:record_id).each do |row|
-          expected_group_id = (row[:record_id] % 10) + 1
-          assert_equal expected_group_id, row[:group_id], "Record #{row[:record_id]} should have been in group #{expected_group_id}"
+          group = Linkage::Group.from_row(row)
+          dataset, _ = result_set.groups_records_datasets(group)
+          assert_equal 10, dataset.count
         end
       end
     end
@@ -73,13 +70,13 @@ module IntegrationTests
           assert_equal Date.civil(1985, 1, i / 2 + 1 + (i % 2 == 0 ? 0 : 10)), row[:dob]
         end
 
-        assert_equal 100, db[:groups_records].count
-        expected_group_id = nil
-        db[:groups_records].order(:record_id).each do |row|
-          v = row[:record_id] % 20
-          expected_group_id = v < 10 ? 1 + 2 * v : 2 * (v % 10 + 1)
-          assert_equal expected_group_id, row[:group_id], "Record #{row[:record_id]} should have been in group #{expected_group_id}"
-        end
+        #assert_equal 100, db[:groups_records].count
+        #expected_group_id = nil
+        #db[:groups_records].order(:record_id).each do |row|
+          #v = row[:record_id] % 20
+          #expected_group_id = v < 10 ? 1 + 2 * v : 2 * (v % 10 + 1)
+          #assert_equal expected_group_id, row[:group_id], "Record #{row[:record_id]} should have been in group #{expected_group_id}"
+        #end
       end
     end
 
@@ -107,12 +104,12 @@ module IntegrationTests
           assert_equal "12345678#{(i * 5) + 3}", row[:ssn]
         end
 
-        assert_equal 20, db[:groups_records].count
-        expected_group_id = nil
-        db[:groups_records].order(:record_id).each do |row|
-          expected_group_id = (row[:record_id] / 5) % 2 + 1
-          assert_equal expected_group_id, row[:group_id], "Record #{row[:record_id]} should have been in group #{expected_group_id}"
-        end
+        #assert_equal 20, db[:groups_records].count
+        #expected_group_id = nil
+        #db[:groups_records].order(:record_id).each do |row|
+          #expected_group_id = (row[:record_id] / 5) % 2 + 1
+          #assert_equal expected_group_id, row[:group_id], "Record #{row[:record_id]} should have been in group #{expected_group_id}"
+        #end
       end
     end
 
@@ -141,12 +138,12 @@ module IntegrationTests
           assert_equal "12345678#{(i * 5) + 3}", row[:ssn]
         end
 
-        assert_equal 20, db[:groups_records].count
-        expected_group_id = nil
-        db[:groups_records].order(:record_id).each do |row|
-          expected_group_id = (row[:record_id] / 5) % 2 + 1
-          assert_equal expected_group_id, row[:group_id], "Record #{row[:record_id]} should have been in group #{expected_group_id}"
-        end
+        #assert_equal 20, db[:groups_records].count
+        #expected_group_id = nil
+        #db[:groups_records].order(:record_id).each do |row|
+          #expected_group_id = (row[:record_id] / 5) % 2 + 1
+          #assert_equal expected_group_id, row[:group_id], "Record #{row[:record_id]} should have been in group #{expected_group_id}"
+        #end
       end
     end
 
@@ -175,12 +172,12 @@ module IntegrationTests
           assert_equal "123456789#{i}", row[:ssn]
         end
 
-        assert_equal 25, db[:groups_records].count
-        expected_group_id = nil
-        db[:groups_records].order(:record_id).each do |row|
-          expected_group_id = row[:record_id] % 5 + 1
-          assert_equal expected_group_id, row[:group_id], "Record #{row[:record_id]} should have been in group #{expected_group_id}"
-        end
+        #assert_equal 25, db[:groups_records].count
+        #expected_group_id = nil
+        #db[:groups_records].order(:record_id).each do |row|
+          #expected_group_id = row[:record_id] % 5 + 1
+          #assert_equal expected_group_id, row[:group_id], "Record #{row[:record_id]} should have been in group #{expected_group_id}"
+        #end
       end
     end
 
@@ -210,12 +207,12 @@ module IntegrationTests
           assert_equal "123456789#{i}", row[:ssn]
         end
 
-        assert_equal 25, db[:groups_records].count
-        expected_group_id = nil
-        db[:groups_records].order(:record_id).each do |row|
-          expected_group_id = row[:record_id] % 5 + 1
-          assert_equal expected_group_id, row[:group_id], "Record #{row[:record_id]} should have been in group #{expected_group_id}"
-        end
+        #assert_equal 25, db[:groups_records].count
+        #expected_group_id = nil
+        #db[:groups_records].order(:record_id).each do |row|
+          #expected_group_id = row[:record_id] % 5 + 1
+          #assert_equal expected_group_id, row[:group_id], "Record #{row[:record_id]} should have been in group #{expected_group_id}"
+        #end
       end
     end
   end

data/test/unit/test_configuration.rb

@@ -135,9 +135,7 @@ class UnitTests::TestConfiguration < Test::Unit::TestCase
     field_2 = stub('field 2', :to_expr => :foo)
     dataset_2.stubs(:field_set).returns({:foo => field_2})
 
-    func_expr = stub('function expression') do
-      expects(:as).with(:trim_foo_foo).returns(self)
-    end
+    func_expr = stub('function expression')
     func = stub('function', :static? => false, :to_expr => func_expr)
     Linkage::Functions::Trim.expects(:new).with(field_1).returns(func)
     merged_field = stub('merged field', :name => :trim_foo_foo)
@@ -147,12 +145,10 @@ class UnitTests::TestConfiguration < Test::Unit::TestCase
     c.configure do
       trim(lhs[:foo]).must == rhs[:foo]
     end
-    dataset_1.expects(:select_more).with(func_expr).returns(dataset_1)
-    dataset_1.expects(:order_more).with(func_expr).returns(dataset_1)
+    dataset_1.expects(:match).with(func_expr, :trim_foo_foo).returns(dataset_1)
     c.expectations[0].apply_to(dataset_1, :lhs)
 
-    dataset_2.expects(:select_more).with(:foo.as(:trim_foo_foo)).returns(dataset_2)
-    dataset_2.expects(:order_more).with(:foo).returns(dataset_2)
+    dataset_2.expects(:match).with(:foo, :trim_foo_foo).returns(dataset_2)
     c.expectations[0].apply_to(dataset_2, :rhs)
   end
 

data/test/unit/test_dataset.rb

@@ -34,4 +34,55 @@ class UnitTests::TestDataset < Test::Unit::TestCase
     @database.expects(:adapter_scheme).returns(:foo)
     assert_equal :foo, ds.adapter_scheme
   end
+
+  test "add match expression" do
+    ds_1 = Linkage::Dataset.new('foo:/bar', "foo", {:foo => 'bar'})
+    @dataset.expects(:clone).returns(@dataset)
+    ds_2 = ds_1.match(:foo)
+    assert_not_same ds_1, ds_2
+    assert_not_equal ds_1.instance_variable_get(:@_match),
+      ds_2.instance_variable_get(:@_match)
+  end
+
+  test "add match expression with alias, then each_group" do
+    ds_1 = Linkage::Dataset.new('foo:/bar', "foo", {:foo => 'bar'})
+    @dataset.expects(:clone).returns(@dataset)
+    ds_2 = ds_1.match(:foo, :aliased_foo)
+    @dataset.expects(:group_and_count).with(:foo.as(:aliased_foo)).returns(@dataset)
+    @dataset.expects(:having).returns(@dataset)
+    @dataset.expects(:each).yields({:aliased_foo => 123, :count => 1})
+    ds_2.each_group { |g| }
+  end
+
+  test "group_by_matches" do
+    ds = Linkage::Dataset.new('foo:/bar', "foo", {:foo => 'bar'})
+
+    @dataset.expects(:clone).returns(@dataset)
+    ds = ds.match(:foo)
+    @dataset.expects(:group).with(:foo).returns(@dataset)
+
+    ds.group_by_matches
+  end
+
+  test "dataset_for_group" do
+    ds = Linkage::Dataset.new('foo:/bar', "foo", {:foo => 'bar'})
+    @dataset.expects(:clone).returns(@dataset)
+    ds = ds.match(:foo, :foo_bar)
+
+    group = stub("group", :values => {:foo_bar => 'baz'})
+    filtered_dataset = stub('filtered dataset')
+    @dataset.expects(:filter).with(:foo => 'baz').returns(filtered_dataset)
+    assert_equal filtered_dataset, ds.dataset_for_group(group)
+  end
+
+  test "dataset_for_group without aliases" do
+    ds = Linkage::Dataset.new('foo:/bar', "foo", {:foo => 'bar'})
+    @dataset.expects(:clone).returns(@dataset)
+    ds = ds.match(:foo)
+
+    group = stub("group", :values => {:foo => 'baz'})
+    filtered_dataset = stub('filtered dataset')
+    @dataset.expects(:filter).with(:foo => 'baz').returns(filtered_dataset)
+    assert_equal filtered_dataset, ds.dataset_for_group(group)
+  end
 end

data/test/unit/test_group.rb

@@ -1,21 +1,10 @@
 require 'helper'
 
 class UnitTests::TestGroup < Test::Unit::TestCase
-  test "matches?" do
-    g = Linkage::Group.new(:test => 'test')
-    assert g.matches?({:test => 'test'})
-    assert !g.matches?({:foo => 'bar'})
-  end
-
-  test "add_record adds a record" do
-    g = Linkage::Group.new(:test => 'test')
-    g.add_record(123)
-    assert_equal [123], g.records
-  end
-
-  test "count returns number of records" do
-    g = Linkage::Group.new(:test => 'test')
-    g.add_record(123)
+  test "initialize" do
+    g = Linkage::Group.new({:test => 'test'}, {:count => 1, :id => 456})
+    assert_equal({:test => 'test'}, g.values)
     assert_equal 1, g.count
+    assert_equal 456, g.id
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: linkage
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-30 00:00:00.000000000 Z
+date: 2012-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -307,7 +307,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1705215583013388953
+      hash: -1901911346636016746
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -316,7 +316,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.18
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Record linkage library