linkage 0.0.0 → 0.0.1

data/VERSION

@@ -1 +1 @@
-0.0.0
+0.0.1

data/lib/linkage/runner/single_threaded.rb

@@ -27,21 +27,26 @@ module Linkage
 
     def group_records
       if config.linkage_type == :self
-        add_groups(group_records_for(@dataset_1), 1)
+        group_records_for(@dataset_1, 1)
       else
-        add_groups(group_records_for(@dataset_1, false), 1)
-        add_groups(group_records_for(@dataset_2, false), 2)
+        group_records_for(@dataset_1, 1, false)
+        group_records_for(@dataset_2, 2, false)
         combine_groups
       end
     end
 
-    def group_records_for(dataset, ignore_empty_groups = true)
-      groups = []
+    # @param [Linkage::Dataset] dataset
+    # @param [Fixnum, nil] dataset_id
+    # @param [Boolean] ignore_empty_groups
+    # @yield [Linkage::Group] If a block is given, yield completed groups to
+    #   the block. Otherwise, call save_group on the group.
+    def group_records_for(dataset, dataset_id = nil, ignore_empty_groups = true, &block)
       current_group = nil
+      block ||= lambda { |group| save_group(current_group, dataset_id) }
       dataset.each do |row|
         if current_group.nil? || !current_group.matches?(row[:values])
           if current_group && (!ignore_empty_groups || current_group.count > 1)
-            groups << current_group
+            block.call(current_group)
           end
           new_group = Group.new(row[:values])
           current_group = new_group
@@ -49,33 +54,33 @@ module Linkage
         current_group.add_record(row[:pk])
       end
       if current_group && (!ignore_empty_groups || current_group.count > 1)
-        groups << current_group
+        block.call(current_group)
       end
-      groups
+      flush_buffers
     end
 
-    def add_groups(groups, dataset_id = nil)
-      return if groups.empty?
-
-      groups_headers = [:id] + groups[0].values.keys
-      groups_buffer = ImportBuffer.new(@uri, :groups, groups_headers, @options)
-
-      groups_records_buffer = ImportBuffer.new(@uri, :groups_records, [:group_id, :dataset, :record_id], @options)
+    def save_group(group, dataset_id = nil)
+      if !@groups_buffer
+        groups_headers = [:id] + group.values.keys
+        @groups_buffer = ImportBuffer.new(@uri, :groups, groups_headers, @options)
+      end
+      @groups_records_buffer ||= ImportBuffer.new(@uri, :groups_records, [:group_id, :dataset, :record_id], @options)
 
-      groups.each_with_index do |group, i|
-        group_id = next_group_id
-        groups_buffer.add([group_id] + group.values.values)
-        group.records.each do |record_id|
-          groups_records_buffer.add([group_id, dataset_id, record_id])
-        end
+      group_id = next_group_id
+      @groups_buffer.add([group_id] + group.values.values)
+      group.records.each do |record_id|
+        @groups_records_buffer.add([group_id, dataset_id, record_id])
       end
-      groups_buffer.flush
-      groups_records_buffer.flush
+    end
+
+    def flush_buffers
+      @groups_buffer.flush if @groups_buffer
+      @groups_records_buffer.flush if @groups_records_buffer
     end
 
     def combine_groups
       # Create a new dataset for the groups table
-      ds = Dataset.new(@uri, :groups, :single_threaded => true)
+      ds = Dataset.new(@uri, :groups, @options)
       ds.fields.each_value do |field|
         # Sort on all fields
         next if field.primary_key?
@@ -83,11 +88,10 @@ module Linkage
         ds.add_select(field)
       end
       ds.add_order(ds.primary_key) # ensure matching groups are sorted by id
-      combined_groups = group_records_for(ds, false)
       database do |db|
         groups_to_delete = []
         db.transaction do  # for speed reasons
-          combined_groups.each do |group|
+          group_records_for(ds, nil, false) do |group|
             if group.count == 1
               # Delete the empty group
               groups_to_delete << group.records[0]
@@ -96,9 +100,9 @@ module Linkage
               # id, delete other groups.
               new_group_id = group.records[0]
               group.records[1..-1].each do |old_group_id|
-                # There can only be a group with max size of 2, but this
-                # adds in future support for matching more than 2 datasets
-                # at once. Code smell?
+                # NOTE: There can only be a group with max size of 2, but
+                #       this adds in future support for matching more than
+                #       2 datasets at once.
                 db[:groups_records].filter(:group_id => old_group_id).
                   update(:group_id => new_group_id)
                 groups_to_delete << old_group_id

data/linkage.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "linkage"
-  s.version = "0.0.0"
+  s.version = "0.0.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jeremy Stephens"]
-  s.date = "2011-09-30"
+  s.date = "2011-10-25"
   s.description = "Wraps Sequel to perform record linkage between one or two datasets"
   s.email = "jeremy.f.stephens@vanderbilt.edu"
   s.extra_rdoc_files = [

data/test/integration/test_cross_linkage.rb

@@ -27,11 +27,11 @@ module IntegrationTests
       conf = ds.link_with(ds) do
         lhs[:foo].must == rhs[:bar]
       end
-      runner = Linkage::SingleThreadedRunner.new(conf, @tmpuri)
+      runner = Linkage::SingleThreadedRunner.new(conf, @tmpuri, :single_threaded => true)
       runner.execute
 
       database do |db|
-        assert_equal 5, db[:groups].count
+        assert_equal 5, db[:groups].count, PP.pp(db[:groups].all, "")
         db[:groups].order(:foo_bar).each_with_index do |row, i|
           assert_equal i, row[:foo_bar]
         end

metadata

@@ -1,206 +1,156 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: linkage
-version: !ruby/object:Gem::Version 
-  hash: 31
+version: !ruby/object:Gem::Version
+  version: 0.0.1
   prerelease: 
-  segments: 
-  - 0
-  - 0
-  - 0
-  version: 0.0.0
 platform: ruby
-authors: 
+authors:
 - Jeremy Stephens
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-09-30 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  version_requirements: *id001
+date: 2011-10-25 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: sequel
-  prerelease: false
+  requirement: &14148960 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-- !ruby/object:Gem::Dependency 
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  prerelease: false
+  version_requirements: *14148960
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: &14147000 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 23
-        segments: 
-        - 1
-        - 0
-        - 0
+      - !ruby/object:Gem::Version
         version: 1.0.0
-  version_requirements: *id002
-  name: bundler
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency 
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  prerelease: false
+  version_requirements: *14147000
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: &14143380 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 7
-        segments: 
-        - 1
-        - 6
-        - 4
+      - !ruby/object:Gem::Version
         version: 1.6.4
-  version_requirements: *id003
-  name: jeweler
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency 
-  requirement: &id004 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  version_requirements: *id004
-  name: rcov
   prerelease: false
-  type: :development
-- !ruby/object:Gem::Dependency 
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  version_requirements: *14143380
+- !ruby/object:Gem::Dependency
+  name: rcov
+  requirement: &14142120 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  version_requirements: *id005
-  name: guard-test
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
   prerelease: false
+  version_requirements: *14142120
+- !ruby/object:Gem::Dependency
+  name: guard-test
+  requirement: &14139060 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-- !ruby/object:Gem::Dependency 
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  prerelease: false
+  version_requirements: *14139060
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: &14137580 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - "="
-      - !ruby/object:Gem::Version 
-        hash: 7
-        segments: 
-        - 2
-        - 3
-        - 2
+    requirements:
+    - - =
+      - !ruby/object:Gem::Version
         version: 2.3.2
-  version_requirements: *id006
-  name: test-unit
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency 
-  requirement: &id007 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  version_requirements: *id007
-  name: mocha
   prerelease: false
-  type: :development
-- !ruby/object:Gem::Dependency 
-  requirement: &id008 !ruby/object:Gem::Requirement 
+  version_requirements: *14137580
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: &14135540 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  version_requirements: *id008
-  name: sqlite3
-  prerelease: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-- !ruby/object:Gem::Dependency 
-  requirement: &id009 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  version_requirements: *id009
-  name: yard
   prerelease: false
-  type: :development
-- !ruby/object:Gem::Dependency 
-  requirement: &id010 !ruby/object:Gem::Requirement 
+  version_requirements: *14135540
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: &14133820 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  version_requirements: *id010
-  name: rake
-  prerelease: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-- !ruby/object:Gem::Dependency 
-  requirement: &id011 !ruby/object:Gem::Requirement 
+  prerelease: false
+  version_requirements: *14133820
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: &14132420 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  version_requirements: *id011
-  name: versionomy
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
   prerelease: false
+  version_requirements: *14132420
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &14130000 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-- !ruby/object:Gem::Dependency 
-  requirement: &id012 !ruby/object:Gem::Requirement 
+  prerelease: false
+  version_requirements: *14130000
+- !ruby/object:Gem::Dependency
+  name: versionomy
+  requirement: &14129120 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  version_requirements: *id012
-  name: guard-yard
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
   prerelease: false
+  version_requirements: *14129120
+- !ruby/object:Gem::Dependency
+  name: guard-yard
+  requirement: &14127660 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
+  prerelease: false
+  version_requirements: *14127660
 description: Wraps Sequel to perform record linkage between one or two datasets
 email: jeremy.f.stephens@vanderbilt.edu
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - LICENSE.txt
 - README.markdown
-files: 
+files:
 - .document
 - .vimrc
 - Gemfile
@@ -236,37 +186,31 @@ files:
 - test/unit/test_single_threaded_runner.rb
 - test/unit/test_utils.rb
 homepage: http://github.com/coupler/linkage
-licenses: 
+licenses:
 - MIT
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
       - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+      hash: 2889761151257020688
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
 rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Sequel-based record linkage
 test_files: []
-