assimilate 0.2.3 → 0.3

data/Rakefile

@@ -5,4 +5,4 @@ require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-# task :default => :spec
+task :default => :spec

data/assimilate.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "bson_ext", "~> 1.6.0"
   gem.add_dependency 'activesupport', "~> 3.2.0"
 
+  gem.add_development_dependency "rake", "~> 0.9.2"
   gem.add_development_dependency "rspec", "~> 2.9.0"
   gem.add_development_dependency "guard-rspec", "~> 0.7.0"
   gem.add_development_dependency "ruby_gntp", "~> 0.3.4"

data/lib/assimilate/extender.rb

@@ -43,11 +43,14 @@ class Assimilate::Extender
 
   # if there is a field to compare on (i.e. a timestamp), then apply the update if the timestamp is newer;
   # otherwise (no timestamp) compare the hashes and apply if there are any differences.
-  def apply_this_update(current_record, new_data)
-    if @comparison_field && current_record[@keyfield]
-      is_newer(current_record[@keyfield], new_data)
+  def apply_this_update?(current_record, new_data)
+    if @comparison_field && current_record[keyfield]
+      is_newer(current_record[keyfield], new_data)
+    elsif keyfield
+      current_record[keyfield] != new_data
     else
-      current_record[@keyfield] != new_data
+      # top-level extension - compare all the attributes to be added
+      new_data.select {|k,v| current_record[k] != v}.any?
     end
   end
 
@@ -60,7 +63,7 @@ class Assimilate::Extender
     # @seen[key] = data
     current_record = @baseline[key]
     if current_record
-      if apply_this_update(current_record, data)
+      if apply_this_update?(current_record, data)
         @changes << key
         @seen[key] = data
       else
@@ -98,11 +101,21 @@ class Assimilate::Extender
   def apply_inserts
     @adds.each do |key|
       data = @seen[key]
-      @catalog.catalog.insert(
-        @domainkey => domain,
-        idfield => key,
-        keyfield => data
-      )
+      if keyfield
+        @catalog.catalog.insert(
+          @domainkey => domain,
+          idfield => key,
+          keyfield => data
+        )
+      else
+        # top-level extension
+        @catalog.catalog.insert(
+          data.merge(
+            @domainkey => domain,
+            idfield => key
+          )
+        )
+      end
     end
   end
 
@@ -110,16 +123,27 @@ class Assimilate::Extender
   def apply_updates
     @changes.each do |key|
       data = @seen[key]
-      @catalog.catalog.update(
-        {
-          @domainkey => domain,
-          idfield => key
-        },
-        {"$set" => {
-            keyfield => data
+      if keyfield
+        @catalog.catalog.update(
+          {
+            @domainkey => domain,
+            idfield => key
+          },
+          {"$set" => {
+              keyfield => data
+            }
           }
-        }
-      )
+        )
+      else
+        # top-level extension
+        @catalog.catalog.update(
+          {
+            @domainkey => domain,
+            idfield => key
+          },
+          {"$set" => data }
+        )
+      end
     end
   end
 

data/lib/assimilate/version.rb

@@ -1,3 +1,3 @@
 module Assimilate
-  VERSION = "0.2.3"
+  VERSION = "0.3"
 end

data/spec/data/birthdates.csv

@@ -0,0 +1,5 @@
+ID,birthdate
+1,1732/02/22
+2,1735/10/30
+6,1751/03/16
+999,1802/03/04

data/spec/lib/extend_spec.rb

@@ -30,6 +30,14 @@ describe "loading extended data" do
     @extender.commit
   end
 
+  def import_toplevel_extended_data(datestamp, filename, opts = {})
+    @extender = @catalog.extend_data(opts.merge(domain: 'testdata', datestamp: datestamp, idfield: 'ID'))
+    Assimilate.slurp(File.dirname(__FILE__) + "/../data/#{filename}") do |rec|
+      @extender << rec
+    end
+    @extender.commit
+  end
+
   describe "into matching catalog entries" do
     before(:all) do
       reset_catalog
@@ -70,6 +78,46 @@ describe "loading extended data" do
     end
   end
 
+  describe "at top level of catalog entries" do
+    before(:all) do
+      reset_catalog
+      import_base_data("123")
+    end
+
+    before(:each) do
+      import_toplevel_extended_data("991", "birthdates.csv")
+    end
+
+    it "should capture changes" do
+      @extender.stats.should == {
+        :baseline_count => 6,
+        :final_count => 7,
+        :distinct_ids => 4,
+        :adds_count => 1,
+        :new_ids => ['999'],
+        :updates_count => 3,
+        :updated_fields => {'birthdate' => 4},
+        :unchanged_count => 0
+      }
+    end
+
+    it "should do no-ops on duplicate load" do
+      # import_extended_data("1002", "dates")
+      lambda {import_toplevel_extended_data("992", "birthdates.csv")}.should_not raise_error
+
+      @extender.stats.should == {
+        :baseline_count => 7,
+        :final_count => 7,
+        :distinct_ids => 4,
+        :adds_count => 0,
+        :new_ids => [],
+        :updates_count => 0,
+        :updated_fields => {},
+        :unchanged_count => 4
+      }
+    end
+  end
+
   describe "updating log entries" do
     before(:all) do
       reset_catalog

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: assimilate
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: '0.3'
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-31 00:00:00.000000000 Z
+date: 2012-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &2154717840 !ruby/object:Gem::Requirement
+  requirement: &2161063380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *2154717840
+  version_requirements: *2161063380
 - !ruby/object:Gem::Dependency
   name: bson_ext
-  requirement: &2154717340 !ruby/object:Gem::Requirement
+  requirement: &2161062840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *2154717340
+  version_requirements: *2161062840
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &2154716880 !ruby/object:Gem::Requirement
+  requirement: &2161062360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,21 @@ dependencies:
         version: 3.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *2154716880
+  version_requirements: *2161062360
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &2161061860 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: *2161061860
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2154716420 !ruby/object:Gem::Requirement
+  requirement: &2161061380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +65,10 @@ dependencies:
         version: 2.9.0
   type: :development
   prerelease: false
-  version_requirements: *2154716420
+  version_requirements: *2161061380
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &2152593120 !ruby/object:Gem::Requirement
+  requirement: &2161060880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +76,10 @@ dependencies:
         version: 0.7.0
   type: :development
   prerelease: false
-  version_requirements: *2152593120
+  version_requirements: *2161060880
 - !ruby/object:Gem::Dependency
   name: ruby_gntp
-  requirement: &2152587980 !ruby/object:Gem::Requirement
+  requirement: &2161060380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,7 +87,7 @@ dependencies:
         version: 0.3.4
   type: :development
   prerelease: false
-  version_requirements: *2152587980
+  version_requirements: *2161060380
 description: Ingest updates from CSV and apply to set of hashes
 email:
 - jmay@pobox.com
@@ -100,6 +111,7 @@ files:
 - lib/assimilate/extender.rb
 - lib/assimilate/version.rb
 - spec/data/batch_input.csv
+- spec/data/birthdates.csv
 - spec/data/dates.csv
 - spec/data/duplicate_input.csv
 - spec/data/logs1.csv
@@ -135,6 +147,7 @@ specification_version: 3
 summary: Review & incorporate changes to a repository of persistent hashes in mongodb.
 test_files:
 - spec/data/batch_input.csv
+- spec/data/birthdates.csv
 - spec/data/dates.csv
 - spec/data/duplicate_input.csv
 - spec/data/logs1.csv