assimilate 0.0.4 → 0.0.5

data/lib/assimilate/batch.rb

@@ -13,7 +13,7 @@ class Assimilate::Batch
     load_baseline
 
     @noops = []
-    @changes = []
+    @changes = {}
     @adds = []
     @deletes = []
     @resolved = false
@@ -45,22 +45,26 @@ class Assimilate::Batch
       if current_record == hash
         @noops << hash
       else
-        @changes << hash
+        @changes[key] = deltas(current_record, hash)
       end
     else
       @adds << hash
     end
   end
 
+  def deltas(h1,h2)
+    (h1.keys | h2.keys).each_with_object({}) {|k,h| h[k] = h2[k] if h1[k] != h2[k]}
+  end
+
   # compute anything needed before we can write updates to permanent store
   # * find records that have been deleted
   def resolve
     if !@resolved
       @deleted_keys = (@baseline.keys - @seen.keys).reject {|k| @baseline[k][@catalog.config[:deletion_marker]]}
 
-      @updated_field_counts = @changes.each_with_object(Hash.new(0)) do |rec,h|
-        key = rec[idfield]
-        diffs = rec.diff(stripped_record_for(key))
+      @updated_field_counts = @changes.each_with_object(Hash.new(0)) do |(key,diffs),h|
+        # key = rec[idfield]
+        # diffs = deltas(stripped_record_for(key), rec)
         diffs.keys.each do |f|
           h[f] += 1
         end
@@ -79,8 +83,8 @@ class Assimilate::Batch
       :new_ids => @adds.map {|rec| rec[idfield]},
       :deletes_count => @deleted_keys.count,
       :deleted_ids => @deleted_keys,
-      :updates_count => @changes.count,
-      :updated_ids => @changes.map {|rec| rec[idfield]},
+      :updates_count => @changes.size,
+      :updated_ids => @changes.keys,
       :unchanged_count => @noops.count,
       :updated_fields => @updated_field_counts
     }
@@ -128,13 +132,13 @@ class Assimilate::Batch
   end
 
   def apply_updates
-    @changes.each do |rec|
+    @changes.each do |key, diffs|
       @catalog.catalog.update(
         {
           @domainkey => domain,
-          idfield => rec[idfield]
+          idfield => key
         },
-        {"$set" => rec}
+        {"$set" => diffs}
       )
     end
   end

data/lib/assimilate/catalog.rb

@@ -51,7 +51,8 @@ class Assimilate::Catalog
   end
 
   def where(params)
-    @catalog.find(params).first.select {|k,v| k !~ /^_/}
+    record = @catalog.find(params).first
+    record && record.select {|k,v| k !~ /^_/}
   end
 
   def active_count

data/lib/assimilate/version.rb

@@ -1,3 +1,3 @@
 module Assimilate
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

data/lib/assimilate.rb

@@ -1,5 +1,5 @@
 require "mongo"
-require "active_support/core_ext" # needed for Hash#diff
+require "active_support/core_ext" # needed for Hash#symbolize_keys!
 require "csv"
 
 require_relative "assimilate/version"

data/spec/data/batch_input.csv

@@ -1,7 +1,7 @@
-ID,name,title
-1,George Washington,President
-2,John Adams,Vice President
-3,Benjamin Franklin,Sage
-4,Aaron Burr,Duelist
-5,Alexander Hamilton,Financier
-6,James Madison,Theorist
+ID,name,title,spouse
+1,George Washington,President,Martha
+2,John Adams,Vice President,Abigail
+3,Benjamin Franklin,Sage,Deborah
+4,Aaron Burr,Duelist,Theodosia
+5,Alexander Hamilton,Financier,Elizabeth
+6,James Madison,Theorist,Dolly

data/spec/data/duplicate_input.csv

@@ -1,7 +1,7 @@
-ID,name,title
-1,George Washington,President
-2,John Adams,Vice President
-3,Benjamin Franklin,Sage
-4,Aaron Burr,Duelist
-5,Alexander Hamilton,Financier
-6,James Madison,Theorist
+ID,name,title,spouse
+1,George Washington,President,Martha
+2,John Adams,Vice President,Abigail
+3,Benjamin Franklin,Sage,Deborah
+4,Aaron Burr,Duelist,Theodosia
+5,Alexander Hamilton,Financier,Elizabeth
+6,James Madison,Theorist,Dolly

data/spec/data/updates.csv

@@ -1,6 +1,6 @@
-ID,name,title
-1,George Washington,President
-2,John Adams,Vice President
-3,Benjamin Franklin,Ambassador
-5,Alexander Hamilton,Financier
-7,Thomas Jefferson,Anti-Federalist
+ID,name,title,spouse
+1,George Washington,President,Martha
+2,John Adams,Vice President,Abigail
+3,Benjamin Franklin,Ambassador,
+5,Alexander Hamilton,Financier,Elizabeth
+7,Thomas Jefferson,Anti-Federalist,

data/spec/lib/batch_spec.rb

@@ -45,15 +45,16 @@ describe "importing file" do
 
     it "should load the records verbatim" do
       @catalog.catalog.count.should == 6
-      @catalog.where('_resource' => 'testdata', 'ID' => '3').should == {'ID' => '3', 'name' => 'Benjamin Franklin', 'title' => 'Sage'}
+      @catalog.where('_resource' => 'testdata', 'ID' => '3').should ==
+        {'ID' => '3', 'name' => 'Benjamin Franklin', 'title' => 'Sage', 'spouse' => 'Deborah'}
     end
 
     it "should refuse to do a duplicate import" do
-      lambda {import_data("123")}.should raise_error(Assimilate::DuplicateImportError)
+      lambda {import_data("123")}.should raise_error(Assimilate::DuplicateImportError, "duplicate batch for datestamp 123")
     end
 
     it "should refuse to re-import same file" do
-      lambda {import_data("234")}.should raise_error(Assimilate::DuplicateImportError)
+      lambda {import_data("234")}.should raise_error(Assimilate::DuplicateImportError, "duplicate batch for file batch_input.csv")
     end
 
     it "should do all no-ops when importing identical data" do
@@ -78,9 +79,7 @@ describe "importing file" do
     before(:all) do
       reset_catalog
       import_data("123")
-    end
 
-    before(:each) do
       import_data("345", "updates.csv")
     end
 
@@ -95,9 +94,14 @@ describe "importing file" do
         :updates_count => 1,
         :updated_ids => ['3'],
         :unchanged_count => 3,
-        :updated_fields => {'title' => 1}
+        :updated_fields => {'title' => 1, 'spouse' => 1}
       }
       @catalog.active_count.should == 5
     end
+
+    it "should handle deleted attributes" do
+      franklin = @catalog.where('ID' => '3')
+      franklin['spouse'].should be_nil
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: assimilate
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-02 00:00:00.000000000 Z
+date: 2012-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &2152558540 !ruby/object:Gem::Requirement
+  requirement: &2157608660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *2152558540
+  version_requirements: *2157608660
 - !ruby/object:Gem::Dependency
   name: bson_ext
-  requirement: &2152557720 !ruby/object:Gem::Requirement
+  requirement: &2157607760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *2152557720
+  version_requirements: *2157607760
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &2152556760 !ruby/object:Gem::Requirement
+  requirement: &2157607080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 3.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *2152556760
+  version_requirements: *2157607080
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2152555800 !ruby/object:Gem::Requirement
+  requirement: &2157606260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 2.9.0
   type: :development
   prerelease: false
-  version_requirements: *2152555800
+  version_requirements: *2157606260
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &2152554060 !ruby/object:Gem::Requirement
+  requirement: &2157605600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 0.7.0
   type: :development
   prerelease: false
-  version_requirements: *2152554060
+  version_requirements: *2157605600
 - !ruby/object:Gem::Dependency
   name: ruby_gntp
-  requirement: &2152550400 !ruby/object:Gem::Requirement
+  requirement: &2157603980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,7 +76,7 @@ dependencies:
         version: 0.3.4
   type: :development
   prerelease: false
-  version_requirements: *2152550400
+  version_requirements: *2157603980
 description: Ingest updates from CSV and apply to set of hashes
 email:
 - jmay@pobox.com