assimilate 0.1.0 → 0.1.1

data/lib/assimilate/batch.rb

@@ -30,8 +30,11 @@ class Assimilate::Batch
     end
   end
 
+  # The stripped record contains only the data values from the source (no internal values with leading underscores).
+  # Any nil values are ignored; these should not be stored but if they do appear in the catalog then don't want
+  # to include them when comparing new records vs. old.
   def stripped_record_for(key)
-    @baseline[key] && @baseline[key].select {|k,v| k !~ /^_/}
+    @baseline[key] && @baseline[key].select {|k,v| k !~ /^_/ && !v.nil?}
   end
 
   def <<(record)
@@ -100,11 +103,14 @@ class Assimilate::Batch
   end
 
   def record_batch
-    raise(Assimilate::DuplicateImportError, "duplicate batch for datestamp #{datestamp}") if @catalog.batches.find(@domainkey => @domain, 'datestamp' => @datestamp).to_a.any?
-    raise(Assimilate::DuplicateImportError, "duplicate batch for file #{@filename}") if @catalog.batches.find(@domainkey => @domain, 'filename' => @filename).to_a.any?
+    # don't want leading underscore on attributes in the batches table
+    dkey = @domainkey.gsub(/^_/,'')
+
+    raise(Assimilate::DuplicateImportError, "duplicate batch for datestamp #{datestamp}") if @catalog.batches.find(dkey => @domain, 'datestamp' => @datestamp).to_a.any?
+    raise(Assimilate::DuplicateImportError, "duplicate batch for file #{@filename}") if @catalog.batches.find(dkey => @domain, 'filename' => @filename).to_a.any?
 
     @catalog.batches.insert({
-      @domainkey => @domain,
+      dkey => @domain,
       'datestamp' => @datestamp,
       'filename' => @filename
       })

data/lib/assimilate/version.rb

@@ -1,3 +1,3 @@
 module Assimilate
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/spec/lib/extend_spec.rb

@@ -16,8 +16,7 @@ describe "loading extended data" do
   def import_base_data(datestamp, filename = "batch_input.csv")
     @batcher = @catalog.start_batch(domain: 'testdata', datestamp: datestamp, idfield: 'ID')
 
-    @records = CSV.read(File.dirname(__FILE__) + "/../data/#{filename}", :headers => true)
-    @records.each do |rec|
+    Assimilate.slurp(File.dirname(__FILE__) + "/../data/#{filename}") do |rec|
       @batcher << rec
     end
     @batcher.commit
@@ -31,8 +30,7 @@ describe "loading extended data" do
 
     def import_extended_data(datestamp, filename)
       @extender = @catalog.extend_data(domain: 'testdata', datastamp: datestamp, idfield: 'ID', key: 'inauguration')
-      @records = CSV.read(File.dirname(__FILE__) + "/../data/#{filename}", :headers => true)
-      @records.each do |rec|
+      Assimilate.slurp(File.dirname(__FILE__) + "/../data/#{filename}") do |rec|
         @extender << rec
       end
       @extender.commit

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: assimilate
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-03 00:00:00.000000000 Z
+date: 2012-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &2157622380 !ruby/object:Gem::Requirement
+  requirement: &2155898600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *2157622380
+  version_requirements: *2155898600
 - !ruby/object:Gem::Dependency
   name: bson_ext
-  requirement: &2157620560 !ruby/object:Gem::Requirement
+  requirement: &2155897820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *2157620560
+  version_requirements: *2155897820
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &2157620020 !ruby/object:Gem::Requirement
+  requirement: &2155897040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 3.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *2157620020
+  version_requirements: *2155897040
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2157619060 !ruby/object:Gem::Requirement
+  requirement: &2155896200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 2.9.0
   type: :development
   prerelease: false
-  version_requirements: *2157619060
+  version_requirements: *2155896200
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &2157618020 !ruby/object:Gem::Requirement
+  requirement: &2155895320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 0.7.0
   type: :development
   prerelease: false
-  version_requirements: *2157618020
+  version_requirements: *2155895320
 - !ruby/object:Gem::Dependency
   name: ruby_gntp
-  requirement: &2157616740 !ruby/object:Gem::Requirement
+  requirement: &2155894100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,7 +76,7 @@ dependencies:
         version: 0.3.4
   type: :development
   prerelease: false
-  version_requirements: *2157616740
+  version_requirements: *2155894100
 description: Ingest updates from CSV and apply to set of hashes
 email:
 - jmay@pobox.com