assimilate 0.0.5 → 0.1.0

data/lib/assimilate/batch.rb

@@ -132,20 +132,23 @@ class Assimilate::Batch
   end
 
   def apply_updates
+    marker = @catalog.config[:update_marker]
     @changes.each do |key, diffs|
       @catalog.catalog.update(
         {
           @domainkey => domain,
           idfield => key
         },
-        {"$set" => diffs}
+        {"$set" => diffs.merge(marker => datestamp)}
       )
     end
   end
 
   def decorate(records)
+    marker = @catalog.config[:insertion_marker]
     records.map do |r|
       r[@domainkey] = @domain
+      r[marker] = datestamp
       r.to_hash
     end
   end

data/lib/assimilate/catalog.rb

@@ -33,10 +33,10 @@ class Assimilate::Catalog
 
   def check_config
     config.symbolize_keys!
-    [:db, :catalog, :batch, :domain, :deletion_marker].each do |key|
+    [:db, :catalog, :batch, :domain, :deletion_marker, :insertion_marker, :update_marker].each do |key|
       raise Assimilate::InvalidConfiguration, "missing required parameter: #{key}" unless config[key]
     end
-    [:domain, :deletion_marker].each do |key|
+    [:domain, :deletion_marker, :insertion_marker, :update_marker].each do |key|
       # enforce leading underscore on internal attributes
       config[key] = "_#{config[key]}" unless config[key] =~ /^_/
     end
@@ -51,8 +51,12 @@ class Assimilate::Catalog
   end
 
   def where(params)
-    record = @catalog.find(params).first
-    record && record.select {|k,v| k !~ /^_/}
+    records = @catalog.find(params).to_a #.map {|rec| rec.select {|k,v| k !~ /^_/}}
+    if records.count == 1
+      records.first
+    else
+      records
+    end
   end
 
   def active_count

data/lib/assimilate/version.rb

@@ -1,3 +1,3 @@
 module Assimilate
-  VERSION = "0.0.5"
+  VERSION = "0.1.0"
 end

data/spec/data/test.yml

@@ -4,3 +4,5 @@ catalog: forefathers
 batch: files
 domain: resource
 deletion_marker: dt_removed
+insertion_marker: first_seen
+update_marker: last_updated

data/spec/lib/batch_spec.rb

@@ -45,8 +45,8 @@ describe "importing file" do
 
     it "should load the records verbatim" do
       @catalog.catalog.count.should == 6
-      @catalog.where('_resource' => 'testdata', 'ID' => '3').should ==
-        {'ID' => '3', 'name' => 'Benjamin Franklin', 'title' => 'Sage', 'spouse' => 'Deborah'}
+      franklin = @catalog.where('_resource' => 'testdata', 'ID' => '3')
+      franklin.keys.sort.should == ['ID', 'name', 'title', 'spouse', '_id', '_resource', '_first_seen'].sort
     end
 
     it "should refuse to do a duplicate import" do
@@ -102,6 +102,7 @@ describe "importing file" do
     it "should handle deleted attributes" do
       franklin = @catalog.where('ID' => '3')
       franklin['spouse'].should be_nil
+      franklin['_last_updated'].should == '345'
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: assimilate
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &2157608660 !ruby/object:Gem::Requirement
+  requirement: &2157622380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *2157608660
+  version_requirements: *2157622380
 - !ruby/object:Gem::Dependency
   name: bson_ext
-  requirement: &2157607760 !ruby/object:Gem::Requirement
+  requirement: &2157620560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *2157607760
+  version_requirements: *2157620560
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &2157607080 !ruby/object:Gem::Requirement
+  requirement: &2157620020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 3.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *2157607080
+  version_requirements: *2157620020
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2157606260 !ruby/object:Gem::Requirement
+  requirement: &2157619060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 2.9.0
   type: :development
   prerelease: false
-  version_requirements: *2157606260
+  version_requirements: *2157619060
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &2157605600 !ruby/object:Gem::Requirement
+  requirement: &2157618020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 0.7.0
   type: :development
   prerelease: false
-  version_requirements: *2157605600
+  version_requirements: *2157618020
 - !ruby/object:Gem::Dependency
   name: ruby_gntp
-  requirement: &2157603980 !ruby/object:Gem::Requirement
+  requirement: &2157616740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,7 +76,7 @@ dependencies:
         version: 0.3.4
   type: :development
   prerelease: false
-  version_requirements: *2157603980
+  version_requirements: *2157616740
 description: Ingest updates from CSV and apply to set of hashes
 email:
 - jmay@pobox.com