assimilate 0.1.1 → 0.2

data/lib/assimilate/extender.rb

@@ -9,6 +9,7 @@ class Assimilate::Extender
     @idfield = args[:idfield]
     @filename = args[:filename]
     @keyfield = args[:key]
+    @comparison_field = args[:compare]
 
     load_baseline
 
@@ -29,6 +30,20 @@ class Assimilate::Extender
     end
   end
 
+  def is_newer(current_data, new_data)
+    new_data[@comparison_field].to_i > current_data[@comparison_field].to_i
+  end
+
+  # if there is a field to compare on (i.e. a timestamp), then apply the update if the timestamp is newer;
+  # otherwise (no timestamp) compare the hashes and apply if there are any differences.
+  def apply_this_update(current_record, new_data)
+    if @comparison_field
+      is_newer(current_record[@keyfield], new_data)
+    else
+      current_record[@keyfield] != new_data
+    end
+  end
+
   def <<(record)
     @seen ||= Hash.new(0)
 
@@ -38,12 +53,12 @@ class Assimilate::Extender
     # @seen[key] = data
     current_record = @baseline[key]
     if current_record
-      if current_record[@keyfield] == data
-        @noops << key
-        @seen[key] = {}
-      else
+      if apply_this_update(current_record, data)
         @changes << key
         @seen[key] = data
+      else
+        @noops << key
+        @seen[key] = {}
       end
     else
       @adds << key

data/lib/assimilate/version.rb

@@ -1,3 +1,3 @@
 module Assimilate
-  VERSION = "0.1.1"
+  VERSION = "0.2"
 end

data/spec/data/logs1.csv

@@ -0,0 +1,5 @@
+timestamp,ID,event
+1201010101,1,something happened
+1201010304,2,important event
+1201010205,6,no comment
+1201000123,5,no comment

data/spec/data/logs2.csv

@@ -0,0 +1,5 @@
+timestamp,ID,event
+1201020101,1,something else happened
+1201020304,2,another important event
+1201020205,6,a comment
+1201020123,5,no comment

data/spec/lib/extend_spec.rb

@@ -22,20 +22,20 @@ describe "loading extended data" do
     @batcher.commit
   end
 
+  def import_extended_data(datestamp, filename, opts = {})
+    @extender = @catalog.extend_data(opts.merge(domain: 'testdata', datestamp: datestamp, idfield: 'ID', key: 'inauguration'))
+    Assimilate.slurp(File.dirname(__FILE__) + "/../data/#{filename}") do |rec|
+      @extender << rec
+    end
+    @extender.commit
+  end
+
   describe "into matching catalog entries" do
     before(:all) do
       reset_catalog
       import_base_data("123")
     end
 
-    def import_extended_data(datestamp, filename)
-      @extender = @catalog.extend_data(domain: 'testdata', datastamp: datestamp, idfield: 'ID', key: 'inauguration')
-      Assimilate.slurp(File.dirname(__FILE__) + "/../data/#{filename}") do |rec|
-        @extender << rec
-      end
-      @extender.commit
-    end
-
     before(:each) do
       import_extended_data("1001", "dates.csv")
     end
@@ -70,6 +70,87 @@ describe "loading extended data" do
     end
   end
 
+  describe "updating log entries" do
+    before(:all) do
+      reset_catalog
+      import_base_data("20120501")
+      import_extended_data("20120505", "logs1.csv")
+    end
+
+
+    before(:each) do
+    end
+
+    it "should capture changes" do
+      @extender.stats.should == {
+        :baseline_count => 6,
+        :final_count => 6,
+        :distinct_ids => 4,
+        :adds_count => 0,
+        :new_ids => [],
+        :updates_count => 4,
+        :updated_fields => {'timestamp' => 4, 'event' => 4},
+        :unchanged_count => 0
+      }
+    end
+
+    it "should load the new events" do
+      lambda {import_extended_data("20120506", "logs2.csv", :compare => 'timestamp')}.should_not raise_error
+
+      @extender.stats.should == {
+        :baseline_count => 6,
+        :final_count => 6,
+        :distinct_ids => 4,
+        :adds_count => 0,
+        :new_ids => [],
+        :updates_count => 4,
+        :updated_fields => {'timestamp' => 4, 'event' => 4},
+        :unchanged_count => 0
+      }
+    end
+  end
+
+  describe "updating log entries in reverse order" do
+    before(:all) do
+      reset_catalog
+      import_base_data("20120501")
+      import_extended_data("20120505", "logs2.csv")
+    end
+
+
+    before(:each) do
+    end
+
+    it "should capture changes" do
+      @extender.stats.should == {
+        :baseline_count => 6,
+        :final_count => 6,
+        :distinct_ids => 4,
+        :adds_count => 0,
+        :new_ids => [],
+        :updates_count => 4,
+        :updated_fields => {'timestamp' => 4, 'event' => 4},
+        :unchanged_count => 0
+      }
+    end
+
+    it "should load the new events" do
+      lambda {import_extended_data("20120506", "logs1.csv", :compare => 'timestamp')}.should_not raise_error
+
+      @extender.stats.should == {
+        :baseline_count => 6,
+        :final_count => 6,
+        :distinct_ids => 4,
+        :adds_count => 0,
+        :new_ids => [],
+        :updates_count => 0,
+        :updated_fields => {},
+        :unchanged_count => 4
+      }
+    end
+  end
+
+
   # test handling of multiple records for same ID in the extended-data file
   # test importing data at top level (no keyfield for sub-attributes)
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: assimilate
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: '0.2'
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-04 00:00:00.000000000 Z
+date: 2012-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &2155898600 !ruby/object:Gem::Requirement
+  requirement: &2157154580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *2155898600
+  version_requirements: *2157154580
 - !ruby/object:Gem::Dependency
   name: bson_ext
-  requirement: &2155897820 !ruby/object:Gem::Requirement
+  requirement: &2157154080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *2155897820
+  version_requirements: *2157154080
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &2155897040 !ruby/object:Gem::Requirement
+  requirement: &2156541740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 3.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *2155897040
+  version_requirements: *2156541740
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2155896200 !ruby/object:Gem::Requirement
+  requirement: &2156539660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 2.9.0
   type: :development
   prerelease: false
-  version_requirements: *2155896200
+  version_requirements: *2156539660
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &2155895320 !ruby/object:Gem::Requirement
+  requirement: &2156536900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 0.7.0
   type: :development
   prerelease: false
-  version_requirements: *2155895320
+  version_requirements: *2156536900
 - !ruby/object:Gem::Dependency
   name: ruby_gntp
-  requirement: &2155894100 !ruby/object:Gem::Requirement
+  requirement: &2156535700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,7 +76,7 @@ dependencies:
         version: 0.3.4
   type: :development
   prerelease: false
-  version_requirements: *2155894100
+  version_requirements: *2156535700
 description: Ingest updates from CSV and apply to set of hashes
 email:
 - jmay@pobox.com
@@ -102,6 +102,8 @@ files:
 - spec/data/batch_input.csv
 - spec/data/dates.csv
 - spec/data/duplicate_input.csv
+- spec/data/logs1.csv
+- spec/data/logs2.csv
 - spec/data/test.yml
 - spec/data/updates.csv
 - spec/lib/batch_spec.rb
@@ -135,6 +137,8 @@ test_files:
 - spec/data/batch_input.csv
 - spec/data/dates.csv
 - spec/data/duplicate_input.csv
+- spec/data/logs1.csv
+- spec/data/logs2.csv
 - spec/data/test.yml
 - spec/data/updates.csv
 - spec/lib/batch_spec.rb