assimilate 0.0.2 → 0.0.3

data/lib/assimilate/batch.rb

@@ -3,6 +3,8 @@ class Assimilate::Batch
 
   def initialize(args)
     @catalog = args[:catalog]
+    @domainkey = @catalog.config[:domain]
+
     @domain = args[:domain]
     @datestamp = args[:datestamp]
     @idfield = args[:idfield]
@@ -18,11 +20,11 @@ class Assimilate::Batch
   end
 
   def load_baseline
-    stored_records = @catalog.catalog.find(@catalog.domainkey => @domain).to_a
+    stored_records = @catalog.catalog.find(@domainkey => @domain).to_a
     @baseline = stored_records.each_with_object({}) do |rec, h|
       key = rec[@idfield]
       if h.include?(key)
-        raise Assimilate::CorruptDataError, "Duplicate records for key [#{key}] in domain [#{@domain}]"
+        raise Assimilate::CorruptDataError, "Duplicate records for key [#{key}] in #{@domainkey} [#{@domain}]"
       end
       h[key] = rec
     end
@@ -54,7 +56,7 @@ class Assimilate::Batch
   # * find records that have been deleted
   def resolve
     if !@resolved
-      @deleted_keys = @baseline.keys - @seen.keys
+      @deleted_keys = (@baseline.keys - @seen.keys).reject {|k| @baseline[k][@catalog.config[:deletion_marker]]}
 
       @updated_field_counts = @changes.each_with_object(Hash.new(0)) do |rec,h|
         key = rec[idfield]
@@ -76,6 +78,7 @@ class Assimilate::Batch
       :adds_count => @adds.count,
       :new_ids => @adds.map {|rec| rec[idfield]},
       :deletes_count => @deleted_keys.count,
+      :deleted_ids => @deleted_keys,
       :updates_count => @changes.count,
       :unchanged_count => @noops.count,
       :updated_fields => @updated_field_counts
@@ -92,11 +95,11 @@ class Assimilate::Batch
   end
 
   def record_batch
-    raise(Assimilate::DuplicateImportError, "duplicate batch for datestamp #{datestamp}") if @catalog.batches.find('domain' => @domain, 'datestamp' => @datestamp).to_a.any?
-    raise(Assimilate::DuplicateImportError, "duplicate batch for file #{@filename}") if @catalog.batches.find('domain' => @domain, 'filename' => @filename).to_a.any?
+    raise(Assimilate::DuplicateImportError, "duplicate batch for datestamp #{datestamp}") if @catalog.batches.find(@domainkey => @domain, 'datestamp' => @datestamp).to_a.any?
+    raise(Assimilate::DuplicateImportError, "duplicate batch for file #{@filename}") if @catalog.batches.find(@domainkey => @domain, 'filename' => @filename).to_a.any?
 
     @catalog.batches.insert({
-      'domain' => @domain,
+      @domainkey => @domain,
       'datestamp' => @datestamp,
       'filename' => @filename
       })
@@ -106,10 +109,10 @@ class Assimilate::Batch
     @deleted_keys.each do |key|
       @catalog.catalog.update(
         {
-          @catalog.domainkey => domain,
+          @domainkey => domain,
           idfield => key
         },
-        {"$set" => {:_dt_removed => datestamp}}
+        {"$set" => {@catalog.config[:deletion_marker] => datestamp}}
     )
     end
   end
@@ -127,7 +130,7 @@ class Assimilate::Batch
     @changes.each do |rec|
       @catalog.catalog.update(
         {
-          @catalog.domainkey => domain,
+          @domainkey => domain,
           idfield => rec[idfield]
         },
         {"$set" => rec}
@@ -137,7 +140,7 @@ class Assimilate::Batch
 
   def decorate(records)
     records.map do |r|
-      r[@catalog.domainkey] = @domain
+      r[@domainkey] = @domain
       r.to_hash
     end
   end

data/lib/assimilate/catalog.rb

@@ -1,28 +1,45 @@
 require "yaml"
 
+# Catalog configuration:
+#   db              name of mongo database
+#   catalog         name of the catalog collection
+#   batch           name of the batches collection (e.g. "files")
+#   domain          key to use for specifying record domains (will be prefixed with _)
+#   deletion_marker key to use to marker records that have disappeared from the source file
+#
 # Records in each catalog acquire the following internal attributes:
-#   _id             Unique ID, assigned by mongo
-#   _[domain]       Domain key, specified with :domainkey attribute when initializing catalog
-#   _dt_first_seen  Batch datestamp reference for when this record was first captured
-#   _dt_last_seen   Batch datestamp reference for when this record was most recently affirmed
-#   _dt_last_update Batch datestamp reference for when this record was most recently altered
-#   _dt_removed     Batch datestamp reference for when this record was removed from input
+#   _id               Unique ID, assigned by mongo
+#   [domain]          Domain key, specified with :domainkey attribute when initializing catalog
+#   _dt_first_seen    Batch datestamp reference for when this record was first captured
+#   _dt_last_seen     Batch datestamp reference for when this record was most recently affirmed
+#   _dt_last_update   Batch datestamp reference for when this record was most recently altered
+#   [deletion_marker] Batch datestamp reference for when this record was removed from input
 #
 # Inbound records must not have attributes named with leading underscores.
 #
 # A "domain" here is a namespace of identifiers.
 
 class Assimilate::Catalog
-  attr_reader :catalog, :batches, :domainkey
+  attr_reader :catalog, :config, :batches
 
   def initialize(args)
     @config = YAML.load(File.open(args[:config]))
+    check_config
+
+    @db = Mongo::Connection.new.db(@config[:db])
+    @catalog = @db.collection(@config[:catalog])
+    @batches = @db.collection(@config[:batch])
+  end
 
-    @db = Mongo::Connection.new.db(@config['db'])
-    @catalog = @db.collection(@config['catalog'])
-    @batches = @db.collection(@config['batch'])
-    @domainkey = @config['domain']
-    @domainkey = "_#{@domainkey}" unless @domainkey =~ /^_/ # enforce leading underscore on internal attributes
+  def check_config
+    config.symbolize_keys!
+    [:db, :catalog, :batch, :domain, :deletion_marker].each do |key|
+      raise Assimilate::InvalidConfiguration, "missing required parameter: #{key}" unless config[key]
+    end
+    [:domain, :deletion_marker].each do |key|
+      # enforce leading underscore on internal attributes
+      config[key] = "_#{config[key]}" unless config[key] =~ /^_/
+    end
   end
 
   def start_batch(args)
@@ -38,6 +55,9 @@ class Assimilate::Catalog
   end
 
   def active_count
-    @catalog.find("_dt_removed" => nil).count
+    @catalog.find(config[:deletion_marker] => nil).count
   end
 end
+
+class Assimilate::InvalidConfiguration < StandardError
+end

data/lib/assimilate/command.rb

@@ -81,7 +81,7 @@ EOT
        Final record count: #{results[:final_count]}
         Unchanged records: #{results[:unchanged_count]}
               New records: #{results[:adds_count]} (#{results[:new_ids].take(10).join(',')})
-                  Deletes: #{results[:deletes_count]}
+                  Deletes: #{results[:deletes_count]} (#{results[:deleted_ids].take(10).join(',')})
                   Updates: #{results[:updates_count]}
 EOT
       if results[:updated_fields].any?

data/lib/assimilate/extender.rb

@@ -3,6 +3,8 @@ class Assimilate::Extender
 
   def initialize(args)
     @catalog = args[:catalog]
+    @domainkey = @catalog.config[:domain]
+
     @domain = args[:domain]
     @idfield = args[:idfield]
     @filename = args[:filename]
@@ -17,11 +19,11 @@ class Assimilate::Extender
   end
 
   def load_baseline
-    stored_records = @catalog.catalog.find(@catalog.domainkey => @domain).to_a
+    stored_records = @catalog.catalog.find(@domainkey => @domain).to_a
     @baseline = stored_records.each_with_object({}) do |rec, h|
       key = rec[@idfield]
       if h.include?(key)
-        raise Assimilate::CorruptDataError, "Duplicate records for key [#{key}] in domain [#{@domain}]"
+        raise Assimilate::CorruptDataError, "Duplicate records for key [#{key}] in #{@domainkey} [#{@domain}]"
       end
       h[key] = rec
     end
@@ -75,7 +77,7 @@ class Assimilate::Extender
     @adds.each do |key|
       data = @seen[key]
       @catalog.catalog.insert(
-        @catalog.domainkey => domain,
+        @domainkey => domain,
         idfield => key,
         keyfield => data
       )
@@ -88,7 +90,7 @@ class Assimilate::Extender
       data = @seen[key]
       @catalog.catalog.update(
         {
-          @catalog.domainkey => domain,
+          @domainkey => domain,
           idfield => key
         },
         {"$set" => {

data/lib/assimilate/version.rb

@@ -1,3 +1,3 @@
 module Assimilate
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/spec/data/test.yml

@@ -3,3 +3,4 @@ db: assimilate-test
 catalog: forefathers
 batch: files
 domain: resource
+deletion_marker: dt_removed

data/spec/lib/batch_spec.rb

@@ -36,6 +36,7 @@ describe "importing file" do
         :adds_count => 6,
         :new_ids => ["1", "2", "3", "4", "5", "6"],
         :deletes_count => 0,
+        :deleted_ids => [],
         :updates_count => 0,
         :unchanged_count => 0,
         :updated_fields => {}
@@ -64,6 +65,7 @@ describe "importing file" do
         :adds_count => 0,
         :new_ids => [],
         :deletes_count => 0,
+        :deleted_ids => [],
         :updates_count => 0,
         :unchanged_count => 6,
         :updated_fields => {}
@@ -89,6 +91,7 @@ describe "importing file" do
         :adds_count => 1,
         :new_ids => ["7"],
         :deletes_count => 2,
+        :deleted_ids => ['4', '6'],
         :updates_count => 1,
         :unchanged_count => 3,
         :updated_fields => {'title' => 1}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: assimilate
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &2151862180 !ruby/object:Gem::Requirement
+  requirement: &2152913700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *2151862180
+  version_requirements: *2152913700
 - !ruby/object:Gem::Dependency
   name: bson_ext
-  requirement: &2151861480 !ruby/object:Gem::Requirement
+  requirement: &2152912480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *2151861480
+  version_requirements: *2152912480
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &2151860560 !ruby/object:Gem::Requirement
+  requirement: &2152911920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 3.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *2151860560
+  version_requirements: *2152911920
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2151859660 !ruby/object:Gem::Requirement
+  requirement: &2152911460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 2.9.0
   type: :development
   prerelease: false
-  version_requirements: *2151859660
+  version_requirements: *2152911460
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &2151856200 !ruby/object:Gem::Requirement
+  requirement: &2152910880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 0.7.0
   type: :development
   prerelease: false
-  version_requirements: *2151856200
+  version_requirements: *2152910880
 - !ruby/object:Gem::Dependency
   name: ruby_gntp
-  requirement: &2151855040 !ruby/object:Gem::Requirement
+  requirement: &2152910140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,7 +76,7 @@ dependencies:
         version: 0.3.4
   type: :development
   prerelease: false
-  version_requirements: *2151855040
+  version_requirements: *2152910140
 description: Ingest updates from CSV and apply to set of hashes
 email:
 - jmay@pobox.com