assimilate 0.3.5 → 0.4.1

data/lib/assimilate.rb

@@ -16,7 +16,12 @@ module Assimilate
       catalog = Catalog.new(:config => opts[:config])
       batcher = catalog.start_batch(opts.merge(:filename => filename))
 
+      headers = nil
       slurp(filename) do |rec|
+        if opts[:subset] && !headers
+          headers = rec.keys
+          batcher.prime(headers)
+        end
         batcher << rec
       end
       if opts[:commit]

data/lib/assimilate/batch.rb

@@ -10,6 +10,9 @@ class Assimilate::Batch
     @idfield = args[:idfield]
     @filename = args[:filename]
 
+    @subset = args[:subset]
+    @suppress_deletes = args[:nodeletes]
+
     load_baseline
 
     @noops = []
@@ -19,6 +22,10 @@ class Assimilate::Batch
     @resolved = false
   end
 
+  def prime(fieldnames)
+    @fields = fieldnames
+  end
+
   def load_baseline
     stored_records = @catalog.catalog.find(@domainkey => @domain, @idfield => {"$exists" => 1}).to_a
     @baseline = stored_records.each_with_object({}) do |rec, h|
@@ -34,7 +41,11 @@ class Assimilate::Batch
   # Any nil values are ignored; these should not be stored but if they do appear in the catalog then don't want
   # to include them when comparing new records vs. old.
   def stripped_record_for(key)
-    @baseline[key] && @baseline[key].select {|k,v| k !~ /^_/ && !v.nil?}
+    if @subset
+      @baseline[key] && @baseline[key].select {|k,v| @fields.include?(k)}
+    else
+      @baseline[key] && @baseline[key].select {|k,v| k !~ /^_/ && !v.nil?}
+    end
   end
 
   def <<(record)
@@ -65,7 +76,7 @@ class Assimilate::Batch
     if !@resolved
       @deleted_keys = (@baseline.keys - @seen.keys).reject {|k| @baseline[k][@catalog.config[:deletion_marker]]}
 
-      @updated_field_counts = @changes.each_with_object(Hash.new(0)) do |(key,diffs),h|
+      @updated_field_counts = @changes.each_with_object(Hash.new(0)) do |(_,diffs),h|
         # key = rec[idfield]
         # diffs = deltas(stripped_record_for(key), rec)
         diffs.keys.each do |f|
@@ -117,14 +128,18 @@ class Assimilate::Batch
   end
 
   def apply_deletes
-    @deleted_keys.each do |key|
-      @catalog.catalog.update(
-        {
-          @domainkey => domain,
-          idfield => key
-        },
-        {"$set" => {@catalog.config[:deletion_marker] => datestamp}}
-    )
+    unless @suppress_deletes
+      @deleted_keys.each do |key|
+        @catalog.catalog.update(
+          {
+            @domainkey => domain,
+            idfield => key
+          },
+          {
+            "$set" => {@catalog.config[:deletion_marker] => datestamp}
+          }
+      )
+      end
     end
   end
 
@@ -145,7 +160,9 @@ class Assimilate::Batch
           @domainkey => domain,
           idfield => key
         },
-        {"$set" => diffs.merge(marker => datestamp)}
+        {
+          "$set" => diffs.merge(marker => datestamp)
+        }
       )
     end
   end

data/lib/assimilate/command.rb

@@ -20,6 +20,14 @@ class Assimilate::Command
         @options[:commit] = true
       end
 
+      opts.on("--subset", "Only consider the fields in the input file") do
+        @options[:subset] = true
+      end
+
+      opts.on("--nodeletes", "Do NOT delete existing records that are not present in the input") do
+        @options[:nodeletes] = true
+      end
+
       opts.on("--key FIELDNAME", String, "(*extend* only; optional) Hash key to store extended attributes under") do |f|
         @options[:key] = f
       end
@@ -42,10 +50,12 @@ class Assimilate::Command
     @command = argv.shift
     filenames = @parser.parse(argv)
 
-    raise OptionParser::MissingArgument, "missing config" unless options[:config]
-    raise OptionParser::MissingArgument, "missing idfield" unless options[:idfield]
-    raise OptionParser::MissingArgument, "missing domain" unless options[:domain]
-    raise "missing filename" unless filenames.any?
+    unless command == 'version'
+      raise OptionParser::MissingArgument, "missing config" unless options[:config]
+      raise OptionParser::MissingArgument, "missing idfield" unless options[:idfield]
+      raise OptionParser::MissingArgument, "missing domain" unless options[:domain]
+      raise "missing filename" unless filenames.any?
+    end
 
     # argv remnants are filenames
     [@command, @options, filenames]
@@ -55,6 +65,9 @@ class Assimilate::Command
     filename = filenames.first
 
     case command
+    when 'version'
+      puts "Assimilate #{Assimilate::VERSION}"
+
     when 'load'
       raise OptionParser::MissingArgument, "missing datestamp" unless options[:datestamp]
 
@@ -92,15 +105,21 @@ EOT
         Unchanged records: #{results[:unchanged_count]}
               New records: #{results[:adds_count]} #{idsummary(results[:new_ids])}
                   Deletes: #{results[:deletes_count]} #{idsummary(results[:deleted_ids])}
+EOT
+      if options[:nodeletes]
+        warn "\t\t\tNO DELETIONS"
+      end
+
+      warn <<-EOT
                   Updates: #{results[:updates_count]} #{idsummary(results[:updated_ids])}
 EOT
       if results[:updated_fields].any?
         $stderr.puts <<-EOT
-            Counts by field:
+   Update counts by field:
 EOT
-        results[:updated_fields].each do |k,v|
+        results[:updated_fields].sort.each do |k,v|
           $stderr.puts <<-EOT
-                        #{k}: #{v}
+                #{k.ljust(30, '.')}#{"%6d" % v}
 EOT
         end
       end

data/lib/assimilate/version.rb

@@ -1,3 +1,3 @@
 module Assimilate
-  VERSION = "0.3.5"
+  VERSION = "0.4.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: assimilate
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.4.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-10 00:00:00.000000000 Z
+date: 2013-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mongo
@@ -178,7 +178,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3776853293402426240
+      hash: 4032073337181857520
 requirements: []
 rubyforge_project: assimilate
 rubygems_version: 1.8.24