assimilate 0.0.1 → 0.0.2

data/.gitignore

@@ -17,3 +17,4 @@ test/version_tmp
 tmp
 .rspec
 .irb_history
+dev.yml

data/bin/assimilate

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require_relative "../lib/assimilate"
+
+assimilator = Assimilate::Command.new
+command, options, filenames = assimilator.parse
+
+assimilator.execute(command, options, filenames)

data/lib/assimilate/batch.rb

@@ -14,6 +14,7 @@ class Assimilate::Batch
     @changes = []
     @adds = []
     @deletes = []
+    @resolved = false
   end
 
   def load_baseline
@@ -52,14 +53,18 @@ class Assimilate::Batch
   # compute anything needed before we can write updates to permanent store
   # * find records that have been deleted
   def resolve
-    @deleted_keys = @baseline.keys - @seen.keys
-
-    @updated_field_counts = @changes.each_with_object(Hash.new(0)) do |rec,h|
-      key = rec[idfield]
-      diffs = rec.diff(stripped_record_for(key))
-      diffs.keys.each do |f|
-        h[f] += 1
+    if !@resolved
+      @deleted_keys = @baseline.keys - @seen.keys
+
+      @updated_field_counts = @changes.each_with_object(Hash.new(0)) do |rec,h|
+        key = rec[idfield]
+        diffs = rec.diff(stripped_record_for(key))
+        diffs.keys.each do |f|
+          h[f] += 1
+        end
       end
+
+      @resolved = true
     end
   end
 
@@ -69,6 +74,7 @@ class Assimilate::Batch
       :baseline_count => @baseline.size,
       :final_count => @baseline.size + @adds.count,
       :adds_count => @adds.count,
+      :new_ids => @adds.map {|rec| rec[idfield]},
       :deletes_count => @deleted_keys.count,
       :updates_count => @changes.count,
       :unchanged_count => @noops.count,
@@ -86,7 +92,9 @@ class Assimilate::Batch
   end
 
   def record_batch
-    raise(Assimilate::DuplicateImportError, "duplicate batch") if @catalog.batches.find('domain' => @domain, 'datestamp' => @datestamp).to_a.any?
+    raise(Assimilate::DuplicateImportError, "duplicate batch for datestamp #{datestamp}") if @catalog.batches.find('domain' => @domain, 'datestamp' => @datestamp).to_a.any?
+    raise(Assimilate::DuplicateImportError, "duplicate batch for file #{@filename}") if @catalog.batches.find('domain' => @domain, 'filename' => @filename).to_a.any?
+
     @catalog.batches.insert({
       'domain' => @domain,
       'datestamp' => @datestamp,
@@ -137,3 +145,6 @@ end
 
 class Assimilate::DuplicateImportError < StandardError
 end
+
+class Assimilate::CorruptDataError < StandardError
+end

data/lib/assimilate/catalog.rb

@@ -29,6 +29,10 @@ class Assimilate::Catalog
     Assimilate::Batch.new(args.merge(:catalog => self))
   end
 
+  def extend_data(args)
+    Assimilate::Extender.new(args.merge(:catalog => self))
+  end
+
   def where(params)
     @catalog.find(params).first.select {|k,v| k !~ /^_/}
   end

data/lib/assimilate/command.rb

@@ -0,0 +1,117 @@
+require "optparse"
+
+class Assimilate::Command
+  attr_reader :command, :options
+
+  def initialize
+    @options = {}
+    @parser = OptionParser.new do |opts|
+      opts.banner = "Usage: assimilate [command] [options]"
+
+      opts.on("--config FILENAME", String, "Catalog database configuration file") do |f|
+        @options[:config] = f
+      end
+
+      opts.on("--id FIELDNAME", String, "Field name to be used for record identifier") do |f|
+        @options[:idfield] = f
+      end
+
+      opts.on("--commit", "Commit changes to database") do
+        @options[:commit] = true
+      end
+
+      opts.on("--key FIELDNAME", String, "(*extend* only) Hash key to store extended attributes under") do |f|
+        @options[:key] = f
+      end
+
+      opts.on("--datestamp DATESTRING", String, "(*load* only) Datestamp to record for file batch operation") do |s|
+        @options[:datestamp] = s
+      end
+
+      opts.on("--domain STRING", String, "Domain value to apply to each record") do |s|
+        @options[:domain] = s
+      end
+    end
+  end
+
+  def parse(argv = ARGV)
+    @command = argv.shift
+    filenames = @parser.parse(argv)
+
+    raise OptionParser::MissingArgument, "missing config" unless options[:config]
+    raise OptionParser::MissingArgument, "missing idfield" unless options[:idfield]
+    raise OptionParser::MissingArgument, "missing domain" unless options[:domain]
+    raise "missing filename" unless filenames.any?
+
+    # argv remnants are filenames
+    [@command, @options, filenames]
+  end
+
+  def execute(command, options, filenames = nil)
+    filename = filenames.first
+
+    case command
+    when 'load'
+      raise OptionParser::MissingArgument, "missing datestamp" unless options[:datestamp]
+
+      results = Assimilate.load(filename, options)
+      logmessage(command, options, results)
+
+    when 'extend'
+      raise OptionParser::MissingArgument, "missing keyfield" unless options[:key]
+
+      results = Assimilate.extend_data(filename, options)
+      logmessage(command, options, results)
+
+
+    else
+      raise "unknown command #{command}"
+    end
+  end
+
+  def logmessage(command, options, results)
+    $stderr.puts <<-EOT
+* assimilate #{command} (#{options.keys.join(', ')})
+EOT
+
+    case command
+    when 'load'
+      $stderr.puts <<-EOT
+    Original record count: #{results[:baseline_count]}
+       Final record count: #{results[:final_count]}
+        Unchanged records: #{results[:unchanged_count]}
+              New records: #{results[:adds_count]} (#{results[:new_ids].take(10).join(',')})
+                  Deletes: #{results[:deletes_count]}
+                  Updates: #{results[:updates_count]}
+EOT
+      if results[:updated_fields].any?
+        $stderr.puts <<-EOT
+            Counts by field:
+EOT
+        results[:updated_fields].each do |k,v|
+          $stderr.puts <<-EOT
+                        #{k}: #{v}
+EOT
+        end
+      end
+    when 'extend'
+      $stderr.puts <<-EOT
+    Original record count: #{results[:baseline_count]}
+       Final record count: #{results[:final_count]}
+          New identifiers: #{results[:adds_count]} #{options[:idfield]} (#{results[:new_ids].take(10).join(',')})
+             Distinct ids: #{results[:distinct_ids]}
+        Unchanged records: #{results[:unchanged_count]}
+                  Updates: #{results[:updates_count]}
+EOT
+      if results[:updated_fields].any?
+        results[:updated_fields].each do |k,v|
+          $stderr.puts <<-EOT
+                        #{options[:key]}.#{k}: #{v}
+EOT
+        end
+      end
+    else
+      $stderr.puts results.inspect
+    end
+  end
+end

data/lib/assimilate/extender.rb

@@ -0,0 +1,103 @@
+class Assimilate::Extender
+  attr_reader :domain, :idfield, :keyfield
+
+  def initialize(args)
+    @catalog = args[:catalog]
+    @domain = args[:domain]
+    @idfield = args[:idfield]
+    @filename = args[:filename]
+    @keyfield = args[:key]
+
+    load_baseline
+
+    @noops = []
+    @changes = []
+    @adds = []
+    @deletes = []
+  end
+
+  def load_baseline
+    stored_records = @catalog.catalog.find(@catalog.domainkey => @domain).to_a
+    @baseline = stored_records.each_with_object({}) do |rec, h|
+      key = rec[@idfield]
+      if h.include?(key)
+        raise Assimilate::CorruptDataError, "Duplicate records for key [#{key}] in domain [#{@domain}]"
+      end
+      h[key] = rec
+    end
+  end
+
+  def <<(record)
+    @seen ||= Hash.new(0)
+
+    hash = record.to_hash
+    key = hash[@idfield]
+    data = hash.reject {|k,v| k == idfield}
+    # @seen[key] = data
+    current_record = @baseline[key]
+    if current_record
+      if current_record[@keyfield] == data
+        @noops << key
+        @seen[key] = {}
+      else
+        @changes << key
+        @seen[key] = data
+      end
+    else
+      @adds << key
+      @seen[key] = data
+    end
+  end
+
+  def stats
+    {
+      :baseline_count => @baseline.size,
+      :final_count => @baseline.size + @adds.count,
+      :distinct_ids => @seen.size,
+      :adds_count => @adds.count,
+      :new_ids => @adds,
+      :updates_count => @changes.count,
+      :updated_fields => @seen.each_with_object(Hash.new(0)) {|(k,hash),memo| hash.each {|k,v| memo[k] += 1}},
+      :unchanged_count => @noops.count
+    }
+  end
+
+  # write all the changes to the catalog
+  def commit
+    apply_inserts
+    apply_updates
+  end
+
+  # an "insert" here means a record for which we have extended data
+  # but does not appear in the current catalog, so we need to create
+  # a stub entry.
+  def apply_inserts
+    @adds.each do |key|
+      data = @seen[key]
+      @catalog.catalog.insert(
+        @catalog.domainkey => domain,
+        idfield => key,
+        keyfield => data
+      )
+    end
+  end
+
+  # "update" means store the extended data in the record (which must exist)
+  def apply_updates
+    @changes.each do |key|
+      data = @seen[key]
+      @catalog.catalog.update(
+        {
+          @catalog.domainkey => domain,
+          idfield => key
+        },
+        {"$set" => {
+            keyfield => data
+          }
+        }
+      )
+    end
+  end
+
+
+end

data/lib/assimilate/version.rb

@@ -1,3 +1,3 @@
 module Assimilate
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/assimilate.rb

@@ -1,11 +1,52 @@
 require "mongo"
 require "active_support/core_ext" # needed for Hash#diff
+require "csv"
 
 require_relative "assimilate/version"
 
 require_relative "assimilate/catalog"
 require_relative "assimilate/batch"
+require_relative "assimilate/extender"
+
+require_relative "assimilate/command"
 
 module Assimilate
-    # Your code goes here...
+  def self.load(filename, opts = {})
+    begin
+      catalog = Catalog.new(:config => opts[:config])
+      batcher = catalog.start_batch(opts.merge(:filename => filename))
+
+      records = CSV.read(filename, :headers => true)
+      records.each do |rec|
+        batcher << rec
+      end
+      if opts[:commit]
+        batcher.commit
+      else
+        $stderr.puts "suppressing data commit"
+      end
+      batcher.stats
+    # TODO explicit handling for Assimilate exceptions - when code is stable
+    # rescue Assimilate::DuplicateImportError => e
+    #   $stderr.puts e.message
+    #   exit 1
+    end
+  end
+
+  def self.extend_data(filename, opts = {})
+    begin
+      catalog = Catalog.new(:config => opts[:config])
+      extender = catalog.extend_data(opts)
+      records = CSV.read(filename, :headers => true)
+      records.each do |rec|
+        extender << rec
+      end
+      if opts[:commit]
+        extender.commit
+      else
+        $stderr.puts "suppressing data commit"
+      end
+      extender.stats
+    end
+  end
 end

data/spec/data/batch_input.csv

@@ -4,3 +4,4 @@ ID,name,title
 3,Benjamin Franklin,Sage
 4,Aaron Burr,Duelist
 5,Alexander Hamilton,Financier
+6,James Madison,Theorist

data/spec/data/dates.csv

@@ -0,0 +1,5 @@
+ID,date
+1,1789/04/30
+2,1797/03/04
+6,1809/03/04
+16,1861/03/04

data/spec/data/duplicate_input.csv

@@ -0,0 +1,7 @@
+ID,name,title
+1,George Washington,President
+2,John Adams,Vice President
+3,Benjamin Franklin,Sage
+4,Aaron Burr,Duelist
+5,Alexander Hamilton,Financier
+6,James Madison,Theorist

data/spec/lib/batch_spec.rb

@@ -14,7 +14,7 @@ describe "importing file" do
   end
 
   def import_data(datestamp, filename = "batch_input.csv")
-    @batcher = @catalog.start_batch(domain: 'testdata', datestamp: datestamp, idfield: 'ID')
+    @batcher = @catalog.start_batch(domain: 'testdata', datestamp: datestamp, filename: filename, idfield: 'ID')
 
     @records = CSV.read(File.dirname(__FILE__) + "/../data/#{filename}", :headers => true)
     @records.each do |rec|
@@ -32,8 +32,9 @@ describe "importing file" do
     it "should return correct import stats" do
       @batcher.stats.should == {
         :baseline_count => 0,
-        :final_count => 5,
-        :adds_count => 5,
+        :final_count => 6,
+        :adds_count => 6,
+        :new_ids => ["1", "2", "3", "4", "5", "6"],
         :deletes_count => 0,
         :updates_count => 0,
         :unchanged_count => 0,
@@ -51,15 +52,20 @@ describe "importing file" do
       lambda {import_data("123")}.should raise_error(Assimilate::DuplicateImportError)
     end
 
+    it "should refuse to re-import same file" do
+      lambda {import_data("234")}.should raise_error(Assimilate::DuplicateImportError)
+    end
+
     it "should do all no-ops when importing identical data" do
-      lambda {import_data("234")}.should_not raise_error
+      lambda {import_data("234", "duplicate_input.csv")}.should_not raise_error
       @batcher.stats.should == {
-        :baseline_count => 5,
-        :final_count => 5,
+        :baseline_count => 6,
+        :final_count => 6,
         :adds_count => 0,
+        :new_ids => [],
         :deletes_count => 0,
         :updates_count => 0,
-        :unchanged_count => 5,
+        :unchanged_count => 6,
         :updated_fields => {}
       }
       @catalog.catalog.count.should == @records.count
@@ -78,10 +84,11 @@ describe "importing file" do
 
     it "should recognize changes" do
       @batcher.stats.should == {
-        :baseline_count => 5,
-        :final_count => 6,
+        :baseline_count => 6,
+        :final_count => 7,
         :adds_count => 1,
-        :deletes_count => 1,
+        :new_ids => ["7"],
+        :deletes_count => 2,
         :updates_count => 1,
         :unchanged_count => 3,
         :updated_fields => {'title' => 1}

data/spec/lib/extend_spec.rb

@@ -0,0 +1,77 @@
+# tests for extending the base records
+
+require "spec_helper"
+
+describe "loading extended data" do
+  before(:all) do
+    @catalog = Assimilate::Catalog.new(:config => File.dirname(__FILE__) + "/../data/test.yml")
+    reset_catalog
+  end
+
+  def reset_catalog
+    @catalog.catalog.remove
+    @catalog.batches.remove
+  end
+
+  def import_base_data(datestamp, filename = "batch_input.csv")
+    @batcher = @catalog.start_batch(domain: 'testdata', datestamp: datestamp, idfield: 'ID')
+
+    @records = CSV.read(File.dirname(__FILE__) + "/../data/#{filename}", :headers => true)
+    @records.each do |rec|
+      @batcher << rec
+    end
+    @batcher.commit
+  end
+
+  describe "into matching catalog entries" do
+    before(:all) do
+      reset_catalog
+      import_base_data("123")
+    end
+
+    def import_extended_data(datestamp, filename)
+      @extender = @catalog.extend_data(domain: 'testdata', datastamp: datestamp, idfield: 'ID', key: 'inauguration')
+      @records = CSV.read(File.dirname(__FILE__) + "/../data/#{filename}", :headers => true)
+      @records.each do |rec|
+        @extender << rec
+      end
+      @extender.commit
+    end
+
+    before(:each) do
+      import_extended_data("1001", "dates.csv")
+    end
+
+    it "should capture changes" do
+      @extender.stats.should == {
+        :baseline_count => 6,
+        :final_count => 7,
+        :distinct_ids => 4,
+        :adds_count => 1,
+        :new_ids => ['16'],
+        :updates_count => 3,
+        :updated_fields => {'date' => 4},
+        :unchanged_count => 0
+      }
+    end
+
+    it "should do no-ops on duplicate load" do
+      # import_extended_data("1002", "dates")
+      lambda {import_extended_data("1002", "dates.csv")}.should_not raise_error
+
+      @extender.stats.should == {
+        :baseline_count => 7,
+        :final_count => 7,
+        :distinct_ids => 4,
+        :adds_count => 0,
+        :new_ids => [],
+        :updates_count => 0,
+        :updated_fields => {},
+        :unchanged_count => 4
+      }
+    end
+  end
+
+  # test handling of multiple records for same ID in the extended-data file
+  # test importing data at top level (no keyfield for sub-attributes)
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: assimilate
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-26 00:00:00.000000000 Z
+date: 2012-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &2157149420 !ruby/object:Gem::Requirement
+  requirement: &2151862180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *2157149420
+  version_requirements: *2151862180
 - !ruby/object:Gem::Dependency
   name: bson_ext
-  requirement: &2157148260 !ruby/object:Gem::Requirement
+  requirement: &2151861480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *2157148260
+  version_requirements: *2151861480
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &2157147300 !ruby/object:Gem::Requirement
+  requirement: &2151860560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 3.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *2157147300
+  version_requirements: *2151860560
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2157146720 !ruby/object:Gem::Requirement
+  requirement: &2151859660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 2.9.0
   type: :development
   prerelease: false
-  version_requirements: *2157146720
+  version_requirements: *2151859660
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &2157146120 !ruby/object:Gem::Requirement
+  requirement: &2151856200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 0.7.0
   type: :development
   prerelease: false
-  version_requirements: *2157146120
+  version_requirements: *2151856200
 - !ruby/object:Gem::Dependency
   name: ruby_gntp
-  requirement: &2156851680 !ruby/object:Gem::Requirement
+  requirement: &2151855040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,11 +76,12 @@ dependencies:
         version: 0.3.4
   type: :development
   prerelease: false
-  version_requirements: *2156851680
+  version_requirements: *2151855040
 description: Ingest updates from CSV and apply to set of hashes
 email:
 - jmay@pobox.com
-executables: []
+executables:
+- assimilate
 extensions: []
 extra_rdoc_files: []
 files:
@@ -91,14 +92,20 @@ files:
 - README.md
 - Rakefile
 - assimilate.gemspec
+- bin/assimilate
 - lib/assimilate.rb
 - lib/assimilate/batch.rb
 - lib/assimilate/catalog.rb
+- lib/assimilate/command.rb
+- lib/assimilate/extender.rb
 - lib/assimilate/version.rb
 - spec/data/batch_input.csv
+- spec/data/dates.csv
+- spec/data/duplicate_input.csv
 - spec/data/test.yml
 - spec/data/updates.csv
 - spec/lib/batch_spec.rb
+- spec/lib/extend_spec.rb
 - spec/spec_helper.rb
 homepage: ''
 licenses: []
@@ -126,8 +133,11 @@ specification_version: 3
 summary: Review & incorporate changes to a repository of persistent hashes in mongodb.
 test_files:
 - spec/data/batch_input.csv
+- spec/data/dates.csv
+- spec/data/duplicate_input.csv
 - spec/data/test.yml
 - spec/data/updates.csv
 - spec/lib/batch_spec.rb
+- spec/lib/extend_spec.rb
 - spec/spec_helper.rb
 has_rdoc: