assimilate 0.0.1

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.rspec
+.irb_history

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in assimilate.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,8 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard 'rspec', :version => 2 do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1].gsub(/assimilate./, '')}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Jason May
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,36 @@
+# Assimilate
+
+Ingest updates from CSV and apply to set of persistent hashes
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'assimilate'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install assimilate
+
+## Usage
+
+    assimilate --config repo.yml filename
+
+or
+
+    > require 'assimilate'
+    > catalog = Assimilate::Catalog.new(:config => configfile)
+    > catalog.start_batch(:filename => filename, :datestamp => datestamp, :idfield => idfield)
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+

data/Rakefile

@@ -0,0 +1,8 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+# task :default => :spec

data/assimilate.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/assimilate/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Jason May"]
+  gem.email         = ["jmay@pobox.com"]
+  gem.description   = %q{Ingest updates from CSV and apply to set of hashes}
+  gem.summary       = %q{Review & incorporate changes to a repository of persistent hashes in mongodb.}
+  gem.homepage      = ""
+  gem.rubyforge_project = "assimilate"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "assimilate"
+  gem.require_paths = ["lib"]
+  gem.version       = Assimilate::VERSION
+
+  gem.add_dependency "mongo", "~> 1.6.0"
+  gem.add_dependency "bson_ext", "~> 1.6.0"
+  gem.add_dependency 'activesupport', "~> 3.2.0"
+
+  gem.add_development_dependency "rspec", "~> 2.9.0"
+  gem.add_development_dependency "guard-rspec", "~> 0.7.0"
+  gem.add_development_dependency "ruby_gntp", "~> 0.3.4"
+end

data/lib/assimilate/batch.rb

@@ -0,0 +1,139 @@
+class Assimilate::Batch
+  attr_reader :domain, :idfield, :datestamp
+
+  def initialize(args)
+    @catalog = args[:catalog]
+    @domain = args[:domain]
+    @datestamp = args[:datestamp]
+    @idfield = args[:idfield]
+    @filename = args[:filename]
+
+    load_baseline
+
+    @noops = []
+    @changes = []
+    @adds = []
+    @deletes = []
+  end
+
+  def load_baseline
+    stored_records = @catalog.catalog.find(@catalog.domainkey => @domain).to_a
+    @baseline = stored_records.each_with_object({}) do |rec, h|
+      key = rec[@idfield]
+      if h.include?(key)
+        raise Assimilate::CorruptDataError, "Duplicate records for key [#{key}] in domain [#{@domain}]"
+      end
+      h[key] = rec
+    end
+  end
+
+  def stripped_record_for(key)
+    @baseline[key] && @baseline[key].select {|k,v| k !~ /^_/}
+  end
+
+  def <<(record)
+    @seen ||= Hash.new(0)
+
+    hash = record.to_hash
+    key = hash[@idfield]
+    @seen[key] += 1
+    current_record = stripped_record_for(key)
+    if current_record
+      if current_record == hash
+        @noops << hash
+      else
+        @changes << hash
+      end
+    else
+      @adds << hash
+    end
+  end
+
+  # compute anything needed before we can write updates to permanent store
+  # * find records that have been deleted
+  def resolve
+    @deleted_keys = @baseline.keys - @seen.keys
+
+    @updated_field_counts = @changes.each_with_object(Hash.new(0)) do |rec,h|
+      key = rec[idfield]
+      diffs = rec.diff(stripped_record_for(key))
+      diffs.keys.each do |f|
+        h[f] += 1
+      end
+    end
+  end
+
+  def stats
+    resolve
+    {
+      :baseline_count => @baseline.size,
+      :final_count => @baseline.size + @adds.count,
+      :adds_count => @adds.count,
+      :deletes_count => @deleted_keys.count,
+      :updates_count => @changes.count,
+      :unchanged_count => @noops.count,
+      :updated_fields => @updated_field_counts
+    }
+  end
+
+  # write the updates to the catalog
+  def commit
+    resolve
+    record_batch
+    apply_deletes
+    apply_inserts
+    apply_updates
+  end
+
+  def record_batch
+    raise(Assimilate::DuplicateImportError, "duplicate batch") if @catalog.batches.find('domain' => @domain, 'datestamp' => @datestamp).to_a.any?
+    @catalog.batches.insert({
+      'domain' => @domain,
+      'datestamp' => @datestamp,
+      'filename' => @filename
+      })
+  end
+
+  def apply_deletes
+    @deleted_keys.each do |key|
+      @catalog.catalog.update(
+        {
+          @catalog.domainkey => domain,
+          idfield => key
+        },
+        {"$set" => {:_dt_removed => datestamp}}
+    )
+    end
+  end
+
+  INSERT_BATCH_SIZE = 1000 # default batch size for bulk loading into mongo
+
+  def apply_inserts
+    @adds.each_slice(INSERT_BATCH_SIZE) do |slice|
+      # mongo insert can't handle CSV::Row objects, must be converted to regular hashes
+      @catalog.catalog.insert(decorate(slice))
+    end
+  end
+
+  def apply_updates
+    @changes.each do |rec|
+      @catalog.catalog.update(
+        {
+          @catalog.domainkey => domain,
+          idfield => rec[idfield]
+        },
+        {"$set" => rec}
+      )
+    end
+  end
+
+  def decorate(records)
+    records.map do |r|
+      r[@catalog.domainkey] = @domain
+      r.to_hash
+    end
+  end
+end
+
+class Assimilate::DuplicateImportError < StandardError
+end

data/lib/assimilate/catalog.rb

@@ -0,0 +1,39 @@
+require "yaml"
+
+# Records in each catalog acquire the following internal attributes:
+#   _id             Unique ID, assigned by mongo
+#   _[domain]       Domain key, specified with :domainkey attribute when initializing catalog
+#   _dt_first_seen  Batch datestamp reference for when this record was first captured
+#   _dt_last_seen   Batch datestamp reference for when this record was most recently affirmed
+#   _dt_last_update Batch datestamp reference for when this record was most recently altered
+#   _dt_removed     Batch datestamp reference for when this record was removed from input
+#
+# Inbound records must not have attributes named with leading underscores.
+#
+# A "domain" here is a namespace of identifiers.
+
+class Assimilate::Catalog
+  attr_reader :catalog, :batches, :domainkey
+
+  def initialize(args)
+    @config = YAML.load(File.open(args[:config]))
+
+    @db = Mongo::Connection.new.db(@config['db'])
+    @catalog = @db.collection(@config['catalog'])
+    @batches = @db.collection(@config['batch'])
+    @domainkey = @config['domain']
+    @domainkey = "_#{@domainkey}" unless @domainkey =~ /^_/ # enforce leading underscore on internal attributes
+  end
+
+  def start_batch(args)
+    Assimilate::Batch.new(args.merge(:catalog => self))
+  end
+
+  def where(params)
+    @catalog.find(params).first.select {|k,v| k !~ /^_/}
+  end
+
+  def active_count
+    @catalog.find("_dt_removed" => nil).count
+  end
+end

data/lib/assimilate/version.rb

@@ -0,0 +1,3 @@
+module Assimilate
+  VERSION = "0.0.1"
+end

data/lib/assimilate.rb

@@ -0,0 +1,11 @@
+require "mongo"
+require "active_support/core_ext" # needed for Hash#diff
+
+require_relative "assimilate/version"
+
+require_relative "assimilate/catalog"
+require_relative "assimilate/batch"
+
+module Assimilate
+    # Your code goes here...
+end

data/spec/data/batch_input.csv

@@ -0,0 +1,6 @@
+ID,name,title
+1,George Washington,President
+2,John Adams,Vice President
+3,Benjamin Franklin,Sage
+4,Aaron Burr,Duelist
+5,Alexander Hamilton,Financier

data/spec/data/test.yml

@@ -0,0 +1,5 @@
+---
+db: assimilate-test
+catalog: forefathers
+batch: files
+domain: resource

data/spec/data/updates.csv

@@ -0,0 +1,6 @@
+ID,name,title
+1,George Washington,President
+2,John Adams,Vice President
+3,Benjamin Franklin,Ambassador
+5,Alexander Hamilton,Financier
+7,Thomas Jefferson,Anti-Federalist

data/spec/lib/batch_spec.rb

@@ -0,0 +1,92 @@
+# batch import tests
+
+require "spec_helper"
+
+describe "importing file" do
+  before(:all) do
+    @catalog = Assimilate::Catalog.new(:config => File.dirname(__FILE__) + "/../data/test.yml")
+    reset_catalog
+  end
+
+  def reset_catalog
+    @catalog.catalog.remove
+    @catalog.batches.remove
+  end
+
+  def import_data(datestamp, filename = "batch_input.csv")
+    @batcher = @catalog.start_batch(domain: 'testdata', datestamp: datestamp, idfield: 'ID')
+
+    @records = CSV.read(File.dirname(__FILE__) + "/../data/#{filename}", :headers => true)
+    @records.each do |rec|
+      @batcher << rec
+    end
+    @batcher.commit
+  end
+
+  describe "into empty catalog" do
+    before :each do
+      reset_catalog
+      import_data("123")
+    end
+
+    it "should return correct import stats" do
+      @batcher.stats.should == {
+        :baseline_count => 0,
+        :final_count => 5,
+        :adds_count => 5,
+        :deletes_count => 0,
+        :updates_count => 0,
+        :unchanged_count => 0,
+        :updated_fields => {}
+      }
+    end
+
+    it "should load the records verbatim" do
+      @catalog.catalog.count.should == @records.count
+      example = @records[rand(@records.count)]
+      @catalog.where('_resource' => 'testdata', 'ID' => example['ID']).should == example.to_hash
+    end
+
+    it "should refuse to do a duplicate import" do
+      lambda {import_data("123")}.should raise_error(Assimilate::DuplicateImportError)
+    end
+
+    it "should do all no-ops when importing identical data" do
+      lambda {import_data("234")}.should_not raise_error
+      @batcher.stats.should == {
+        :baseline_count => 5,
+        :final_count => 5,
+        :adds_count => 0,
+        :deletes_count => 0,
+        :updates_count => 0,
+        :unchanged_count => 5,
+        :updated_fields => {}
+      }
+      @catalog.catalog.count.should == @records.count
+    end
+  end
+
+  describe "into existing catalog" do
+    before(:all) do
+      reset_catalog
+      import_data("123")
+    end
+
+    before(:each) do
+      import_data("345", "updates.csv")
+    end
+
+    it "should recognize changes" do
+      @batcher.stats.should == {
+        :baseline_count => 5,
+        :final_count => 6,
+        :adds_count => 1,
+        :deletes_count => 1,
+        :updates_count => 1,
+        :unchanged_count => 3,
+        :updated_fields => {'title' => 1}
+      }
+      @catalog.active_count.should == @records.count
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+require 'rspec/autorun'
+require "tempfile"
+require "csv"
+
+require File.expand_path('../../lib/assimilate', __FILE__)
+
+RSpec.configure do |config|
+end

metadata

@@ -0,0 +1,133 @@
+--- !ruby/object:Gem::Specification
+name: assimilate
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Jason May
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-04-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: &2157149420 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *2157149420
+- !ruby/object:Gem::Dependency
+  name: bson_ext
+  requirement: &2157148260 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *2157148260
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: &2157147300 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *2157147300
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &2157146720 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.9.0
+  type: :development
+  prerelease: false
+  version_requirements: *2157146720
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: &2157146120 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+  type: :development
+  prerelease: false
+  version_requirements: *2157146120
+- !ruby/object:Gem::Dependency
+  name: ruby_gntp
+  requirement: &2156851680 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.4
+  type: :development
+  prerelease: false
+  version_requirements: *2156851680
+description: Ingest updates from CSV and apply to set of hashes
+email:
+- jmay@pobox.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Guardfile
+- LICENSE
+- README.md
+- Rakefile
+- assimilate.gemspec
+- lib/assimilate.rb
+- lib/assimilate/batch.rb
+- lib/assimilate/catalog.rb
+- lib/assimilate/version.rb
+- spec/data/batch_input.csv
+- spec/data/test.yml
+- spec/data/updates.csv
+- spec/lib/batch_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: assimilate
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Review & incorporate changes to a repository of persistent hashes in mongodb.
+test_files:
+- spec/data/batch_input.csv
+- spec/data/test.yml
+- spec/data/updates.csv
+- spec/lib/batch_spec.rb
+- spec/spec_helper.rb
+has_rdoc: