arcopy 0.0.1

data/lib/replicate/emitter.rb

@@ -0,0 +1,54 @@
+module Replicate
+  # Base class for Dumper / Loader classes. Manages a list of callback listeners
+  # and dispatches to each when #emit is called.
+  class Emitter
+    # Yields self to the block and calls #complete when block is finished.
+    def initialize
+      @listeners = []
+      if block_given?
+        yield self
+        complete
+      end
+    end
+
+    # Register a listener to be called for each loaded object with the
+    # type, id, attributes, object structure. Listeners are executed in the
+    # reverse order of which they were registered. Listeners registered later
+    # modify the view of listeners registered earlier.
+    #
+    # p     - An optional Proc object. Must respond to call.
+    # block - An optional block.
+    #
+    # Returns nothing.
+    def listen(p=nil, &block)
+      @listeners.unshift p if p
+      @listeners.unshift block if block
+    end
+
+    # Sugar for creating a listener with an object instance. Instances of the
+    # class must respond to call(type, id, attrs, object).
+    #
+    # klass - The class to create. Must respond to new.
+    # args  - Arguments to pass to new in addition to self.
+    #
+    # Returns the object created.
+    def use(klass, *args, &block)
+      instance = klass.new(self, *args, &block)
+      listen instance
+      instance
+    end
+
+    # Emit an object event to each listener.
+    #
+    # Returns the object.
+    def emit(type, id, attributes, object)
+      @listeners.each { |p| p.call(type, id, attributes, object) }
+      object
+    end
+
+    # Notify all listeners that processing is complete.
+    def complete
+      @listeners.each { |p| p.complete if p.respond_to?(:complete) }
+    end
+  end
+end

data/lib/replicate/loader.rb

@@ -0,0 +1,157 @@
+module Replicate
+  # Load replicants in a streaming fashion.
+  #
+  # The Loader reads [type, id, attributes] replicant tuples and creates
+  # objects in the current environment.
+  #
+  # Objects are expected to arrive in order such that a record referenced via
+  # foreign key always precedes the referencing record. The Loader maintains a
+  # mapping of primary keys from the dump system to the current environment.
+  # This mapping is used to properly establish new foreign key values on all
+  # records inserted.
+  class Loader < Emitter
+
+    # Stats hash.
+    attr_reader :stats
+
+    def initialize
+      @keymap = Hash.new { |hash,k| hash[k] = {} }
+      @stats  = Hash.new { |hash,k| hash[k] = 0 }
+      super
+    end
+
+    # Register a filter to write status information to the given stream. By
+    # default, a single line is used to report object counts while the dump is
+    # in progress; dump counts for each class are written when complete. The
+    # verbose and quiet options can be used to increase or decrease
+    # verbository.
+    #
+    # out - An IO object to write to, like stderr.
+    # verbose - Whether verbose output should be enabled.
+    # quiet - Whether quiet output should be enabled.
+    #
+    # Returns the Replicate::Status object.
+    def log_to(out=$stderr, verbose=false, quiet=false)
+      use Replicate::Status, 'load', out, verbose, quiet
+    end
+
+    # Feed a single replicant tuple into the loader.
+    #
+    # type  - The class to create. Must respond to load_replicant.
+    # id    - The remote system's id for this object.
+    # attrs - Hash of primitively typed objects.
+    #
+    # Returns the need object resulting from the load operation.
+    def feed(type, id, attrs)
+      type = type.to_s
+      object = load(type, id, attrs)
+      @stats[type] += 1
+      emit type, id, attrs, object
+    end
+
+    # Read multiple [type, id, attrs] replicant tuples from io and call the
+    # feed method to load and filter the object.
+    def read(io)
+      ActiveRecord::Base.transaction do
+        begin
+          if ActiveRecord::Base.connection.instance_values["config"][:adapter] == "postgresql"
+            ActiveRecord::Base.connection.execute("set constraints all deferred")
+          end
+          while object = Marshal.load(io)
+            type, id, attrs = object
+            feed type, id, attrs
+          end
+        rescue EOFError
+        end
+      end
+    end
+
+    # Load an individual replicant into the underlying datastore.
+    #
+    # type  - Model class name as a String.
+    # id    - Primary key id of the record on the dump system. This must be
+    #         translated to the local system and stored in the keymap.
+    # attrs - Hash of attributes to set on the new record.
+    #
+    # Returns the new object instance.
+    def load(type, id, attributes)
+      model_class = constantize(type)
+      translate_ids type, id, attributes
+      begin
+        new_id, instance = model_class.load_replicant(type, id, attributes)
+      rescue => boom
+        warn "error: loading #{type} #{id} #{boom.class} #{boom}"
+        raise
+      end
+      register_id instance, type, id, new_id
+      instance
+    end
+
+    # Translate remote system id references in the attributes hash to their
+    # local system id values. The attributes hash may include special id
+    # values like this:
+    #     { 'title'         => 'hello there',
+    #       'repository_id' => [:id, 'Repository', 1234],
+    #       'label_ids'     => [:id, 'Label', [333, 444, 555, 666, ...]]
+    #       ... }
+    # These values are translated to local system ids. All object
+    # references must be loaded prior to the referencing object.
+    def translate_ids(type, id, attributes)
+      attributes.each do |key, value|
+        next unless value.is_a?(Array) && value[0] == :id
+        referenced_type, value = value[1].to_s, value[2]
+        local_ids =
+          Array(value).map do |remote_id|
+            if local_id = @keymap[referenced_type][remote_id]
+              local_id
+            else
+              warn "warn: #{referenced_type}(#{remote_id}) not in keymap, " +
+                   "referenced by #{type}(#{id})##{key}"
+            end
+          end
+        if value.is_a?(Array)
+          attributes[key] = local_ids
+        else
+          attributes[key] = local_ids[0]
+        end
+      end
+    end
+
+    # Register an id in the keymap. Every object loaded must be stored here so
+    # that key references can be resolved.
+    def register_id(object, type, remote_id, local_id)
+      @keymap[type.to_s][remote_id] = local_id
+      c = object.class
+      while !['Object', 'ActiveRecord::Base'].include?(c.name)
+        @keymap[c.name][remote_id] = local_id
+        c = c.superclass
+      end
+    end
+
+    # Turn a string into an object by traversing constants. Identical to
+    # ActiveSupport's String#constantize implementation.
+    if Module.method(:const_get).arity == 1
+      # 1.8 implementation doesn't have the inherit argument to const_defined?
+      def constantize(string)
+        string.split('::').inject ::Object do |namespace, name|
+          if namespace.const_defined?(name)
+            namespace.const_get(name)
+          else
+            namespace.const_missing(name)
+          end
+        end
+      end
+    else
+      # 1.9 implement has the inherit argument to const_defined?. Use it!
+      def constantize(string)
+        string.split('::').inject ::Object do |namespace, name|
+          if namespace.const_defined?(name, false)
+            namespace.const_get(name)
+          else
+            namespace.const_missing(name)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/replicate/object.rb

@@ -0,0 +1,57 @@
+module Replicate
+  # Simple OpenStruct style object that supports the dump and load protocols.
+  # Useful in tests and also when you want to dump an object that doesn't
+  # implement the dump and load methods.
+  #
+  #     >> object = Replicate::Object.new :name => 'Joe', :age => 24
+  #     >> object.age
+  #     >> 24
+  #     >> object.attributes
+  #     { 'name' => 'Joe', 'age' => 24 }
+  #
+  class Object
+    attr_accessor :id
+    attr_accessor :attributes
+
+    def initialize(id=nil, attributes={})
+      attributes, id = id, nil if id.is_a?(Hash)
+      @id = id || self.class.generate_id
+      self.attributes = attributes
+    end
+
+    def attributes=(hash)
+      @attributes = {}
+      hash.each { |key, value| write_attribute key, value }
+    end
+
+    def [](key)
+      @attributes[key.to_s]
+    end
+
+    def []=(key, value)
+      @attributes[key.to_s] = value
+    end
+
+    def write_attribute(key, value)
+      meta = (class<<self;self;end)
+      meta.send(:define_method, key) { value }
+      meta.send(:define_method, "#{key}=") { |val| write_attribute(key, val) }
+      @attributes[key.to_s] = value
+      value
+    end
+
+    def dump_replicant(dumper, opts={})
+      dumper.write self.class, @id, @attributes, self
+    end
+
+    def self.load_replicant(type, id, attrs)
+      object = new(generate_id, attrs)
+      [object.id, object]
+    end
+
+    def self.generate_id
+      @last_id ||= 0
+      @last_id += 1
+    end
+  end
+end

data/lib/replicate/status.rb

@@ -0,0 +1,54 @@
+module Replicate
+  # Replicate filter that writes info to the console. Used by the Dumper and
+  # Loader to get basic console output.
+  class Status
+    def initialize(dumper, prefix, out, verbose=false, quiet=false)
+      @dumper = dumper
+      @prefix = prefix
+      @out = out
+      @verbose = verbose
+      @quiet = quiet
+      @count = 0
+    end
+
+    def call(type, id, attrs, object)
+      @count += 1
+      if @verbose
+        verbose_log type, id, attrs, object
+      elsif !@quiet
+        normal_log type, id, attrs, object
+      end
+    end
+
+    def verbose_log(type, id, attrs, object)
+      desc_attr = %w[name login email number title].find { |k| attrs.key?(k) }
+      desc = desc_attr ? attrs[desc_attr] : id
+      @out.puts "#{@prefix}: %-30s %s" % [type.sub('Replicate::', ''), desc]
+    end
+
+    def normal_log(type, id, attrs, object)
+      @out.write "  %sing: %4d objects    \r" % [@prefix, @count]
+    end
+
+    def complete
+      dump_stats if !@quiet
+    end
+
+    def dump_stats(stats=@dumper.stats.dup)
+      @out.puts "==> #{@prefix}ed #{@count} total objects:    "
+      width = 0
+      stats.keys.each do |key|
+        class_name = format_class_name(key)
+        stats[class_name] = stats.delete(key)
+        width = class_name.size if class_name.size > width
+      end
+      stats.to_a.sort_by { |k,n| k }.each do |class_name, count|
+        @out.write "%-#{width + 1}s %5d\n" % [class_name, count]
+      end
+    end
+
+    def format_class_name(class_name)
+      class_name.sub(/Replicate::/, '')
+    end
+  end
+end

data/test/active_record_test.rb

@@ -0,0 +1,589 @@
+$VERBOSE = nil
+require_relative 'test_helper'
+require 'minitest/around/unit'
+require 'stringio'
+require 'active_record'
+require 'active_record/version'
+version = ActiveRecord::VERSION::STRING
+warn "Using activerecord #{version}"
+require 'replicate'
+
+# create the sqlite db on disk
+dbfile = File.expand_path('../db', __FILE__)
+File.unlink dbfile if File.exist?(dbfile)
+ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => dbfile)
+# load schema
+ActiveRecord::Migration.verbose = false
+ActiveRecord::Schema.define do
+  create_table "users", :force => true do |t|
+    t.string   "login"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+  create_table "profiles", :force => true do |t|
+    t.integer  "user_id"
+    t.string   "name"
+    t.string   "homepage"
+  end
+
+  create_table "emails", :force => true do |t|
+    t.integer  "user_id"
+    t.string   "email"
+    t.datetime "created_at"
+  end
+
+  if version[0,3] > '2.2'
+    create_table "domains", :force => true do |t|
+      t.string "host"
+    end
+
+    create_table "web_pages", :force => true do |t|
+      t.string "url"
+      t.string "domain_host"
+    end
+  end
+
+  create_table "notes", :force => true do |t|
+    t.integer "notable_id"
+    t.string  "notable_type"
+  end
+
+  create_table "namespaced", :force => true
+end
+
+# models
+class User < ActiveRecord::Base
+  has_one  :profile, :dependent => :destroy
+  has_many :emails,  -> { order('id') }, :dependent => :destroy
+  has_many :notes,   :as => :notable
+  replicate_natural_key :login
+end
+
+class Profile < ActiveRecord::Base
+  belongs_to :user
+  replicate_natural_key :user_id
+end
+
+class Email < ActiveRecord::Base
+  belongs_to :user
+  replicate_natural_key :user_id, :email
+end
+
+class WebPage < ActiveRecord::Base
+  belongs_to :domain, :foreign_key => 'domain_host', :primary_key => 'host'
+end
+
+class Domain < ActiveRecord::Base
+  replicate_natural_key :host
+end
+
+class Note < ActiveRecord::Base
+  belongs_to :notable, :polymorphic => true
+end
+
+class User::Namespaced < ActiveRecord::Base
+  self.table_name = "namespaced"
+end
+
+# The test case loads some fixture data once and uses transaction rollback to
+# reset fixture state for each test's setup.
+class ActiveRecordTest < Minitest::Test
+  i_suck_and_my_tests_are_order_dependent!
+
+  def around(&block)
+    ActiveRecord::Base.connection.transaction do
+      self.class.fixtures
+
+      @rtomayko = User.find_by_login('rtomayko')
+      @kneath   = User.find_by_login('kneath')
+      @tmm1     = User.find_by_login('tmm1')
+
+      User.replicate_associations = []
+
+      @dumper = Replicate::Dumper.new
+      @loader = Replicate::Loader.new
+      yield
+      raise ActiveRecord::Rollback
+    end
+  end
+
+  def self.fixtures
+    user = User.create! :login => 'rtomayko'
+    user.create_profile :name => 'Ryan Tomayko', :homepage => 'http://tomayko.com'
+    user.emails.create! :email => 'ryan@github.com'
+    user.emails.create! :email => 'rtomayko@gmail.com'
+
+    user = User.create! :login => 'kneath'
+    user.create_profile :name => 'Kyle Neath', :homepage => 'http://warpspire.com'
+    user.emails.create! :email => 'kyle@github.com'
+
+    user = User.create! :login => 'tmm1'
+    user.create_profile :name => 'tmm1', :homepage => 'https://github.com/tmm1'
+
+    github = Domain.create! :host => 'github.com'
+    WebPage.create! :url => 'http://github.com/about', :domain => github
+  end
+
+  def test_extension_modules_loaded
+    assert User.respond_to?(:load_replicant)
+    assert User.new.respond_to?(:dump_replicant)
+  end
+
+  def test_auto_dumping_belongs_to_associations
+    objects = []
+    @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+    rtomayko = User.find_by_login('rtomayko')
+    @dumper.dump rtomayko&.profile
+
+    assert_equal 2, objects.size
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'User', type
+    assert_equal rtomayko.id, id
+    assert_equal 'rtomayko', attrs['login']
+    assert_equal rtomayko.created_at, attrs['created_at']
+    assert_equal rtomayko, obj
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'Profile', type
+    assert_equal rtomayko.profile.id, id
+    assert_equal 'Ryan Tomayko', attrs['name']
+    assert_equal rtomayko.profile, obj
+  end
+
+  def test_omit_dumping_of_attribute
+    objects = []
+    @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+    User.replicate_omit_attributes :created_at
+    rtomayko = User.find_by_login('rtomayko')
+    @dumper.dump rtomayko
+
+    assert_equal 2, objects.size
+
+    _type, _id, attrs, _obj = objects.shift
+    assert_nil attrs['created_at']
+  end
+
+  def test_omit_dumping_of_association
+    objects = []
+    @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+    User.replicate_omit_attributes :profile
+    rtomayko = User.find_by_login('rtomayko')
+    @dumper.dump rtomayko
+
+    assert_equal 1, objects.size
+
+    type, _id, _attrs, _obj = objects.shift
+    assert_equal 'User', type
+  end
+
+  if ActiveRecord::VERSION::STRING[0, 3] > '2.2'
+    def test_dump_and_load_non_standard_foreign_key_association
+      objects = []
+      @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+      github_about_page = WebPage.find_by_url('http://github.com/about')
+      assert_equal "github.com", github_about_page&.domain&.host
+      @dumper.dump github_about_page
+
+      WebPage.delete_all
+      Domain.delete_all
+
+      # load everything back up
+      objects.each { |type, id, attrs, obj| @loader.feed type, id, attrs }
+
+      github_about_page = WebPage.find_by_url('http://github.com/about')
+      assert_equal "github.com", github_about_page.domain_host
+      assert_equal "github.com", github_about_page.domain.host
+    end
+  end
+
+  def test_auto_dumping_has_one_associations
+    objects = []
+    @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+    rtomayko = User.find_by_login('rtomayko')
+    @dumper.dump rtomayko
+
+    assert_equal 2, objects.size
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'User', type
+    assert_equal rtomayko.id, id
+    assert_equal 'rtomayko', attrs['login']
+    assert_equal rtomayko.created_at, attrs['created_at']
+    assert_equal rtomayko, obj
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'Profile', type
+    assert_equal rtomayko.profile.id, id
+    assert_equal 'Ryan Tomayko', attrs['name']
+    assert_equal [:id, 'User', rtomayko.id], attrs['user_id']
+    assert_equal rtomayko.profile, obj
+  end
+
+  def test_auto_dumping_does_not_fail_on_polymorphic_associations
+    objects = []
+    @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+    rtomayko = User.find_by_login('rtomayko')
+    note = Note.create!(:notable => rtomayko)
+    @dumper.dump note
+
+    assert_equal 3, objects.size
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'User', type
+    assert_equal rtomayko.id, id
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'Profile', type
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'Note', type
+    assert_equal note.id, id
+    assert_equal note.notable_type, attrs['notable_type']
+    assert_equal attrs["notable_id"], [:id, 'User', rtomayko.id]
+    assert_equal note, obj
+  end
+
+  def test_dumping_has_many_associations
+    objects = []
+    @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+    User.replicate_associations :emails
+    rtomayko = User.find_by_login('rtomayko')
+    @dumper.dump rtomayko
+
+    assert_equal 4, objects.size
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'User', type
+    assert_equal rtomayko.id, id
+    assert_equal 'rtomayko', attrs['login']
+    assert_equal rtomayko.created_at, attrs['created_at']
+    assert_equal rtomayko, obj
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'Profile', type
+    assert_equal rtomayko.profile.id, id
+    assert_equal 'Ryan Tomayko', attrs['name']
+    assert_equal rtomayko.profile, obj
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'Email', type
+    assert_equal 'ryan@github.com', attrs['email']
+    assert_equal [:id, 'User', rtomayko.id], attrs['user_id']
+    assert_equal rtomayko.emails.first, obj
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'Email', type
+    assert_equal 'rtomayko@gmail.com', attrs['email']
+    assert_equal [:id, 'User', rtomayko.id], attrs['user_id']
+    assert_equal rtomayko.emails.last, obj
+  end
+
+  def test_dumping_associations_at_dump_time
+    objects = []
+    @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+    rtomayko = User.find_by_login('rtomayko')
+    @dumper.dump rtomayko, :associations => [:emails], :omit => [:profile]
+
+    assert_equal 3, objects.size
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'User', type
+    assert_equal rtomayko.id, id
+    assert_equal 'rtomayko', attrs['login']
+    assert_equal rtomayko.created_at, attrs['created_at']
+    assert_equal rtomayko, obj
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'Email', type
+    assert_equal 'ryan@github.com', attrs['email']
+    assert_equal [:id, 'User', rtomayko.id], attrs['user_id']
+    assert_equal rtomayko.emails.first, obj
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'Email', type
+    assert_equal 'rtomayko@gmail.com', attrs['email']
+    assert_equal [:id, 'User', rtomayko.id], attrs['user_id']
+    assert_equal rtomayko.emails.last, obj
+  end
+
+  def test_dumping_many_associations_at_dump_time
+    objects = []
+    @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+    users = User.where(:login => %w[rtomayko kneath])
+    @dumper.dump users, :associations => [:emails], :omit => [:profile]
+
+    assert_equal 5, objects.size
+    assert_equal ['Email', 'Email', 'Email', 'User', 'User'], objects.map { |type,_,_| type }.sort
+  end
+
+  def test_omit_attributes_at_dump_time
+    objects = []
+    @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+    rtomayko = User.find_by_login('rtomayko')
+    @dumper.dump rtomayko, :omit => [:created_at]
+
+    type, _id, attrs, _obj = objects.shift
+    assert_equal 'User', type
+    assert attrs['updated_at']
+    assert_nil attrs['created_at']
+  end
+
+  def test_dumping_polymorphic_associations
+    objects = []
+    @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+    User.replicate_associations :notes
+    rtomayko = User.find_by_login('rtomayko')
+    note = Note.create!(:notable => rtomayko)
+    @dumper.dump rtomayko
+
+    assert_equal 3, objects.size
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'User', type
+    assert_equal rtomayko.id, id
+    assert_equal 'rtomayko', attrs['login']
+    assert_equal rtomayko.created_at, attrs['created_at']
+    assert_equal rtomayko, obj
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'Profile', type
+    assert_equal rtomayko.profile.id, id
+    assert_equal 'Ryan Tomayko', attrs['name']
+    assert_equal rtomayko.profile, obj
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'Note', type
+    assert_equal note.notable_type, attrs['notable_type']
+    assert_equal [:id, 'User', rtomayko.id], attrs['notable_id']
+    assert_equal rtomayko.notes.first, obj
+
+  end
+
+  def test_dumping_empty_polymorphic_associations
+    objects = []
+    @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+    note = Note.create!()
+    @dumper.dump note
+
+    assert_equal 1, objects.size
+
+    type, _id, attrs, _obj = objects.shift
+    assert_equal 'Note', type
+    assert_nil attrs['notable_type']
+    assert_nil attrs['notable_id']
+  end
+
+  def test_dumps_polymorphic_namespaced_associations
+    objects = []
+    @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+    note = Note.create! :notable => User::Namespaced.create!
+    @dumper.dump note
+
+    assert_equal 2, objects.size
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'User::Namespaced', type
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'Note', type
+  end
+
+  def test_skips_belongs_to_information_if_omitted
+    objects = []
+    @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+    Profile.replicate_omit_attributes :user
+    @dumper.dump @rtomayko&.profile
+
+    assert_equal 1, objects.size
+    _type, _id, attrs, _obj = objects.shift
+    assert_equal @rtomayko.profile.user_id, attrs["user_id"]
+  end
+
+  def test_loading_everything
+    objects = []
+    @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+    # dump all users and associated objects and destroy
+    User.replicate_associations :emails
+    dumped_users = {}
+    %w[rtomayko kneath tmm1].each do |login|
+      user = User.find_by_login(login)
+      @dumper.dump user
+      user&.destroy
+      dumped_users[login] = user
+    end
+    assert_equal 9, objects.size
+
+    # insert another record to ensure id changes for loaded records
+    sr = User.create!(:login => 'sr')
+    sr.create_profile :name => 'Simon Rozet'
+    sr.emails.create :email => 'sr@github.com'
+
+    # load everything back up
+    objects.each { |type, id, attrs, obj| @loader.feed type, id, attrs }
+
+    # verify attributes are set perfectly again
+    user = User.find_by_login('rtomayko')
+    assert_equal 'rtomayko', user.login
+    assert_equal dumped_users['rtomayko'].created_at, user.created_at
+    assert_equal dumped_users['rtomayko'].updated_at, user.updated_at
+    assert_equal 'Ryan Tomayko', user.profile.name
+    assert_equal 2, user.emails.size
+
+    # make sure everything was recreated
+    %w[rtomayko kneath tmm1].each do |login|
+      user = User.find_by_login(login)
+      assert user
+      assert user&.profile
+      assert !user.emails.empty?, "#{login} has no emails" if login != 'tmm1'
+    end
+  end
+
+  def test_loading_with_existing_records
+    objects = []
+    @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+    # dump all users and associated objects and destroy
+    User.replicate_associations :emails
+    dumped_users = {}
+    %w[rtomayko kneath tmm1].each do |login|
+      user = User.find_by_login(login)
+      user&.profile&.update_attribute :name, 'CHANGED'
+      @dumper.dump user
+      dumped_users[login] = user
+    end
+    assert_equal 9, objects.size
+
+    # load everything back up
+    objects.each { |type, id, attrs, obj| @loader.feed type, id, attrs }
+
+    # ensure additional objects were not created
+    assert_equal 3, User.count
+
+    # verify attributes are set perfectly again
+    user = User.find_by_login('rtomayko')
+    assert_equal 'rtomayko', user.login
+    assert_equal dumped_users['rtomayko'].created_at, user.created_at
+    assert_equal dumped_users['rtomayko'].updated_at, user.updated_at
+    assert_equal 'CHANGED', user.profile.name
+    assert_equal 2, user.emails.size
+
+    # make sure everything was recreated
+    %w[rtomayko kneath tmm1].each do |login|
+      user = User.find_by_login(login)
+      assert user
+      assert user&.profile
+      assert_equal 'CHANGED', user.profile.name
+      assert !user.emails.empty?, "#{login} has no emails" if login != 'tmm1'
+    end
+  end
+
+  def test_loading_with_replicating_id
+    objects = []
+    @dumper.listen do |type, id, attrs, obj|
+      objects << [type, id, attrs, obj] if type == 'User'
+    end
+
+    dumped_users = {}
+    %w[rtomayko kneath tmm1].each do |login|
+      user = User.find_by_login(login)
+      @dumper.dump user
+      dumped_users[login] = user
+    end
+    assert_equal 3, objects.size
+
+    User.destroy_all
+    User.replicate_id = false
+
+    # load everything back up
+    objects.each { |type, id, attrs, obj| User.load_replicant type, id, attrs }
+
+    user = User.find_by_login('rtomayko')
+    assert_operator dumped_users['rtomayko'].id, :!=,  user.id
+
+    User.destroy_all
+    User.replicate_id = true
+
+    # load everything back up
+    objects.each { |type, id, attrs, obj| User.load_replicant type, id, attrs }
+
+    user = User.find_by_login('rtomayko')
+    assert_equal dumped_users['rtomayko'].id, user.id
+  end
+
+  def test_loader_saves_without_validations
+    # note when a record is saved with validations
+    ran_validations = false
+    User.class_eval { validate { ran_validations = true } }
+
+    # check our assumptions
+    user = User.create(:login => 'defunkt')
+    assert ran_validations, "should run validations here"
+    ran_validations = false
+
+    # load one and verify validations are not run
+    user = nil
+    @loader.listen { |type, id, attrs, obj| user = obj }
+    @loader.feed 'User', 1, 'login' => 'rtomayko'
+    assert user
+    assert !ran_validations, 'validations should not run on save'
+  end
+
+  def test_loader_saves_without_callbacks
+    # note when a record is saved with callbacks
+    callbacks = false
+    User.class_eval { after_save { callbacks = true } }
+    User.class_eval { after_create { callbacks = true } }
+    User.class_eval { after_update { callbacks = true } }
+    User.class_eval { after_commit { callbacks = true } }
+
+    # check our assumptions
+    user = User.create(:login => 'defunkt')
+    assert callbacks, "should run callbacks here"
+    callbacks = false
+
+    # load one and verify validations are not run
+    user = nil
+    @loader.listen { |type, id, attrs, obj| user = obj }
+    @loader.feed 'User', 1, 'login' => 'rtomayko'
+    assert user
+    assert !callbacks, 'callbacks should not run on save'
+  end
+
+  def test_loader_saves_without_updating_created_at_timestamp
+    timestamp = Time.at((Time.now - (24 * 60 * 60)).to_i)
+    user = nil
+    @loader.listen { |type, id, attrs, obj| user = obj }
+    @loader.feed 'User', 23, 'login' => 'brianmario', 'created_at' => timestamp
+    assert_equal timestamp, user.created_at
+    user = User.find(user.id)
+    assert_equal timestamp, user.created_at
+  end
+
+  def test_loader_saves_without_updating_updated_at_timestamp
+    timestamp = Time.at((Time.now - (24 * 60 * 60)).to_i)
+    user = nil
+    @loader.listen { |type, id, attrs, obj| user = obj }
+    @loader.feed 'User', 29, 'login' => 'rtomayko', 'updated_at' => timestamp
+    assert_equal timestamp, user.updated_at
+    user = User.find(user.id)
+    assert_equal timestamp, user.updated_at
+  end
+  
+end