replicate 1.0

data/lib/replicate/emitter.rb

@@ -0,0 +1,54 @@
+module Replicate
+  # Base class for Dumper / Loader classes. Manages a list of callback listeners
+  # and dispatches to each when #emit is called.
+  class Emitter
+    # Yields self to the block and calls #complete when block is finished.
+    def initialize
+      @listeners = []
+      if block_given?
+        yield self
+        complete
+      end
+    end
+
+    # Register a listener to be called for each loaded object with the
+    # type, id, attributes, object structure. Listeners are executed in the
+    # reverse order of which they were registered. Listeners registered later
+    # modify the view of listeners registered earlier.
+    #
+    # p     - An optional Proc object. Must respond to call.
+    # block - An optional block.
+    #
+    # Returns nothing.
+    def listen(p=nil, &block)
+      @listeners.unshift p if p
+      @listeners.unshift block if block
+    end
+
+    # Sugar for creating a listener with an object instance. Instances of the
+    # class must respond to call(type, id, attrs, object).
+    #
+    # klass - The class to create. Must respond to new.
+    # args  - Arguments to pass to new in addition to self.
+    #
+    # Returns the object created.
+    def use(klass, *args, &block)
+      instance = klass.new(self, *args, &block)
+      listen instance
+      instance
+    end
+
+    # Emit an object event to each listener.
+    #
+    # Returns the object.
+    def emit(type, id, attributes, object)
+      @listeners.each { |p| p.call(type, id, attributes, object) }
+      object
+    end
+
+    # Notify all listeners that processing is complete.
+    def complete
+      @listeners.each { |p| p.complete if p.respond_to?(:complete) }
+    end
+  end
+end

data/lib/replicate/loader.rb

@@ -0,0 +1,133 @@
+module Replicate
+  # Load replicants in a streaming fashion.
+  #
+  # The Loader reads [type, id, attributes] replicant tuples and creates
+  # objects in the current environment.
+  #
+  # Objects are expected to arrive in order such that a record referenced via
+  # foreign key always precedes the referencing record. The Loader maintains a
+  # mapping of primary keys from the dump system to the current environment.
+  # This mapping is used to properly establish new foreign key values on all
+  # records inserted.
+  class Loader < Emitter
+
+    # Stats hash.
+    attr_reader :stats
+
+    def initialize
+      @keymap = Hash.new { |hash,k| hash[k] = {} }
+      @stats  = Hash.new { |hash,k| hash[k] = 0 }
+      super
+    end
+
+    # Register a filter to write status information to the given stream. By
+    # default, a single line is used to report object counts while the dump is
+    # in progress; dump counts for each class are written when complete. The
+    # verbose and quiet options can be used to increase or decrease
+    # verbository.
+    #
+    # out - An IO object to write to, like stderr.
+    # verbose - Whether verbose output should be enabled.
+    # quiet - Whether quiet output should be enabled.
+    #
+    # Returns the Replicate::Status object.
+    def log_to(out=$stderr, verbose=false, quiet=false)
+      use Replicate::Status, 'load', out, verbose, quiet
+    end
+
+    # Feed a single replicant tuple into the loader.
+    #
+    # type  - The class to create. Must respond to load_replicant.
+    # id    - The remote system's id for this object.
+    # attrs - Hash of primitively typed objects.
+    #
+    # Returns the need object resulting from the load operation.
+    def feed(type, id, attrs)
+      type = type.to_s
+      object = load(type, id, attrs)
+      @stats[type] += 1
+      emit type, id, attrs, object
+    end
+
+    # Read multiple [type, id, attrs] replicant tuples from io and call the
+    # feed method to load and filter the object.
+    def read(io)
+      begin
+        while object = Marshal.load(io)
+          type, id, attrs = object
+          feed type, id, attrs
+        end
+      rescue EOFError
+      end
+    end
+
+    # Load an individual replicant into the underlying datastore.
+    #
+    # type  - Model class name as a String.
+    # id    - Primary key id of the record on the dump system. This must be
+    #         translated to the local system and stored in the keymap.
+    # attrs - Hash of attributes to set on the new record.
+    #
+    # Returns the new object instance.
+    def load(type, id, attributes)
+      model_class = constantize(type)
+      translate_ids attributes
+      begin
+        new_id, instance = model_class.load_replicant(type, id, attributes)
+      rescue => boom
+        warn "error: loading #{type} #{id} #{boom.class} #{boom}"
+        raise
+      end
+      register_id instance, type, id, new_id
+      instance
+    end
+
+    # Translate remote system id references in the attributes hash to their
+    # local system id values. The attributes hash may include special id
+    # values like this:
+    #     { 'title'         => 'hello there',
+    #       'repository_id' => [:id, 'Repository', 1234],
+    #       'label_ids'     => [:id, 'Label', [333, 444, 555, 666, ...]]
+    #       ... }
+    # These values are translated to local system ids. All object
+    # references must be loaded prior to the referencing object.
+    def translate_ids(attributes)
+      attributes.each do |key, value|
+        next unless value.is_a?(Array) && value[0] == :id
+        type, value = value[1].to_s, value[2]
+        local_ids =
+          Array(value).map do |remote_id|
+            if local_id = @keymap[type][remote_id]
+              local_id
+            else
+              warn "error: #{type} #{remote_id} missing from keymap"
+            end
+          end
+        if value.is_a?(Array)
+          attributes[key] = local_ids
+        else
+          attributes[key] = local_ids[0]
+        end
+      end
+    end
+
+    # Register an id in the keymap. Every object loaded must be stored here so
+    # that key references can be resolved.
+    def register_id(object, type, remote_id, local_id)
+      @keymap[type.to_s][remote_id] = local_id
+      c = object.class
+      while !['Object', 'ActiveRecord::Base'].include?(c.name)
+        @keymap[c.name][remote_id] = local_id
+        c = c.superclass
+      end
+    end
+
+
+    # Turn a string into an object by traversing constants.
+    def constantize(string)
+      namespace = Object
+      string.split('::').each { |name| namespace = namespace.const_get(name) }
+      namespace
+    end
+  end
+end

data/lib/replicate/object.rb

@@ -0,0 +1,57 @@
+module Replicate
+  # Simple OpenStruct style object that supports the dump and load protocols.
+  # Useful in tests and also when you want to dump an object that doesn't
+  # implement the dump and load methods.
+  #
+  #     >> object = Replicate::Object.new :name => 'Joe', :age => 24
+  #     >> object.age
+  #     >> 24
+  #     >> object.attributes
+  #     { 'name' => 'Joe', 'age' => 24 }
+  #
+  class Object
+    attr_accessor :id
+    attr_accessor :attributes
+
+    def initialize(id=nil, attributes={})
+      attributes, id = id, nil if id.is_a?(Hash)
+      @id = id || self.class.generate_id
+      self.attributes = attributes
+    end
+
+    def attributes=(hash)
+      @attributes = {}
+      hash.each { |key, value| write_attribute key, value }
+    end
+
+    def [](key)
+      @attributes[key.to_s]
+    end
+
+    def []=(key, value)
+      @attributes[key.to_s] = value
+    end
+
+    def write_attribute(key, value)
+      meta = (class<<self;self;end)
+      meta.send(:define_method, key) { value }
+      meta.send(:define_method, "#{key}=") { |val| write_attribute(key, val) }
+      @attributes[key.to_s] = value
+      value
+    end
+
+    def dump_replicant(dumper)
+      dumper.write self.class, @id, @attributes, self
+    end
+
+    def self.load_replicant(type, id, attrs)
+      object = new(generate_id, attrs)
+      [object.id, object]
+    end
+
+    def self.generate_id
+      @last_id ||= 0
+      @last_id += 1
+    end
+  end
+end

data/lib/replicate/status.rb

@@ -0,0 +1,54 @@
+module Replicate
+  # Replicate filter that writes info to the console. Used by the Dumper and
+  # Loader to get basic console output.
+  class Status
+    def initialize(dumper, prefix, out, verbose=false, quiet=false)
+      @dumper = dumper
+      @prefix = prefix
+      @out = out
+      @verbose = verbose
+      @quiet = quiet
+      @count = 0
+    end
+
+    def call(type, id, attrs, object)
+      @count += 1
+      if @verbose
+        verbose_log type, id, attrs, object
+      elsif !@quiet
+        normal_log type, id, attrs, object
+      end
+    end
+
+    def verbose_log(type, id, attrs, object)
+      desc_attr = %w[name login email number title].find { |k| attrs.key?(k) }
+      desc = desc_attr ? attrs[desc_attr] : id
+      @out.puts "#{@prefix}: %-30s %s" % [type.sub('Replicate::', ''), desc]
+    end
+
+    def normal_log(type, id, attrs, object)
+      @out.write "==> #{@prefix}ing: #{@count} objects      \r"
+    end
+
+    def complete
+      dump_stats if !@quiet
+    end
+
+    def dump_stats(stats=@dumper.stats.dup)
+      @out.puts "==> #{@prefix}ed #{@count} total objects:    "
+      width = 0
+      stats.keys.each do |key|
+        class_name = format_class_name(key)
+        stats[class_name] = stats.delete(key)
+        width = class_name.size if class_name.size > width
+      end
+      stats.to_a.sort_by { |k,n| k }.each do |class_name, count|
+        @out.write "%-#{width + 1}s %5d\n" % [class_name, count]
+      end
+    end
+
+    def format_class_name(class_name)
+      class_name.sub(/Replicate::/, '')
+    end
+  end
+end

data/test/active_record_test.rb

@@ -0,0 +1,253 @@
+require 'test/unit'
+require 'stringio'
+require 'active_record'
+require 'replicate'
+
+dbfile = File.expand_path('../db', __FILE__)
+File.unlink dbfile if File.exist?(dbfile)
+ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => dbfile)
+
+ActiveRecord::Migration.verbose = false
+ActiveRecord::Schema.define do
+  create_table "users", :force => true do |t|
+    t.string   "login"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+  create_table "profiles", :force => true do |t|
+    t.integer  "user_id"
+    t.string   "name"
+    t.string   "homepage"
+  end
+
+  create_table "emails", :force => true do |t|
+    t.integer  "user_id"
+    t.string   "email"
+    t.datetime "created_at"
+  end
+end
+
+class User < ActiveRecord::Base
+  has_one  :profile, :dependent => :destroy
+  has_many :emails,  :dependent => :destroy, :order => 'id'
+
+  replicate_natural_key :login
+end
+
+class Profile < ActiveRecord::Base
+  belongs_to :user
+
+  replicate_natural_key :user_id
+end
+
+class Email < ActiveRecord::Base
+  belongs_to :user
+
+  replicate_natural_key :user_id, :email
+end
+
+class ActiveRecordTest < Test::Unit::TestCase
+  def setup
+    self.class.fixtures
+    ActiveRecord::Base.connection.increment_open_transactions
+    ActiveRecord::Base.connection.begin_db_transaction
+
+    @rtomayko = User.find_by_login('rtomayko')
+    @kneath   = User.find_by_login('kneath')
+    @tmm1     = User.find_by_login('tmm1')
+
+    User.replicate_associations = []
+
+    @dumper = Replicate::Dumper.new
+    @loader = Replicate::Loader.new
+  end
+
+  def teardown
+    ActiveRecord::Base.connection.rollback_db_transaction
+    ActiveRecord::Base.connection.decrement_open_transactions
+  end
+
+  def self.fixtures
+    return if @fixtures
+    @fixtures = true
+    user = User.create! :login => 'rtomayko'
+    user.create_profile :name => 'Ryan Tomayko', :homepage => 'http://tomayko.com'
+    user.emails.create! :email => 'ryan@github.com'
+    user.emails.create! :email => 'rtomayko@gmail.com'
+
+    user = User.create! :login => 'kneath'
+    user.create_profile :name => 'Kyle Neath', :homepage => 'http://warpspire.com'
+    user.emails.create! :email => 'kyle@github.com'
+
+    user = User.create! :login => 'tmm1'
+    user.create_profile :name => 'tmm1', :homepage => 'https://github.com/tmm1'
+  end
+
+  def test_extension_modules_loaded
+    assert User.respond_to?(:load_replicant)
+    assert User.new.respond_to?(:dump_replicant)
+  end
+
+  def test_auto_dumping_belongs_to_associations
+    objects = []
+    @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+    rtomayko = User.find_by_login('rtomayko')
+    @dumper.dump rtomayko.profile
+
+    assert_equal 2, objects.size
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'User', type
+    assert_equal rtomayko.id, id
+    assert_equal 'rtomayko', attrs['login']
+    assert_equal rtomayko.created_at, attrs['created_at']
+    assert_equal rtomayko, obj
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'Profile', type
+    assert_equal rtomayko.profile.id, id
+    assert_equal 'Ryan Tomayko', attrs['name']
+    assert_equal rtomayko.profile, obj
+  end
+
+  def test_auto_dumping_has_one_associations
+    objects = []
+    @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+    rtomayko = User.find_by_login('rtomayko')
+    @dumper.dump rtomayko
+
+    assert_equal 2, objects.size
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'User', type
+    assert_equal rtomayko.id, id
+    assert_equal 'rtomayko', attrs['login']
+    assert_equal rtomayko.created_at, attrs['created_at']
+    assert_equal rtomayko, obj
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'Profile', type
+    assert_equal rtomayko.profile.id, id
+    assert_equal 'Ryan Tomayko', attrs['name']
+    assert_equal [:id, 'User', rtomayko.id], attrs['user_id']
+    assert_equal rtomayko.profile, obj
+  end
+
+  def test_dumping_has_many_associations
+    objects = []
+    @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+    User.replicate_associations :emails
+    rtomayko = User.find_by_login('rtomayko')
+    @dumper.dump rtomayko
+
+    assert_equal 4, objects.size
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'User', type
+    assert_equal rtomayko.id, id
+    assert_equal 'rtomayko', attrs['login']
+    assert_equal rtomayko.created_at, attrs['created_at']
+    assert_equal rtomayko, obj
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'Profile', type
+    assert_equal rtomayko.profile.id, id
+    assert_equal 'Ryan Tomayko', attrs['name']
+    assert_equal rtomayko.profile, obj
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'Email', type
+    assert_equal 'ryan@github.com', attrs['email']
+    assert_equal [:id, 'User', rtomayko.id], attrs['user_id']
+    assert_equal rtomayko.emails.first, obj
+
+    type, id, attrs, obj = objects.shift
+    assert_equal 'Email', type
+    assert_equal 'rtomayko@gmail.com', attrs['email']
+    assert_equal [:id, 'User', rtomayko.id], attrs['user_id']
+    assert_equal rtomayko.emails.last, obj
+  end
+
+  def test_loading_everything
+    objects = []
+    @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+    # dump all users and associated objects and destroy
+    User.replicate_associations :emails
+    dumped_users = {}
+    %w[rtomayko kneath tmm1].each do |login|
+      user = User.find_by_login(login)
+      @dumper.dump user
+      user.destroy
+      dumped_users[login] = user
+    end
+    assert_equal 9, objects.size
+
+    # insert another record to ensure id changes for loaded records
+    sr = User.create!(:login => 'sr')
+    sr.create_profile :name => 'Simon Rozet'
+    sr.emails.create :email => 'sr@github.com'
+
+    # load everything back up
+    objects.each { |type, id, attrs, obj| @loader.feed type, id, attrs }
+
+    # verify attributes are set perfectly again
+    user = User.find_by_login('rtomayko')
+    assert_equal 'rtomayko', user.login
+    assert_equal dumped_users['rtomayko'].created_at, user.created_at
+    assert_equal dumped_users['rtomayko'].updated_at, user.updated_at
+    assert_equal 'Ryan Tomayko', user.profile.name
+    assert_equal 2, user.emails.size
+
+    # make sure everything was recreated
+    %w[rtomayko kneath tmm1].each do |login|
+      user = User.find_by_login(login)
+      assert_not_nil user
+      assert_not_nil user.profile
+      assert !user.emails.empty?, "#{login} has no emails" if login != 'tmm1'
+    end
+  end
+
+  def test_loading_with_existing_records
+    objects = []
+    @dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
+
+    # dump all users and associated objects and destroy
+    User.replicate_associations :emails
+    dumped_users = {}
+    %w[rtomayko kneath tmm1].each do |login|
+      user = User.find_by_login(login)
+      user.profile.update_attribute :name, 'CHANGED'
+      @dumper.dump user
+      dumped_users[login] = user
+    end
+    assert_equal 9, objects.size
+
+    # load everything back up
+    objects.each { |type, id, attrs, obj| @loader.feed type, id, attrs }
+
+    # ensure additional objects were not created
+    assert_equal 3, User.count
+
+    # verify attributes are set perfectly again
+    user = User.find_by_login('rtomayko')
+    assert_equal 'rtomayko', user.login
+    assert_equal dumped_users['rtomayko'].created_at, user.created_at
+    assert_equal dumped_users['rtomayko'].updated_at, user.updated_at
+    assert_equal 'CHANGED', user.profile.name
+    assert_equal 2, user.emails.size
+
+    # make sure everything was recreated
+    %w[rtomayko kneath tmm1].each do |login|
+      user = User.find_by_login(login)
+      assert_not_nil user
+      assert_not_nil user.profile
+      assert_equal 'CHANGED', user.profile.name
+      assert !user.emails.empty?, "#{login} has no emails" if login != 'tmm1'
+    end
+  end
+end