replicate 1.0

data/COPYING

@@ -0,0 +1,18 @@
+Copyright (c) 2011 Ryan Tomayko <http://tomayko.com/about>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to
+deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,99 @@
+# Replicate
+
+Dump and load relational objects between Ruby environments.
+
+The project was started at GitHub to ease the process of getting real production
+data into staging and development environments. We have a custom command that
+uses the replicate machinery to dump entire repository data (including
+associated objects like issues, pull requests, commit comments, etc.) from
+production and load it into the current environment. This is excessively useful
+for troubleshooting issues, support requests, and exception reports.
+
+## Synopsis
+
+Dumping objects:
+
+    $ replicate -r config/environment -d 'User.find(1)' > user.dump
+    ==> dumped 4 total objects:
+    Profile        1
+    User           1
+    UserEmail      2
+
+Loading objects:
+
+    $ replicate -r config/environment -l < user.dump
+    ==> loaded 4 total objects:
+    Profile        1
+    User           1
+    UserEmail      2
+
+Dumping and loading over SSH:
+
+    $ remote_command="replicate -r /app/config/environment -d 'User.find(1234)'"
+    $ ssh example.org "$remote_command" |replicate -r config/environment -l
+
+## ActiveRecord
+
+*NOTE: Replicate has been tested only under ActiveRecord 2.2. Support for
+ActiveRecord 3.x is planned.*
+
+Basic support for dumping and loading ActiveRecord objects is included. When an
+object is dumped, all `belongs_to` and `has_one` associations are automatically
+followed and included in the dump. You can mark `has_many` and
+`has_and_belongs_to_many` associations for automatic inclusion using the
+`replicate_associations` macro:
+
+    class User < ActiveRecord::Base
+      belongs_to :profile
+      has_many   :email_addresses
+
+      replicate_associations :email_addresses
+    end
+
+By default, the loader attempts to create a new record for all objects. This can
+lead to unique constraint errors when a record already exists with matching
+attributes. To update existing records instead of always creating new ones,
+define a natural key for the model using the `replicate_natural_key` macro:
+
+    class User < ActiveRecord::Base
+      belongs_to :profile
+      has_many   :email_addresses
+
+      replicate_natural_key :login
+      replicate_associations :email_addresses
+    end
+
+Multiple attribute names may be specified to define a compound key.
+
+## Custom Objects
+
+Other object types may be included in the dump stream so long as they implement
+the `dump_replicant` and `load_replicant` methods.
+
+The dump side calls `#dump_replicant(dumper)` on each object. The method must
+call `dumper.write()` with the class name, id, and hash of primitively typed
+attributes for the object:
+
+    class User
+      attr_reader   :id
+      attr_accessor :name, :email
+
+      def dump_replicant(dumper)
+        attributes { 'name' => name, 'email' => email }
+        dumper.write self.class, id, attributes
+      end
+    end
+
+The load side calls `::load_replicant(type, id, attributes)` on the class to
+load each object into the current environment. The method must return an
+`[id, object]` tuple:
+
+    class User
+      def self.load_replicant(type, id, attributes)
+        user = User.new
+        user.name  = attributes['name']
+        user.email = attributes['email']
+        user.save!
+        [user.id, user]
+      end
+    end

data/Rakefile

@@ -0,0 +1,6 @@
+task :default => :test
+
+desc "Run tests"
+task :test do
+  sh "testrb test/*_test.rb", :verbose => false
+end

data/bin/replicate

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+#/        script/replicate [-r <lib>] --dump "<ruby>" > objects.dump
+#/        script/replicate [-r <lib>] --load < objects.dump
+#/ Dump and load objects between environments. The --dump form writes to stdout
+#/ the objects returned by evaluating "<ruby>", which must be a valid Ruby
+#/ expression. The --load form reads dump data from stdin and loads into the
+#/ current environment.
+#/
+#/ Options:
+#/   -r, --require        Require the library. Often used with 'config/environment'.
+#/   -d, --dump           Dump the repository and all related objects to stdout.
+#/   -l, --load           Load dump file data from stdin.
+#/
+#/   -v, --verbose        Write more status output.
+#/   -q, --quiet          Write less status output.
+$stderr.sync = true
+require 'optparse'
+
+# default options
+mode     = nil
+verbose  = false
+quiet    = false
+out      = $stdout
+
+# parse arguments
+file = __FILE__
+usage = lambda { exec "grep ^#/<'#{file}'|cut -c4-" }
+ARGV.options do |opts|
+  opts.on("-d", "--dump")      { mode = :dump }
+  opts.on("-l", "--load")      { mode = :load }
+  opts.on("-r", "--require=f") { |file| require file }
+  opts.on("-v", "--verbose")   { verbose = true }
+  opts.on("-q", "--quiet")     { quiet = true }
+  opts.on_tail("-h", "--help", &usage)
+  opts.parse!
+end
+
+# load rails environment and replicator lib.
+require 'replicate'
+
+# hack to enable AR query cache
+if defined?(ActiveRecord::Base)
+  ActiveRecord::ConnectionAdapters::QueryCache.
+    send :attr_writer, :query_cache, :query_cache_enabled
+  ActiveRecord::Base.connection.send(:query_cache=, {})
+  ActiveRecord::Base.connection.send(:query_cache_enabled=, true)
+end
+
+# dump mode means we're reading records from the database here and writing to
+# stdout. the database should not be modified at all by this operation.
+if mode == :dump
+  usage.call if ARGV.empty? || ARGV[0].empty?
+  objects = eval(ARGV[0])
+  Replicate::Dumper.new do |dumper|
+    dumper.marshal_to out
+    dumper.log_to $stderr, verbose, quiet
+    dumper.dump objects
+  end
+
+# load mode means we're reading objects from stdin and creating them under
+# the current environment.
+elsif mode == :load
+  Replicate::Loader.new do |loader|
+    loader.log_to $stderr, verbose, quiet
+    loader.read $stdin
+  end
+
+# mode not set means no -l or -d arg was given. show usage and bail.
+else
+  usage.call
+end

data/lib/replicate.rb

@@ -0,0 +1,10 @@
+module Replicate
+  autoload :Emitter, 'replicate/emitter'
+  autoload :Dumper,  'replicate/dumper'
+  autoload :Loader,  'replicate/loader'
+  autoload :Object,  'replicate/object'
+  autoload :Status,  'replicate/status'
+  autoload :AR,      'replicate/active_record'
+
+  AR if defined?(::ActiveRecord::Base)
+end

data/lib/replicate/active_record.rb

@@ -0,0 +1,217 @@
+module Replicate
+  # ActiveRecord::Base instance methods used to dump replicant objects for the
+  # record and all 1:1 associations. This module implements the replicant_id
+  # and dump_replicant methods using AR's reflection API to determine
+  # relationships with other objects.
+  module AR
+    # Mixin for the ActiveRecord instance.
+    module InstanceMethods
+      # Replicate::Dumper calls this method on objects to trigger dumping a
+      # replicant object tuple. The default implementation dumps all belongs_to
+      # associations, then self, then all has_one associations, then any
+      # has_many or has_and_belongs_to_many associations declared with the
+      # replicate_associations macro.
+      #
+      # dumper - Dumper object whose #write method must be called with the
+      #          type, id, and attributes hash.
+      #
+      # Returns nothing.
+      def dump_replicant(dumper)
+        dump_all_association_replicants dumper, :belongs_to
+        dumper.write self.class.to_s, id, replicant_attributes, self
+        dump_all_association_replicants dumper, :has_one
+        self.class.replicate_associations.each do |association|
+          dump_association_replicants dumper, association
+        end
+      end
+
+      # Attributes hash used to persist this object. This consists of simply
+      # typed values (no complex types or objects) with the exception of special
+      # foreign key values. When an attribute value is [:id, "SomeClass:1234"],
+      # the loader will handle translating the id value to the local system's
+      # version of the same object.
+      def replicant_attributes
+        attributes = self.attributes.dup
+        self.class.reflect_on_all_associations(:belongs_to).each do |reflection|
+          foreign_key = (reflection.options[:foreign_key] || "#{reflection.name}_id").to_s
+          if id = attributes[foreign_key]
+            attributes[foreign_key] = [:id, reflection.klass.to_s, id]
+          end
+        end
+        attributes
+      end
+
+      # The replicant id is a two tuple containing the class and object id. This
+      # is used by Replicant::Dumper to determine if the object has already been
+      # dumped or not.
+      def replicant_id
+        [self.class.name, id]
+      end
+
+      # Dump all associations of a given type.
+      #
+      # dumper           - The Dumper object used to dump additional objects.
+      # association_type - :has_one, :belongs_to, :has_many
+      #
+      # Returns nothing.
+      def dump_all_association_replicants(dumper, association_type)
+        self.class.reflect_on_all_associations(association_type).each do |reflection|
+          next if (dependent = __send__(reflection.name)).nil?
+          case dependent
+          when ActiveRecord::Base, Array
+            dumper.dump(dependent)
+          else
+            warn "warn: #{self.class}##{reflection.name} #{association_type} association " \
+                 "unexpectedly returned a #{dependent.class}. skipping."
+          end
+        end
+      end
+
+      # Dump objects associated with an AR object through an association name.
+      #
+      # object      - AR object instance.
+      # association - Name of the association whose objects should be dumped.
+      #
+      # Returns nothing.
+      def dump_association_replicants(dumper, association)
+        if reflection = self.class.reflect_on_association(association)
+          objects = __send__(reflection.name)
+          dumper.dump(objects)
+          if reflection.macro == :has_and_belongs_to_many
+            dump_has_and_belongs_to_many_replicant(dumper, reflection)
+          end
+        else
+          warn "error: #{self.class}##{association} is invalid"
+        end
+      end
+
+      # Dump the special Habtm object used to establish many-to-many
+      # relationships between objects that have already been dumped. Note that
+      # this object and all objects referenced must have already been dumped
+      # before calling this method.
+      def dump_has_and_belongs_to_many_replicant(dumper, reflection)
+        dumper.dump Habtm.new(self, reflection)
+      end
+    end
+
+    # Mixin for the ActiveRecord class.
+    module ClassMethods
+      # Set and retrieve list of association names that should be dumped when
+      # objects of this class are dumped. This method may be called multiple
+      # times to add associations.
+      def replicate_associations(*names)
+        self.replicate_associations += names if names.any?
+        @replicate_associations || superclass.replicate_associations
+      end
+
+      # Set the list of association names to dump to the specific set of values.
+      def replicate_associations=(names)
+        @replicate_associations = names.uniq.map { |name| name.to_sym }
+      end
+
+      # Compound key used during load to locate existing objects for update.
+      # When no natural key is defined, objects are created new.
+      #
+      # attribute_names - Macro style setter.
+      def replicate_natural_key(*attribute_names)
+        self.replicate_natural_key = attribute_names if attribute_names.any?
+        @replicate_natural_key || superclass.replicate_natural_key
+      end
+
+      # Set the compound key used to locate existing objects for update when
+      # loading. When not set, loading will always create new records.
+      #
+      # attribute_names - Array of attribute name symbols
+      def replicate_natural_key=(attribute_names)
+        @replicate_natural_key = attribute_names
+      end
+
+      # Load an individual record into the database. If the models defines a
+      # replicate_natural_key then an existing record will be updated if found
+      # instead of a new record being created.
+      #
+      # type  - Model class name as a String.
+      # id    - Primary key id of the record on the dump system. This must be
+      #         translated to the local system and stored in the keymap.
+      # attrs - Hash of attributes to set on the new record.
+      #
+      # Returns the ActiveRecord object instance for the new record.
+      def load_replicant(type, id, attributes)
+        instance = replicate_find_existing_record(attributes) || new
+        create_or_update_replicant instance, attributes
+      end
+
+      # Locate an existing record using the replicate_natural_key attribute
+      # values.
+      #
+      # Returns the existing record if found, nil otherwise.
+      def replicate_find_existing_record(attributes)
+        return if replicate_natural_key.empty?
+        conditions = {}
+        replicate_natural_key.each do |attribute_name|
+          conditions[attribute_name] = attributes[attribute_name.to_s]
+        end
+        find(:first, :conditions => conditions)
+      end
+
+      # Update an AR object's attributes and persist to the database without
+      # running validations or callbacks.
+      def create_or_update_replicant(instance, attributes)
+        def instance.callback(*args);end # Rails 2.x hack to disable callbacks.
+
+        attributes.each do |key, value|
+          next if key == primary_key
+          instance.write_attribute key, value
+        end
+
+        instance.save false
+        [instance.id, instance]
+      end
+    end
+
+    # Special object used to dump the list of associated ids for a
+    # has_and_belongs_to_many association. The object includes attributes for
+    # locating the source object and writing the list of ids to the appropriate
+    # association method.
+    class Habtm
+      def initialize(object, reflection)
+        @object = object
+        @reflection = reflection
+      end
+
+      def id
+      end
+
+      def attributes
+        ids = @object.__send__("#{@reflection.name.to_s.singularize}_ids")
+        {
+          'id'         => [:id, @object.class.to_s, @object.id],
+          'class'      => @object.class.to_s,
+          'ref_class'  => @reflection.klass.to_s,
+          'ref_name'   => @reflection.name.to_s,
+          'collection' => [:id, @reflection.klass.to_s, ids]
+        }
+      end
+
+      def dump_replicant(dumper)
+        type = self.class.name
+        id   = "#{@object.class.to_s}:#{@reflection.name}:#{@object.id}"
+        dumper.write type, id, attributes, self
+      end
+
+      def self.load_replicant(type, id, attrs)
+        object = attrs['class'].constantize.find(attrs['id'])
+        ids    = attrs['collection']
+        object.__send__("#{attrs['ref_name'].to_s.singularize}_ids=", ids)
+        [id, new(object, nil)]
+      end
+    end
+
+    # Load active record and install the extension methods.
+    require 'active_record'
+    ::ActiveRecord::Base.send :include, InstanceMethods
+    ::ActiveRecord::Base.send :extend,  ClassMethods
+    ::ActiveRecord::Base.replicate_associations = []
+    ::ActiveRecord::Base.replicate_natural_key  = []
+  end
+end

data/lib/replicate/dumper.rb

@@ -0,0 +1,109 @@
+module Replicate
+  # Dump replicants in a streaming fashion.
+  #
+  # The Dumper takes objects and generates one or more replicant objects. A
+  # replicant has the form [type, id, attributes] and describes exactly one
+  # addressable record in a datastore. The type and id identify the model
+  # class name and model primary key id. The attributes Hash is a set of attribute
+  # name to primitively typed object value mappings.
+  #
+  # Example dump session:
+  #
+  #     >> Replicate::Dumper.new do |dumper|
+  #     >>   dumper.marshal_to $stdout
+  #     >>   dumper.log_to $stderr
+  #     >>   dumper.dump User.find(1234)
+  #     >> end
+  #
+  class Dumper < Emitter
+    # Create a new Dumper.
+    #
+    # io     - IO object to write marshalled replicant objects to.
+    # block  - Dump context block. If given, the end of the block's execution
+    #          is assumed to be the end of the dump stream.
+    def initialize(io=nil)
+      @memo = Hash.new { |hash,k| hash[k] = {} }
+      super() do
+        marshal_to io if io
+        yield self if block_given?
+      end
+    end
+
+    # Register a filter to write marshalled data to the given IO object.
+    def marshal_to(io)
+      listen { |type, id, attrs, obj| Marshal.dump([type, id, attrs], io) }
+    end
+
+    # Register a filter to write status information to the given stream. By
+    # default, a single line is used to report object counts while the dump is
+    # in progress; dump counts for each class are written when complete. The
+    # verbose and quiet options can be used to increase or decrease
+    # verbosity.
+    #
+    # out - An IO object to write to, like stderr.
+    # verbose - Whether verbose output should be enabled.
+    # quiet - Whether quiet output should be enabled.
+    #
+    # Returns the Replicate::Status object.
+    def log_to(out=$stderr, verbose=false, quiet=false)
+      use Replicate::Status, 'dump', out, verbose, quiet
+    end
+
+    # Dump one or more objects to the internal array or provided dump
+    # stream. This method guarantees that the same object will not be dumped
+    # more than once.
+    #
+    # objects - ActiveRecord object instances.
+    #
+    # Returns nothing.
+    def dump(*objects)
+      objects = objects[0] if objects.size == 1 && objects[0].respond_to?(:to_ary)
+      objects.each do |object|
+        next if object.nil? || dumped?(object)
+        if object.respond_to?(:dump_replicant)
+          object.dump_replicant(self)
+        else
+          raise NoMethodError, "#{object.class} must respond to #dump_replicant"
+        end
+      end
+    end
+
+    # Check if object has been written yet.
+    def dumped?(object)
+      if object.respond_to?(:replicant_id)
+        type, id = object.replicant_id
+      elsif object.is_a?(Array)
+        type, id = object
+      else
+        return false
+      end
+      @memo[type.to_s][id]
+    end
+
+    # Called exactly once per unique type and id. Emits to all listeners.
+    #
+    # type       - The model class name as a String.
+    # id         - The record's id. Usually an integer.
+    # attributes - All model attributes.
+    # object     - The object this dump is generated for.
+    #
+    # Returns the object.
+    def write(type, id, attributes, object)
+      type = type.to_s
+      return if dumped?([type, id])
+      @memo[type][id] = true
+
+      emit type, id, attributes, object
+    end
+
+    # Retrieve dumped object counts for all classes.
+    #
+    # Returns a Hash of { class_name => count } where count is the number of
+    # objects dumped with a class of class_name.
+    def stats
+      stats = {}
+      @memo.each { |class_name, items| stats[class_name] = items.size }
+      stats
+    end
+  end
+end