arcopy 0.0.1

data/lib/replicate.rb

@@ -0,0 +1,24 @@
+module Replicate
+  autoload :Emitter, 'replicate/emitter'
+  autoload :Dumper,  'replicate/dumper'
+  autoload :Loader,  'replicate/loader'
+  autoload :Object,  'replicate/object'
+  autoload :Status,  'replicate/status'
+  autoload :AR,      'replicate/active_record'
+
+  # Determine if this is a production looking environment. Used in bin/replicate
+  # to safeguard against loading in production.
+  def self.production_environment?
+    if defined?(Rails) && Rails.respond_to?(:env)
+      Rails.env.to_s == 'production'
+    elsif defined?(RAILS_ENV)
+      RAILS_ENV == 'production'
+    elsif ENV['RAILS_ENV']
+      ENV['RAILS_ENV'] == 'production'
+    elsif ENV['RACK_ENV']
+      ENV['RACK_ENV'] == 'production'
+    end
+  end
+
+  AR if defined?(::ActiveRecord::Base)
+end

data/lib/replicate/active_record.rb

@@ -0,0 +1,347 @@
+require 'active_record'
+
+module Replicate
+  # ActiveRecord::Base instance methods used to dump replicant objects for the
+  # record and all 1:1 associations. This module implements the replicant_id
+  # and dump_replicant methods using AR's reflection API to determine
+  # relationships with other objects.
+  module AR
+    # Mixin for the ActiveRecord instance.
+    module InstanceMethods
+      # Replicate::Dumper calls this method on objects to trigger dumping a
+      # replicant object tuple. The default implementation dumps all belongs_to
+      # associations, then self, then all has_one associations, then any
+      # has_many or has_and_belongs_to_many associations declared with the
+      # replicate_associations macro.
+      #
+      # dumper - Dumper object whose #write method must be called with the
+      #          type, id, and attributes hash.
+      #
+      # Returns nothing.
+      def dump_replicant(dumper, opts={})
+        @replicate_opts = opts
+        @replicate_opts[:associations] ||= []
+        @replicate_opts[:omit] ||= []
+        dump_all_association_replicants dumper, :belongs_to
+        dumper.write self.class.to_s, id, replicant_attributes, self
+        dump_all_association_replicants dumper, :has_one
+        included_associations.each do |association|
+          dump_association_replicants dumper, association
+        end
+      end
+
+      # List of associations to explicitly include when dumping this object.
+      def included_associations
+        (self.class.replicate_associations + @replicate_opts[:associations]).uniq
+      end
+
+      # List of attributes and associations to omit when dumping this object.
+      def omitted_attributes
+        (self.class.replicate_omit_attributes + @replicate_opts[:omit]).uniq
+      end
+
+      # Attributes hash used to persist this object. This consists of simply
+      # typed values (no complex types or objects) with the exception of special
+      # foreign key values. When an attribute value is [:id, "SomeClass:1234"],
+      # the loader will handle translating the id value to the local system's
+      # version of the same object.
+      def replicant_attributes
+        attributes = self.attributes.dup
+
+        omitted_attributes.each { |omit| attributes.delete(omit.to_s) }
+        self.class.reflect_on_all_associations(:belongs_to).each do |reflection|
+          next if omitted_attributes.include?(reflection.name)
+          if info = replicate_reflection_info(reflection)
+            if replicant_id = info[:replicant_id]
+              foreign_key = info[:foreign_key].to_s
+              attributes[foreign_key] = [:id, *replicant_id]
+            end
+          end
+        end
+
+        attributes
+      end
+
+      # Retrieve information on a reflection's associated class and various
+      # keys.
+      #
+      # Returns an info hash with these keys:
+      #   :class - The class object the association points to.
+      #   :primary_key  - The string primary key column name.
+      #   :foreign_key  - The string foreign key column name.
+      #   :replicant_id - The [classname, id] tuple identifying the record.
+      #
+      # Returns nil when the reflection can not be linked to a model.
+      def replicate_reflection_info(reflection)
+        options = reflection.options
+        if options[:polymorphic]
+          reference_class = attributes[reflection.foreign_type]
+          return if reference_class.nil?
+
+          klass = reference_class.constantize
+          primary_key = klass.primary_key
+          foreign_key = "#{reflection.name}_id"
+        else
+          klass = reflection.klass
+          primary_key = (options[:primary_key] || klass.primary_key).to_s
+          foreign_key = (options[:foreign_key] || "#{reflection.name}_id").to_s
+        end
+
+        info = {
+          :class       => klass,
+          :primary_key => primary_key,
+          :foreign_key => foreign_key
+        }
+
+        if primary_key == klass.primary_key
+          if id = attributes[foreign_key]
+            info[:replicant_id] = [klass.to_s, id]
+          else
+            # nil value in association reference
+          end
+        else
+          # association uses non-primary-key foreign key. no special key
+          # conversion needed.
+        end
+
+        info
+      end
+
+      # The replicant id is a two tuple containing the class and object id. This
+      # is used by Replicant::Dumper to determine if the object has already been
+      # dumped or not.
+      def replicant_id
+        [self.class.name, id]
+      end
+
+      # Dump all associations of a given type.
+      #
+      # dumper           - The Dumper object used to dump additional objects.
+      # association_type - :has_one, :belongs_to, :has_many
+      #
+      # Returns nothing.
+      def dump_all_association_replicants(dumper, association_type)
+        self.class.reflect_on_all_associations(association_type).each do |reflection|
+          next if omitted_attributes.include?(reflection.name)
+
+          # bail when this object has already been dumped
+          next if (info = replicate_reflection_info(reflection)) &&
+            (replicant_id = info[:replicant_id]) &&
+            dumper.dumped?(replicant_id)
+
+          next if (dependent = __send__(reflection.name)).nil?
+
+          case dependent
+          when ActiveRecord::Base, Array
+            dumper.dump(dependent)
+
+            # clear reference to allow GC
+            if respond_to?(:association)
+              association(reflection.name).reset
+            elsif respond_to?(:association_instance_set, true)
+              association_instance_set(reflection.name, nil)
+            end
+          else
+            warn "warn: #{self.class}##{reflection.name} #{association_type} association " \
+                 "unexpectedly returned a #{dependent.class}. skipping."
+          end
+        end
+      end
+
+      # Dump objects associated with an AR object through an association name.
+      #
+      # object      - AR object instance.
+      # association - Name of the association whose objects should be dumped.
+      #
+      # Returns nothing.
+      def dump_association_replicants(dumper, association)
+        if reflection = self.class.reflect_on_association(association)
+          objects = __send__(reflection.name)
+          dumper.dump(objects)
+          if reflection.macro == :has_and_belongs_to_many
+            dump_has_and_belongs_to_many_replicant(dumper, reflection)
+          end
+          object = __send__(reflection.name) # clear to allow GC
+          if object.respond_to?(:reset)
+            object.reset
+          end
+        else
+          warn "error: #{self.class}##{association} is invalid"
+        end
+      end
+
+      # Dump the special Habtm object used to establish many-to-many
+      # relationships between objects that have already been dumped. Note that
+      # this object and all objects referenced must have already been dumped
+      # before calling this method.
+      def dump_has_and_belongs_to_many_replicant(dumper, reflection)
+        dumper.dump Habtm.new(self, reflection)
+      end
+    end
+
+    # Mixin for the ActiveRecord class.
+    module ClassMethods
+      # Set and retrieve list of association names that should be dumped when
+      # objects of this class are dumped. This method may be called multiple
+      # times to add associations.
+      def replicate_associations(*names)
+        self.replicate_associations += names if names.any?
+        @replicate_associations || superclass.replicate_associations
+      end
+
+      # Set the list of association names to dump to the specific set of values.
+      def replicate_associations=(names)
+        @replicate_associations = names.uniq.map { |name| name.to_sym }
+      end
+
+      # Compound key used during load to locate existing objects for update.
+      # When no natural key is defined, objects are created new.
+      #
+      # attribute_names - Macro style setter.
+      def replicate_natural_key(*attribute_names)
+        self.replicate_natural_key = attribute_names if attribute_names.any?
+        @replicate_natural_key || superclass.replicate_natural_key
+      end
+
+      # Set the compound key used to locate existing objects for update when
+      # loading. When not set, loading will always create new records.
+      #
+      # attribute_names - Array of attribute name symbols
+      def replicate_natural_key=(attribute_names)
+        @replicate_natural_key = attribute_names
+      end
+
+      # Set or retrieve whether replicated object should keep its original id.
+      # When not set, replicated objects will be created with new id.
+      def replicate_id(boolean=nil)
+        self.replicate_id = boolean unless boolean.nil?
+        @replicate_id.nil? ? superclass.replicate_id : @replicate_id
+      end
+
+      # Set flag for replicating original id.
+      def replicate_id=(boolean)
+        self.replicate_natural_key = [self.primary_key.to_sym] if boolean
+        @replicate_id = boolean
+      end
+
+      # Set which, if any, attributes should not be dumped. Also works for
+      # associations.
+      #
+      # attribute_names - Macro style setter.
+      def replicate_omit_attributes(*attribute_names)
+        self.replicate_omit_attributes = attribute_names if attribute_names.any?
+        @replicate_omit_attributes || superclass.replicate_omit_attributes
+      end
+
+      # Set which, if any, attributes should not be dumped. Also works for
+      # associations.
+      #
+      # attribute_names - Array of attribute name symbols
+      def replicate_omit_attributes=(attribute_names)
+        @replicate_omit_attributes = attribute_names
+      end
+
+      # Load an individual record into the database. If the models defines a
+      # replicate_natural_key then an existing record will be updated if found
+      # instead of a new record being created.
+      #
+      # type  - Model class name as a String.
+      # id    - Primary key id of the record on the dump system. This must be
+      #         translated to the local system and stored in the keymap.
+      # attrs - Hash of attributes to set on the new record.
+      #
+      # Returns the ActiveRecord object instance for the new record.
+      def load_replicant(type, id, attributes)
+        instance = replicate_find_existing_record(attributes) || new
+        create_or_update_replicant instance, attributes
+      end
+
+      # Locate an existing record using the replicate_natural_key attribute
+      # values.
+      #
+      # Returns the existing record if found, nil otherwise.
+      def replicate_find_existing_record(attributes)
+        return if replicate_natural_key.empty?
+        conditions = {}
+        replicate_natural_key.each do |attribute_name|
+          conditions[attribute_name] = attributes[attribute_name.to_s]
+        end
+        where(conditions).first
+      end
+
+      # Update an AR object's attributes and persist to the database without
+      # running validations or callbacks.
+      #
+      # Returns the [id, object] tuple for the newly replicated objected.
+      def create_or_update_replicant(instance, attributes)
+        # write replicated attributes to the instance
+        attributes.each do |key, value|
+          next if key == primary_key and not replicate_id
+          instance.send :write_attribute, key, value
+        end
+
+        # save the instance bypassing all callbacks and validations
+        replicate_disable_callbacks instance
+        instance.save :validate => false
+
+        [instance.send(instance.class.primary_key), instance]
+      end
+
+      # Disable all callbacks on an ActiveRecord::Base instance. Only the
+      # instance is effected. There is no way to re-enable callbacks once
+      # they've been disabled on an object.
+      def replicate_disable_callbacks(instance)
+        def instance.run_callbacks(*args)
+          yield if block_given?
+        end
+      end
+
+    end
+
+    # Special object used to dump the list of associated ids for a
+    # has_and_belongs_to_many association. The object includes attributes for
+    # locating the source object and writing the list of ids to the appropriate
+    # association method.
+    class Habtm
+      def initialize(object, reflection)
+        @object = object
+        @reflection = reflection
+      end
+
+      def id
+      end
+
+      def attributes
+        ids = @object.__send__("#{@reflection.name.to_s.singularize}_ids")
+        {
+          'id'         => [:id, @object.class.to_s, @object.id],
+          'class'      => @object.class.to_s,
+          'ref_class'  => @reflection.klass.to_s,
+          'ref_name'   => @reflection.name.to_s,
+          'collection' => [:id, @reflection.klass.to_s, ids]
+        }
+      end
+
+      def dump_replicant(dumper, opts={})
+        type = self.class.name
+        id   = "#{@object.class.to_s}:#{@reflection.name}:#{@object.id}"
+        dumper.write type, id, attributes, self
+      end
+
+      def self.load_replicant(type, id, attrs)
+        object = attrs['class'].constantize.find(attrs['id'])
+        ids    = attrs['collection']
+        object.__send__("#{attrs['ref_name'].to_s.singularize}_ids=", ids)
+        [id, new(object, nil)]
+      end
+    end
+
+    # Load active record and install the extension methods.
+    ::ActiveRecord::Base.send :include, InstanceMethods
+    ::ActiveRecord::Base.send :extend,  ClassMethods
+    ::ActiveRecord::Base.replicate_associations = []
+    ::ActiveRecord::Base.replicate_natural_key  = []
+    ::ActiveRecord::Base.replicate_omit_attributes  = []
+    ::ActiveRecord::Base.replicate_id           = false
+  end
+end

data/lib/replicate/dumper.rb

@@ -0,0 +1,142 @@
+module Replicate
+  # Dump replicants in a streaming fashion.
+  #
+  # The Dumper takes objects and generates one or more replicant objects. A
+  # replicant has the form [type, id, attributes] and describes exactly one
+  # addressable record in a datastore. The type and id identify the model
+  # class name and model primary key id. The attributes Hash is a set of attribute
+  # name to primitively typed object value mappings.
+  #
+  # Example dump session:
+  #
+  #     >> Replicate::Dumper.new do |dumper|
+  #     >>   dumper.marshal_to $stdout
+  #     >>   dumper.log_to $stderr
+  #     >>   dumper.dump User.find(1234)
+  #     >> end
+  #
+  class Dumper < Emitter
+    # Create a new Dumper.
+    #
+    # io     - IO object to write marshalled replicant objects to.
+    # block  - Dump context block. If given, the end of the block's execution
+    #          is assumed to be the end of the dump stream.
+    def initialize(io=nil)
+      @memo = Hash.new { |hash,k| hash[k] = {} }
+      super() do
+        marshal_to io if io
+        yield self if block_given?
+      end
+    end
+
+    # Register a filter to write marshalled data to the given IO object.
+    def marshal_to(io)
+      listen { |type, id, attrs, obj| Marshal.dump([type, id, attrs], io) }
+    end
+
+    # Register a filter to write status information to the given stream. By
+    # default, a single line is used to report object counts while the dump is
+    # in progress; dump counts for each class are written when complete. The
+    # verbose and quiet options can be used to increase or decrease
+    # verbosity.
+    #
+    # out - An IO object to write to, like stderr.
+    # verbose - Whether verbose output should be enabled.
+    # quiet - Whether quiet output should be enabled.
+    #
+    # Returns the Replicate::Status object.
+    def log_to(out=$stderr, verbose=false, quiet=false)
+      use Replicate::Status, 'dump', out, verbose, quiet
+    end
+
+    # Load a dump script. This evals the source of the file in the context
+    # of a special object with a #dump method that forwards to this instance.
+    # Dump scripts are useful when you want to dump a lot of stuff. Call the
+    # dump method as many times as necessary to dump all objects.
+    def load_script(path)
+      dumper = self
+      object = ::Object.new
+      meta = (class<<object;self;end)
+      [:dump, :load_script].each do |method|
+        meta.send(:define_method, method) { |*args| dumper.send(method, *args) }
+      end
+      file = find_file(path)
+      object.instance_eval File.read(file), file, 0
+    end
+
+    # Dump one or more objects to the internal array or provided dump
+    # stream. This method guarantees that the same object will not be dumped
+    # more than once.
+    #
+    # objects - ActiveRecord object instances.
+    #
+    # Returns nothing.
+    def dump(*objects)
+      opts = if objects.last.is_a? Hash
+        objects.pop
+      else
+        {}
+      end
+      objects = objects[0] if objects.size == 1 && objects[0].respond_to?(:to_ary)
+      objects.each do |object|
+        next if object.nil? || dumped?(object)
+        if object.respond_to?(:dump_replicant)
+          args = [self]
+          args << opts unless object.method(:dump_replicant).arity == 1
+          object.dump_replicant(*args)
+        else
+          raise NoMethodError, "#{object.class} must respond to #dump_replicant"
+        end
+      end
+    end
+
+    # Check if object has been written yet.
+    def dumped?(object)
+      if object.respond_to?(:replicant_id)
+        type, id = object.replicant_id
+      elsif object.is_a?(Array)
+        type, id = object
+      else
+        return false
+      end
+      @memo[type.to_s][id]
+    end
+
+    # Called exactly once per unique type and id. Emits to all listeners.
+    #
+    # type       - The model class name as a String.
+    # id         - The record's id. Usually an integer.
+    # attributes - All model attributes.
+    # object     - The object this dump is generated for.
+    #
+    # Returns the object.
+    def write(type, id, attributes, object)
+      type = type.to_s
+      return if dumped?([type, id])
+      @memo[type][id] = true
+
+      emit type, id, attributes, object
+    end
+
+    # Retrieve dumped object counts for all classes.
+    #
+    # Returns a Hash of { class_name => count } where count is the number of
+    # objects dumped with a class of class_name.
+    def stats
+      stats = {}
+      @memo.each { |class_name, items| stats[class_name] = items.size }
+      stats
+    end
+
+    protected
+    def find_file(path)
+      path = "#{path}.rb" unless path =~ /\.rb$/
+      return path if File.exists? path
+      $LOAD_PATH.each do |prefix|
+        full_path = File.expand_path(path, prefix)
+        return full_path if File.exists? full_path
+      end
+      false
+    end
+  end
+end