migratrix 0.0.9 → 0.8.1

data/lib/migratrix/migratrix.rb

@@ -1,79 +1,12 @@
 # Main "App" or Driver class for Migrating. Responsible for loading
 # and integrating all the parts of a migration.
 module Migratrix
-  include ::Migratrix::Loggable
-
-  def self.migrate!(name, options={})
-    ::Migratrix::Migratrix.migrate(name, options)
-  end
-
-  def self.reload_migration(name)
-    ::Migratrix::Migratrix.reload_migration(name)
-  end
-
-  def self.logger
-    ::Migratrix::Migratrix.logger
-  end
-
-  def self.logger=(new_logger)
-    ::Migratrix::Migratrix.logger = new_logger
-  end
-
-  def self.log_to(stream)
-    ::Migratrix::Migratrix.log_to(stream)
-  end
-
   class Migratrix
     include ::Migratrix::Loggable
 
     def initialize
     end
 
-    def self.migrate(name, options={})
-      migratrix = self.new()
-      migration = migratrix.create_migration(name, options)
-      migration.migrate
-      migratrix
-    end
-
-    # Loads #{name}_migration.rb from migrations path, instantiates
-    # #{Name}Migration with options, and returns it.
-    def create_migration(name, options={})
-      options = filter_options(options)
-      klass_name = migration_name(name)
-      unless loaded?(klass_name)
-        raise MigrationAlreadyExists.new("Migratrix cannot instantiate class Migratrix::#{klass_name} because it already exists") if ::Migratrix.const_defined?(klass_name)
-        reload_migration name
-        raise MigrationNotDefined.new("Expected migration file #{filename} to define Migratrix::#{klass_name} but it did not") unless ::Migratrix.const_defined?(klass_name)
-        register_migration(klass_name, "Migratrix::#{klass_name}".constantize)
-      end
-      fetch_migration(klass_name).new(options)
-    end
-
-    def migration_name(name)
-      name = name.to_s
-      name = if name.plural?
-               name.classify.pluralize
-             else
-               name.classify
-             end
-      name + "Migration"
-    end
-
-    def filter_options(hash)
-      Hash[valid_options.map {|v| hash.key?(v) ? [v, hash[v]] : nil }.compact]
-    end
-
-    def valid_options
-      %w(limit offset order where)
-    end
-
-    def reload_migration(name)
-      filename = migrations_path + "#{name}_migration.rb"
-      raise MigrationFileNotFound.new("Migratrix cannot find migration file #{filename}") unless File.exists?(filename)
-      load filename
-    end
-
     # ----------------------------------------------------------------------
     # Logger singleton; tries to hook into Rails.logger if it exists (it
     # won't if you log anything during startup because Migratrix is
@@ -93,15 +26,15 @@ module Migratrix
 
     def self.init_logger
       return Rails.logger if Rails.logger
-      @@logger = create_logger($stdout)
+      @logger = create_logger($stdout)
     end
 
     def self.logger
-      @@logger ||= self.init_logger
+      @logger ||= self.init_logger
     end
 
     def self.logger=(new_logger)
-      @@logger = new_logger
+      @logger = new_logger
     end
     # ----------------------------------------------------------------------
 
@@ -109,30 +42,56 @@ module Migratrix
 
     # ----------------------------------------------------------------------
     # Candidate for exract class? MigrationRegistry?
-    def loaded?(name)
-      registered_migrations.key? name.to_s
+    def self.registry
+      @registry ||= Hash[[:extractions,:loads,:migrations,:transforms].map {|key| [key, Registry.new]}]
     end
 
-    def register_migration(name, klass)
-      registered_migrations[name.to_s] = klass
+    # --------------------
+    # extractions
+    def self.extractions
+      registry[:extractions]
     end
 
-    def fetch_migration(name)
-      registered_migrations.fetch name.to_s
+    def self.register_extraction(class_name, klass, options={})
+      self.extractions.register(class_name, klass, options)
     end
 
-    def registered_migrations
-      @@registered_migrations ||= {}
+    def self.extraction(class_name, extraction_name, options={})
+      self.extractions.class_for(class_name).new(extraction_name, options)
+    end
+    # --------------------
+
+    # --------------------
+    # transforms
+    def self.transforms
+      registry[:transforms]
     end
-    # End MigrationRegistry
-    # ----------------------------------------------------------------------
 
-    def migrations_path
-      @migrations_path ||= ::Migratrix.default_migrations_path
+    def self.register_transform(name, klass, options={})
+      self.transforms.register(name, klass, options)
     end
 
-    def migrations_path=(new_path)
-      @migrations_path = Pathname.new new_path
+    def self.transform(transform_name, class_name, options={})
+      self.transforms.class_for(class_name).new(transform_name, options)
     end
+    # --------------------
+
+    # --------------------
+    # loads
+    def self.loads
+      registry[:loads]
+    end
+
+    def self.register_load(name, klass, options={})
+      self.loads.register(name, klass, options)
+    end
+
+    def self.load(load_name, class_name, options={})
+      self.loads.class_for(class_name).new(load_name, options)
+    end
+    # --------------------
+
+    # End MigrationRegistry
+    # ----------------------------------------------------------------------
   end
 end

data/lib/migratrix/registry.rb

@@ -0,0 +1,20 @@
+module Migratrix
+  # Basically a place to store our factories
+  class Registry
+    def register(name, klass, init_options)
+      registry[name] = [klass, init_options]
+    end
+
+    def class_for(name)
+      registry.fetch(name).first
+    end
+
+    def registered?(name)
+      registry.key?(name)
+    end
+
+    def registry
+      @registry ||= {}
+    end
+  end
+end

data/lib/migratrix/transforms/map.rb

@@ -0,0 +1,57 @@
+module Migratrix
+  module Transforms
+    # Map is a transform that maps attributes from the source object
+    # to the target object.
+    #
+    # :transform: a hash with dst => src keys, where dst is an
+    # attribute on the transformed target object and src is either an
+    # attribute on the source object or a Proc that receives the
+    # entire extracted row and returns a value to be set.
+    #
+    # TODO: Right now map makes a lot of hard-coded assumptions as a
+    # result of the primary test case. Notably that target is a Hash,
+    # final class is a Hash keyed by transformed_object[:id], etc.
+    #
+    # TODO: Figure out how to do both of these strategies with Map:
+    #
+    # # Create object and then modify it sequentially
+    # new_object = target.new
+    # map.each do |dst, src|
+    #   new_object[dst] = extracted_item[src]
+    # end
+    #
+    # # Build up creation params and then new the object
+    # hash = Hash.new
+    # map.each do [dst, src]
+    #   hash[dst] = extracted_item[src]
+    # end
+    # new_object = target.new(hash)
+    class Map < Transform
+      attr_accessor :map
+
+      def initialize(name, options={})
+        super
+      end
+
+      def create_transformed_collection
+        Hash.new
+      end
+
+      def create_new_object(extracted_row)
+        Hash.new
+      end
+
+      def apply_attribute(object, attribute_or_apply, value)
+        object[attribute_or_apply] = value
+      end
+
+      def extract_attribute(object, attribute_or_extract)
+        object[attribute_or_extract]
+      end
+
+      def store_transformed_object(object, collection)
+        collection[object[:id]] = object
+      end
+    end
+  end
+end

data/lib/migratrix/transforms/transform.rb

@@ -0,0 +1,268 @@
+module Migratrix
+  module Transforms
+    # Transform base class. A transform takes a collection of
+    # extracted objects and returns a collection of transformed
+    # objects. To do this, it needs to know the following:
+    #
+    # 1. What kind of collection to create. (Array? Hash? Set?
+    # Custom?)
+    # 2. How to transform one extracted object into a transformed
+    # object.
+    # 2.1 How to create the object that will hold the transformed
+    # object's attributes (might be the transformed object's class,
+    # might be a hash of attributes used to create the object itself,
+    # you might even try loading the target object from the
+    # destination db to see if it's already been migrated--and if so,
+    # can it be skipped or does it need to be updated?)
+    # 2.2 What attributes to pull off the extracted object
+    # 2.3 HOW to pull an attribute off the extracted object
+    # 2.4 How to transform each attribute
+    # 2.5 What attributes to store on the target object
+    # 2.6 HOW to store an attribute on the target object
+    # 2.7 How to finalize the transformation
+    # 2.7.1 Did you create the target object by class and have you
+    # been updating it all along? Then yay, this step is a no-op and
+    # you're already done.
+    # 2.7.2 If you're doing a merge transformation, and you haven't
+    # already checked the destination db for an existing record, now's
+    # the time to try to load that sucker and, if successful, to merge
+    # it with your migrated values.
+    # 2.7.3 A common Rails optimization is to create an attributes
+    # hash and new the ActiveRecord object in one call. This is
+    # faster* than creating a blank object and updating its attributes
+    # individually.
+    # 2.8 How to store the finalized object in the collection.
+    # (hash[id]=obj? set << obj?)
+    #
+    # And remember, all of this is just to handle ONE record, albeit
+    # admittedly in the most complicated way possible . For sql->sql
+    # migrations the transformation step is pretty simple. (It doesn't
+    # exist, ha ha. A cross-database INSERT SELECT means that the
+    # extract, transform and load all take place in the extract step.)
+    #
+    # * DANGER DANGER DANGER This is a completely unsubstantiated
+    # optimization claim. I'm *sure* it's faster, though, so that's
+    # good enough, right? TODO: BENCHMARK THIS OR SOMETHING WHATEVER
+    class Transform
+      include ::Migratrix::Loggable
+      include ::Migratrix::ValidOptions
+      attr_accessor :name, :options, :extraction, :transformations
+
+      set_valid_options(
+                :apply_attribute,
+                :extract_attribute,
+                :extraction,
+                :final_class,
+                :finalize_object,
+                :store_transform,
+                :target,
+                :transform,
+                :transform_class,
+                :transform_collection
+                )
+
+      def initialize(name, options={})
+        @name = name
+        @options = options.symbolize_keys
+        @transformations = options[:transform]
+      end
+
+      # Name of the extraction to use. If omitted, returns our name.
+      def extraction
+        options[:extraction] || name
+      end
+
+      # This transform method has strategy magic at every turn. I
+      # expect it to be slow, but we can optimize it later, e.g. by
+      # rolling out a define_method or similar for all of the constant
+      # parts.
+      #
+      # ----------------------------------------------------------------------
+      # Map's strategy, as used by PetsMigration
+      #
+      # create_transformed_collection -> Hash.new
+      # create_new_object -> Hash.new
+      # transformation -> {:id => :id, :name => :name }
+      # extract_attribute -> object[attribute_or_extract]
+      # apply_attribute -> object[attribute] = attribute_or_apply
+      # finalize_object -> no-op
+      # store_transformed_object -> collection[object[:id]] = object
+      # ----------------------------------------------------------------------
+      #
+      # ----------------------------------------------------------------------
+      # Default strategy:
+      #
+      # create_transformed_collection -> Array.new
+      # create_new_object -> @options[:target].new
+      # transformations -> MUST BE SUPPLIED BY CHILD CLASS, e.g. Map's {:dst => :src} hash
+      # extract_attribute -> attribute_or_extract.is_a?(Proc) ? attribute_or_extract.call(object) : object.send(attribute_or_extract)
+      # apply_attribute -> object.send("#{attribute_or_apply}=", value)
+      # finalize_object -> no-op
+      # store_transformed_object -> collection << transformed_object
+      # ----------------------------------------------------------------------
+      #
+      #
+      # Now, can we represent these two strategies as configurations
+      # in the migration? For example, here's one way of representing
+      # PetTypeMigration's strategy, which in the Load step must save
+      # off a YAML dump of a hash of all the pet objects (with just id
+      # and name)  keyed
+      #
+      # set_transform :repetition_types, :map,
+      #               :map => {:id => :id, :name => :name },
+      #               :extract_method => :index,
+      #               :apply_method => :index,
+      #               :new_class => Hash,
+      #               :collection => Hash,
+      #               :store_method => lambda {|item, hash| hash[item[:id]] = item },
+      #
+      # So the backing magic is this:
+      #
+      # - We expect :new_class to respond to new() and give us a new,
+      #   blank object.
+      #
+      # - Ditto for :collection; it should respond to new()
+      #
+      # - extract_method :index means attr = extracted_object[:name]
+      #
+      # - apply_method :index means the same thing
+      #
+      # - store_method is the only weirdness here, and it might
+      #   actually make sense to have names for the most obvious
+      #   strategies, like :store_by => { :index => :id } (An array
+      #   could use :store_by => :push, Set by :add, etc.)
+      #
+      # - The :final_class option is optional; its presence interacts
+      #   with the also-optional :finalize option.
+      #
+      # - The :finalize option is optional; its presence interacts
+      #   with the also-optional :final_class option, as follows:
+      #
+      #   Both Missing: final_object = new_object
+      #   :final_class only: final_obj = FinalClass.new(new_obj)
+      #   :finalize only: final_obj = finalize(new_obj)
+      #   Both Present: final_obj = FinalClass.new(finalize(obj))
+      #
+      #   Examples/Notes:
+      #
+      #   - To build ActiveRecord objects quickly, use :new_class =>
+      #     Hash, :final_class => ModelClass, :finalize => nil.
+      #
+      #   - If your source row is a hash (e.g. select_all or a YAML
+      #     load) that needs no transformation, but has more columns
+      #     than your ActiveRecord model can accept, you could use a
+      #     copy transform (basically a map transform without the map
+      #     step; new_object = extracted_object) with something like
+      #     :new_class => Hash, :final_class => ModelClass, :finalize
+      #     => lambda {|hsh| hsh.dup.keep_if? {|k,v|
+      #     k.in?(ModelClass.new.attributes.keys)}}
+      #     (That would be HORRIBLY inefficient but there would be
+      #     ways to memoize with e.g. before_finalize { @keys =
+      #     ModelClass.new.attributes.keys })
+      #
+      # Advanced Magic:
+      # - new_class, collection can be procs returning a new object
+      # - extract_method, apply_method can be procs taking |obj, attr|
+      #   and |obj, attr, value|
+      #
+      # Super Advanced Magic (YAGNI):
+      # - instead of procs, take blocks and use define_method on them
+      #   so they're faster.
+      def transform(extracted_objects)
+        info "Transform #{name} started transform."
+        transformed_collection = create_transformed_collection
+        extracted_objects.each do |extracted_object|
+          new_object = create_new_object(extracted_object)
+          transformations.each do |attribute_or_apply, attribute_or_extract|
+            apply_attribute(new_object, attribute_or_apply, extract_attribute(extracted_object, attribute_or_extract))
+          end
+          transformed_object = finalize_object(new_object)
+          store_transformed_object(transformed_object, transformed_collection)
+        end
+        info "Transform #{name} finished transform."
+        transformed_collection
+      end
+
+
+      def create_transformed_collection
+        raise NotImplementedError unless options[:transform_collection]
+        option = options[:transform_collection]
+        case option
+        when Proc
+          option.call
+        when Class
+          option.new
+        else
+          raise TypeError
+        end
+      end
+
+      def create_new_object(extracted_row)
+        # TODO: this should work like finalize, taking a create and an init.
+        raise NotImplementedError unless options[:transform_class]
+        option = options[:transform_class]
+        case option
+        when Proc
+          option.call(extracted_row)
+        when Class
+          option.new # laaame--should receive extracted_row, see todo above
+        else
+          raise TypeError
+        end
+      end
+
+      def apply_attribute(object, attribute_or_apply, value)
+        raise NotImplementedError unless options[:apply_attribute]
+        option = options[:apply_attribute]
+        case option
+        when Proc
+          option.call(object, attribute_or_apply, value)
+        when Symbol
+          object.send(option, attribute_or_apply, value)
+        else
+          raise TypeError
+        end
+      end
+
+      def extract_attribute(object, attribute_or_extract)
+        raise NotImplementedError unless options[:extract_attribute]
+        option = options[:extract_attribute]
+        case option
+        when Proc
+          option.call(object, attribute_or_extract)
+        when Symbol
+          object.send(option, attribute_or_extract)
+        else
+          raise TypeError
+        end
+      end
+
+
+      #   Both Missing: final_object = new_object
+      #   :final_class only: final_obj = FinalClass.new(new_obj)
+      #   :finalize only: final_obj = finalize(new_obj)
+      #   Both Present: final_obj = FinalClass.new(finalize(obj))
+      def finalize_object(new_object)
+        return new_object unless options[:final_class] || options[:finalize_object]
+        raise TypeError if options[:finalize_object] && !options[:finalize_object].is_a?(Proc)
+        raise TypeError if options[:final_class] && !options[:final_class].is_a?(Class)
+        new_object = options[:finalize_object].call(new_object) if options[:finalize_object]
+        new_object = options[:final_class].new(new_object) if options[:final_class]
+        new_object
+      end
+
+      def store_transformed_object(object, collection)
+        raise NotImplementedError unless options[:store_transformed_object]
+        option = options[:store_transformed_object]
+        case option
+        when Proc
+          option.call(object, collection)
+        when Symbol
+          collection.send(option, object)
+        else
+          raise TypeError
+        end
+      end
+    end
+  end
+end