migratrix 0.0.9 → 0.8.1

data/lib/migratrix.rb

@@ -6,19 +6,75 @@ module Migratrix
   APP=Pathname.new(__FILE__).dirname + "migratrix"
   EXT=Pathname.new(__FILE__).dirname + "patches"
 
-  def self.default_migrations_path
-    Rails.root + 'db/legacy'
-  end
-
   require EXT + 'string_ext'
   require EXT + 'object_ext'
   require EXT + 'andand'
   require APP + 'loggable'
+  require APP + 'valid_options'
   require APP + 'exceptions'
+  require APP + 'registry'
   require APP + 'migration'
   require APP + 'migratrix'
 
-  require APP + 'extractors/extractor'
-  require APP + 'extractors/active_record'
+  require APP + 'extractions/extraction'
+  require APP + 'extractions/active_record'
+
+  require APP + 'transforms/transform'
+  require APP + 'transforms/map'
+
+  require APP + 'loads/load'
+  require APP + 'loads/yaml'
+#  require APP + 'loads/csv'
+#  require APP + 'loads/active_record'
+
+
+  include ::Migratrix::Loggable
+
+  def self.register_extraction(name, klass, init_options={})
+    ::Migratrix::Migratrix.register_extraction(name, klass, init_options)
+  end
+
+  def self.extractions
+    ::Migratrix::Migratrix.extractions
+  end
+
+  def self.register_transform(name, klass, init_options={})
+    ::Migratrix::Migratrix.register_transform(name, klass, init_options)
+  end
+
+  def self.transforms
+    ::Migratrix::Migratrix.transforms
+  end
+
+  def self.register_load(name, klass, init_options={})
+    ::Migratrix::Migratrix.register_load(name, klass, init_options)
+  end
+
+  def self.loads
+    ::Migratrix::Migratrix.loads
+  end
+
+  def self.logger
+    ::Migratrix::Migratrix.logger
+  end
+
+  def self.logger=(new_logger)
+    ::Migratrix::Migratrix.logger = new_logger
+  end
+
+  def self.log_to(stream)
+    ::Migratrix::Migratrix.log_to(stream)
+  end
+
+  # ----------------------------------------------------------------------
+  # Register in-gem Components
+  register_extraction :extraction, Extractions::Extraction
+  register_extraction :active_record, Extractions::ActiveRecord
+
+  register_transform :transform, Transforms::Transform
+  register_transform :map, Transforms::Map
+
+  register_load :load, Loads::Load
+  register_load :yaml, Loads::Yaml
 end
 

data/lib/migratrix/exceptions.rb

@@ -4,6 +4,9 @@ module Migratrix
   class MigrationAlreadyExists < Exception; end
   class MigrationFileNotFound < Exception; end
   class MigrationNotDefined < Exception; end
-  class ExtractorSourceUndefined < Exception; end
+  class ExtractionNotDefined < Exception; end
+  class TransformNotDefined < Exception; end
+  class LoadNotDefined < Exception; end
+  class ExtractionSourceUndefined < Exception; end
 end
 

data/lib/migratrix/{extractors → extractions}/active_record.rb

@@ -1,9 +1,8 @@
 module Migratrix
-  module Extractors
-    # Extractor that expects to be pointed at an ActiveRecord class.
-    class ActiveRecord < Extractor
-      # TODO: Raise TypeError in initialize unless Extractor source is
-      # an ActiveRecord model
+  module Extractions
+    # Extraction that expects to be pointed at an ActiveRecord class.
+    class ActiveRecord < Extraction
+      set_valid_options :fetchall
 
       def source=(new_source)
         raise TypeError.new(":source is of type must be an ActiveRecord model class (must inherit from ActiveRecord::Base)") unless is_ar?(new_source)
@@ -14,8 +13,8 @@ module Migratrix
         source.is_a?(Class) && source.ancestors.include?(::ActiveRecord::Base)
       end
 
-      def obtain_source(source)
-        raise ExtractorSourceUndefined unless source
+      def obtain_source(source, options={})
+        raise ExtractionSourceUndefined unless source
         raise TypeError.new(":source is of type must be an ActiveRecord model class (must inherit from ActiveRecord::Base)") unless is_ar?(source)
         source
       end
@@ -41,12 +40,17 @@ module Migratrix
         if source.is_a?(::ActiveRecord::Relation)
           source.to_sql
         else
-          source.handle_where(1).to_sql
+          handle_where(source, 1).to_sql
         end
       end
 
-      def execute_extract(source)
-        source.all
+      def execute_extract(src, options={})
+        return src.all if options['fetchall']
+        ret = if src.is_a?(::ActiveRecord::Relation)
+                src
+              else
+                handle_where(src, 1)
+              end
       end
     end
   end

data/lib/migratrix/{extractors/extractor.rb → extractions/extraction.rb}

@@ -1,35 +1,36 @@
-#require 'active_model/attribute_methods'
-
 module Migratrix
-  module Extractors
+  module Extractions
     # base class for extraction
-    class Extractor
+    class Extraction
       include ::Migratrix::Loggable
-#      include ActiveModel::AttributeMethods
+      include ::Migratrix::ValidOptions
+
+      attr_accessor :name, :source, :options
 
-      attr_accessor :source, :options
+      set_valid_options :limit, :offset, :order, :where
 
-      def initialize(options={})
+      def initialize(name, options={})
         @options = options.deep_copy
         self.source = options[:source] if options[:source]
       end
 
       def extract(options={})
-        options = @options.merge(options)
+        options = @options.merge(options).symbolize_keys
 
         # TODO: Raise error if self.abstract? DANGER/NOTE that this is
         # the "default strategy" for extraction, and may need to be
         # extracted to a strategy object.
 
-        src = obtain_source(self.source)
-        src = handle_where(src, options["where"]) if options["where"]
-        src = handle_order(src, options["order"]) if options["order"]
-        src = handle_limit(src, options["limit"]) if options["limit"]
-        src = handle_offset(src, options["offset"]) if options["offset"]
-        execute_extract(src)
+        src = obtain_source(self.source, options)
+        src = handle_where(src, options[:where]) if options[:where]
+        src = handle_order(src, options[:order]) if options[:order]
+        src = handle_limit(src, options[:limit]) if options[:limit]
+        src = handle_offset(src, options[:offset]) if options[:offset]
+        execute_extract(src, options)
       end
 
-      # = extraction filter methods
+
+    # = extraction filter methods
       #
       # The handle_* methods receive a source and return a source and
       # must be chainable. For example, source might come in as an
@@ -45,11 +46,11 @@ module Migratrix
 
       # First step in extraction is to take the given source and turn
       # it into something that the filter chain can used. The
-      # ActiveRecord extractor uses a legacy model class as its source
-      # so it can simply return its source. A CSV or Yaml extractor
+      # ActiveRecord extraction uses a legacy model class as its source
+      # so it can simply return its source. A CSV or Yaml extraction
       # here might need to read the entire file contents and returns
       # the full, unfiltered data source.
-      def obtain_source(source)
+      def obtain_source(source, options={})
         raise NotImplementedError
       end
 
@@ -74,13 +75,13 @@ module Migratrix
       end
 
       # Constructs the query, if applicable. May not exist or make
-      # sense for non-SQL and/or non-ActiveRecord extractors.
+      # sense for non-SQL and/or non-ActiveRecord extractions.
       def to_query(source)
         raise NotImplementedError
       end
 
       # Execute the extraction and return the result set.
-      def execute_extract(source)
+      def execute_extract(source, options={})
         raise NotImplementedError
       end
     end

data/lib/migratrix/loads/load.rb

@@ -0,0 +1,43 @@
+module Migratrix
+  module Loads
+    class Load
+      include ::Migratrix::Loggable
+      include ::Migratrix::ValidOptions
+
+      attr_accessor :name, :options
+
+      set_valid_options :transform
+
+      def initialize(name, options={})
+        @name = name
+        @options = options.symbolize_keys
+      end
+
+      # Default strategy: call save() on every transformed_object.
+      def load(transformed_objects)
+        transformed_objects.each do |transformed_object|
+          transformed_object.save
+        end
+      end
+
+      # Name of the transform to use. If omitted, returns our name.
+      def transform
+        options[:transform] || name
+      end
+
+
+      # # Prepare for load. Here is where you might want to truncate
+      # # database tables, clear out target files, etc.
+      # def before_load
+      #   raise NotImplementedError
+      # end
+
+      # # Clean up after load. If you opened a file pointer in
+      # # before_load, now's a good time to close it.
+      # # TODO: Use the active model hooks to do this
+      # def after_load
+      #   raise NotImplementedError
+      # end
+    end
+  end
+end

data/lib/migratrix/loads/yaml.rb

@@ -0,0 +1,15 @@
+require 'yaml'
+
+module Migratrix
+  module Loads
+    class Yaml < Load
+      set_valid_options :filename
+
+      def load(transformed_items)
+        File.open(options[:filename], 'w') do |file|
+          file.puts transformed_items.to_yaml
+        end
+      end
+    end
+  end
+end

data/lib/migratrix/migration.rb

@@ -6,46 +6,132 @@ module Migratrix
   class Migration
     include ::Migratrix::Loggable
     include ActiveModel::AttributeMethods
+    include Migratrix::ValidOptions
 
-    cattr_accessor :extractor
     attr_accessor :options
+    set_valid_options :console
 
     def initialize(options={})
-      @options = options.deep_copy
+      @options = options.deep_copy.symbolize_keys
+      Migratrix.log_to($stdout) if @options[:console]
     end
 
-    # Adds an extractor to the extractors chain.
-    def self.set_extractor(name, options={})
-      # klassify this name
-      raise NotImplementedError.new("Migratrix currently only supports ActiveRecord extractor.") unless name == :active_record
-      @@extractor = ::Migratrix::Extractors::ActiveRecord.new(options)
+    # TODO: Technically, we need to ask our extractions, transformers
+    # and loaders for THEIR valid options as well. limit, offset,
+    # order and where are all extraction-only options, and fetchall is
+    # an ActiveRecord-specific option
+    def self.valid_options
+      opts = super # wacky, I know, but the extended ValidOptions module is in the super chain. (I <3 Ruby)
+      if extractions
+        extractions.each do |name, extraction|
+          opts += extraction.valid_options
+        end
+      end
+      if transforms
+        transforms.each do |name, transform|
+          opts += transform.valid_options
+        end
+      end
+#       if loads
+#         loads.each do |name, load|
+#           opts += load.valid_options
+#         end
+#       end
+      opts.uniq.sort
     end
 
-    def extractor
-      @@extractor
+    # TODO: THIS IS HUGE DUPLICATION, REFACTOR REFACTOR REFACTOR
+
+    # extraction crap
+    def self.set_extraction(extraction_name, class_name, options={})
+      extractions[extraction_name] = Migratrix.extraction(class_name, extraction_name, options)
+    end
+
+    def self.extend_extraction(extraction_name, options={})
+      migration = ancestors.detect {|k| k.respond_to?(:extractions) && k.extractions[extraction_name]}
+      raise ExtractionNotDefined.new("Could not extend extractar '%s'; no parent Migration defines it" % extraction_name) unless migration
+      extraction = migration.extractions[extraction_name]
+      extractions[extraction_name] = extraction.class.new(extraction_name, extraction.options.merge(options))
+    end
+
+    def self.extractions
+      @extractions ||= {}
+    end
+
+    def extractions
+      self.class.extractions
+    end
+
+    # transform crap
+    def self.set_transform(name, type, options={})
+      transforms[name] = Migratrix.transform(name, type, options)
+    end
+
+    def self.extend_transform(transform_name, options={})
+      migration = ancestors.detect {|k| k.respond_to?(:transforms) && k.transforms[transform_name]}
+      raise TransformNotDefined.new("Could not extend extractar '%s'; no parent Migration defines it" % transform_name) unless migration
+      transform = migration.transforms[transform_name]
+      transforms[transform_name] = transform.class.new(transform_name, transform.options.merge(options))
     end
 
-    # OKAY, NEW RULE: You get ONE Extractor per Migration. You're
-    # allowed to have multiple transform/load chains to the
-    # extraction, but extractors? ONE.
+    def self.transforms
+      @transforms ||= {}
+    end
+
+    def transforms
+      self.class.transforms
+    end
+
+    # load crap
+    def self.set_load(name, type, options={})
+      loads[name] = Migratrix.load(name, type, options)
+    end
+
+    def self.extend_load(load_name, options={})
+      migration = ancestors.detect {|k| k.respond_to?(:loads) && k.loads[load_name]}
+      raise LoadNotDefined.new("Could not extend extractar '%s'; no parent Migration defines it" % load_name) unless migration
+      load = migration.loads[load_name]
+      loads[load_name] = load.class.new(load_name, load.options.merge(options))
+    end
+
+    def self.loads
+      @loads ||= {}
+    end
+
+    def loads
+      self.class.loads
+    end
 
-    # default extraction method; simply assigns @extractor.extract to
-    # @extracted_items. If you override this method, you should
-    # populate @extracted_items if you want the default transform
-    # and/or load to work correctly.
     def extract
-      extractor.extract(options)
+      extracted_items = {}
+      extractions.each do |name, extraction|
+        extracted_items[name] = extraction.extract(options)
+      end
+      extracted_items
     end
 
-    # Transforms source data into outputs
-    def transform
-      # run the chain of transforms
+    # Transforms source data into outputs. @transformed_items is a
+    # hash of name => transformed_items.
+    #
+    def transform(extracted_items)
+      transformed_items = { }
+      transforms.each do |name, transform|
+        transformed_items[transform.name] = transform.transform extracted_items[transform.extraction]
+      end
+      transformed_items
     end
 
     # Saves the migrated data by "loading" it into our database or
-    # other data sink.
-    def load
-      # run the chain of loads
+    # other data sink. Loaders have their own names, and by default
+    # they depend on a transformed_items key of the same name, but you
+    # may override this behavior by setting :source => :name or
+    # possibly :source => [:name1, :name2, etc].
+    def load(transformed_items)
+      loaded_items = { }
+      loads.each do |name, load|
+        loaded_items[load.name] = load.load transformed_items[load.transform]
+      end
+      loaded_items
     end
 
     # Perform the migration
@@ -54,12 +140,14 @@ module Migratrix
     # strategy. YAGNI: Rails 3 lets us defer the querying until we get
     # to the transform step, and then it's batched for us under the
     # hood. ...assuming, of course, we change the ActiveRecord
-    # extractor's execute_extract method to return source instead of
+    # extraction's execute_extract method to return source instead of
     # all, but now the
     def migrate
-      @extracted_items = extract
-      transform
-      load
+      # This fn || @var API lets you write a method and either set the
+      # @var or return the value.
+      @extracted_items = extract || @extracted_items
+      @transformed_items = transform(@extracted_items) || @transformed_items
+      load @transformed_items
     end
   end
 end