abstract_importer 1.2.0.rc1 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: df7b7a0811459f30502cf09d03741312919b9f54
-  data.tar.gz: 116840924377d0562121169b715b9fd64f49fde1
+  metadata.gz: 6545777a4f5b26c7f43754894478c1ee2b38db44
+  data.tar.gz: defa4ee62445d06bfdadaf9453b2c54f19829c90
 SHA512:
-  metadata.gz: a9c6b45e74533ba5a1ef07432d677834057649b66656c07c61594ea34e91bb449e5346167fb69cad7d7c464a1c48899e420f8f73a051a22b731afeecfab1db49
-  data.tar.gz: b64e4b346bbc2f566c5686b95edde07f3f49f86be9a971916ac870a3d583038ef31da07cad2ed4e7d58f9f976c6f4de92343dfd93c7449841523cafd1488176d
+  metadata.gz: 607c5f129b8ac2742ad76f8c4a964b3ee24f5304b7d1701409bc5e3b0a5bf3eca0dda5d00d5f8556383d7ec32ba3786259946d55d3eee9f88709b83028c595fc
+  data.tar.gz: edaecfc589c1844af50cf53239940ebfc17052be632fa2bf563fc8bcd74d8e33521b157b9caa73c3285c3685d4e724bda514749cefa9f3fc51ba85d787b4810e

data/README.md

@@ -107,6 +107,7 @@ AbstractImporter optionally takes a hash of settings as a third argument:
  * `:dry_run` (default: `false`) when set to `true`, goes through all the steps except creating the records
  * `:io` (default: `$stderr`) an IO object that is passed to the reporter
  * `:reporter` (default: `AbstractImporter::Reporter.new(io)`) performs logging in response to various events
+ * `:strategy` allows you to use alternate import strategies for particular collections (See below)
 
 
 
@@ -147,7 +148,7 @@ The complete list of callbacks is below.
 
 `before_build` allows a callback to modify the hash of attributes before it is passed to `ActiveRecord::Relation#build`.
 
-##### before_create
+##### before_create, before_update, before_save
 
 `before_create` allows a callback to modify a record before `save` is called on it.
 
@@ -155,7 +156,7 @@ The complete list of callbacks is below.
 
 `rescue` (like `before_create`) is called with a record just before `save` is called. Unlike `before_create`, `rescue` is only called if the record does not pass validations.
 
-##### after_create
+##### after_create, after_update, after_save
 
 `after_create` is called with the original hash of attributes and the newly-saved record right after it is successfully saved.
 
@@ -169,6 +170,23 @@ The complete list of callbacks is below.
 
 
 
+### Strategies
+
+The importer's default strategy is to skip records that have already been imported and create records one-by-one as ActiveRecord objects.
+
+But AbstractImporter supports alternate strategies which you can specify per collection like this:
+
+```ruby
+summary = MyImport.new(parent, data_source, strategy: {students: :replace}).perform!
+```
+
+The following alternate strategies are built in:
+
+##### replace
+
+Replaces records that have already been imported rather than skipping them.
+
+
 
 ## Contributing
 

data/abstract_importer.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "activerecord"
   
   spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "minitest", "~> 4.7"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rails"
   spec.add_development_dependency "sqlite3"

data/lib/abstract_importer/base.rb

@@ -1,6 +1,6 @@
 require 'abstract_importer/import_options'
 require 'abstract_importer/import_plan'
-require 'abstract_importer/reporter'
+require 'abstract_importer/reporters'
 require 'abstract_importer/collection'
 require 'abstract_importer/collection_importer'
 require 'abstract_importer/id_map'
@@ -29,13 +29,16 @@ module AbstractImporter
       @parent       = parent
       
       io            = options.fetch(:io, $stderr)
-      @reporter     = Reporter.new(io, Rails.env.production?)
+      @reporter     = default_reporter(io)
       @dry_run      = options.fetch(:dry_run, false)
       
       @id_map       = IdMap.new
       @results      = {}
       @import_plan  = self.class.import_plan.to_h
       @atomic       = options.fetch(:atomic, false)
+      @strategies   = options.fetch(:strategy, {})
+      @skip         = Array(options[:skip])
+      @only         = Array(options[:only]) if options.key?(:only)
       @collections  = []
     end
     
@@ -80,12 +83,24 @@ module AbstractImporter
     end
     
     def import_collection(collection)
+      return if skip? collection
       results[collection.name] = CollectionImporter.new(self, collection).perform!
     end
     
     def teardown
     end
     
+    def skip?(collection)
+      return true if skip.member?(collection.name)
+      return true if only && !only.member?(collection.name)
+      false
+    end
+    
+    def strategy_for(collection)
+      strategy_name = @strategies.fetch collection.name, :default
+      AbstractImporter::Strategies.const_get :"#{strategy_name.capitalize}Strategy"
+    end
+    
     
     
     
@@ -119,7 +134,7 @@ module AbstractImporter
     
   private
     
-    attr_reader :collections, :import_plan
+    attr_reader :collections, :import_plan, :skip, :only
     
     def verify_source!
       import_plan.keys.each do |collection|
@@ -199,5 +214,13 @@ module AbstractImporter
       end
     end
     
+    def default_reporter(io)
+      case ENV["IMPORT_REPORTER"].to_s.downcase
+      when "none"        then Reporters::NullReporter.new(io)
+      when "performance" then Reporters::PerformanceReporter.new(io)
+      else                    Reporters::DebugReporter.new(io)
+      end
+    end
+    
   end
 end

data/lib/abstract_importer/collection.rb

@@ -1,4 +1,19 @@
 module AbstractImporter
   class Collection < Struct.new(:name, :model, :table_name, :scope, :options)
+
+    def association_attrs
+      return @assocation_attrs if defined?(@assocation_attrs)
+
+      # Instead of calling `tenant.people.build(__)`, we'll reflect on the
+      # association to find its foreign key and its owner's id, so that we
+      # can call `Person.new(__.merge(tenant_id: id))`.
+      @assocation_attrs = {}
+      assocation = scope.instance_variable_get(:@association)
+      unless assocation.is_a?(ActiveRecord::Associations::HasManyThroughAssociation)
+        @assocation_attrs.merge!(assocation.reflection.foreign_key.to_sym => assocation.owner.id)
+      end
+      @assocation_attrs.freeze
+    end
+
   end
 end

data/lib/abstract_importer/collection_importer.rb

@@ -1,18 +1,22 @@
+require "abstract_importer/strategies"
+
 module AbstractImporter
   class CollectionImporter
     
     def initialize(importer, collection)
       @importer = importer
       @collection = collection
+      @strategy = importer.strategy_for(collection).new(self)
     end
     
-    attr_reader :importer, :collection, :summary
+    attr_reader :importer, :collection, :summary, :strategy
     
     delegate :name,
              :table_name,
              :model,
              :scope,
              :options,
+             :association_attrs,
              :to => :collection
     
     delegate :dry_run?,
@@ -32,7 +36,9 @@ module AbstractImporter
       
       invoke_callback(:before_all)
       summary.ms = Benchmark.ms do
-        each_new_record &method(:process_record)
+        each_new_record do |attributes|
+          strategy.process_record(attributes)
+        end
       end
       invoke_callback(:after_all)
       
@@ -100,46 +106,14 @@ module AbstractImporter
     
     
     
-    
-    
     def each_new_record
       source.public_send(name).each do |attrs|
         yield attrs.dup
       end
     end
     
-    def process_record(hash)
-      summary.total += 1
-      
-      if already_imported?(hash)
-        summary.already_imported += 1
-        return
-      end
-      
-      remap_foreign_keys!(hash)
-      
-      if redundant_record?(hash)
-        summary.redundant += 1
-        return
-      end
-      
-      if create_record(hash)
-        summary.created += 1
-      else
-        summary.invalid += 1
-      end
-    rescue ::AbstractImporter::Skip
-      summary.skipped += 1
-    end
-    
-    
-    
     
     
-    def already_imported?(hash)
-      id_map.contains? table_name, hash[:id]
-    end
-    
     def remap_foreign_keys!(hash)
       @mappings.each do |proc|
         proc.call(hash)
@@ -158,41 +132,6 @@ module AbstractImporter
     
     
     
-    
-    
-    def create_record(hash)
-      record = build_record(hash)
-      
-      return true if dry_run?
-      
-      invoke_callback(:before_create, record)
-      
-      # rescue_callback has one shot to fix things
-      invoke_callback(:rescue, record) unless record.valid?
-      
-      if record.valid? && record.save
-        invoke_callback(:after_create, hash, record)
-        id_map << record
-        
-        reporter.record_created(record)
-        true
-      else
-        
-        reporter.record_failed(record, hash)
-        false
-      end
-    end
-    
-    def build_record(hash)
-      hash = invoke_callback(:before_build, hash) || hash
-      
-      legacy_id = hash.delete(:id)
-      
-      scope.build hash.merge(legacy_id: legacy_id)
-    end
-    
-    
-    
     def invoke_callback(callback, *args)
       callback_name = :"#{callback}_callback"
       callback = options.public_send(callback_name)

data/lib/abstract_importer/import_options.rb

@@ -1,40 +1,26 @@
 module AbstractImporter
   class ImportOptions
-    attr_reader :finder_callback,
-                :rescue_callback,
-                :before_build_callback,
-                :before_create_callback,
-                :after_create_callback,
-                :before_all_callback,
-                :after_all_callback
-                
-    def finder(sym=nil, &block)
-      @finder_callback = sym || block
+    CALLBACKS = [ :finder,
+                  :rescue,
+                  :before_build,
+                  :before_create,
+                  :before_update,
+                  :before_save,
+                  :after_create,
+                  :after_update,
+                  :after_save,
+                  :before_all,
+                  :after_all ]
+
+    CALLBACKS.each do |callback|
+      attr_reader :"#{callback}_callback"
+
+      class_eval <<-RUBY, __FILE__, __LINE__ + 1
+      def #{callback}(sym=nil, &block)
+        @#{callback}_callback = sym || block
+      end
+      RUBY
     end
-    
-    def before_build(sym=nil, &block)
-      @before_build_callback = sym || block
-    end
-    
-    def before_create(sym=nil, &block)
-      @before_create_callback = sym || block
-    end
-    
-    def after_create(sym=nil, &block)
-      @after_create_callback = sym || block
-    end
-    
-    def rescue(sym=nil, &block)
-      @rescue_callback = sym || block
-    end
-    
-    def before_all(sym=nil, &block)
-      @before_all_callback = sym || block
-    end
-    
-    def after_all(sym=nil, &block)
-      @after_all_callback = sym || block
-    end
-    
+
   end
 end

data/lib/abstract_importer/reporters.rb

@@ -0,0 +1,4 @@
+require 'abstract_importer/reporters/base_reporter'
+require 'abstract_importer/reporters/debug_reporter'
+require 'abstract_importer/reporters/null_reporter'
+require 'abstract_importer/reporters/performance_reporter'

data/lib/abstract_importer/reporters/base_reporter.rb

@@ -0,0 +1,72 @@
+module AbstractImporter
+  module Reporters
+    class BaseReporter
+      attr_reader :io
+      
+      def initialize(io)
+        @io = io
+      end
+      
+      
+      
+      def start_all(importer)
+        io.puts "Importing #{importer.describe_source} to #{importer.describe_destination}\n"
+      end
+      
+      def finish_all(importer, ms)
+        io.puts "\n\nFinished in #{distance_of_time(ms)}"
+      end
+      
+      def finish_setup(ms)
+        io.puts "Setup took #{distance_of_time(ms)}\n"
+      end
+      
+      def start_collection(collection)
+        io.puts "\n#{("="*80)}\nImporting #{collection.name}\n#{("="*80)}\n"
+      end
+      
+      def finish_collection(collection, summary)
+      end
+      
+      
+      
+      def record_created(record)
+      end
+      
+      def record_failed(record, hash)
+      end
+      
+      
+      
+      def count_notice(message)
+      end
+      
+      def count_error(message)
+      end
+      
+      
+      
+    protected
+      
+      def distance_of_time(milliseconds)
+        milliseconds = milliseconds.to_i
+        seconds = milliseconds / 1000
+        milliseconds %= 1000
+        minutes = seconds / 60
+        seconds %= 60
+        hours = minutes / 60
+        minutes %= 60
+        days = hours / 24
+        hours %= 24
+        
+        time = []
+        time << "#{days} days" unless days.zero?
+        time << "#{hours} hours" unless hours.zero?
+        time << "#{minutes} minutes" unless minutes.zero?
+        time << "#{seconds}.#{milliseconds.to_s.rjust(3, "0")} seconds"
+        time.join(", ")
+      end
+      
+    end
+  end
+end

data/lib/abstract_importer/reporters/debug_reporter.rb

@@ -0,0 +1,131 @@
+module AbstractImporter
+  module Reporters
+    class DebugReporter < BaseReporter
+      attr_reader :invalid_params
+      
+      def initialize(io)
+        super
+        @notices = {}
+        @errors  = {}
+        @invalid_params = {}
+      end
+      
+      
+      
+      def production?
+        Rails.env.production?
+      end
+      
+      
+      
+      def start_all(importer)
+        super
+      end
+      
+      def finish_all(importer, ms)
+        print_invalid_params
+        super
+      end
+      
+      
+      
+      def finish_setup(ms)
+        super
+      end
+      
+      
+      
+      def start_collection(collection)
+        super
+        @notices = {}
+        @errors  = {}
+      end
+      
+      def finish_collection(collection, summary)
+        print_summary summary, collection.name
+        print_messages @notices, "Notices"
+        print_messages @errors,  "Errors"
+      end
+      
+      
+      
+      def record_created(record)
+        io.print "." unless production?
+      end
+      
+      def record_failed(record, hash)
+        io.print "×" unless production?
+        
+        error_messages = invalid_params[record.class.name] ||= {}
+        record.errors.full_messages.each do |error_message|
+          error_messages[error_message] = hash unless error_messages.key?(error_message)
+          count_error(error_message)
+        end
+      end
+      
+      
+      
+      def status(s)
+        io.puts s
+      end
+      
+      def stat(s)
+        io.puts "  #{s}"
+      end
+      alias :info :stat
+      
+      def file(s)
+        io.puts s.inspect
+      end
+      
+      
+      
+      def count_notice(message)
+        return if production?
+        @notices[message] = (@notices[message] || 0) + 1
+      end
+      
+      def count_error(message)
+        @errors[message] = (@errors[message] || 0) + 1
+      end
+      
+      
+      
+    private
+      
+      def print_invalid_params
+        return if invalid_params.empty?
+        status "\n\n\n#{("="*80)}\nExamples of invalid hashes\n#{("="*80)}"
+        invalid_params.each do |model_name, errors|
+          status "\n\n--#{model_name}#{("-"*(78 - model_name.length))}"
+          errors.each do |error_message, hash|
+            status "\n  #{error_message}:\n    #{hash.inspect}"
+          end
+        end
+      end
+      
+      def print_summary(summary, plural)
+        stat "\n  #{summary.total} #{plural} were found"
+        if summary.total > 0
+          stat "#{summary.already_imported} #{plural} were imported previously"
+          stat "#{summary.redundant} #{plural} would create duplicates and will not be imported"
+          stat "#{summary.invalid} #{plural} were invalid"
+          stat "#{summary.skipped} #{plural} were skipped"
+          stat "#{summary.created} #{plural} were imported"
+          stat "#{distance_of_time(summary.ms)} elapsed (#{summary.average_ms.to_i}ms each)"
+        else
+          stat "#{distance_of_time(summary.ms)} elapsed"
+        end
+      end
+      
+      def print_messages(array, caption)
+        return if array.empty?
+        status "\n--#{caption}#{("-"*(78-caption.length))}\n\n"
+        array.each do |message, count|
+          stat "#{count} × #{message}"
+        end
+      end
+      
+    end
+  end
+end