abstract_importer 1.2.1 → 1.3.0

data/lib/abstract_importer/id_map.rb

@@ -1,32 +1,40 @@
 module AbstractImporter
   class IdMap
-    
+
     class IdNotMappedError < StandardError; end
-    
+
     def initialize
       @id_map = Hash.new { |hash, key| hash[key] = {} }
     end
-    
-    
-    
-    def init(table_name, map)
+
+
+
+    def init(table_name, query)
       table_name = table_name.to_sym
-      @id_map[table_name] = map
+      @id_map[table_name] = {}
+      merge! table_name, query
     end
-    
+
+    def merge!(table_name, query)
+      table_name = table_name.to_sym
+      @id_map[table_name].merge!(query
+        .pluck(:id, :legacy_id)
+        .each_with_object({}) { |(id, legacy_id), map| map[legacy_id] = id })
+    end
+
     def get(table_name)
       @id_map[table_name.to_sym].dup
     end
     alias :[] :get
-    
+
     def contains?(table_name, id)
       @id_map[table_name.to_sym].key?(id)
     end
-    
+
     def <<(record)
       register(record: record)
     end
-    
+
     def register(options={})
       if options.key?(:record)
         record = options[:record]
@@ -35,17 +43,17 @@ module AbstractImporter
       table_name = options[:table_name] if options.key?(:table_name)
       legacy_id = options[:legacy_id] if options.key?(:legacy_id)
       record_id = options[:record_id] if options.key?(:record_id)
-      
+
       table_name = table_name.to_sym
       @id_map[table_name][legacy_id] = record_id
     end
-    
+
     def apply!(legacy_id, depends_on)
       return nil if legacy_id.blank?
       id_map = @id_map[depends_on]
       raise IdNotMappedError.new unless id_map.key?(legacy_id)
       id_map[legacy_id]
     end
-    
+
   end
 end

data/lib/abstract_importer/import_options.rb

@@ -2,7 +2,9 @@ module AbstractImporter
   class ImportOptions
     CALLBACKS = [ :finder,
                   :rescue,
+                  :rescue_batch,
                   :before_build,
+                  :before_batch,
                   :before_create,
                   :before_update,
                   :before_save,

data/lib/abstract_importer/import_plan.rb

@@ -1,17 +1,17 @@
 module AbstractImporter
   class ImportPlan
-    
+
     def initialize
       @plan = {} # <-- requires Ruby 1.9's ordered hash
     end
-    
+
     def to_h
       @plan.dup
     end
-    
+
     def method_missing(plural, &block)
       @plan[plural] = block
     end
-    
+
   end
 end

data/lib/abstract_importer/reporters/base_reporter.rb

@@ -2,52 +2,52 @@ module AbstractImporter
   module Reporters
     class BaseReporter
       attr_reader :io
-      
+
       def initialize(io)
         @io = io
       end
-      
-      
-      
+
+
+
       def start_all(importer)
         io.puts "Importing #{importer.describe_source} to #{importer.describe_destination}\n"
       end
-      
+
       def finish_all(importer, ms)
         io.puts "\n\nFinished in #{distance_of_time(ms)}"
       end
-      
+
       def finish_setup(ms)
         io.puts "Setup took #{distance_of_time(ms)}\n"
       end
-      
+
       def start_collection(collection)
         io.puts "\n#{("="*80)}\nImporting #{collection.name}\n#{("="*80)}\n"
       end
-      
+
       def finish_collection(collection, summary)
       end
-      
-      
-      
+
+
+
       def record_created(record)
       end
-      
+
       def record_failed(record, hash)
       end
-      
-      
-      
+
+
+
       def count_notice(message)
       end
-      
+
       def count_error(message)
       end
-      
-      
-      
+
+
+
     protected
-      
+
       def distance_of_time(milliseconds)
         milliseconds = milliseconds.to_i
         seconds = milliseconds / 1000
@@ -58,7 +58,7 @@ module AbstractImporter
         minutes %= 60
         days = hours / 24
         hours %= 24
-        
+
         time = []
         time << "#{days} days" unless days.zero?
         time << "#{hours} hours" unless hours.zero?
@@ -66,7 +66,7 @@ module AbstractImporter
         time << "#{seconds}.#{milliseconds.to_s.rjust(3, "0")} seconds"
         time.join(", ")
       end
-      
+
     end
   end
 end

data/lib/abstract_importer/reporters/debug_reporter.rb

@@ -2,97 +2,97 @@ module AbstractImporter
   module Reporters
     class DebugReporter < BaseReporter
       attr_reader :invalid_params
-      
+
       def initialize(io)
         super
         @notices = {}
         @errors  = {}
         @invalid_params = {}
       end
-      
-      
-      
+
+
+
       def production?
         Rails.env.production?
       end
-      
-      
-      
+
+
+
       def start_all(importer)
         super
       end
-      
+
       def finish_all(importer, ms)
         print_invalid_params
         super
       end
-      
-      
-      
+
+
+
       def finish_setup(ms)
         super
       end
-      
-      
-      
+
+
+
       def start_collection(collection)
         super
         @notices = {}
         @errors  = {}
       end
-      
+
       def finish_collection(collection, summary)
         print_summary summary, collection.name
         print_messages @notices, "Notices"
         print_messages @errors,  "Errors"
       end
-      
-      
-      
+
+
+
       def record_created(record)
         io.print "." unless production?
       end
-      
+
       def record_failed(record, hash)
         io.print "×" unless production?
-        
+
         error_messages = invalid_params[record.class.name] ||= {}
         record.errors.full_messages.each do |error_message|
           error_messages[error_message] = hash unless error_messages.key?(error_message)
           count_error(error_message)
         end
       end
-      
-      
-      
+
+
+
       def status(s)
         io.puts s
       end
-      
+
       def stat(s)
         io.puts "  #{s}"
       end
       alias :info :stat
-      
+
       def file(s)
         io.puts s.inspect
       end
-      
-      
-      
+
+
+
       def count_notice(message)
         return if production?
         @notices[message] = (@notices[message] || 0) + 1
       end
-      
+
       def count_error(message)
         @errors[message] = (@errors[message] || 0) + 1
       end
-      
-      
-      
+
+
+
     private
-      
+
       def print_invalid_params
         return if invalid_params.empty?
         status "\n\n\n#{("="*80)}\nExamples of invalid hashes\n#{("="*80)}"
@@ -103,7 +103,7 @@ module AbstractImporter
           end
         end
       end
-      
+
       def print_summary(summary, plural)
         stat "\n  #{summary.total} #{plural} were found"
         if summary.total > 0
@@ -117,7 +117,7 @@ module AbstractImporter
           stat "#{distance_of_time(summary.ms)} elapsed"
         end
       end
-      
+
       def print_messages(array, caption)
         return if array.empty?
         status "\n--#{caption}#{("-"*(78-caption.length))}\n\n"
@@ -125,7 +125,7 @@ module AbstractImporter
           stat "#{count} × #{message}"
         end
       end
-      
+
     end
   end
 end

data/lib/abstract_importer/reporters/null_reporter.rb

@@ -1,19 +1,19 @@
 module AbstractImporter
   module Reporters
     class NullReporter < BaseReporter
-      
+
       def start_all(importer)
       end
-      
+
       def finish_all(importer, ms)
       end
-      
+
       def finish_setup(ms)
       end
-      
+
       def start_collection(collection)
       end
-      
+
     end
   end
 end

data/lib/abstract_importer/reporters/performance_reporter.rb

@@ -4,50 +4,50 @@ module AbstractImporter
   module Reporters
     class PerformanceReporter < BaseReporter
       attr_reader :sample_size
-      
+
       def initialize(io, options={})
         super io
         @sample_size = options.fetch(:sample_size, 50)
         ObjectSpace.trace_object_allocations_start
       end
-      
-      
+
+
       def start_collection(collection)
         super
         @collection = collection
         @major_gc_runs = GC.stat[:major_gc_count]
         @i = 0
       end
-      
+
       def finish_collection(collection, summary)
         @collection = nil
         return if @i.zero?
         find_objects_holding_onto_references_to_a collection.model
       end
-      
+
       def record_created(record)
         print_stats if @i % sample_size == 0
         @i += 1
       end
-      
+
       def record_failed(record, hash)
         print_stats if @i % sample_size == 0
         @i += 1
       end
-      
-      
+
+
       def print_stats
         stats = GC.stat
         objects = ObjectSpace.count_objects
         puts "gc[minor]: #{stats[:minor_gc_count]}, gc[major]: #{stats[:major_gc_count]}, objects: #{objects[:TOTAL] - objects[:FREE]}, memsize: #{(ObjectSpace.memsize_of_all / 1048576.0).round(3)}MB, #{collection.name}: #{ObjectSpace.each_object(collection.model).count}"
       end
-      
+
     private
       attr_reader :collection
-      
+
       def find_objects_holding_onto_references_to_a(model)
         GC.start
-        
+
         # After GC.start, all models in this collection should be
         # garbage-collected unless there is a memory leak. Find one
         # of the uncollected objects and figure out what is holding
@@ -58,11 +58,11 @@ module AbstractImporter
           return
         end
         puts "\e[33mThere are #{ObjectSpace.each_object(model).count} #{model.name.tableize.gsub("_", " ")} still in memory\e[0m"
-        
+
         example_klass = example.class.name
         example_id = example.object_id
         example = nil
-        
+
         # Search through all objects to find ones that hold a reference
         # to the model that hasn't been garbage-collected.
         print "\e[90m"
@@ -72,7 +72,7 @@ module AbstractImporter
         ObjectSpace.each_object do |o|
           pbar.inc
           next if ObjectSpace.reachable_objects_from(o).none? { |oo| oo.object_id == example_id }
-          
+
           message = "#{o.class.name}"
           case o
           when Array
@@ -84,12 +84,12 @@ module AbstractImporter
           end
           message << " [#{ObjectSpace.allocation_sourcefile(o)}" <<
                      ":#{ObjectSpace.allocation_sourceline(o)}]"
-          
+
           objects_of_holding.push(message)
         end
         pbar.finish
         print "\e[0m"
-        
+
         if objects_of_holding.none?
           puts "\e[95mNo objects are holding a reference to the first one\e[0m"
         else
@@ -97,7 +97,7 @@ module AbstractImporter
                "\e[35m#{objects_of_holding.join("\n")}\e[0m"
         end
       end
-      
+
     end
   end
 end