abstract_importer 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6545777a4f5b26c7f43754894478c1ee2b38db44
-  data.tar.gz: defa4ee62445d06bfdadaf9453b2c54f19829c90
+  metadata.gz: aef74ea6f4a839ec7a9fc349cf07cb44a53001e4
+  data.tar.gz: 8c2a8da38743766e795050d5fe67b1102ec44f60
 SHA512:
-  metadata.gz: 607c5f129b8ac2742ad76f8c4a964b3ee24f5304b7d1701409bc5e3b0a5bf3eca0dda5d00d5f8556383d7ec32ba3786259946d55d3eee9f88709b83028c595fc
-  data.tar.gz: edaecfc589c1844af50cf53239940ebfc17052be632fa2bf563fc8bcd74d8e33521b157b9caa73c3285c3685d4e724bda514749cefa9f3fc51ba85d787b4810e
+  metadata.gz: 5215f833089ae23c768c3b9763438e72d43030cb87c8e5d99eb85c61e1736d837c4f4d356a7b7b8708ac3de8136b18f4fb3b9bc2653c7e6c981bebc015b35bfa
+  data.tar.gz: 62ab5e5d9c9b8d984a0bfa098bea336238491ec6546783ae41757dc9922d6f5c7c491a1a3c537fce0bd46d95fb29fc7a8eaa9900ad647f8c66307c8fcbd9e856

data/.ruby-version

@@ -0,0 +1 @@
+2.1.2

data/.travis.yml

@@ -1,8 +1,10 @@
 # .travis.yml
 language: ruby
 rvm:
-  - 2.0.0
-env:
-  - DB=sqlite
+  - 2.1.2
+before_install:
+  - sudo apt-add-repository -y ppa:travis-ci/sqlite3
+  - sudo apt-get -y update
+  - sudo apt-get install sqlite3=3.7.15.1-1~travis1
 script:
   - bundle exec rake test

data/abstract_importer.gemspec

@@ -17,7 +17,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
   
-  spec.add_dependency "activerecord"
+  spec.add_dependency "activerecord", ">= 4.0"
+  spec.add_dependency "activerecord-insert_many", "~> 0.1.1"
   
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "minitest", "~> 4.7"

data/lib/abstract_importer/base.rb

@@ -9,29 +9,29 @@ require 'abstract_importer/summary'
 
 module AbstractImporter
   class Base
-    
+
     class << self
       def import
         yield @import_plan = ImportPlan.new
       end
-      
+
       def depends_on(*dependencies)
         @dependencies = dependencies
       end
-      
+
       attr_reader :import_plan, :dependencies
     end
-    
-    
-    
+
+
+
     def initialize(parent, source, options={})
       @source       = source
       @parent       = parent
-      
+
       io            = options.fetch(:io, $stderr)
       @reporter     = default_reporter(io)
       @dry_run      = options.fetch(:dry_run, false)
-      
+
       @id_map       = IdMap.new
       @results      = {}
       @import_plan  = self.class.import_plan.to_h
@@ -41,171 +41,159 @@ module AbstractImporter
       @only         = Array(options[:only]) if options.key?(:only)
       @collections  = []
     end
-    
+
     attr_reader :source, :parent, :reporter, :id_map, :results
-    
+
     def atomic?
       @atomic
     end
-    
+
     def dry_run?
       @dry_run
     end
-    
-    
-    
-    
-    
+
+
+
+
+
     def perform!
       reporter.start_all(self)
-      
+
       ms = Benchmark.ms do
         setup
       end
       reporter.finish_setup(ms)
-      
+
       ms = Benchmark.ms do
         with_transaction do
           collections.each &method(:import_collection)
         end
       end
-      
+
       teardown
       reporter.finish_all(self, ms)
       results
     end
-    
+
     def setup
       verify_source!
       verify_parent!
       instantiate_collections!
       prepopulate_id_map!
     end
-    
+
     def import_collection(collection)
       return if skip? collection
       results[collection.name] = CollectionImporter.new(self, collection).perform!
     end
-    
+
     def teardown
     end
-    
+
     def skip?(collection)
       return true if skip.member?(collection.name)
       return true if only && !only.member?(collection.name)
       false
     end
-    
+
     def strategy_for(collection)
       strategy_name = @strategies.fetch collection.name, :default
       AbstractImporter::Strategies.const_get :"#{strategy_name.capitalize}Strategy"
     end
-    
-    
-    
-    
-    
+
+
+
+
+
     def describe_source
       source.to_s
     end
-    
+
     def describe_destination
       parent.to_s
     end
-    
-    
-    
-    
-    
+
+
+
+
+
     def remap_foreign_key?(plural, foreign_key)
       true
     end
-    
+
     def map_foreign_key(legacy_id, plural, foreign_key, depends_on)
       id_map.apply!(legacy_id, depends_on)
     rescue IdMap::IdNotMappedError
       record_no_id_in_map_error(legacy_id, plural, foreign_key, depends_on)
       nil
     end
-    
-    
-    
-    
-    
+
+
+
+
+
   private
-    
+
     attr_reader :collections, :import_plan, :skip, :only
-    
+
     def verify_source!
       import_plan.keys.each do |collection|
         next if source.respond_to?(collection)
-        
+
         raise "#{source.class} does not respond to `#{collection}`; " <<
-              "but #{self.class} plans to import records with that name" 
+              "but #{self.class} plans to import records with that name"
       end
     end
-    
+
     def verify_parent!
       import_plan.keys.each do |collection|
         next if parent.respond_to?(collection)
-        
+
         raise "#{parent.class} does not have a collection named `#{collection}`; " <<
               "but #{self.class} plans to import records with that name"
       end
     end
-    
+
     def instantiate_collections!
       @collections = import_plan.map do |name, block|
         reflection = parent.class.reflect_on_association(name)
         model = reflection.klass
         table_name = model.table_name
         scope = parent.public_send(name)
-        
+
         options = ImportOptions.new
         instance_exec(options, &block) if block
-        
+
         Collection.new(name, model, table_name, scope, options)
       end
     end
-    
+
     def dependencies
       @dependencies ||= Array(self.class.dependencies).map do |name|
         reflection = parent.class.reflect_on_association(name)
         model = reflection.klass
         table_name = model.table_name
         scope = parent.public_send(name)
-        
+
         Collection.new(name, model, table_name, scope, nil)
       end
     end
-    
+
     def prepopulate_id_map!
       (collections + dependencies).each do |collection|
-        query = collection.scope.where("#{collection.table_name}.legacy_id IS NOT NULL")
-        map = values_of(query, :id, :legacy_id) \
-          .each_with_object({}) { |(id, legacy_id), map| map[legacy_id] = id }
-        
-        id_map.init collection.table_name, map
-      end
-    end
-    
-    def values_of(query, *columns)
-      if Rails.version < "4.0.0"
-        query = query.select(columns.map { |column| "#{query.table_name}.#{column}" }.join(", "))
-        ActiveRecord::Base.connection.select_rows(query.to_sql)
-      else
-        query.pluck(*columns)
+        id_map.init collection.table_name, collection.scope
+          .where("#{collection.table_name}.legacy_id IS NOT NULL")
       end
     end
-    
-    
-    
+
+
+
     def record_no_id_in_map_error(legacy_id, plural, foreign_key, depends_on)
       reporter.count_notice "#{plural}.#{foreign_key} will be nil: a #{depends_on.to_s.singularize} with the legacy id #{legacy_id} was not mapped."
     end
-    
-    
-    
+
+
+
     def with_transaction(&block)
       if atomic?
         ActiveRecord::Base.transaction(requires_new: true, &block)
@@ -213,7 +201,7 @@ module AbstractImporter
         block.call
       end
     end
-    
+
     def default_reporter(io)
       case ENV["IMPORT_REPORTER"].to_s.downcase
       when "none"        then Reporters::NullReporter.new(io)
@@ -221,6 +209,6 @@ module AbstractImporter
       else                    Reporters::DebugReporter.new(io)
       end
     end
-    
+
   end
 end

data/lib/abstract_importer/collection_importer.rb

@@ -2,15 +2,15 @@ require "abstract_importer/strategies"
 
 module AbstractImporter
   class CollectionImporter
-    
+
     def initialize(importer, collection)
       @importer = importer
       @collection = collection
       @strategy = importer.strategy_for(collection).new(self)
     end
-    
+
     attr_reader :importer, :collection, :summary, :strategy
-    
+
     delegate :name,
              :table_name,
              :model,
@@ -18,7 +18,7 @@ module AbstractImporter
              :options,
              :association_attrs,
              :to => :collection
-    
+
     delegate :dry_run?,
              :parent,
              :source,
@@ -27,46 +27,47 @@ module AbstractImporter
              :id_map,
              :map_foreign_key,
              :to => :importer
-    
-    
-    
+
+
+
     def perform!
       reporter.start_collection(self)
       prepare!
-      
+
       invoke_callback(:before_all)
       summary.ms = Benchmark.ms do
         each_new_record do |attributes|
           strategy.process_record(attributes)
         end
       end
+      strategy.flush
       invoke_callback(:after_all)
-      
+
       reporter.finish_collection(self, summary)
       summary
     end
-    
-    
-    
+
+
+
     def prepare!
       @summary = Summary.new
       @mappings = prepare_mappings!
     end
-    
+
     def prepare_mappings!
       mappings = []
       model.reflect_on_all_associations.each do |association|
-        
+
         # We only want the associations where this record
         # has foreign keys that refer to another
         next unless association.macro == :belongs_to
-        
+
         # We support skipping some mappings entirely. I believe
         # this is largely to cut down on verbosity in the log
         # files and should be refactored to another place in time.
         foreign_key = association.foreign_key.to_sym
         next unless remap_foreign_key?(name, foreign_key)
-        
+
         if association.options[:polymorphic]
           mappings << prepare_polymorphic_mapping!(association)
         else
@@ -75,11 +76,11 @@ module AbstractImporter
       end
       mappings
     end
-    
+
     def prepare_mapping!(association)
       depends_on = association.table_name.to_sym
       foreign_key = association.foreign_key.to_sym
-      
+
       Proc.new do |attrs|
         if attrs.key?(foreign_key)
           attrs[foreign_key] = map_foreign_key(attrs[foreign_key], name, foreign_key, depends_on)
@@ -88,11 +89,11 @@ module AbstractImporter
         end
       end
     end
-    
+
     def prepare_polymorphic_mapping!(association)
       foreign_key = association.foreign_key.to_sym
       foreign_type = association.foreign_key.gsub(/_id$/, "_type").to_sym
-      
+
       Proc.new do |attrs|
         if attrs.key?(foreign_key) && attrs.key?(foreign_type)
           foreign_model = attrs[foreign_type]
@@ -103,23 +104,23 @@ module AbstractImporter
         end
       end
     end
-    
-    
-    
+
+
+
     def each_new_record
       source.public_send(name).each do |attrs|
         yield attrs.dup
       end
     end
-    
-    
-    
+
+
+
     def remap_foreign_keys!(hash)
       @mappings.each do |proc|
         proc.call(hash)
       end
     end
-    
+
     def redundant_record?(hash)
       existing_record = invoke_callback(:finder, hash)
       if existing_record
@@ -129,9 +130,9 @@ module AbstractImporter
         false
       end
     end
-    
-    
-    
+
+
+
     def invoke_callback(callback, *args)
       callback_name = :"#{callback}_callback"
       callback = options.public_send(callback_name)
@@ -139,9 +140,9 @@ module AbstractImporter
       callback = importer.method(callback) if callback.is_a?(Symbol)
       callback.call(*args)
     end
-    
+
   end
-  
+
   class Skip < StandardError; end
-  
+
 end