abstract_importer 1.3.4 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 51e786a9666c22acd429d7f492d0cc84744cd222
-  data.tar.gz: feea5423d323820bd47c5b701c11187b88d3ff94
+  metadata.gz: a64bcc8c97755427d62459b7f1089c22d3806501
+  data.tar.gz: d8ef7bf828aa70c7294dd77f8d50a93857324a57
 SHA512:
-  metadata.gz: 0840d9026d26c679f66e8f038d00480f77d3d6228ead03b25dd905fd855387121ba30db9036239cd77ac8a83d8933087976afcd265d7f192404da210099d97d3
-  data.tar.gz: 63bdb142056eefdf38eb3585d98fdce6b7c009286c10d8c81cafc1b2fccc4e185ab3fd41b462d84714626396e242293d44978c3ce634846e6621478aac6447f0
+  metadata.gz: 830be8e79c0c5fbc3c8cc7fa372fc808f689dc78bc8518143a0395ad13a5dc1d7cc88ad6ebdbc81c1427688e19a40f5c5902282fa6126975ced79e5a034f4f25
+  data.tar.gz: 596609a81ff049f029baad8492aa8b4902c20e31e6a53237b02011c0a4cb7ed3d77070f93e27849b5c70f980852eb5747fc4d09667c9d49068a102a6f1108008

data/.ruby-version

@@ -1 +1 @@
-2.1.2
+2.3.0

data/.travis.yml

@@ -1,14 +1,17 @@
-# .travis.yml
 language: ruby
 rvm:
-  - 2.1.2
+  - 2.3.1
 
-# https://github.com/travis-ci/travis-ci/issues/5239
-before_install:
-  - gem update bundler
-  - sudo apt-add-repository -y ppa:travis-ci/sqlite3
-  - sudo apt-get -y update
-  - sudo apt-get install sqlite3=3.7.15.1-1~travis1
+# Use Postgres 9.5
+# https://www.brandur.org/fragments/postgres-95-travis
+dist: trusty
+sudo: required
+addons:
+  postgresql: "9.5"
 
-script:
-  - bundle exec rake test
+before_install: gem update bundler
+script: bundle exec rake test
+
+# To stop Travis from running tests for a new commit,
+# add the following to your commit message: [ci skip]
+# You should add this when you edit documentation or comments, etc.

data/Rakefile

@@ -1,9 +1,8 @@
 require "bundler/gem_tasks"
-require 'rake/testtask'
+require "rake/testtask"
 
 Rake::TestTask.new(:test) do |t|
-  t.libs << 'lib'
-  t.libs << 'test'
-  t.pattern = 'test/**/*_test.rb'
-  t.verbose = false
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
 end

data/abstract_importer.gemspec

@@ -1,7 +1,7 @@
 # coding: utf-8
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'abstract_importer/version'
+require "abstract_importer/version"
 
 Gem::Specification.new do |spec|
   spec.name          = "abstract_importer"
@@ -16,20 +16,21 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
-  
-  spec.add_dependency "activerecord", ">= 4.0"
-  spec.add_dependency "activerecord-insert_many", ">= 0.1.1"
-  
+
+  spec.add_dependency "activerecord", ">= 5.0"
+  spec.add_dependency "activesupport", ">= 5.0"
+  spec.add_dependency "activerecord-insert_many", ">= 0.4.0"
+  spec.add_dependency "progressbar"
+
   spec.add_development_dependency "bundler", "~> 1.3"
-  spec.add_development_dependency "minitest", "~> 4.7"
+  spec.add_development_dependency "minitest-reporters"
+  spec.add_development_dependency "minitest-reporters-turn_reporter"
   spec.add_development_dependency "rake"
-  spec.add_development_dependency "rails"
   spec.add_development_dependency "sqlite3"
-  spec.add_development_dependency "turn"
+  spec.add_development_dependency "pg"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "rr"
   spec.add_development_dependency "database_cleaner"
   spec.add_development_dependency "simplecov"
   spec.add_development_dependency "shoulda-context"
-  
 end

data/lib/abstract_importer.rb

@@ -1,2 +1,2 @@
-require 'abstract_importer/base'
-require 'abstract_importer/version'
+require "abstract_importer/base"
+require "abstract_importer/version"

data/lib/abstract_importer/base.rb

@@ -1,13 +1,15 @@
-require 'abstract_importer/import_options'
-require 'abstract_importer/import_plan'
-require 'abstract_importer/reporters'
-require 'abstract_importer/collection'
-require 'abstract_importer/collection_importer'
-require 'abstract_importer/id_map'
-require 'abstract_importer/summary'
+require "active_support/core_ext/module/delegation"
+require "abstract_importer/import_options"
+require "abstract_importer/import_plan"
+require "abstract_importer/reporters"
+require "abstract_importer/collection"
+require "abstract_importer/collection_importer"
+require "abstract_importer/id_map"
+require "abstract_importer/summary"
 
 
 module AbstractImporter
+  class IdNotMappedError < StandardError; end
   class Base
 
     class << self
@@ -27,22 +29,51 @@ module AbstractImporter
     def initialize(parent, source, options={})
       @source       = source
       @parent       = parent
+      @options      = options
 
       io            = options.fetch(:io, $stderr)
-      @reporter     = default_reporter(io)
+      @reporter     = default_reporter(options, io)
       @dry_run      = options.fetch(:dry_run, false)
 
-      @id_map       = IdMap.new
-      @results      = {}
+      id_map        = options.fetch(:id_map, true)
       @import_plan  = self.class.import_plan.to_h
       @atomic       = options.fetch(:atomic, false)
       @strategies   = options.fetch(:strategy, {})
+      @generate_id  = options[:generate_id]
       @skip         = Array(options[:skip])
       @only         = Array(options[:only]) if options.key?(:only)
       @collections  = []
+
+      @use_id_map, @id_map = id_map.is_a?(IdMap) ? [true, id_map] : [id_map, IdMap.new]
+
+      verify_source!
+      verify_parent!
+      instantiate_collections!
+
+      @collection_importers = []
+      collections.each do |collection|
+        next if skip? collection
+        @collection_importers.push CollectionImporter.new(self, collection)
+      end
     end
 
-    attr_reader :source, :parent, :reporter, :id_map, :results
+    attr_reader :source,
+                :parent,
+                :reporter,
+                :id_map,
+                :collections,
+                :import_plan,
+                :skip,
+                :only,
+                :collection_importers,
+                :options,
+                :generate_id
+
+    def use_id_map_for?(collection)
+      collection = find_collection(collection) if collection.is_a?(Symbol)
+      return false unless collection
+      @use_id_map && collection.has_legacy_id?
+    end
 
     def atomic?
       @atomic
@@ -57,48 +88,70 @@ module AbstractImporter
 
 
     def perform!
-      reporter.start_all(self)
+      {}.tap do |results|
+        reporter.start_all(self)
 
-      ms = Benchmark.ms do
-        setup
-      end
-      reporter.finish_setup(ms)
+        ms = Benchmark.ms do
+          setup
+        end
+        reporter.finish_setup(self, ms)
+
+        ms = Benchmark.ms do
+          with_transaction do
+            collection_importers.each do |importer|
+              results[importer.name] = importer.perform!
+            end
+          end
+        end
 
-      ms = Benchmark.ms do
-        with_transaction do
-          collections.each &method(:import_collection)
+        ms = Benchmark.ms do
+          teardown
         end
-      end
+        reporter.finish_teardown(self, ms)
 
-      teardown
-      reporter.finish_all(self, ms)
-      results
+        reporter.finish_all(self, ms)
+      end
     end
 
     def setup
-      verify_source!
-      verify_parent!
-      instantiate_collections!
       prepopulate_id_map!
     end
 
-    def import_collection(collection)
-      return if skip? collection
-      results[collection.name] = CollectionImporter.new(self, collection).perform!
+    def count_collection(collection)
+      collection_name = collection.respond_to?(:name) ? collection.name : collection
+      collection_counts[collection_name]
+    end
+
+    def collection_counts
+      @collection_counts ||= Hash.new do |counts, collection_name|
+        counts[collection_name] = if self.source.respond_to?(:"#{collection_name}_count")
+          self.source.public_send(:"#{collection_name}_count")
+        else
+          self.source.public_send(collection_name).count
+        end
+      end
     end
 
     def teardown
     end
 
     def skip?(collection)
-      return true if skip.member?(collection.name)
-      return true if only && !only.member?(collection.name)
+      collection_name = collection.respond_to?(:name) ? collection.name : collection
+      return true if skip.member?(collection_name)
+      return true if only && !only.member?(collection_name)
       false
     end
 
-    def strategy_for(collection)
+    def strategy_for(collection_importer)
+      collection = collection_importer.collection
       strategy_name = @strategies.fetch collection.name, :default
-      AbstractImporter::Strategies.const_get :"#{strategy_name.capitalize}Strategy"
+      strategy_options = {}
+      if strategy_name.is_a?(Hash)
+        strategy_options = strategy_name
+        strategy_name = strategy_name[:name]
+      end
+      strategy_klass = AbstractImporter::Strategies.const_get :"#{strategy_name.capitalize}Strategy"
+      strategy_klass.new(collection_importer, strategy_options)
     end
 
 
@@ -122,10 +175,11 @@ module AbstractImporter
     end
 
     def map_foreign_key(legacy_id, plural, foreign_key, depends_on)
-      id_map.apply!(legacy_id, depends_on)
-    rescue IdMap::IdNotMappedError
-      record_no_id_in_map_error(legacy_id, plural, foreign_key, depends_on)
-      nil
+      return nil if legacy_id.nil?
+      return legacy_id unless use_id_map_for?(depends_on)
+      id_map.apply!(depends_on, legacy_id)
+    rescue KeyError
+      raise IdNotMappedError, "#{plural}.#{foreign_key} will be nil: a #{depends_on.to_s.singularize} with the legacy id #{legacy_id} was not mapped."
     end
 
 
@@ -134,8 +188,6 @@ module AbstractImporter
 
   private
 
-    attr_reader :collections, :import_plan, :skip, :only
-
     def verify_source!
       import_plan.keys.each do |collection|
         next if source.respond_to?(collection)
@@ -152,6 +204,13 @@ module AbstractImporter
         raise "#{parent.class} does not have a collection named `#{collection}`; " <<
               "but #{self.class} plans to import records with that name"
       end
+
+      Array(self.class.dependencies).each do |collection|
+        next if parent.respond_to?(collection)
+
+        raise "#{parent.class} does not have a collection named `#{collection}`; " <<
+              "but #{self.class} declares it as a dependency"
+      end
     end
 
     def instantiate_collections!
@@ -179,17 +238,20 @@ module AbstractImporter
       end
     end
 
+    def find_collection(name)
+      collections.find { |collection| collection.name == name } ||
+      dependencies.find { |collection| collection.name == name }
+    end
+
     def prepopulate_id_map!
       (collections + dependencies).each do |collection|
-        id_map.init collection.table_name, collection.scope
-          .where("#{collection.table_name}.legacy_id IS NOT NULL")
+        next unless use_id_map_for?(collection)
+        prepopulate_id_map_for!(collection)
       end
     end
 
-
-
-    def record_no_id_in_map_error(legacy_id, plural, foreign_key, depends_on)
-      reporter.count_notice "#{plural}.#{foreign_key} will be nil: a #{depends_on.to_s.singularize} with the legacy id #{legacy_id} was not mapped."
+    def prepopulate_id_map_for!(collection)
+      id_map.init collection.table_name, collection.scope
     end
 
 
@@ -202,14 +264,17 @@ module AbstractImporter
       end
     end
 
-    def default_reporter(io)
-      case ENV["IMPORT_REPORTER"].to_s.downcase
+    def default_reporter(options, io)
+      reporter = options.fetch(:reporter, ENV["IMPORT_REPORTER"])
+      return reporter if reporter.is_a?(AbstractImporter::Reporters::BaseReporter)
+
+      case reporter.to_s.downcase
       when "none"        then Reporters::NullReporter.new(io)
       when "performance" then Reporters::PerformanceReporter.new(io)
       when "debug"       then Reporters::DebugReporter.new(io)
       when "dot"         then Reporters::DotReporter.new(io)
       else
-        if Rails.env.production?
+        if ENV["RAILS_ENV"] == "production"
           Reporters::DebugReporter.new(io)
         else
           Reporters::DotReporter.new(io)

data/lib/abstract_importer/collection.rb

@@ -2,17 +2,24 @@ module AbstractImporter
   class Collection < Struct.new(:name, :model, :table_name, :scope, :options)
 
     def association_attrs
-      return @assocation_attrs if defined?(@assocation_attrs)
+      return @association_attrs if defined?(@association_attrs)
 
       # Instead of calling `tenant.people.build(__)`, we'll reflect on the
       # association to find its foreign key and its owner's id, so that we
       # can call `Person.new(__.merge(tenant_id: id))`.
-      @assocation_attrs = {}
-      assocation = scope.instance_variable_get(:@association)
-      unless assocation.is_a?(ActiveRecord::Associations::HasManyThroughAssociation)
-        @assocation_attrs.merge!(assocation.reflection.foreign_key.to_sym => assocation.owner.id)
+      @association_attrs = {}
+      association = scope.instance_variable_get(:@association)
+      unless association.is_a?(ActiveRecord::Associations::HasManyThroughAssociation)
+        @association_attrs.merge!(association.reflection.foreign_key.to_sym => association.owner.id)
       end
-      @assocation_attrs.freeze
+      if association.reflection.inverse_of && association.reflection.inverse_of.polymorphic?
+        @association_attrs.merge!(association.reflection.inverse_of.foreign_type.to_sym => association.owner.class.name)
+      end
+      @association_attrs.freeze
+    end
+
+    def has_legacy_id?
+      @has_legacy_id ||= model.column_names.member?("legacy_id")
     end
 
   end

data/lib/abstract_importer/collection_importer.rb

@@ -6,7 +6,7 @@ module AbstractImporter
     def initialize(importer, collection)
       @importer = importer
       @collection = collection
-      @strategy = importer.strategy_for(collection).new(self)
+      @strategy = importer.strategy_for(self)
     end
 
     attr_reader :importer, :collection, :summary, :strategy
@@ -17,15 +17,18 @@ module AbstractImporter
              :scope,
              :options,
              :association_attrs,
+             :has_legacy_id?,
              :to => :collection
 
     delegate :dry_run?,
              :parent,
              :source,
              :reporter,
+             :use_id_map_for?,
              :remap_foreign_key?,
              :id_map,
              :map_foreign_key,
+             :generate_id,
              :to => :importer
 
 

data/lib/abstract_importer/id_map.rb

@@ -1,24 +1,16 @@
 module AbstractImporter
   class IdMap
 
-    class IdNotMappedError < StandardError; end
-
     def initialize
       @id_map = Hash.new { |hash, key| hash[key] = {} }
     end
 
-
-
-    def init(table_name, query)
-      table_name = table_name.to_sym
-      @id_map[table_name] = {}
-      merge! table_name, query
-    end
-
-    def merge!(table_name, query)
+    def merge!(table_name, map)
       table_name = table_name.to_sym
-      @id_map[table_name].merge! Hash[query.pluck(:legacy_id, :id)]
+      map = Hash[map.where.not(legacy_id: nil).pluck(:legacy_id, :id)] unless map.is_a?(Hash)
+      @id_map[table_name].merge! map
     end
+    alias :init :merge!
 
     def get(table_name)
       @id_map[table_name.to_sym].dup
@@ -46,11 +38,21 @@ module AbstractImporter
       @id_map[table_name][legacy_id] = record_id
     end
 
-    def apply!(legacy_id, depends_on)
+    def apply!(depends_on, legacy_id)
       return nil if legacy_id.blank?
-      id_map = @id_map[depends_on]
-      raise IdNotMappedError.new unless id_map.key?(legacy_id)
-      id_map[legacy_id]
+      @id_map[depends_on].fetch(legacy_id)
+    end
+
+    def tables
+      @id_map.keys
+    end
+
+    def dup
+      IdMap.new.tap do |other|
+        tables.each do |table|
+          other.init table, get(table)
+        end
+      end
     end
 
   end