abstract_importer 1.3.4 → 1.4.0

data/lib/abstract_importer/import_options.rb

@@ -2,7 +2,6 @@ module AbstractImporter
   class ImportOptions
     CALLBACKS = [ :finder,
                   :rescue,
-                  :rescue_batch,
                   :before_build,
                   :before_batch,
                   :before_create,

data/lib/abstract_importer/reporters.rb

@@ -1,5 +1,6 @@
-require 'abstract_importer/reporters/base_reporter'
-require 'abstract_importer/reporters/debug_reporter'
-require 'abstract_importer/reporters/dot_reporter'
-require 'abstract_importer/reporters/null_reporter'
-require 'abstract_importer/reporters/performance_reporter'
+require "abstract_importer/reporters/base_reporter"
+require "abstract_importer/reporters/debug_reporter"
+require "abstract_importer/reporters/dot_reporter"
+require "abstract_importer/reporters/null_reporter"
+require "abstract_importer/reporters/performance_reporter"
+require "abstract_importer/reporters/progress_reporter"

data/lib/abstract_importer/reporters/base_reporter.rb

@@ -17,8 +17,10 @@ module AbstractImporter
         io.puts "\n\nFinished in #{distance_of_time(ms)}"
       end
 
-      def finish_setup(ms)
-        io.puts "Setup took #{distance_of_time(ms)}\n"
+      def finish_setup(importer, ms)
+      end
+
+      def finish_teardown(importer, ms)
       end
 
       def start_collection(collection)
@@ -36,6 +38,9 @@ module AbstractImporter
       def record_failed(record, hash)
       end
 
+      def batch_inserted(size)
+      end
+
 
 
       def count_notice(message)

data/lib/abstract_importer/reporters/debug_reporter.rb

@@ -12,12 +12,6 @@ module AbstractImporter
 
 
 
-      def production?
-        Rails.env.production?
-      end
-
-
-
       def start_all(importer)
         super
       end
@@ -29,7 +23,7 @@ module AbstractImporter
 
 
 
-      def finish_setup(ms)
+      def finish_setup(importer, ms)
         super
       end
 
@@ -75,7 +69,6 @@ module AbstractImporter
 
 
       def count_notice(message)
-        return if production?
         @notices[message] = (@notices[message] || 0) + 1
       end
 

data/lib/abstract_importer/reporters/dot_reporter.rb

@@ -13,6 +13,11 @@ module AbstractImporter
         super
       end
 
+      def batch_inserted(size)
+        io.print "." * size
+        super
+      end
+
 
     end
   end

data/lib/abstract_importer/reporters/null_reporter.rb

@@ -8,7 +8,7 @@ module AbstractImporter
       def finish_all(importer, ms)
       end
 
-      def finish_setup(ms)
+      def finish_setup(importer, ms)
       end
 
       def start_collection(collection)

data/lib/abstract_importer/reporters/progress_reporter.rb

@@ -0,0 +1,42 @@
+require "progressbar"
+
+module AbstractImporter
+  module Reporters
+    class ProgressReporter < BaseReporter
+
+      def finish_setup(importer, ms)
+        total = importer.collections.reduce(0) do |total, collection|
+          total + importer.count_collection(collection)
+        end
+        @pbar = ProgressBar.new("progress", total)
+      end
+
+      def finish_all(importer, ms)
+        pbar.finish
+        io.puts "Finished in #{distance_of_time(ms)}"
+      end
+
+      def start_collection(collection)
+        # Say nothing
+      end
+
+
+
+      def record_created(record)
+        pbar.inc
+      end
+
+      def record_failed(record, hash)
+        pbar.inc
+      end
+
+      def batch_inserted(size)
+        pbar.inc size
+      end
+
+    protected
+      attr_reader :pbar
+
+    end
+  end
+end

data/lib/abstract_importer/strategies.rb

@@ -1,3 +1,4 @@
 require "abstract_importer/strategies/default_strategy"
 require "abstract_importer/strategies/replace_strategy"
 require "abstract_importer/strategies/insert_strategy"
+require "abstract_importer/strategies/upsert_strategy"

data/lib/abstract_importer/strategies/base.rb

@@ -7,15 +7,22 @@ module AbstractImporter
                :remap_foreign_keys!,
                :redundant_record?,
                :invoke_callback,
+               :use_id_map_for?,
                :dry_run?,
                :id_map,
                :scope,
                :reporter,
                :association_attrs,
+               :generate_id,
                to: :collection
 
-      def initialize(collection)
+      def initialize(collection, options={})
         @collection = collection
+        @remap_ids = options.fetch(:id_map, use_id_map_for?(collection))
+      end
+
+      def remap_ids?
+        @remap_ids
       end
 
       def process_record(hash)
@@ -32,9 +39,12 @@ module AbstractImporter
       def prepare_attributes(hash)
         hash = invoke_callback(:before_build, hash) || hash
 
-        legacy_id = hash.delete(:id)
+        if remap_ids?
+          hash = hash.merge(legacy_id: hash.delete(:id))
+          hash[:id] = generate_id.call if generate_id
+        end
 
-        hash.merge(legacy_id: legacy_id).merge(association_attrs)
+        hash.merge(association_attrs)
       end
 
     end

data/lib/abstract_importer/strategies/default_strategy.rb

@@ -44,7 +44,7 @@ module AbstractImporter
         if record.valid? && record.save
           invoke_callback(:after_create, hash, record)
           invoke_callback(:after_save, hash, record)
-          id_map << record
+          id_map << record if remap_ids?
 
           reporter.record_created(record)
           clean_record(record)

data/lib/abstract_importer/strategies/insert_strategy.rb

@@ -5,10 +5,10 @@ module AbstractImporter
   module Strategies
     class InsertStrategy < Base
 
-      def initialize(collection)
+      def initialize(collection, options={})
         super
         @batch = []
-        @batch_size = 250
+        @batch_size = options.fetch(:batch_size, 250)
       end
 
 
@@ -27,8 +27,7 @@ module AbstractImporter
           return
         end
 
-        @batch << prepare_attributes(hash)
-        flush if @batch.length >= @batch_size
+        add_to_batch prepare_attributes(hash)
 
       rescue ::AbstractImporter::Skip
         summary.skipped += 1
@@ -38,24 +37,37 @@ module AbstractImporter
       def flush
         invoke_callback(:before_batch, @batch)
 
-        begin
-          tries = (tries || 0) + 1
-          collection.scope.insert_many(@batch)
-        rescue
-          raise if tries > 1
-          invoke_callback(:rescue_batch, @batch)
-          retry
-        end
+        insert_batch(@batch)
 
-        ids = collection.scope.where(legacy_id: @batch.map { |hash| hash[:legacy_id] })
-        id_map.merge! collection.table_name, ids
+        id_map_record_batch(@batch) if remap_ids?
 
-        summary.created += ids.length
+        summary.created += @batch.length
+        reporter.batch_inserted(@batch.length)
 
         @batch = []
       end
 
 
+      def insert_batch(batch)
+        collection.scope.insert_many(batch)
+      end
+
+
+      def add_to_batch(attributes)
+        @batch << attributes
+        legacy_id, id = attributes.values_at(:legacy_id, :id)
+        id_map.merge! collection.table_name, legacy_id => id if id && legacy_id
+        flush if @batch.length >= @batch_size
+      end
+
+
+      def id_map_record_batch(batch)
+        return if generate_id
+        id_map.merge! collection.table_name,
+          collection.scope.where(legacy_id: @batch.map { |hash| hash[:legacy_id] })
+      end
+
+
     end
   end
 end

data/lib/abstract_importer/strategies/replace_strategy.rb

@@ -37,7 +37,7 @@ module AbstractImporter
       def update_record(hash)
         hash = invoke_callback(:before_build, hash) || hash
 
-        record = scope.find_by(legacy_id: hash.delete(:id))
+        record = remap_ids? ? scope.find_by(legacy_id: hash.delete(:id)) : scope.find_by(id: hash[:id])
         record.attributes = hash
 
         return true if dry_run?

data/lib/abstract_importer/strategies/upsert_strategy.rb

@@ -0,0 +1,25 @@
+require "abstract_importer/strategies/insert_strategy"
+require "activerecord/insert_many"
+
+module AbstractImporter
+  module Strategies
+    class UpsertStrategy < InsertStrategy
+
+
+      # We won't skip any records for already being imported
+      def already_imported?(hash)
+        false
+      end
+
+
+      def insert_batch(batch)
+        collection.scope.insert_many(batch, on_conflict: {
+          column: remap_ids? ? :legacy_id : :id,
+          do: :update
+        })
+      end
+
+
+    end
+  end
+end

data/lib/abstract_importer/summary.rb

@@ -1,8 +1,8 @@
 module AbstractImporter
   class Summary < Struct.new(:total, :redundant, :created, :already_imported, :invalid, :ms, :skipped)
 
-    def initialize
-      super(0,0,0,0,0,0,0)
+    def initialize(a=0, b=0, c=0, d=0, e=0, f=0, g=0)
+      super(a,b,c,d,e,f,g)
     end
 
     def average_ms
@@ -10,5 +10,16 @@ module AbstractImporter
       ms / total
     end
 
+    def +(other)
+      Summary.new(
+        total + other.total,
+        redundant + other.redundant,
+        created + other.created,
+        already_imported + other.already_imported,
+        invalid + other.invalid,
+        ms + other.ms,
+        skipped + other.skipped)
+    end
+
   end
 end

data/lib/abstract_importer/version.rb

@@ -1,3 +1,3 @@
 module AbstractImporter
-  VERSION = "1.3.4"
+  VERSION = "1.4.0"
 end

data/test/callback_test.rb

@@ -63,9 +63,9 @@ class CallbackTest < ActiveSupport::TestCase
     end
 
     should "should be invoked after the record is created" do
-      mock(importer).callback({name: "Harry Potter"}, satisfy(&:persisted?)).once
-      mock(importer).callback({name: "Ron Weasley"}, satisfy(&:persisted?)).once
-      mock(importer).callback({name: "Hermione Granger"}, satisfy(&:persisted?)).once
+      mock(importer).callback(hash_including(name: "Harry Potter"), satisfy(&:persisted?)).once
+      mock(importer).callback(hash_including(name: "Ron Weasley"), satisfy(&:persisted?)).once
+      mock(importer).callback(hash_including(name: "Hermione Granger"), satisfy(&:persisted?)).once
       import!
     end
   end

data/test/id_map_test.rb

@@ -0,0 +1,18 @@
+require "test_helper"
+
+class IdMapTest < ActiveSupport::TestCase
+
+  context ".dup" do
+    should "create an independent copy of an IdMap" do
+      map1 = AbstractImporter::IdMap.new
+      map1.merge! :examples, { "a" => 1 }
+
+      map2 = map1.dup
+      assert_equal({ "a" => 1 }, map2.get(:examples))
+
+      map1.merge! :examples, { "b" => 2 }
+      assert_equal({ "a" => 1 }, map2.get(:examples))
+    end
+  end
+
+end

data/test/insert_strategy_test.rb

@@ -1,7 +1,7 @@
 require "test_helper"
 
 
-class ImporterTest < ActiveSupport::TestCase
+class InsertStrategyTest < ActiveSupport::TestCase
 
   setup do
     options.merge!(strategy: {students: :insert})
@@ -68,25 +68,42 @@ class ImporterTest < ActiveSupport::TestCase
     end
   end
 
-  context "When the import would create a duplicate record" do
+  context "When the imported records belong to a parent polymorphically" do
     setup do
+      @account = Owl.create!(name: "Pigwidgeon")
       plan do |import|
-        import.students do |options|
-          options.rescue_batch do |batch|
-            names = parent.students.pluck :name
-            batch.reject! { |student| names.member? student[:name] }
-          end
-        end
+        import.abilities
       end
-      account.students.create!(name: "Ron Weasley")
     end
 
-    should "not import existing records twice" do
+    should "import records just fine" do
+      pet = @account
       import!
-      assert_equal 3, account.students.count
+      assert_equal [["Owl", pet.id]], Ability.pluck(:pet_type, :pet_id)
     end
   end
 
 
 
+  context "Given an ID generator" do
+    setup do
+      plan do |import|
+        import.students
+      end
+
+      id = 0
+      options.merge!(generate_id: -> { id += 1 })
+    end
+
+    should "insert the records with the specified IDs" do
+      import!
+      assert_equal [1, 2, 3], account.students.pluck(:id)
+    end
+
+    should "map the specified IDs to the legacy_ids" do
+      import!
+      assert_equal ({ 456 => 1, 457 => 2, 458 => 3 }), importer.id_map.get(:students)
+    end
+  end
+
 end

data/test/support/mock_data_source.rb

@@ -52,5 +52,11 @@ class MockDataSource
     end
   end
 
+  def abilities
+    Enumerator.new do |e|
+      e.yield id: 701, name: "Hyperactivity"
+    end
+  end
+
 
 end