abstract_importer 1.2.1 → 1.3.0

data/lib/abstract_importer/strategies.rb

@@ -1,2 +1,3 @@
 require "abstract_importer/strategies/default_strategy"
 require "abstract_importer/strategies/replace_strategy"
+require "abstract_importer/strategies/insert_strategy"

data/lib/abstract_importer/strategies/base.rb

@@ -11,6 +11,7 @@ module AbstractImporter
                :id_map,
                :scope,
                :reporter,
+               :association_attrs,
                to: :collection
 
       def initialize(collection)
@@ -25,6 +26,17 @@ module AbstractImporter
         id_map.contains? collection.table_name, hash[:id]
       end
 
+      def flush
+      end
+
+      def prepare_attributes(hash)
+        hash = invoke_callback(:before_build, hash) || hash
+
+        legacy_id = hash.delete(:id)
+
+        hash.merge(legacy_id: legacy_id).merge(association_attrs)
+      end
+
     end
   end
 end

data/lib/abstract_importer/strategies/default_strategy.rb

@@ -58,13 +58,7 @@ module AbstractImporter
       end
 
       def build_record(hash)
-        hash = invoke_callback(:before_build, hash) || hash
-
-        legacy_id = hash.delete(:id)
-
-        collection.model.new(hash
-          .merge(legacy_id: legacy_id)
-          .merge(collection.association_attrs))
+        collection.model.new prepare_attributes(hash)
       end
 
       def clean_record(record)

data/lib/abstract_importer/strategies/insert_strategy.rb

@@ -0,0 +1,58 @@
+require "abstract_importer/strategies/base"
+require "activerecord/insert_many"
+
+module AbstractImporter
+  module Strategies
+    class InsertStrategy < Base
+
+      def initialize(collection)
+        super
+        @batch = []
+        @batch_size = 250
+      end
+
+
+      def process_record(hash)
+        summary.total += 1
+
+        if already_imported?(hash)
+          summary.already_imported += 1
+          return
+        end
+
+        remap_foreign_keys!(hash)
+
+        if redundant_record?(hash)
+          summary.redundant += 1
+          return
+        end
+
+        @batch << prepare_attributes(hash)
+        flush if @batch.length >= @batch_size
+
+      rescue ::AbstractImporter::Skip
+        summary.skipped += 1
+      end
+
+
+      def flush
+        invoke_callback(:before_batch, @batch)
+
+        begin
+          tries = (tries || 0) + 1
+          collection.scope.insert_many(@batch)
+        rescue
+          raise if tries > 1
+          invoke_callback(:rescue_batch, @batch)
+          retry
+        end
+
+        id_map.merge! collection.table_name, collection.scope
+          .where(legacy_id: @batch.map { |hash| hash[:legacy_id] })
+        @batch = []
+      end
+
+
+    end
+  end
+end

data/lib/abstract_importer/summary.rb

@@ -1,14 +1,14 @@
 module AbstractImporter
   class Summary < Struct.new(:total, :redundant, :created, :already_imported, :invalid, :ms, :skipped)
-    
+
     def initialize
       super(0,0,0,0,0,0,0)
     end
-    
+
     def average_ms
       return nil if total == 0
       ms / total
     end
-    
+
   end
 end

data/lib/abstract_importer/version.rb

@@ -1,3 +1,3 @@
 module AbstractImporter
-  VERSION = "1.2.1"
+  VERSION = "1.3.0"
 end

data/test/callback_test.rb

@@ -2,9 +2,9 @@ require "test_helper"
 
 
 class CallbackTest < ActiveSupport::TestCase
-  
-  
-  
+
+
+
   context "before_build" do
     setup do
       plan do |import|
@@ -13,12 +13,12 @@ class CallbackTest < ActiveSupport::TestCase
         end
       end
     end
-    
+
     should "should be invoked on the incoming attributes" do
       import!
       assert_equal ["Harry", "Ron", "Hermione"], account.students.pluck(:name)
     end
-    
+
     should "allow you to skip certain records" do
       plan do |import|
         import.students do |options|
@@ -27,15 +27,15 @@ class CallbackTest < ActiveSupport::TestCase
           end
         end
       end
-      
+
       import!
       assert_equal ["Ron Weasley", "Hermione Granger"], account.students.pluck(:name)
       assert_equal 1, results[:students].skipped
     end
   end
-  
-  
-  
+
+
+
   context "before_create" do
     setup do
       plan do |import|
@@ -44,15 +44,15 @@ class CallbackTest < ActiveSupport::TestCase
         end
       end
     end
-    
+
     should "should be invoked on imported records before they are saved" do
       import!
       assert_equal ["Gryffindor"], account.students.pluck(:house).uniq
     end
   end
-  
-  
-  
+
+
+
   context "after_create" do
     setup do
       plan do |import|
@@ -61,7 +61,7 @@ class CallbackTest < ActiveSupport::TestCase
         end
       end
     end
-    
+
     should "should be invoked after the record is created" do
       mock(importer).callback({name: "Harry Potter"}, satisfy(&:persisted?)).once
       mock(importer).callback({name: "Ron Weasley"}, satisfy(&:persisted?)).once
@@ -69,9 +69,9 @@ class CallbackTest < ActiveSupport::TestCase
       import!
     end
   end
-  
-  
-  
+
+
+
   context "rescue" do
     setup do
       plan do |import|
@@ -80,15 +80,15 @@ class CallbackTest < ActiveSupport::TestCase
         end
       end
     end
-    
+
     should "should be given a chance to amend an invalid record" do
       import!
       assert_equal ["godrics-hollow", "azkaban"], account.locations.pluck(:slug)
     end
   end
-  
-  
-  
+
+
+
   context "before_all" do
     setup do
       plan do |import|
@@ -97,15 +97,15 @@ class CallbackTest < ActiveSupport::TestCase
         end
       end
     end
-    
+
     should "should be invoked before the collection has been imported" do
       mock(importer).callback.once
       import!
     end
   end
-  
-  
-  
+
+
+
   context "after_all" do
     setup do
       plan do |import|
@@ -114,13 +114,13 @@ class CallbackTest < ActiveSupport::TestCase
         end
       end
     end
-    
+
     should "should be invoked after the collection has been imported" do
       mock(importer).callback.once
       import!
     end
   end
-  
-  
-  
+
+
+
 end

data/test/{importer_test.rb → default_strategy_test.rb}

@@ -1,30 +1,30 @@
 require "test_helper"
 
 
-class ImporterTest < ActiveSupport::TestCase
-  
-  
-  
+class DefaultStrategyTest < ActiveSupport::TestCase
+
+
+
   context "with a simple data source" do
     setup do
       plan do |import|
         import.students
       end
     end
-    
+
     should "import the given records" do
       import!
       assert_equal ["Harry Potter", "Ron Weasley", "Hermione Granger"], account.students.pluck(:name)
     end
-    
+
     should "record their legacy_id" do
       import!
       assert_equal [456, 457, 458], account.students.pluck(:legacy_id)
     end
   end
-  
-  
-  
+
+
+
   context "with a complex data source" do
     setup do
       plan do |import|
@@ -32,19 +32,19 @@ class ImporterTest < ActiveSupport::TestCase
         import.parents
       end
     end
-    
+
     should "preserve mappings" do
       import!
       harry = account.students.find_by_name("Harry Potter")
       assert_equal ["James Potter", "Lily Potter"], harry.parents.pluck(:name)
     end
-    
+
     should "preserve mappings even when a record was previously imported" do
       harry = account.students.create!(name: "Harry Potter", legacy_id: 456)
       import!
       assert_equal ["James Potter", "Lily Potter"], harry.parents.pluck(:name)
     end
-    
+
     context "when {atomic: true}" do
       should "rollback the whole import if an part fails" do
         mock(importer).atomic? { true }
@@ -54,7 +54,7 @@ class ImporterTest < ActiveSupport::TestCase
         assert_equal 0, account.students.count, "Expected students to have been rolled back"
       end
     end
-    
+
     context "when {atomic: false}" do
       should "not rollback the whole import if an part fails" do
         mock(importer).atomic? { false }
@@ -65,9 +65,9 @@ class ImporterTest < ActiveSupport::TestCase
       end
     end
   end
-  
-  
-  
+
+
+
   context "with a dependency" do
     setup do
       depends_on :students
@@ -75,16 +75,16 @@ class ImporterTest < ActiveSupport::TestCase
         import.parents
       end
     end
-    
+
     should "preserve mappings when a dependency was imported by another importer" do
       harry = account.students.create!(name: "Harry Potter", legacy_id: 456)
       import!
       assert_equal ["James Potter", "Lily Potter"], harry.parents.pluck(:name)
     end
   end
-  
-  
-  
+
+
+
   context "when a finder is specified" do
     setup do
       plan do |import|
@@ -94,22 +94,22 @@ class ImporterTest < ActiveSupport::TestCase
         import.parents
       end
     end
-    
+
     should "not import redundant records" do
       account.students.create!(name: "Ron Weasley", legacy_id: nil)
       import!
       assert_equal 3, account.students.count
     end
-    
+
     should "preserve mappings" do
       harry = account.students.create!(name: "Harry Potter", legacy_id: nil)
       import!
       assert_equal ["James Potter", "Lily Potter"], harry.parents.pluck(:name)
     end
   end
-  
-  
-  
+
+
+
   context "with a more complex data source" do
     setup do
       plan do |import|
@@ -124,16 +124,16 @@ class ImporterTest < ActiveSupport::TestCase
         import.grades
       end
     end
-    
+
     should "preserve mappings" do
       import!
       ron = account.students.find_by_name "Ron Weasley"
       assert_equal ["Advanced Potions: Acceptable", "History of Magic: Troll"], ron.report_card
     end
   end
-  
-  
-  
+
+
+
   context "with polymorphic associations" do
     setup do
       plan do |import|
@@ -142,7 +142,7 @@ class ImporterTest < ActiveSupport::TestCase
         import.students
       end
     end
-    
+
     should "preserve mappings" do
       import!
       assert_equal 2, account.students.map(&:pet).compact.count, "Expected two students to still be linked to their pets upon import"
@@ -150,30 +150,25 @@ class ImporterTest < ActiveSupport::TestCase
       assert_kind_of Cat, account.students.find_by_name("Hermione Granger").pet, "Expected Harry's pet to be a Cat"
     end
   end
-  
-  
-  
-  context "When we use the default strategy" do
+
+
+
+  context "When records already exist" do
     setup do
       plan do |import|
         import.students
       end
+      account.students.create!(name: "Ron Weasley", legacy_id: 457)
     end
-    
-    context "and records already exist" do
-      setup do
-        account.students.create!(name: "Ron Weasley", legacy_id: 457)
-      end
-      
-      should "not import existing records twice" do
-        import!
-        assert_equal 3, account.students.count
-      end
+
+    should "not import existing records twice" do
+      import!
+      assert_equal 3, account.students.count
     end
   end
-  
-  
-  
+
+
+
   context "When we specify collections to skip" do
     setup do
       plan do |import|
@@ -181,24 +176,24 @@ class ImporterTest < ActiveSupport::TestCase
         import.parents
       end
     end
-    
+
     context "using :skip" do
       setup do
         options.merge!(skip: :parents)
       end
-      
+
       should "not import the named collections" do
         import!
         assert_equal 3, account.students.length
         assert_equal 0, account.parents.length
       end
     end
-    
+
     context "using :only" do
       setup do
         options.merge!(only: [:students])
       end
-      
+
       should "import only the named collections" do
         import!
         assert_equal 3, account.students.length
@@ -206,30 +201,7 @@ class ImporterTest < ActiveSupport::TestCase
       end
     end
   end
-  
-  
-  
-  context "When we use the :replace strategy" do
-    setup do
-      options.merge!(strategy: {students: :replace})
-      plan do |import|
-        import.students
-      end
-    end
-    
-    context "and records already exist" do
-      setup do
-        account.students.create!(name: "Ron Weasley", legacy_id: 457)
-      end
-      
-      should "reimport the existing records" do
-        import!
-        assert_equal "Gryffindor", account.students.find_by_name("Ron Weasley").house,
-          "Expected Ron's record to have been replaced with one that has a house"
-      end
-    end
-  end
-  
-  
-  
+
+
+
 end