abstract_importer 1.5.6 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 2273e7488017981f0caa7c51ae1bfc7fa2bf8dd3
-  data.tar.gz: 0b2cf66310053405eace82238f10bc42b8af8dbf
+SHA256:
+  metadata.gz: f1a26d52ad8e17f1b2539d5abbcf996a4c80ac157139a988fefd4a417c4fe03e
+  data.tar.gz: 388a9123f67dd9b52c35ae50a8added3c6995d2b8d71d703269dd317e13e8918
 SHA512:
-  metadata.gz: 1d72f4017de21ff358da2f080289cb470b695ac0ee404bf2cd0a3fc6d04b71e1eb8ae38b3d3f98a93b574cd77737ffab17d3f63d1d4e30117c9e2edf18ff7d15
-  data.tar.gz: fc0233f350e80133e8e699a17a95217a546b1de174baf7680ad925fde166a7637733ab509d6bf0bf174547038f54d496bfece1e9a55b6f9260514ca23791e81c
+  metadata.gz: 19531db43b5cd36a2831384fcef16092930b23cc9fa622c1aaa1b1bdc193d73df0d824deda4bdc5ce427d80272bd1d48cd972efea15011187fffa1c5720cb4cf
+  data.tar.gz: 14d9f5e0d630fc2e15be68bd83ef61ca610bf4cec4442af820cfa429915020320d88416af4c8f2fc5d7779daa8b83900c7b0be7e4a87bc21aa73985dd0a58c17

data/.ruby-version

@@ -1 +1 @@
-2.3.1
+2.6.3

data/.travis.yml

@@ -1,6 +1,6 @@
 language: ruby
 rvm:
-  - 2.3.1
+  - 2.6.3
 
 # Use Postgres 9.5
 # https://www.brandur.org/fragments/postgres-95-travis

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+## v1.6.0 (2019 Sept 23)
+* BREAKING: Updated to use Rails 6 and its `insert_all` and `upsert_all` methods and arguments

data/README.md

@@ -187,6 +187,23 @@ The following alternate strategies are built in:
 
 Replaces records that have already been imported rather than skipping them.
 
+##### insert
+
+Bulk inserts records rather than creating them one-by-one as ActiveRecord objects, skipping those that have already been imported.
+
+##### upsert
+
+Bulk inserts records, but updates records that have already been imported rather than skipping them.
+
+
+
+### Important Note on Upgrading from 1.5.x to 1.6.0
+
+With the jump to 1.6.0, `abstract_importer` has dropped reliance upon the `activerecord-insert_many` gem in favor of taking advantage of Rails 6's built-in `insert_all` and `upsert_all` for the insert and upsert strategies. With the move from the gem to relying on Rails, the syntax to specify what constitutes a conflict/duplicate record has been updated to align more closely with that of Rails. In particular:
+
+* Instead of specifying an `on_conflict: { do: :update }` clause, you should specify `on_duplicate: :update` to opt-in to upsert behavior.
+* Rather than listing columns in `on_conflict: { columns: %i{legacy_id id} }`, you should use `unique_by: %i{legacy_id id}` to specify the index or columns by which a duplicate or conflict is defined. The option follows the behavior of Rails' `upsert_all`, so refer to the Rails 6 documentation for more information.
+
 
 
 ## Contributing

data/abstract_importer.gemspec

@@ -17,9 +17,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "activerecord", ">= 5.0"
-  spec.add_dependency "activesupport", ">= 5.0"
-  spec.add_dependency "activerecord-insert_many", ">= 0.4.4"
+  spec.add_dependency "activerecord", ">= 6.0"
+  spec.add_dependency "activesupport", ">= 6.0"
   spec.add_dependency "progressbar"
 
   spec.add_development_dependency "bundler"

data/lib/abstract_importer/strategies/insert_strategy.rb

@@ -1,5 +1,4 @@
 require "abstract_importer/strategies/base"
-require "activerecord/insert_many"
 
 module AbstractImporter
   module Strategies
@@ -9,7 +8,8 @@ module AbstractImporter
         super
         @batch = []
         @batch_size = options.fetch(:batch_size, 250)
-        @insert_options = options.slice(:on_conflict)
+        @bulk_operation = options[:on_duplicate] == :update ? :upsert_all : :insert_all
+        @insert_options = options.slice(:unique_by)
         @insert_options.merge!(returning: [:legacy_id, :id]) if remap_ids?
       end
 
@@ -52,7 +52,9 @@ module AbstractImporter
 
 
       def insert_batch(batch)
-        result = collection.scope.insert_many(batch, @insert_options)
+        return if batch.empty?
+
+        result = collection.scope.public_send(@bulk_operation, batch, @insert_options)
         add_batch_to_id_map(result) if remap_ids?
       end
 
@@ -66,13 +68,26 @@ module AbstractImporter
 
 
       def add_batch_to_id_map(result)
-        map = result.each_with_object({}) do |attrs, map|
+        map = cast_result(result, collection.table_name).each_with_object({}) do |attrs, map|
           map[attrs.fetch("legacy_id")] = attrs.fetch("id")
         end
         id_map.merge! collection.table_name, map
       end
 
 
+      def cast_result(result, table_name)
+        types_by_column = result.columns.each_with_object({}) do |column_name, types|
+          types[column_name] = collection.scope.connection.lookup_cast_type_from_column(collection.scope.columns.find { |column| column.name == column_name })
+        end
+
+        result.to_a.map { |row|
+          Hash[row.map { |column_name, value|
+            [ column_name, types_by_column[column_name].deserialize(value) ]
+          }]
+        }
+      end
+
+
     end
   end
 end

data/lib/abstract_importer/strategies/upsert_strategy.rb

@@ -1,5 +1,4 @@
 require "abstract_importer/strategies/insert_strategy"
-require "activerecord/insert_many"
 
 module AbstractImporter
   module Strategies
@@ -7,7 +6,8 @@ module AbstractImporter
 
       def initialize(collection, options={})
         super
-        @insert_options.reverse_merge!(on_conflict: { column: remap_ids? ? (association_attrs.keys + [:legacy_id]) : :id, do: :update })
+        @bulk_operation = :upsert_all
+        @insert_options.reverse_merge!(unique_by: remap_ids? ? (association_attrs.keys + %i{legacy_id}) : :id)
       end
 
       # We won't skip any records for already being imported

data/lib/abstract_importer/version.rb

@@ -1,3 +1,3 @@
 module AbstractImporter
-  VERSION = "1.5.6"
+  VERSION = "1.6.0"
 end

data/test/insert_strategy_test.rb

@@ -17,7 +17,7 @@ class InsertStrategyTest < ActiveSupport::TestCase
     end
 
     should "import the records in batches" do
-      mock.proxy(Student).insert_many(satisfy { |arg| arg.length == 3 }, anything)
+      mock.proxy(Student).insert_all(satisfy { |arg| arg.length == 3 }, anything)
       import!
       assert_equal [456, 457, 458], account.students.pluck(:legacy_id)
     end
@@ -54,6 +54,22 @@ class InsertStrategyTest < ActiveSupport::TestCase
     end
   end
 
+  context "With an empty data source" do
+    setup do
+      plan do |import|
+        import.students
+      end
+      @data_source = OpenStruct.new
+      @data_source.students = []
+    end
+
+    should "still be able to import" do
+      assert_nothing_raised do
+        import!
+      end
+    end
+  end
+
   context "When records already exist" do
     setup do
       plan do |import|

data/test/upsert_strategy_test.rb

@@ -17,7 +17,7 @@ class UpsertStrategyTest < ActiveSupport::TestCase
     end
 
     should "import the records in batches" do
-      mock.proxy(Student).insert_many(satisfy { |arg| arg.length == 3 }, anything)
+      mock.proxy(Student).upsert_all(satisfy { |arg| arg.length == 3 }, anything)
       import!
       assert_equal [456, 457, 458], account.students.pluck(:legacy_id)
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: abstract_importer
 version: !ruby/object:Gem::Version
-  version: 1.5.6
+  version: 1.6.0
 platform: ruby
 authors:
 - Bob Lail
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-16 00:00:00.000000000 Z
+date: 2019-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -16,42 +16,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
-- !ruby/object:Gem::Dependency
-  name: activerecord-insert_many
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.4.4
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.4.4
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: progressbar
   requirement: !ruby/object:Gem::Requirement
@@ -230,6 +216,7 @@ files:
 - ".gitignore"
 - ".ruby-version"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -288,8 +275,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Provides services for the mass-import of complex relational data