abstract_importer 1.5.5 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 94acabc53243134c9f39de95063b48ec9c0cb297
-  data.tar.gz: 99ccb1ae5001d2c729b486bf59a95ed686c5ce93
+SHA256:
+  metadata.gz: 14a8a74653aaaf12e7dfa92c53d45220c23ffcab5899a6a39d0094f6ae5b80b2
+  data.tar.gz: be3ee5d19164f9ef5fb8b3cbecb7d9ab07abd39dca0ae15684d3a2b407cb1b9c
 SHA512:
-  metadata.gz: 184aa28e55ca225a549454446b9fdf60f64c747226645811fc6c364a7f94253bee9c3015bd29a5b9c6f18ec80d4a9bf4566371917812fb7afc5101a01b5bb778
-  data.tar.gz: ba21046ac751e42b1f45b68712eff7c32b034bd44829ce0d801166af13a7d5eb459b0075b631a6a6701e6755966c944e630f90facc9ca403f6b2b600c606f93b
+  metadata.gz: 8cb7cd6d795367ed66d09fd6bd36f09c50c5cbbd663fe3ed9a84e44582bdf1b6dc66c570b77e6c1e57e9adb7babcf845ba2ceb0b4a36cf7de99bf6ecb84c25cf
+  data.tar.gz: 9db676b2fdd24b9200ddcf102cac0eccf3c8b527f3ce97d171e2526fb5132592eea1eb8ba399a300df15ec187bdd0d57122b56d8b1c979c2c02a6dce7ae2f1ba

data/.github/workflows/ci.yml

@@ -0,0 +1,29 @@
+name: Tests
+on: [push]
+
+jobs:
+  ruby:
+    name: Tests
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:10.11
+        env:
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: abstract_importer_test
+        ports:
+          - 5432:5432
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.6
+          bundler-cache: true
+      - name: Run the Tests
+        env:
+          DATABASE_URL: postgres://postgres:password@localhost:5432/abstract_importer_test
+        run: bundle exec rake test
+

data/.ruby-version

@@ -1 +1 @@
-2.3.1
+2.6.3

data/CHANGELOG.md

@@ -0,0 +1,8 @@
+## v1.8.0 (2021 Dec 10)
+* Added `prepare_scope` as another callback that can be used when importing a collection
+
+## v1.7.0 (2021 Nov 19)
+* Updated to handle `insert_all` and `upsert_all` to `has_many through:` relations in Rails 6.1+
+
+## v1.6.0 (2019 Sept 23)
+* BREAKING: Updated to use Rails 6 and its `insert_all` and `upsert_all` methods and arguments

data/README.md

@@ -187,6 +187,23 @@ The following alternate strategies are built in:
 
 Replaces records that have already been imported rather than skipping them.
 
+##### insert
+
+Bulk inserts records rather than creating them one-by-one as ActiveRecord objects, skipping those that have already been imported.
+
+##### upsert
+
+Bulk inserts records, but updates records that have already been imported rather than skipping them.
+
+
+
+### Important Note on Upgrading from 1.5.x to 1.6.0
+
+With the jump to 1.6.0, `abstract_importer` has dropped reliance upon the `activerecord-insert_many` gem in favor of taking advantage of Rails 6's built-in `insert_all` and `upsert_all` for the insert and upsert strategies. With the move from the gem to relying on Rails, the syntax to specify what constitutes a conflict/duplicate record has been updated to align more closely with that of Rails. In particular:
+
+* Instead of specifying an `on_conflict: { do: :update }` clause, you should specify `on_duplicate: :update` to opt-in to upsert behavior.
+* Rather than listing columns in `on_conflict: { columns: %i{legacy_id id} }`, you should use `unique_by: %i{legacy_id id}` to specify the index or columns by which a duplicate or conflict is defined. The option follows the behavior of Rails' `upsert_all`, so refer to the Rails 6 documentation for more information.
+
 
 
 ## Contributing

data/abstract_importer.gemspec

@@ -17,9 +17,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "activerecord", ">= 5.0"
-  spec.add_dependency "activesupport", ">= 5.0"
-  spec.add_dependency "activerecord-insert_many", ">= 0.4.4"
+  spec.add_dependency "activerecord", ">= 6.0"
+  spec.add_dependency "activesupport", ">= 6.0"
   spec.add_dependency "progressbar"
 
   spec.add_development_dependency "bundler"
@@ -27,7 +26,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "minitest-reporters-turn_reporter"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "sqlite3"
-  spec.add_development_dependency "pg", "~> 0.18"
+  spec.add_development_dependency "pg"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "rr"
   spec.add_development_dependency "database_cleaner"

data/lib/abstract_importer/import_options.rb

@@ -2,6 +2,7 @@ module AbstractImporter
   class ImportOptions
     CALLBACKS = [ :finder,
                   :rescue,
+                  :prepare_scope,
                   :before_build,
                   :before_batch,
                   :before_create,

data/lib/abstract_importer/strategies/insert_strategy.rb

@@ -1,5 +1,4 @@
 require "abstract_importer/strategies/base"
-require "activerecord/insert_many"
 
 module AbstractImporter
   module Strategies
@@ -9,7 +8,8 @@ module AbstractImporter
         super
         @batch = []
         @batch_size = options.fetch(:batch_size, 250)
-        @insert_options = options.slice(:on_conflict)
+        @bulk_operation = options[:on_duplicate] == :update ? :upsert_all : :insert_all
+        @insert_options = options.slice(:unique_by)
         @insert_options.merge!(returning: [:legacy_id, :id]) if remap_ids?
       end
 
@@ -52,7 +52,16 @@ module AbstractImporter
 
 
       def insert_batch(batch)
-        result = collection.scope.insert_many(batch, @insert_options)
+        return if batch.empty?
+
+        scope = collection.scope
+        if scope.respond_to?(:proxy_association) && scope.proxy_association.reflection.through_reflection?
+          scope = scope.klass
+        end
+
+        scope = invoke_callback(:prepare_scope, scope) || scope
+
+        result = scope.public_send(@bulk_operation, batch, @insert_options)
         add_batch_to_id_map(result) if remap_ids?
       end
 
@@ -66,13 +75,26 @@ module AbstractImporter
 
 
       def add_batch_to_id_map(result)
-        map = result.each_with_object({}) do |attrs, map|
+        map = cast_result(result, collection.table_name).each_with_object({}) do |attrs, map|
           map[attrs.fetch("legacy_id")] = attrs.fetch("id")
         end
         id_map.merge! collection.table_name, map
       end
 
 
+      def cast_result(result, table_name)
+        types_by_column = result.columns.each_with_object({}) do |column_name, types|
+          types[column_name] = collection.scope.connection.lookup_cast_type_from_column(collection.scope.columns.find { |column| column.name == column_name })
+        end
+
+        result.to_a.map { |row|
+          Hash[row.map { |column_name, value|
+            [ column_name, types_by_column[column_name].deserialize(value) ]
+          }]
+        }
+      end
+
+
     end
   end
 end

data/lib/abstract_importer/strategies/upsert_strategy.rb

@@ -1,5 +1,4 @@
 require "abstract_importer/strategies/insert_strategy"
-require "activerecord/insert_many"
 
 module AbstractImporter
   module Strategies
@@ -7,7 +6,8 @@ module AbstractImporter
 
       def initialize(collection, options={})
         super
-        @insert_options.merge!(on_conflict: { column: remap_ids? ? (association_attrs.keys + [:legacy_id]) : :id, do: :update })
+        @bulk_operation = :upsert_all
+        @insert_options.reverse_merge!(unique_by: remap_ids? ? (association_attrs.keys + %i{legacy_id}) : :id)
       end
 
       # We won't skip any records for already being imported

data/lib/abstract_importer/version.rb

@@ -1,3 +1,3 @@
 module AbstractImporter
-  VERSION = "1.5.5"
+  VERSION = "1.8.0"
 end

data/test/default_strategy_test.rb

@@ -147,7 +147,7 @@ class DefaultStrategyTest < ActiveSupport::TestCase
       import!
       assert_equal 2, account.students.map(&:pet).compact.count, "Expected two students to still be linked to their pets upon import"
       assert_kind_of Owl, account.students.find_by_name("Harry Potter").pet, "Expected Harry's pet to be an Owl"
-      assert_kind_of Cat, account.students.find_by_name("Hermione Granger").pet, "Expected Harry's pet to be a Cat"
+      assert_kind_of Cat, account.students.find_by_name("Hermione Granger").pet, "Expected Hermione's pet to be a Cat"
     end
   end
 

data/test/insert_strategy_test.rb

@@ -4,7 +4,7 @@ require "test_helper"
 class InsertStrategyTest < ActiveSupport::TestCase
 
   setup do
-    options.merge!(strategy: {students: :insert})
+    options.merge!(strategy: {students: :insert, perils: :insert})
   end
 
 
@@ -17,7 +17,7 @@ class InsertStrategyTest < ActiveSupport::TestCase
     end
 
     should "import the records in batches" do
-      mock.proxy(Student).insert_many(satisfy { |arg| arg.length == 3 }, anything)
+      mock.proxy(Student).insert_all(satisfy { |arg| arg.length == 3 }, anything)
       import!
       assert_equal [456, 457, 458], account.students.pluck(:legacy_id)
     end
@@ -54,6 +54,22 @@ class InsertStrategyTest < ActiveSupport::TestCase
     end
   end
 
+  context "With an empty data source" do
+    setup do
+      plan do |import|
+        import.students
+      end
+      @data_source = OpenStruct.new
+      @data_source.students = []
+    end
+
+    should "still be able to import" do
+      assert_nothing_raised do
+        import!
+      end
+    end
+  end
+
   context "When records already exist" do
     setup do
       plan do |import|
@@ -83,6 +99,20 @@ class InsertStrategyTest < ActiveSupport::TestCase
     end
   end
 
+  context "When importing a has_many through: relationship" do
+    setup do
+      plan do |import|
+        import.locations
+        import.perils
+      end
+    end
+
+    should "handle has_many, through: relations" do
+      import!
+      assert_equal 1, account.perils.count
+    end
+  end
+
 
 
   context "Given an ID generator" do

data/test/support/mock_data_source.rb

@@ -58,5 +58,11 @@ class MockDataSource
     end
   end
 
+  def perils
+    Enumerator.new do |e|
+      e.yield id: 801, name: "Dementors", location_id: 6
+    end
+  end
+
 
 end

data/test/support/mock_objects.rb

@@ -17,6 +17,7 @@ class Parent < ActiveRecord::Base
 end
 
 class Location < ActiveRecord::Base
+  has_many :perils
   validates :slug, format: {with: /\A[a-z0-9\-]+\z/}
 end
 
@@ -37,6 +38,8 @@ class Account < ActiveRecord::Base
   has_many :locations
   has_many :cats
   has_many :owls
+
+  has_many :perils, through: :locations
 end
 
 class Cat < ActiveRecord::Base
@@ -52,3 +55,7 @@ end
 class Ability < ActiveRecord::Base
   belongs_to :pet, inverse_of: :abilities, polymorphic: true
 end
+
+class Peril < ActiveRecord::Base
+  belongs_to :location
+end

data/test/support/schema.rb

@@ -72,4 +72,11 @@ ActiveRecord::Schema.define(:version => 1) do
     t.index    "legacy_id", :unique => true
   end
 
+  create_table "perils", :force => true do |t|
+    t.string    "name"
+    t.integer   "location_id"
+    t.integer   "legacy_id"
+    t.index     "legacy_id", :unique => true
+  end
+
 end

data/test/test_helper.rb

@@ -22,11 +22,15 @@ require "minitest/autorun"
 
 system "psql -c 'create database abstract_importer_test'"
 
-ActiveRecord::Base.establish_connection(
-  adapter: "postgresql",
-  host: "localhost",
-  database: "abstract_importer_test",
-  verbosity: "quiet")
+if ENV["DATABASE_URL"]
+  ActiveRecord::Base.establish_connection(ENV["DATABASE_URL"])
+else
+  ActiveRecord::Base.establish_connection(
+    adapter: "postgresql",
+    host: "localhost",
+    database: "abstract_importer_test",
+    verbosity: "quiet")
+end
 
 load File.join(File.dirname(__FILE__), "support", "schema.rb")
 

data/test/upsert_strategy_test.rb

@@ -17,7 +17,7 @@ class UpsertStrategyTest < ActiveSupport::TestCase
     end
 
     should "import the records in batches" do
-      mock.proxy(Student).insert_many(satisfy { |arg| arg.length == 3 }, anything)
+      mock.proxy(Student).upsert_all(satisfy { |arg| arg.length == 3 }, anything)
       import!
       assert_equal [456, 457, 458], account.students.pluck(:legacy_id)
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: abstract_importer
 version: !ruby/object:Gem::Version
-  version: 1.5.5
+  version: 1.8.0
 platform: ruby
 authors:
 - Bob Lail
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-23 00:00:00.000000000 Z
+date: 2021-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -16,42 +16,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
-- !ruby/object:Gem::Dependency
-  name: activerecord-insert_many
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.4.4
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.4.4
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: progressbar
   requirement: !ruby/object:Gem::Requirement
@@ -140,16 +126,16 @@ dependencies:
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.18'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.18'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -227,9 +213,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".ruby-version"
-- ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -288,8 +275,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Provides services for the mass-import of complex relational data

data/.travis.yml

@@ -1,17 +0,0 @@
-language: ruby
-rvm:
-  - 2.3.1
-
-# Use Postgres 9.5
-# https://www.brandur.org/fragments/postgres-95-travis
-dist: trusty
-sudo: required
-addons:
-  postgresql: "9.5"
-
-before_install: gem update bundler
-script: bundle exec rake test
-
-# To stop Travis from running tests for a new commit,
-# add the following to your commit message: [ci skip]
-# You should add this when you edit documentation or comments, etc.