activerecord-import 1.0.8 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b32b91e70dc3ec25461a37092b774709300798912b930aeb9e7ae57dba41230d
-  data.tar.gz: 5b64d6bc14cae05785300a72cd830e552cd5b21d0dbb304e4e527f69de9865c5
+  metadata.gz: b3360334cbd71089351c8211214bb4aedf0bd3e65305513b17c9fb09e3f1cf19
+  data.tar.gz: 2b97e3a1c6c2b39970dd3bb2a00a608d6e105cddac19d3f9ebaf28f8f30ece69
 SHA512:
-  metadata.gz: 04a16537b5cd4ef535e7ed1c49397b1313d2717f5b60ed43cdee42f220d3a0094d0b4cce6241470c892a184b09d74b28ece67d28506508964f962c9282aa763c
-  data.tar.gz: d84f9ffea3a04a2ca3b2bb3dfc05307a90876d5c7e9f8e155605dbc7a6bbcec8cd3cfefc7aac82a7e14de0ae5334cdeba3ac6467c00206ccd15e016aebe8854e
+  metadata.gz: 4585ba6ff2300d94fbd646d5f5a07cc254311cb9a47eacc7b1266eab8df5d9a53ca3018232e6c2517f05e1aef0dafaff20942bd1150ef518d688940f795bf2a9
+  data.tar.gz: 532506d134f067323d0490a913739089e55bf17893f602c752e5e963af2d7d799f5200a72cc48e7c780992e9f82c68273dc7af308034664ba898ed5292e199c4

data/.github/workflows/test.yaml

@@ -0,0 +1,73 @@
+name: Test
+on: [push, pull_request]
+jobs:
+  test:
+    services:
+      postgres:
+        image: postgis/postgis:10-2.5
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - 2.6
+        env:
+          - AR_VERSION: 6.1
+          - AR_VERSION: 6.0
+          - AR_VERSION: 5.2
+          - AR_VERSION: 5.1
+        include:
+          - ruby: 2.4
+            env:
+              AR_VERSION: 5.0
+          - ruby: 2.4
+            env:
+              AR_VERSION: 4.2
+    runs-on: ubuntu-latest
+    env:
+      AR_VERSION: ${{ matrix.env.AR_VERSION }}
+      DB_DATABASE: activerecord_import_test
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Setup Bundler 1.x for Ruby 2.3
+        if: ${{ matrix.ruby == '2.3' }}
+        run: echo "BUNDLER_VERSION=1.17.3" >> $GITHUB_ENV
+      - name: Set up databases
+        run: |
+          sudo /etc/init.d/mysql start
+          mysql -e 'CREATE DATABASE ${{ env.DB_DATABASE }} CHARACTER SET utf8 COLLATE utf8_general_ci;' -u root -proot
+          psql -h localhost -U postgres -c 'create database ${{ env.DB_DATABASE }};'
+          psql -h localhost -U postgres -d ${{ env.DB_DATABASE }} -c 'create extension if not exists hstore;'
+          psql -h localhost -U postgres -c 'create extension if not exists postgis;'
+          psql -h localhost -U postgres -c 'create extension if not exists "uuid-ossp";'
+          cp test/github/database.yml test/database.yml
+        env:
+          PGPASSWORD: postgres
+      - name: Install dependencies
+        run : AR_VERSION=${{ env.AR_VERSION }} bundle install
+      - name: Run tests
+        run: |
+          bundle exec rake test:mysql2
+          bundle exec rake test:mysql2_makara
+          bundle exec rake test:mysql2spatial
+          bundle exec rake test:postgis
+          bundle exec rake test:postgresql
+          bundle exec rake test:postgresql_makara
+          bundle exec rake test:seamless_database_pool
+          bundle exec rake test:spatialite
+          bundle exec rake test:sqlite3
+      - name: Run Rubocop
+        run: bundle exec rubocop

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+## Changes in 1.2.0
+
+### Fixes
+
+* Update JDBC MySQL adapter to use mysql2 connection adapter. Thanks to @terencechow via \##744.
+* Fix importing STI models with ActiveRecord 6. Thanks to @clemens1483 via \##743.
+* Use polymorphic_name instead of base_class.name for imports. Thanks to @kmhajjar via \##741.
+* Fix compatibility issue with composite primary keys. Thanks to @dlanileonardo via \##737.
+* Prevent double validation of associations on recursive import.
+
+## Changes in 1.1.0
+
+### New Features
+
+*  Add batch progress reporting. Thanks to @gee-forr via \##729.
+
 ## Changes in 1.0.8
 
 ### Fixes
@@ -6,6 +22,8 @@
 
 ## Changes in 1.0.7
 
+### New Features
+
 * Use @@max_allowed_packet session variable instead of querying SHOW VARIABLES. Thanks to @diclophis via \#706.
 * Add option :track_validation_failures. When this is set to true, failed_instances will be an array of arrays, with each inner array having the form [:index_in_dataset, :object_with_errors]. Thanks to @rorymckinley  via \#684.
 
@@ -19,7 +37,7 @@
 
 * Handle after_initialize callbacks. Thanks to @AhMohsen46 via \#691 and
   \#692.
-* Fix regression introduced in 1.0.4. Explicity allow adapters to
+* Fix regression introduced in 1.0.4. Explicitly allow adapters to
   support on duplicate key update. Thanks to @dsobiera, @jkowens via \#696.
 
 ## Changes in 1.0.5
@@ -28,7 +46,7 @@
 
 * Allow serialized attributes to be returned from import. Thanks to @timanovsky, @jkowens via \#660.
 * Return ActiveRecord::Connection from
-  ActiveREcord::Base#establish_connection. Thanks to @reverentF via
+  ActiveRecord::Base#establish_connection. Thanks to @reverentF via
 \#663.
 * Support PostgreSQL array. Thanks to @ujihisa via \#669.
 * Skip loading association ids when column changed. Thanks to @Aristat
@@ -83,7 +101,7 @@
 * Fix import issue for models with Postgresql json/jsonb fields. Thanks
   to @stokarenko via \#594.
 * Fix issue importing models with timestamps that contain timezone
-  information. Thaks to @dekaikiwi, @jkowens via \#598.
+  information. Thanks to @dekaikiwi, @jkowens via \#598.
 * Ignore :no_returning when using :recursive option. Thanks to @dgollahon, @jkowens
   via \#599.
 

data/Gemfile

@@ -26,7 +26,7 @@ platforms :ruby do
 end
 
 platforms :jruby do
-  gem "jdbc-mysql"
+  gem 'jdbc-mysql', '< 8', require: false
   gem "jdbc-postgres"
   gem "activerecord-jdbcsqlite3-adapter",    "~> 1.3"
   gem "activerecord-jdbcmysql-adapter",      "~> 1.3"

data/README.markdown

@@ -1,4 +1,4 @@
-# Activerecord-Import [![Build Status](https://travis-ci.org/zdennis/activerecord-import.svg?branch=master)](https://travis-ci.org/zdennis/activerecord-import)
+# Activerecord-Import ![Build Status](https://github.com/zdennis/activerecord-import/actions/workflows/test.yaml/badge.svg)
 
 Activerecord-Import is a library for bulk inserting data using ActiveRecord.
 
@@ -231,6 +231,18 @@ columns = [ :title ]
 Book.import columns, books, batch_size: 2
 ```
 
+If your import is particularly large or slow (possibly due to [callbacks](#callbacks)) whilst batch importing, you might want a way to report back on progress. This is supported by passing a callable as the `batch_progress` option. e.g:
+
+```ruby
+my_proc = ->(rows_size, num_batches, current_batch_number, batch_duration_in_secs) {
+  # Using the arguments provided to the callable, you can
+  # send an email, post to a websocket,
+  # update slack, alert if import is taking too long, etc.
+}
+
+Book.import columns, books, batch_size: 2, batch_progress: my_proc
+```
+
 #### Recursive
 
 NOTE: This only works with PostgreSQL and ActiveRecord objects. This won't work with
@@ -254,8 +266,8 @@ Key                       | Options               | Default            | Descrip
 ------------------------- | --------------------- | ------------------ | -----------
 :validate                 | `true`/`false`        | `true`             | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
 :validate_uniqueness      | `true`/`false`        | `false`            | Whether or not to run uniqueness validations, has potential pitfalls, use with caution (requires `>= v0.27.0`).
-:validate_with_context    | `Symbol`              |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones. 
-:track_validation_failures| `true`/`false`        | `false`            | When this is set to true, `failed_instances` will be an array of arrays, with each inner array having the form `[:index_in_dataset, :object_with_errors]` 
+:validate_with_context    | `Symbol`              |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones.
+:track_validation_failures| `true`/`false`        | `false`            | When this is set to true, `failed_instances` will be an array of arrays, with each inner array having the form `[:index_in_dataset, :object_with_errors]`
 :on_duplicate_key_ignore  | `true`/`false`        | `false`            | Allows skipping records with duplicate keys. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-ignore) for more details.
 :ignore                   | `true`/`false`        | `false`            | Alias for :on_duplicate_key_ignore.
 :on_duplicate_key_update  | :all, `Array`, `Hash` | N/A                | Allows upsert logic to be used. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-update) for more details.
@@ -380,7 +392,7 @@ Book.import books, validate_uniqueness: true
 
 ### Return Info
 
-The `import` method returns a `Result` object that responds to `failed_instances` and `num_inserts`. Additionally, for users of Postgres, there will be two arrays `ids` and `results` that can be accessed`.
+The `import` method returns a `Result` object that responds to `failed_instances` and `num_inserts`. Additionally, for users of Postgres, there will be two arrays `ids` and `results` that can be accessed.
 
 ```ruby
 articles = [

data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb

@@ -1,6 +1,6 @@
-require "active_record/connection_adapters/mysql_adapter"
-require "activerecord-import/adapters/mysql_adapter"
+require "active_record/connection_adapters/mysql2_adapter"
+require "activerecord-import/adapters/mysql2_adapter"
 
-class ActiveRecord::ConnectionAdapters::MysqlAdapter
-  include ActiveRecord::Import::MysqlAdapter
+class ActiveRecord::ConnectionAdapters::Mysql2Adapter
+  include ActiveRecord::Import::Mysql2Adapter
 end

data/lib/activerecord-import/import.rb

@@ -49,7 +49,7 @@ module ActiveRecord::Import #:nodoc:
           associations = klass.reflect_on_all_associations(:belongs_to)
           associations.each do |assoc|
             if (index = attrs.index(assoc.name))
-              key = assoc.foreign_key.to_sym
+              key = assoc.foreign_key.is_a?(Array) ? assoc.foreign_key.map(&:to_sym) : assoc.foreign_key.to_sym
               attrs[index] = key unless attrs.include?(key)
             end
           end
@@ -734,7 +734,7 @@ class ActiveRecord::Base
         set_attributes_and_mark_clean(models, return_obj, timestamps, options)
 
         # if there are auto-save associations on the models we imported that are new, import them as well
-        import_associations(models, options.dup) if options[:recursive]
+        import_associations(models, options.dup.merge(validate: false)) if options[:recursive]
       end
 
       return_obj
@@ -775,21 +775,22 @@ class ActiveRecord::Base
       unless scope_columns.blank?
         scope_columns.zip(scope_values).each do |name, value|
           name_as_sym = name.to_sym
-          next if column_names.include?(name_as_sym)
-
-          is_sti = (name_as_sym == inheritance_column.to_sym && self < base_class)
-          value = Array(value).first if is_sti
-
+          next if column_names.include?(name_as_sym) || name_as_sym == inheritance_column.to_sym
           column_names << name_as_sym
           array_of_attributes.each { |attrs| attrs << value }
         end
       end
 
+      if finder_needs_type_condition?
+        unless column_names.include?(inheritance_column.to_sym)
+          column_names << inheritance_column.to_sym
+          array_of_attributes.each { |attrs| attrs << sti_name }
+        end
+      end
+
       columns = column_names.each_with_index.map do |name, i|
         column = columns_hash[name.to_s]
-
         raise ActiveRecord::Import::MissingColumnError.new(name.to_s, i) if column.nil?
-
         column
       end
 
@@ -805,17 +806,29 @@ class ActiveRecord::Base
       if supports_import?
         # generate the sql
         post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
+        import_size = values_sql.size
+
+        batch_size = options[:batch_size] || import_size
+        run_proc = options[:batch_size].to_i.positive? && options[:batch_progress].respond_to?( :call )
+        progress_proc = options[:batch_progress]
+        current_batch = 0
+        batches = (import_size / batch_size.to_f).ceil
 
-        batch_size = options[:batch_size] || values_sql.size
         values_sql.each_slice(batch_size) do |batch_values|
+          batch_started_at = Time.now.to_i
+
           # perform the inserts
           result = connection.insert_many( [insert_sql, post_sql_statements].flatten,
             batch_values,
             options,
             "#{model_name} Create Many" )
+
           number_inserted += result.num_inserts
           ids += result.ids
           results += result.results
+          current_batch += 1
+
+          progress_proc.call(import_size, batches, current_batch, Time.now.to_i - batch_started_at) if run_proc
         end
       else
         transaction(requires_new: true) do
@@ -949,8 +962,13 @@ class ActiveRecord::Base
         changed_objects.each do |child|
           child.public_send("#{association_reflection.foreign_key}=", model.id)
           # For polymorphic associations
+          association_name = if model.class.respond_to?(:polymorphic_name)
+            model.class.polymorphic_name
+          else
+            model.class.base_class
+          end
           association_reflection.type.try do |type|
-            child.public_send("#{type}=", model.class.base_class.name)
+            child.public_send("#{type}=", association_name)
           end
         end
         associated_objects_by_class[model.class.name][association_reflection.name].concat changed_objects

data/lib/activerecord-import/version.rb

@@ -1,5 +1,5 @@
 module ActiveRecord
   module Import
-    VERSION = "1.0.8".freeze
+    VERSION = "1.2.0".freeze
   end
 end

data/test/{travis → github}/database.yml

@@ -1,7 +1,8 @@
 common: &common
   username: root
-  password:
+  password: root
   encoding: utf8
+  collation: utf8_general_ci
   host:     localhost
   database: activerecord_import_test
 
@@ -37,6 +38,7 @@ oracle:
 postgresql: &postgresql
   <<: *common
   username: postgres
+  password: postgres
   adapter: postgresql
   min_messages: warning
 

data/test/import_test.rb

@@ -169,7 +169,17 @@ describe "#import" do
       assert_difference "Dictionary.count", +1 do
         Dictionary.import dictionaries
       end
-      assert_equal "Dictionary", Dictionary.first.type
+      assert_equal "Dictionary", Dictionary.last.type
+    end
+
+    it "should import arrays successfully" do
+      columns = [:author_name, :title]
+      values =  [["Noah Webster", "Webster's Dictionary"]]
+
+      assert_difference "Dictionary.count", +1 do
+        Dictionary.import columns, values
+      end
+      assert_equal "Dictionary", Dictionary.last.type
     end
   end
 
@@ -405,6 +415,15 @@ describe "#import" do
         assert_equal 3, result.num_inserts if Topic.supports_import?
       end
     end
+
+    it "should accept and call an optional callable to run after each batch" do
+      lambda_called = 0
+
+      my_proc = ->(_row_count, _batches, _batch, _duration) { lambda_called += 1 }
+      Topic.import Build(10, :topics), batch_size: 4, batch_progress: my_proc
+
+      assert_equal 3, lambda_called
+    end
   end
 
   context "with :synchronize option" do
@@ -652,6 +671,14 @@ describe "#import" do
           assert_equal [val1, val2], scope.map(&column).sort
         end
 
+        context "for cards and decks" do
+          it "works when the polymorphic name is different than base class name" do
+            deck = Deck.create(id: 1, name: 'test')
+            deck.cards.import [:id, :deck_type], [[1, 'PlayingCard']]
+            assert_equal deck.cards.first.deck_type, "PlayingCard"
+          end
+        end
+
         it "works importing array of hashes" do
           scope.import [{ column => val1 }, { column => val2 }]
 

data/test/models/card.rb

@@ -0,0 +1,3 @@
+class Card < ActiveRecord::Base
+  belongs_to :deck, polymorphic: true
+end

data/test/models/deck.rb

@@ -0,0 +1,6 @@
+class Deck < ActiveRecord::Base
+  has_many :cards
+  def self.polymorphic_name
+    "PlayingCard"
+  end
+end

data/test/models/playing_card.rb

@@ -0,0 +1,2 @@
+class PlayingCard < ActiveRecord::Base
+end

data/test/schema/generic_schema.rb

@@ -52,6 +52,20 @@ ActiveRecord::Schema.define do
     t.string :name
   end
 
+  create_table :cards, force: :cascade do |t|
+    t.string :name
+    t.string :deck_type
+    t.integer :deck_id
+  end
+
+  create_table :decks, force: :cascade do |t|
+    t.string :name
+  end
+
+  create_table :playing_cards, force: :cascade do |t|
+    t.string :name
+  end
+
   create_table :books, force: :cascade do |t|
     t.string :title, null: false
     t.string :publisher, null: false, default: 'Default Publisher'

data/test/test_helper.rb

@@ -1,4 +1,5 @@
 require 'pathname'
+require 'rake'
 test_dir = Pathname.new File.dirname(__FILE__)
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-import
 version: !ruby/object:Gem::Version
-  version: 1.0.8
+  version: 1.2.0
 platform: ruby
 authors:
 - Zach Dennis
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-20 00:00:00.000000000 Z
+date: 2021-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -45,10 +45,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yaml"
 - ".gitignore"
 - ".rubocop.yml"
 - ".rubocop_todo.yml"
-- ".travis.yml"
 - Brewfile
 - CHANGELOG.md
 - Gemfile
@@ -114,6 +114,7 @@ files:
 - test/adapters/spatialite.rb
 - test/adapters/sqlite3.rb
 - test/database.yml.sample
+- test/github/database.yml
 - test/import_test.rb
 - test/jdbcmysql/import_test.rb
 - test/jdbcpostgresql/import_test.rb
@@ -125,11 +126,14 @@ files:
 - test/models/bike_maker.rb
 - test/models/book.rb
 - test/models/car.rb
+- test/models/card.rb
 - test/models/chapter.rb
+- test/models/deck.rb
 - test/models/dictionary.rb
 - test/models/discount.rb
 - test/models/end_note.rb
 - test/models/group.rb
+- test/models/playing_card.rb
 - test/models/promotion.rb
 - test/models/question.rb
 - test/models/rule.rb
@@ -164,7 +168,6 @@ files:
 - test/support/sqlite3/import_examples.rb
 - test/synchronize_test.rb
 - test/test_helper.rb
-- test/travis/database.yml
 - test/value_sets_bytes_parser_test.rb
 - test/value_sets_records_parser_test.rb
 homepage: https://github.com/zdennis/activerecord-import
@@ -186,7 +189,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.0.9
 signing_key:
 specification_version: 4
 summary: Bulk insert extension for ActiveRecord
@@ -205,6 +208,7 @@ test_files:
 - test/adapters/spatialite.rb
 - test/adapters/sqlite3.rb
 - test/database.yml.sample
+- test/github/database.yml
 - test/import_test.rb
 - test/jdbcmysql/import_test.rb
 - test/jdbcpostgresql/import_test.rb
@@ -216,11 +220,14 @@ test_files:
 - test/models/bike_maker.rb
 - test/models/book.rb
 - test/models/car.rb
+- test/models/card.rb
 - test/models/chapter.rb
+- test/models/deck.rb
 - test/models/dictionary.rb
 - test/models/discount.rb
 - test/models/end_note.rb
 - test/models/group.rb
+- test/models/playing_card.rb
 - test/models/promotion.rb
 - test/models/question.rb
 - test/models/rule.rb
@@ -255,6 +262,5 @@ test_files:
 - test/support/sqlite3/import_examples.rb
 - test/synchronize_test.rb
 - test/test_helper.rb
-- test/travis/database.yml
 - test/value_sets_bytes_parser_test.rb
 - test/value_sets_records_parser_test.rb

data/.travis.yml

@@ -1,76 +0,0 @@
-language: ruby
-cache: bundler
-rvm:
-  - 2.5.8
-
-env:
-  global:
-    # https://github.com/discourse/discourse/blob/master/.travis.yml
-    - RUBY_GC_MALLOC_LIMIT=50000000
-  matrix:
-    - AR_VERSION=5.1
-    - AR_VERSION=5.2
-    - AR_VERSION=6.0
-    - AR_VERSION=6.1
-
-matrix:
-  include:
-    - rvm: 2.3.8
-      env: AR_VERSION=3.2
-    - rvm: 2.3.8
-      env: AR_VERSION=4.0
-    - rvm: 2.3.8
-      env: AR_VERSION=4.1
-    - rvm: 2.3.8
-      env: AR_VERSION=4.2
-    - rvm: 2.3.8
-      env: AR_VERSION=5.0
-
-  fast_finish: true
-
-addons:
-  postgresql: "10"
-  apt:
-    sources:
-      - travis-ci/sqlite3
-      - mysql-5.7-trusty
-    packages:
-      - sqlite3
-      - mysql-server
-      - mysql-client
-      - postgresql-10-postgis-2.4
-
-before_install:
-  - sudo apt-get update
-  - gem update --system
-  - sudo mysql -e "use mysql; update user set authentication_string=PASSWORD('') where User='root'; update user set plugin='mysql_native_password';FLUSH PRIVILEGES;"
-  - sudo mysql_upgrade
-  - sudo service mysql restart   
-
-before_script:
-  - mysql -e 'create database activerecord_import_test;'
-  - psql -c 'create database activerecord_import_test;' -U postgres
-  - psql activerecord_import_test -c 'create extension if not exists hstore;' -U postgres
-  - psql -c 'create extension if not exists postgis;' -U postgres
-  - psql -c 'create extension if not exists "uuid-ossp";' -U postgres
-  - cp test/travis/database.yml test/database.yml
-
-script:
-  - bundle exec rake test:mysql2
-  - bundle exec rake test:mysql2_makara
-  - bundle exec rake test:mysql2spatial
-  - bundle exec rake test:postgis
-  - bundle exec rake test:postgresql
-  - bundle exec rake test:postgresql_makara
-  - bundle exec rake test:seamless_database_pool
-  - bundle exec rake test:spatialite
-  - bundle exec rake test:sqlite3
-  - bundle exec rubocop
-
-dist: xenial
-
-services:
-  - mysql
-  - postgresql
-
-sudo: required