activerecord-import 1.0.3 → 1.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b9bf4dc4ecf93b15a493cc34134158d7df3d6391b2e6c7fafbe79b630df48ea3
-  data.tar.gz: ab5f7f4707598308dc9a2e7d4be5816f1d5674f63cea58e48b29f6ed87fa059a
+  metadata.gz: b32b91e70dc3ec25461a37092b774709300798912b930aeb9e7ae57dba41230d
+  data.tar.gz: 5b64d6bc14cae05785300a72cd830e552cd5b21d0dbb304e4e527f69de9865c5
 SHA512:
-  metadata.gz: 05c4ad6281d6f0f574ee30cffed790763fb393fdf906ecc20bb8b4504f640ef9734ea1a43284f9821e543f8900aa3023bf66bcf455afcb2419e15828da618a05
-  data.tar.gz: cdc67be0552c3b32a980598a784964df98549a27d428b89ef3ed7385acb4c75681946e010e714770ac81ff076088c0293e26bc04bbc2e15aecd42bf86c3aecc8
+  metadata.gz: 04a16537b5cd4ef535e7ed1c49397b1313d2717f5b60ed43cdee42f220d3a0094d0b4cce6241470c892a184b09d74b28ece67d28506508964f962c9282aa763c
+  data.tar.gz: d84f9ffea3a04a2ca3b2bb3dfc05307a90876d5c7e9f8e155605dbc7a6bbcec8cd3cfefc7aac82a7e14de0ae5334cdeba3ac6467c00206ccd15e016aebe8854e

data/.gitignore

@@ -24,6 +24,7 @@ pkg
 log/*.log
 test.db
 test/database.yml
+benchmarks/log/
 
 .ruby-*
 .bundle/

data/.travis.yml

@@ -1,7 +1,7 @@
 language: ruby
 cache: bundler
 rvm:
-  - 2.5.5
+  - 2.5.8
 
 env:
   global:
@@ -11,6 +11,7 @@ env:
     - AR_VERSION=5.1
     - AR_VERSION=5.2
     - AR_VERSION=6.0
+    - AR_VERSION=6.1
 
 matrix:
   include:
@@ -28,7 +29,7 @@ matrix:
   fast_finish: true
 
 addons:
-  postgresql: "9.5"
+  postgresql: "10"
   apt:
     sources:
       - travis-ci/sqlite3
@@ -37,13 +38,14 @@ addons:
       - sqlite3
       - mysql-server
       - mysql-client
-      - postgresql-9.5-postgis-2.4
+      - postgresql-10-postgis-2.4
 
 before_install:
+  - sudo apt-get update
   - gem update --system
   - sudo mysql -e "use mysql; update user set authentication_string=PASSWORD('') where User='root'; update user set plugin='mysql_native_password';FLUSH PRIVILEGES;"
   - sudo mysql_upgrade
-  - sudo service mysql restart
+  - sudo service mysql restart   
 
 before_script:
   - mysql -e 'create database activerecord_import_test;'

data/CHANGELOG.md

@@ -1,3 +1,48 @@
+## Changes in 1.0.8
+
+### Fixes
+
+* Use correct method for clearing query cache. Thanks to @EtienneDepaulis via \##719.
+
+## Changes in 1.0.7
+
+* Use @@max_allowed_packet session variable instead of querying SHOW VARIABLES. Thanks to @diclophis via \#706.
+* Add option :track_validation_failures. When this is set to true, failed_instances will be an array of arrays, with each inner array having the form [:index_in_dataset, :object_with_errors]. Thanks to @rorymckinley  via \#684.
+
+### Fixes
+
+* Prevent mass-assignment errors in Rails strict mode. Thanks to @diclophis via \##709.
+
+## Changes in 1.0.6
+
+### Fixes
+
+* Handle after_initialize callbacks. Thanks to @AhMohsen46 via \#691 and
+  \#692.
+* Fix regression introduced in 1.0.4. Explicity allow adapters to
+  support on duplicate key update. Thanks to @dsobiera, @jkowens via \#696.
+
+## Changes in 1.0.5
+
+### Fixes
+
+* Allow serialized attributes to be returned from import. Thanks to @timanovsky, @jkowens via \#660.
+* Return ActiveRecord::Connection from
+  ActiveREcord::Base#establish_connection. Thanks to @reverentF via
+\#663.
+* Support PostgreSQL array. Thanks to @ujihisa via \#669.
+* Skip loading association ids when column changed. Thanks to @Aristat
+  via \#673.
+
+## Changes in 1.0.4
+
+### Fixes
+
+* Use prepend pattern for ActiveRecord::Base#establish_connection patching. Thanks to @dombesz via \#648.
+* Fix NoMethodError when using PostgreSQL ENUM types. Thanks to @sebcoetzee via \#651.
+* Fix issue updating optimistic lock in Postgres. Thanks to @timanovsky
+  via \#656.
+
 ## Changes in 1.0.3
 
 ### New Features
@@ -253,7 +298,7 @@
   Thanks to @jkowens via \#301.
 * Allow for custom timestamp columns. Thanks to @mojidabckuu, @jkowens
   via \#401.
- 
+
 ### Fixes
 
 * Fix ActiveRecord 5 issue coercing boolean values when serializing
@@ -265,7 +310,7 @@
 
 * Fix issue where PostgreSQL cannot recognize columns if names
   include mixed case characters. Thanks to @hugobgranja via \#379.
-* Fix an issue for ActiveRecord 5 where serialized fields with 
+* Fix an issue for ActiveRecord 5 where serialized fields with
   default values were not being typecast. Thanks to @whistlerbrk,
   @jkowens via \#386.
 * Add option :force_single_insert for MySQL to make sure a single

data/Gemfile

@@ -6,8 +6,11 @@ version = ENV['AR_VERSION'].to_f
 
 mysql2_version = '0.3.0'
 mysql2_version = '0.4.0' if version >= 4.2
+mysql2_version = '0.5.0' if version >= 6.1
 sqlite3_version = '1.3.0'
 sqlite3_version = '1.4.0' if version >= 6.0
+pg_version = '0.9'
+pg_version = '1.1' if version >= 6.1
 
 group :development, :test do
   gem 'rubocop', '~> 0.40.0'
@@ -17,7 +20,7 @@ end
 # Database Adapters
 platforms :ruby do
   gem "mysql2",                 "~> #{mysql2_version}"
-  gem "pg",                     "~> 0.9"
+  gem "pg",                     "~> #{pg_version}"
   gem "sqlite3",                "~> #{sqlite3_version}"
   gem "seamless_database_pool", "~> 1.0.20"
 end
@@ -47,6 +50,7 @@ end
 
 platforms :ruby do
   gem "pry-byebug"
+  gem "pry", "~> 0.12.0"
   gem "rb-readline"
 end
 

data/LICENSE

@@ -1,56 +1,21 @@
-Ruby is copyrighted free software by Yukihiro Matsumoto <matz@netlab.jp>.
-You can redistribute it and/or modify it under either the terms of the
-2-clause BSDL (see the file BSDL), or the conditions below:
-
-  1. You may make and give away verbatim copies of the source form of the
-     software without restriction, provided that you duplicate all of the
-     original copyright notices and associated disclaimers.
-
-  2. You may modify your copy of the software in any way, provided that
-     you do at least ONE of the following:
-
-       a) place your modifications in the Public Domain or otherwise
-          make them Freely Available, such as by posting said
-	  modifications to Usenet or an equivalent medium, or by allowing
-	  the author to include your modifications in the software.
-
-       b) use the modified software only within your corporation or
-          organization.
-
-       c) give non-standard binaries non-standard names, with
-          instructions on where to get the original software distribution.
-
-       d) make other distribution arrangements with the author.
-
-  3. You may distribute the software in object code or binary form,
-     provided that you do at least ONE of the following:
-
-       a) distribute the binaries and library files of the software,
-	  together with instructions (in the manual page or equivalent)
-	  on where to get the original distribution.
-
-       b) accompany the distribution with the machine-readable source of
-	  the software.
-
-       c) give non-standard binaries non-standard names, with
-          instructions on where to get the original software distribution.
-
-       d) make other distribution arrangements with the author.
-
-  4. You may modify and include the part of the software into any other
-     software (possibly commercial).  But some files in the distribution
-     are not written by the author, so that they are not under these terms.
-
-     For the list of those files and their copying conditions, see the
-     file LEGAL.
-
-  5. The scripts and library files supplied as input to or produced as 
-     output from the software do not automatically fall under the
-     copyright of the software, but belong to whomever generated them, 
-     and may be sold commercially, and may be aggregated with this
-     software.
-
-  6. THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
-     IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
-     WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-     PURPOSE.
+The MIT License (MIT)
+
+Copyright (c) 2020 Zach Dennis <zach.dennis@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.markdown

@@ -60,6 +60,8 @@ The gem provides the following high-level features:
 
 #### Introduction
 
+This gem adds an `import` method (or `bulk_import`, for compatibility with gems like `elasticsearch-model`; see [Conflicts With Other Gems](#conflicts-with-other-gems)) to ActiveRecord classes.
+
 Without `activerecord-import`, you'd write something like this:
 
 ```ruby
@@ -231,7 +233,8 @@ Book.import columns, books, batch_size: 2
 
 #### Recursive
 
-NOTE: This only works with PostgreSQL.
+NOTE: This only works with PostgreSQL and ActiveRecord objects. This won't work with
+hashes or arrays as recursive inputs.
 
 Assume that Books <code>has_many</code> Reviews.
 
@@ -247,19 +250,21 @@ Book.import books, recursive: true
 
 ### Options
 
-Key                     | Options               | Default            | Description
------------------------ | --------------------- | ------------------ | -----------
-:validate               | `true`/`false`        | `true`             | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
-:validate_uniqueness    | `true`/`false`        | `false`            | Whether or not to run uniqueness validations, has potential pitfalls, use with caution (requires `>= v0.27.0`).
-:on_duplicate_key_ignore| `true`/`false`        | `false`            | Allows skipping records with duplicate keys. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-ignore) for more details.
-:ignore                 | `true`/`false`        | `false`            | Alias for :on_duplicate_key_ignore.
-:on_duplicate_key_update| :all, `Array`, `Hash` | N/A                | Allows upsert logic to be used. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-update) for more details.
-:synchronize            | `Array`               | N/A                | An array of ActiveRecord instances. This synchronizes existing instances in memory with updates from the import.
-:timestamps             | `true`/`false`        | `true`             | Enables/disables timestamps on imported records.
-:recursive              | `true`/`false`        | `false`            | Imports has_many/has_one associations (PostgreSQL only).
-:batch_size             | `Integer`             | total # of records | Max number of records to insert per import
-:raise_error            | `true`/`false`        | `false`            | Throws an exception if there are invalid records. `import!` is a shortcut for this.
-
+Key                       | Options               | Default            | Description
+------------------------- | --------------------- | ------------------ | -----------
+:validate                 | `true`/`false`        | `true`             | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
+:validate_uniqueness      | `true`/`false`        | `false`            | Whether or not to run uniqueness validations, has potential pitfalls, use with caution (requires `>= v0.27.0`).
+:validate_with_context    | `Symbol`              |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones. 
+:track_validation_failures| `true`/`false`        | `false`            | When this is set to true, `failed_instances` will be an array of arrays, with each inner array having the form `[:index_in_dataset, :object_with_errors]` 
+:on_duplicate_key_ignore  | `true`/`false`        | `false`            | Allows skipping records with duplicate keys. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-ignore) for more details.
+:ignore                   | `true`/`false`        | `false`            | Alias for :on_duplicate_key_ignore.
+:on_duplicate_key_update  | :all, `Array`, `Hash` | N/A                | Allows upsert logic to be used. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-update) for more details.
+:synchronize              | `Array`               | N/A                | An array of ActiveRecord instances. This synchronizes existing instances in memory with updates from the import.
+:timestamps               | `true`/`false`        | `true`             | Enables/disables timestamps on imported records.
+:recursive                | `true`/`false`        | `false`            | Imports has_many/has_one associations (PostgreSQL only).
+:batch_size               | `Integer`             | total # of records | Max number of records to insert per import
+:raise_error              | `true`/`false`        | `false`            | Raises an exception at the first invalid record. This means there will not be a result object returned. The `import!` method is a shortcut for this.
+:all_or_none              | `true`/`false`        | `false`            | Will not import any records if there is a record with validation errors.
 
 #### Duplicate Key Ignore
 
@@ -525,7 +530,7 @@ require 'activerecord-import'
 ### Load Path Setup
 To understand how rubygems loads code you can reference the following:
 
-  http://guides.rubygems.org/patterns/#loading_code
+  http://guides.rubygems.org/patterns/#loading-code
 
 And an example of how active_record dynamically load adapters:
 
@@ -612,7 +617,7 @@ You can triage issues which may include reproducing bug reports or asking for vi
 
 # License
 
-This is licensed under the ruby license.
+This is licensed under the MIT license.
 
 # Author
 

data/activerecord-import.gemspec

@@ -6,8 +6,8 @@ Gem::Specification.new do |gem|
   gem.email         = ["zach.dennis@gmail.com"]
   gem.summary       = "Bulk insert extension for ActiveRecord"
   gem.description   = "A library for bulk inserting data using ActiveRecord."
-  gem.homepage      = "http://github.com/zdennis/activerecord-import"
-  gem.license       = "Ruby"
+  gem.homepage      = "https://github.com/zdennis/activerecord-import"
+  gem.license       = "MIT"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = ActiveRecord::Import::VERSION
 
-  gem.required_ruby_version = ">= 1.9.2"
+  gem.required_ruby_version = ">= 2.0.0"
 
   gem.add_runtime_dependency "activerecord", ">= 3.2"
   gem.add_development_dependency "rake"

data/gemfiles/6.0.gemfile

@@ -1 +1,2 @@
 gem 'activerecord', '~> 6.0.0'
+gem 'composite_primary_keys', '~> 12.0'

data/gemfiles/6.1.gemfile

@@ -1 +1 @@
-gem 'activerecord', '~> 6.1.0.alpha', github: "rails/rails"
+gem 'activerecord', '~> 6.1.0'

data/lib/activerecord-import/adapters/abstract_adapter.rb

@@ -59,6 +59,12 @@ module ActiveRecord::Import::AbstractAdapter
       post_sql_statements
     end
 
+    def increment_locking_column!(table_name, results, locking_column)
+      if locking_column.present?
+        results << "\"#{locking_column}\"=#{table_name}.\"#{locking_column}\"+1"
+      end
+    end
+
     def supports_on_duplicate_key_update?
       false
     end

data/lib/activerecord-import/adapters/mysql_adapter.rb

@@ -56,9 +56,9 @@ module ActiveRecord::Import::MysqlAdapter
   # in a single packet
   def max_allowed_packet # :nodoc:
     @max_allowed_packet ||= begin
-      result = execute( "SHOW VARIABLES like 'max_allowed_packet'" )
+      result = execute( "SELECT @@max_allowed_packet" )
       # original Mysql gem responds to #fetch_row while Mysql2 responds to #first
-      val = result.respond_to?(:fetch_row) ? result.fetch_row[1] : result.first[1]
+      val = result.respond_to?(:fetch_row) ? result.fetch_row[0] : result.first[0]
       val.to_i
     end
   end
@@ -102,7 +102,7 @@ module ActiveRecord::Import::MysqlAdapter
       qc = quote_column_name( column )
       "#{table_name}.#{qc}=VALUES(#{qc})"
     end
-    increment_locking_column!(results, table_name, locking_column)
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',' )
   end
 
@@ -112,7 +112,7 @@ module ActiveRecord::Import::MysqlAdapter
       qc2 = quote_column_name( column2 )
       "#{table_name}.#{qc1}=VALUES( #{qc2} )"
     end
-    increment_locking_column!(results, table_name, locking_column)
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',')
   end
 
@@ -121,9 +121,9 @@ module ActiveRecord::Import::MysqlAdapter
     exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('Duplicate entry')
   end
 
-  def increment_locking_column!(results, table_name, locking_column)
+  def increment_locking_column!(table_name, results, locking_column)
     if locking_column.present?
-      results << "#{table_name}.`#{locking_column}`=`#{locking_column}`+1"
+      results << "`#{locking_column}`=#{table_name}.`#{locking_column}`+1"
     end
   end
 end

data/lib/activerecord-import/adapters/postgresql_adapter.rb

@@ -28,7 +28,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
       else
         select_values( sql2insert, *args )
       end
-      query_cache.clear if query_cache_enabled
+      clear_query_cache if query_cache_enabled
     end
 
     if options[:returning].blank?
@@ -158,7 +158,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
       qc = quote_column_name( column )
       "#{qc}=EXCLUDED.#{qc}"
     end
-    increment_locking_column!(results, locking_column)
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',' )
   end
 
@@ -168,7 +168,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
       qc2 = quote_column_name( column2 )
       "#{qc1}=EXCLUDED.#{qc2}"
     end
-    increment_locking_column!(results, locking_column)
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',' )
   end
 
@@ -203,12 +203,6 @@ module ActiveRecord::Import::PostgreSQLAdapter
     true
   end
 
-  def increment_locking_column!(results, locking_column)
-    if locking_column.present?
-      results << "\"#{locking_column}\"=EXCLUDED.\"#{locking_column}\"+1"
-    end
-  end
-
   private
 
   def database_version

data/lib/activerecord-import/adapters/sqlite3_adapter.rb

@@ -92,7 +92,7 @@ module ActiveRecord::Import::SQLite3Adapter
 
   # Returns a generated ON CONFLICT DO UPDATE statement given the passed
   # in +args+.
-  def sql_for_on_duplicate_key_update( _table_name, *args ) # :nodoc:
+  def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
     arg, primary_key, locking_column = args
     arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
     return unless arg.is_a?( Hash )
@@ -113,9 +113,9 @@ module ActiveRecord::Import::SQLite3Adapter
 
     sql << "#{conflict_target}DO UPDATE SET "
     if columns.is_a?( Array )
-      sql << sql_for_on_duplicate_key_update_as_array( locking_column, columns )
+      sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, columns )
     elsif columns.is_a?( Hash )
-      sql << sql_for_on_duplicate_key_update_as_hash( locking_column, columns )
+      sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, columns )
     elsif columns.is_a?( String )
       sql << columns
     else
@@ -127,22 +127,22 @@ module ActiveRecord::Import::SQLite3Adapter
     sql
   end
 
-  def sql_for_on_duplicate_key_update_as_array( locking_column, arr ) # :nodoc:
+  def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
     results = arr.map do |column|
       qc = quote_column_name( column )
       "#{qc}=EXCLUDED.#{qc}"
     end
-    increment_locking_column!(results, locking_column)
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',' )
   end
 
-  def sql_for_on_duplicate_key_update_as_hash( locking_column, hsh ) # :nodoc:
+  def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
     results = hsh.map do |column1, column2|
       qc1 = quote_column_name( column1 )
       qc2 = quote_column_name( column2 )
       "#{qc1}=EXCLUDED.#{qc2}"
     end
-    increment_locking_column!(results, locking_column)
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',' )
   end
 
@@ -166,12 +166,6 @@ module ActiveRecord::Import::SQLite3Adapter
     exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key')
   end
 
-  def increment_locking_column!(results, locking_column)
-    if locking_column.present?
-      results << "\"#{locking_column}\"=EXCLUDED.\"#{locking_column}\"+1"
-    end
-  end
-
   private
 
   def database_version

data/lib/activerecord-import/import.rb

@@ -245,16 +245,17 @@ class ActiveRecord::Associations::CollectionAssociation
   alias import bulk_import unless respond_to? :import
 end
 
+module ActiveRecord::Import::Connection
+  def establish_connection(args = nil)
+    conn = super(args)
+    ActiveRecord::Import.load_from_connection_pool connection_pool
+    conn
+  end
+end
+
 class ActiveRecord::Base
   class << self
-    def establish_connection_with_activerecord_import(*args)
-      conn = establish_connection_without_activerecord_import(*args)
-      ActiveRecord::Import.load_from_connection_pool connection_pool
-      conn
-    end
-
-    alias establish_connection_without_activerecord_import establish_connection
-    alias establish_connection establish_connection_with_activerecord_import
+    prepend ActiveRecord::Import::Connection
 
     # Returns true if the current database connection adapter
     # supports import functionality, otherwise returns false.
@@ -546,7 +547,7 @@ class ActiveRecord::Base
     alias import! bulk_import! unless ActiveRecord::Base.respond_to? :import!
 
     def import_helper( *args )
-      options = { validate: true, timestamps: true }
+      options = { validate: true, timestamps: true, track_validation_failures: false }
       options.merge!( args.pop ) if args.last.is_a? Hash
       # making sure that current model's primary key is used
       options[:primary_key] = primary_key
@@ -581,7 +582,7 @@ class ActiveRecord::Base
           if respond_to?(:timestamp_attributes_for_update, true)
             send(:timestamp_attributes_for_update).map(&:to_sym)
           else
-            new.send(:timestamp_attributes_for_update_in_model)
+            allocate.send(:timestamp_attributes_for_update_in_model)
           end
         end
 
@@ -702,14 +703,18 @@ class ActiveRecord::Base
             # keep track of the instance and the position it is currently at. if this fails
             # validation we'll use the index to remove it from the array_of_attributes
             arr.each_with_index do |hsh, i|
-              model = new
-              hsh.each_pair { |k, v| model[k] = v }
+              # utilize block initializer syntax to prevent failure when 'mass_assignment_sanitizer = :strict'
+              model = new do |m|
+                hsh.each_pair { |k, v| m[k] = v }
+              end
+
               next if validator.valid_model?(model)
               raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
+
               array_of_attributes[i] = nil
               failure = model.dup
               failure.errors.send(:initialize_dup, model.errors)
-              failed_instances << failure
+              failed_instances << (options[:track_validation_failures] ? [i, failure] : failure )
             end
             array_of_attributes.compact!
           end
@@ -807,7 +812,7 @@ class ActiveRecord::Base
           result = connection.insert_many( [insert_sql, post_sql_statements].flatten,
             batch_values,
             options,
-            "#{model_name} Create Many Without Validations Or Callbacks" )
+            "#{model_name} Create Many" )
           number_inserted += result.num_inserts
           ids += result.ids
           results += result.results
@@ -841,6 +846,19 @@ class ActiveRecord::Base
         end
       end
 
+      deserialize_value = lambda do |column, value|
+        column = columns_hash[column]
+        return value unless column
+        if respond_to?(:type_caster)
+          type = type_for_attribute(column.name)
+          type.deserialize(value)
+        elsif column.respond_to?(:type_cast_from_database)
+          column.type_cast_from_database(value)
+        else
+          value
+        end
+      end
+
       if models.size == import_result.results.size
         columns = Array(options[:returning])
         single_column = "#{columns.first}=" if columns.size == 1
@@ -848,10 +866,12 @@ class ActiveRecord::Base
           model = models[index]
 
           if single_column
-            model.send(single_column, result)
+            val = deserialize_value.call(columns.first, result)
+            model.send(single_column, val)
           else
             columns.each_with_index do |column, col_index|
-              model.send("#{column}=", result[col_index])
+              val = deserialize_value.call(column, result[col_index])
+              model.send("#{column}=", val)
             end
           end
         end
@@ -872,10 +892,12 @@ class ActiveRecord::Base
 
     # Sync belongs_to association ids with foreign key field
     def load_association_ids(model)
+      changed_columns = model.changed
       association_reflections = model.class.reflect_on_all_associations(:belongs_to)
       association_reflections.each do |association_reflection|
         column_name = association_reflection.foreign_key
         next if association_reflection.options[:polymorphic]
+        next if changed_columns.include?(column_name)
         association = model.association(association_reflection.name)
         association = association.target
         next if association.blank? || model.public_send(column_name).present?
@@ -955,7 +977,7 @@ class ActiveRecord::Base
           elsif column
             if respond_to?(:type_caster)                                         # Rails 5.0 and higher
               type = type_for_attribute(column.name)
-              val = type.type == :boolean ? type.cast(val) : type.serialize(val)
+              val = !type.respond_to?(:subtype) && type.type == :boolean ? type.cast(val) : type.serialize(val)
               connection_memo.quote(val)
             elsif column.respond_to?(:type_cast_from_user)                       # Rails 4.2
               connection_memo.quote(column.type_cast_from_user(val), column)
@@ -964,7 +986,7 @@ class ActiveRecord::Base
                 val = serialized_attributes[column.name].dump(val)
               end
               # Fixes #443 to support binary (i.e. bytea) columns on PG
-              val = column.type_cast(val) unless column.type.to_sym == :binary
+              val = column.type_cast(val) unless column.type && column.type.to_sym == :binary
               connection_memo.quote(val, column)
             end
           else
@@ -983,7 +1005,7 @@ class ActiveRecord::Base
         timestamp_columns[:create] = timestamp_attributes_for_create_in_model
         timestamp_columns[:update] = timestamp_attributes_for_update_in_model
       else
-        instance = new
+        instance = allocate
         timestamp_columns[:create] = instance.send(:timestamp_attributes_for_create_in_model)
         timestamp_columns[:update] = instance.send(:timestamp_attributes_for_update_in_model)
       end

data/lib/activerecord-import/value_sets_parser.rb

@@ -1,3 +1,5 @@
+require 'active_support/core_ext/array'
+
 module ActiveRecord::Import
   class ValueSetTooLargeError < StandardError
     attr_reader :size

data/lib/activerecord-import/version.rb

@@ -1,5 +1,5 @@
 module ActiveRecord
   module Import
-    VERSION = "1.0.3".freeze
+    VERSION = "1.0.8".freeze
   end
 end

data/test/import_test.rb

@@ -252,6 +252,16 @@ describe "#import" do
         end
       end
 
+      it "should index the failed instances by their poistion in the set if `track_failures` is true" do
+        index_offset = valid_values.length
+        results = Topic.import columns, valid_values + invalid_values, validate: true, track_validation_failures: true
+        assert_equal invalid_values.size, results.failed_instances.size
+        invalid_values.each_with_index do |value_set, index|
+          assert_equal index + index_offset, results.failed_instances[index].first
+          assert_equal value_set.first, results.failed_instances[index].last.title
+        end
+      end
+
       it "should set ids in valid models if adapter supports setting primary key of imported objects" do
         if ActiveRecord::Base.supports_setting_primary_key_of_imported_objects?
           Topic.import (invalid_models + valid_models), validate: true
@@ -900,4 +910,33 @@ describe "#import" do
       end
     end
   end
+  describe "importing model with after_initialize callback" do
+    let(:columns) { %w(name size) }
+    let(:valid_values) { [%w("Deer", "Small"), %w("Monkey", "Medium")] }
+    let(:invalid_values) do
+      [
+        { name: "giraffe", size: "Large" },
+        { size: "Medium" } # name is missing
+      ]
+    end
+    context "with validation checks turned off" do
+      it "should import valid data" do
+        Animal.import(columns, valid_values, validate: false)
+        assert_equal 2, Animal.count
+      end
+      it "should raise ArgumentError" do
+        assert_raise(ArgumentError) { Animal.import(invalid_values, validate: false) }
+      end
+    end
+
+    context "with validation checks turned on" do
+      it "should import valid data" do
+        Animal.import(columns, valid_values, validate: true)
+        assert_equal 2, Animal.count
+      end
+      it "should raise ArgumentError" do
+        assert_raise(ArgumentError) { Animal.import(invalid_values, validate: true) }
+      end
+    end
+  end
 end

data/test/models/animal.rb

@@ -0,0 +1,6 @@
+class Animal < ActiveRecord::Base
+  after_initialize :validate_name_presence, if: :new_record?
+  def validate_name_presence
+    raise ArgumentError if name.nil?
+  end
+end

data/test/schema/postgresql_schema.rb

@@ -3,8 +3,20 @@ ActiveRecord::Schema.define do
   execute('CREATE extension IF NOT EXISTS "pgcrypto";')
   execute('CREATE extension IF NOT EXISTS "uuid-ossp";')
 
+  # create ENUM if it does not exist yet
+  begin
+    execute('CREATE TYPE vendor_type AS ENUM (\'wholesaler\', \'retailer\');')
+  rescue ActiveRecord::StatementInvalid => e
+    # since PostgreSQL does not support IF NOT EXISTS when creating a TYPE,
+    # rescue the error and check the error class
+    raise unless e.cause.is_a? PG::DuplicateObject
+    execute('ALTER TYPE vendor_type ADD VALUE IF NOT EXISTS \'wholesaler\';')
+    execute('ALTER TYPE vendor_type ADD VALUE IF NOT EXISTS \'retailer\';')
+  end
+
   create_table :vendors, id: :uuid, force: :cascade do |t|
     t.string :name, null: true
+    t.text :hours
     t.text :preferences
 
     if t.respond_to?(:json)
@@ -29,6 +41,8 @@ ActiveRecord::Schema.define do
       t.text :json_data
     end
 
+    t.column :vendor_type, :vendor_type
+
     t.datetime :created_at
     t.datetime :updated_at
   end

data/test/support/postgresql/import_examples.rb

@@ -127,6 +127,15 @@ def should_support_postgresql_import_functionality
         end
       end
 
+      context "when a returning column is a serialized attribute" do
+        let(:vendor) { Vendor.new(hours: { monday: '8-5' }) }
+        let(:result) { Vendor.import([vendor], returning: %w(hours)) }
+
+        it "creates records" do
+          assert_difference("Vendor.count", +1) { result }
+        end
+      end
+
       context "when primary key and returning overlap" do
         let(:result) { Book.import(books, returning: %w(id title)) }
 
@@ -260,6 +269,17 @@ def should_support_postgresql_import_functionality
     end
   end
 
+  describe "with enum field" do
+    let(:vendor_type) { "retailer" }
+    it "imports the correct values for enum fields" do
+      vendor = Vendor.new(name: 'Vendor 1', vendor_type: vendor_type)
+      assert_difference "Vendor.count", +1 do
+        Vendor.import [vendor]
+      end
+      assert_equal(vendor_type, Vendor.first.vendor_type)
+    end
+  end
+
   describe "with binary field" do
     let(:binary_value) { "\xE0'c\xB2\xB0\xB3Bh\\\xC2M\xB1m\\I\xC4r".force_encoding('ASCII-8BIT') }
     it "imports the correct values for binary fields" do

data/test/support/shared_examples/on_duplicate_key_update.rb

@@ -73,6 +73,16 @@ def should_support_basic_on_duplicate_key_update
             assert_equal user.name, users[i].name + ' Rothschild'
             assert_equal 1, user.lock_version
           end
+          updated_values2 = User.all.map do |user|
+            user.name += ' jr.'
+            { id: user.id, name: user.name }
+          end
+          User.import(updated_values2, on_duplicate_key_update: [:name])
+          assert User.count == updated_values2.length
+          User.all.each_with_index do |user, i|
+            assert_equal user.name, users[i].name + ' Rothschild jr.'
+            assert_equal 2, user.lock_version
+          end
         end
 
         it 'upsert optimistic lock columns other than lock_version by model' do

data/test/support/shared_examples/recursive_import.rb

@@ -138,6 +138,15 @@ def should_support_recursive_import
       books.each do |book|
         assert_equal book.topic_id, second_new_topic.id
       end
+
+      books.each { |book| book.topic_id = nil }
+      assert_no_difference "Book.count", books.size do
+        Book.import books, validate: false, on_duplicate_key_update: [:topic_id]
+      end
+
+      books.each do |book|
+        assert_equal book.topic_id, nil
+      end
     end
 
     unless ENV["SKIP_COMPOSITE_PK"]

data/test/test_helper.rb

@@ -48,7 +48,15 @@ adapter = ENV["ARE_DB"] || "sqlite3"
 FileUtils.mkdir_p 'log'
 ActiveRecord::Base.logger = Logger.new("log/test.log")
 ActiveRecord::Base.logger.level = Logger::DEBUG
-ActiveRecord::Base.configurations["test"] = YAML.load_file(test_dir.join("database.yml"))[adapter]
+
+if ENV['AR_VERSION'].to_f >= 6.0
+  yaml_config = YAML.load_file(test_dir.join("database.yml"))[adapter]
+  config = ActiveRecord::DatabaseConfigurations::HashConfig.new("test", adapter, yaml_config)
+  ActiveRecord::Base.configurations.configurations << config
+else
+  ActiveRecord::Base.configurations["test"] = YAML.load_file(test_dir.join("database.yml"))[adapter]
+end
+
 ActiveRecord::Base.default_timezone = :utc
 
 require "activerecord-import"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-import
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.8
 platform: ruby
 authors:
 - Zach Dennis
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-10-09 00:00:00.000000000 Z
+date: 2021-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -67,7 +67,7 @@ files:
 - benchmarks/models/test_innodb.rb
 - benchmarks/models/test_memory.rb
 - benchmarks/models/test_myisam.rb
-- benchmarks/schema/mysql_schema.rb
+- benchmarks/schema/mysql2_schema.rb
 - gemfiles/3.2.gemfile
 - gemfiles/4.0.gemfile
 - gemfiles/4.1.gemfile
@@ -121,6 +121,7 @@ files:
 - test/makara_postgis/import_test.rb
 - test/models/account.rb
 - test/models/alarm.rb
+- test/models/animal.rb
 - test/models/bike_maker.rb
 - test/models/book.rb
 - test/models/car.rb
@@ -166,11 +167,11 @@ files:
 - test/travis/database.yml
 - test/value_sets_bytes_parser_test.rb
 - test/value_sets_records_parser_test.rb
-homepage: http://github.com/zdennis/activerecord-import
+homepage: https://github.com/zdennis/activerecord-import
 licenses:
-- Ruby
+- MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -178,16 +179,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.2
+      version: 2.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
-signing_key: 
+rubygems_version: 3.0.8
+signing_key:
 specification_version: 4
 summary: Bulk insert extension for ActiveRecord
 test_files:
@@ -212,6 +212,7 @@ test_files:
 - test/makara_postgis/import_test.rb
 - test/models/account.rb
 - test/models/alarm.rb
+- test/models/animal.rb
 - test/models/bike_maker.rb
 - test/models/book.rb
 - test/models/car.rb