activerecord-import 1.0.4 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cbf4c01d6a3f043ed541f7331d6a19e651cbc035fcc3030b17f1e741a6d4e8b7
-  data.tar.gz: a740491de16c78d4d94ccb428c69342fd8a809f235ce9d42fc41237a4d15c7e6
+  metadata.gz: 4c61743fafaad0de04ccf2c6bff4439fe66aed03e951e5c42e4f7926cf8dac39
+  data.tar.gz: c58d8992957e546b73bb7f7b90eaf2a741f007d2109a16db0e3b3d71b376e1da
 SHA512:
-  metadata.gz: 48e62637f6493cd5446cd78aee3db3b4a58e646fb0bc07d3583a662f89f52b03c2a079c83fc2cae807c5577f922e1e3219354cf36dff8ae8cc81ae68266e711f
-  data.tar.gz: 2f40f5fd61975589f8af932a3dbf5b6d14d2cfc6884ebe7019c85956c8c380d3faebffb1b600a7fa04536fe262ed40688946c8d35b8cf5b1bfedd1ddff20af5a
+  metadata.gz: 98638b63235eae1c16f27d3be932b8ce4e8f236b24350766792a4fc563d199cb1d048d25efa5d22abf3a21017fa07b9474c3168a4be6c1ba5aaa6e0843d584a3
+  data.tar.gz: 5c62bed2684f1b6d60d7e26b8ad13c6b7173a64835eeca9474082122425ab1f340f9063868224eda2539878210887701022e1ff6e6a37593872682bf90768d3a

data/.gitignore

@@ -24,6 +24,7 @@ pkg
 log/*.log
 test.db
 test/database.yml
+benchmarks/log/
 
 .ruby-*
 .bundle/

data/.travis.yml

@@ -1,7 +1,7 @@
 language: ruby
 cache: bundler
 rvm:
-  - 2.5.5
+  - 2.5.8
 
 env:
   global:
@@ -11,6 +11,7 @@ env:
     - AR_VERSION=5.1
     - AR_VERSION=5.2
     - AR_VERSION=6.0
+    - AR_VERSION=6.1
 
 matrix:
   include:
@@ -28,7 +29,7 @@ matrix:
   fast_finish: true
 
 addons:
-  postgresql: "9.5"
+  postgresql: "10"
   apt:
     sources:
       - travis-ci/sqlite3
@@ -37,13 +38,14 @@ addons:
       - sqlite3
       - mysql-server
       - mysql-client
-      - postgresql-9.5-postgis-2.4
+      - postgresql-10-postgis-2.4
 
 before_install:
+  - sudo apt-get update
   - gem update --system
   - sudo mysql -e "use mysql; update user set authentication_string=PASSWORD('') where User='root'; update user set plugin='mysql_native_password';FLUSH PRIVILEGES;"
   - sudo mysql_upgrade
-  - sudo service mysql restart
+  - sudo service mysql restart   
 
 before_script:
   - mysql -e 'create database activerecord_import_test;'

data/CHANGELOG.md

@@ -1,3 +1,47 @@
+## Changes in 1.1.0
+
+### New Features
+
+*  Add batch progress reporting. Thanks to @gee-forr via \##729.
+
+## Changes in 1.0.8
+
+### Fixes
+
+* Use correct method for clearing query cache. Thanks to @EtienneDepaulis via \##719.
+
+## Changes in 1.0.7
+
+### New Features
+
+* Use @@max_allowed_packet session variable instead of querying SHOW VARIABLES. Thanks to @diclophis via \#706.
+* Add option :track_validation_failures. When this is set to true, failed_instances will be an array of arrays, with each inner array having the form [:index_in_dataset, :object_with_errors]. Thanks to @rorymckinley  via \#684.
+
+### Fixes
+
+* Prevent mass-assignment errors in Rails strict mode. Thanks to @diclophis via \##709.
+
+## Changes in 1.0.6
+
+### Fixes
+
+* Handle after_initialize callbacks. Thanks to @AhMohsen46 via \#691 and
+  \#692.
+* Fix regression introduced in 1.0.4. Explicity allow adapters to
+  support on duplicate key update. Thanks to @dsobiera, @jkowens via \#696.
+
+## Changes in 1.0.5
+
+### Fixes
+
+* Allow serialized attributes to be returned from import. Thanks to @timanovsky, @jkowens via \#660.
+* Return ActiveRecord::Connection from
+  ActiveREcord::Base#establish_connection. Thanks to @reverentF via
+\#663.
+* Support PostgreSQL array. Thanks to @ujihisa via \#669.
+* Skip loading association ids when column changed. Thanks to @Aristat
+  via \#673.
+
 ## Changes in 1.0.4
 
 ### Fixes
@@ -262,7 +306,7 @@
   Thanks to @jkowens via \#301.
 * Allow for custom timestamp columns. Thanks to @mojidabckuu, @jkowens
   via \#401.
- 
+
 ### Fixes
 
 * Fix ActiveRecord 5 issue coercing boolean values when serializing
@@ -274,7 +318,7 @@
 
 * Fix issue where PostgreSQL cannot recognize columns if names
   include mixed case characters. Thanks to @hugobgranja via \#379.
-* Fix an issue for ActiveRecord 5 where serialized fields with 
+* Fix an issue for ActiveRecord 5 where serialized fields with
   default values were not being typecast. Thanks to @whistlerbrk,
   @jkowens via \#386.
 * Add option :force_single_insert for MySQL to make sure a single

data/Gemfile

@@ -6,8 +6,11 @@ version = ENV['AR_VERSION'].to_f
 
 mysql2_version = '0.3.0'
 mysql2_version = '0.4.0' if version >= 4.2
+mysql2_version = '0.5.0' if version >= 6.1
 sqlite3_version = '1.3.0'
 sqlite3_version = '1.4.0' if version >= 6.0
+pg_version = '0.9'
+pg_version = '1.1' if version >= 6.1
 
 group :development, :test do
   gem 'rubocop', '~> 0.40.0'
@@ -17,7 +20,7 @@ end
 # Database Adapters
 platforms :ruby do
   gem "mysql2",                 "~> #{mysql2_version}"
-  gem "pg",                     "~> 0.9"
+  gem "pg",                     "~> #{pg_version}"
   gem "sqlite3",                "~> #{sqlite3_version}"
   gem "seamless_database_pool", "~> 1.0.20"
 end
@@ -47,6 +50,7 @@ end
 
 platforms :ruby do
   gem "pry-byebug"
+  gem "pry", "~> 0.12.0"
   gem "rb-readline"
 end
 

data/LICENSE

@@ -1,56 +1,21 @@
-Ruby is copyrighted free software by Yukihiro Matsumoto <matz@netlab.jp>.
-You can redistribute it and/or modify it under either the terms of the
-2-clause BSDL (see the file BSDL), or the conditions below:
-
-  1. You may make and give away verbatim copies of the source form of the
-     software without restriction, provided that you duplicate all of the
-     original copyright notices and associated disclaimers.
-
-  2. You may modify your copy of the software in any way, provided that
-     you do at least ONE of the following:
-
-       a) place your modifications in the Public Domain or otherwise
-          make them Freely Available, such as by posting said
-	  modifications to Usenet or an equivalent medium, or by allowing
-	  the author to include your modifications in the software.
-
-       b) use the modified software only within your corporation or
-          organization.
-
-       c) give non-standard binaries non-standard names, with
-          instructions on where to get the original software distribution.
-
-       d) make other distribution arrangements with the author.
-
-  3. You may distribute the software in object code or binary form,
-     provided that you do at least ONE of the following:
-
-       a) distribute the binaries and library files of the software,
-	  together with instructions (in the manual page or equivalent)
-	  on where to get the original distribution.
-
-       b) accompany the distribution with the machine-readable source of
-	  the software.
-
-       c) give non-standard binaries non-standard names, with
-          instructions on where to get the original software distribution.
-
-       d) make other distribution arrangements with the author.
-
-  4. You may modify and include the part of the software into any other
-     software (possibly commercial).  But some files in the distribution
-     are not written by the author, so that they are not under these terms.
-
-     For the list of those files and their copying conditions, see the
-     file LEGAL.
-
-  5. The scripts and library files supplied as input to or produced as 
-     output from the software do not automatically fall under the
-     copyright of the software, but belong to whomever generated them, 
-     and may be sold commercially, and may be aggregated with this
-     software.
-
-  6. THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
-     IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
-     WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-     PURPOSE.
+The MIT License (MIT)
+
+Copyright (c) 2020 Zach Dennis <zach.dennis@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.markdown

@@ -60,6 +60,8 @@ The gem provides the following high-level features:
 
 #### Introduction
 
+This gem adds an `import` method (or `bulk_import`, for compatibility with gems like `elasticsearch-model`; see [Conflicts With Other Gems](#conflicts-with-other-gems)) to ActiveRecord classes.
+
 Without `activerecord-import`, you'd write something like this:
 
 ```ruby
@@ -229,9 +231,22 @@ columns = [ :title ]
 Book.import columns, books, batch_size: 2
 ```
 
+If your import is particularly large or slow (possibly due to [callbacks](#callbacks)) whilst batch importing, you might want a way to report back on progress. This is supported by passing a callable as the `batch_progress` option. e.g:
+
+```ruby
+my_proc = ->(rows_size, num_batches, current_batch_number, batch_duration_in_secs) {
+  # Using the arguments provided to the callable, you can
+  # send an email, post to a websocket,
+  # update slack, alert if import is taking too long, etc.
+}
+
+Book.import columns, books, batch_size: 2, batch_progress: my_proc
+```
+
 #### Recursive
 
-NOTE: This only works with PostgreSQL.
+NOTE: This only works with PostgreSQL and ActiveRecord objects. This won't work with
+hashes or arrays as recursive inputs.
 
 Assume that Books <code>has_many</code> Reviews.
 
@@ -247,20 +262,21 @@ Book.import books, recursive: true
 
 ### Options
 
-Key                     | Options               | Default            | Description
------------------------ | --------------------- | ------------------ | -----------
-:validate               | `true`/`false`        | `true`             | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
-:validate_uniqueness    | `true`/`false`        | `false`            | Whether or not to run uniqueness validations, has potential pitfalls, use with caution (requires `>= v0.27.0`).
-:validate_with_context  | `Symbol`              |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones. 
-:on_duplicate_key_ignore| `true`/`false`        | `false`            | Allows skipping records with duplicate keys. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-ignore) for more details.
-:ignore                 | `true`/`false`        | `false`            | Alias for :on_duplicate_key_ignore.
-:on_duplicate_key_update| :all, `Array`, `Hash` | N/A                | Allows upsert logic to be used. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-update) for more details.
-:synchronize            | `Array`               | N/A                | An array of ActiveRecord instances. This synchronizes existing instances in memory with updates from the import.
-:timestamps             | `true`/`false`        | `true`             | Enables/disables timestamps on imported records.
-:recursive              | `true`/`false`        | `false`            | Imports has_many/has_one associations (PostgreSQL only).
-:batch_size             | `Integer`             | total # of records | Max number of records to insert per import
-:raise_error            | `true`/`false`        | `false`            | Raises an exception at the first invalid record. This means there will not be a result object returned. The `import!` method is a shortcut for this.
-:all_or_none            | `true`/`false`        | `false`            | Will not import any records if there is a record with validation errors.
+Key                       | Options               | Default            | Description
+------------------------- | --------------------- | ------------------ | -----------
+:validate                 | `true`/`false`        | `true`             | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
+:validate_uniqueness      | `true`/`false`        | `false`            | Whether or not to run uniqueness validations, has potential pitfalls, use with caution (requires `>= v0.27.0`).
+:validate_with_context    | `Symbol`              |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones. 
+:track_validation_failures| `true`/`false`        | `false`            | When this is set to true, `failed_instances` will be an array of arrays, with each inner array having the form `[:index_in_dataset, :object_with_errors]` 
+:on_duplicate_key_ignore  | `true`/`false`        | `false`            | Allows skipping records with duplicate keys. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-ignore) for more details.
+:ignore                   | `true`/`false`        | `false`            | Alias for :on_duplicate_key_ignore.
+:on_duplicate_key_update  | :all, `Array`, `Hash` | N/A                | Allows upsert logic to be used. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-update) for more details.
+:synchronize              | `Array`               | N/A                | An array of ActiveRecord instances. This synchronizes existing instances in memory with updates from the import.
+:timestamps               | `true`/`false`        | `true`             | Enables/disables timestamps on imported records.
+:recursive                | `true`/`false`        | `false`            | Imports has_many/has_one associations (PostgreSQL only).
+:batch_size               | `Integer`             | total # of records | Max number of records to insert per import
+:raise_error              | `true`/`false`        | `false`            | Raises an exception at the first invalid record. This means there will not be a result object returned. The `import!` method is a shortcut for this.
+:all_or_none              | `true`/`false`        | `false`            | Will not import any records if there is a record with validation errors.
 
 #### Duplicate Key Ignore
 
@@ -526,7 +542,7 @@ require 'activerecord-import'
 ### Load Path Setup
 To understand how rubygems loads code you can reference the following:
 
-  http://guides.rubygems.org/patterns/#loading_code
+  http://guides.rubygems.org/patterns/#loading-code
 
 And an example of how active_record dynamically load adapters:
 
@@ -613,7 +629,7 @@ You can triage issues which may include reproducing bug reports or asking for vi
 
 # License
 
-This is licensed under the ruby license.
+This is licensed under the MIT license.
 
 # Author
 

data/activerecord-import.gemspec

@@ -6,8 +6,8 @@ Gem::Specification.new do |gem|
   gem.email         = ["zach.dennis@gmail.com"]
   gem.summary       = "Bulk insert extension for ActiveRecord"
   gem.description   = "A library for bulk inserting data using ActiveRecord."
-  gem.homepage      = "http://github.com/zdennis/activerecord-import"
-  gem.license       = "Ruby"
+  gem.homepage      = "https://github.com/zdennis/activerecord-import"
+  gem.license       = "MIT"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = ActiveRecord::Import::VERSION
 
-  gem.required_ruby_version = ">= 1.9.2"
+  gem.required_ruby_version = ">= 2.0.0"
 
   gem.add_runtime_dependency "activerecord", ">= 3.2"
   gem.add_development_dependency "rake"

data/gemfiles/6.0.gemfile

@@ -1 +1,2 @@
 gem 'activerecord', '~> 6.0.0'
+gem 'composite_primary_keys', '~> 12.0'

data/gemfiles/6.1.gemfile

@@ -1 +1 @@
-gem 'activerecord', '~> 6.1.0.alpha', github: "rails/rails"
+gem 'activerecord', '~> 6.1.0'

data/lib/activerecord-import/adapters/abstract_adapter.rb

@@ -66,7 +66,7 @@ module ActiveRecord::Import::AbstractAdapter
     end
 
     def supports_on_duplicate_key_update?
-      true
+      false
     end
   end
 end

data/lib/activerecord-import/adapters/mysql_adapter.rb

@@ -1,5 +1,6 @@
 module ActiveRecord::Import::MysqlAdapter
   include ActiveRecord::Import::ImportSupport
+  include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
 
   NO_MAX_PACKET = 0
   QUERY_OVERHEAD = 8 # This was shown to be true for MySQL, but it's not clear where the overhead is from.
@@ -55,9 +56,9 @@ module ActiveRecord::Import::MysqlAdapter
   # in a single packet
   def max_allowed_packet # :nodoc:
     @max_allowed_packet ||= begin
-      result = execute( "SHOW VARIABLES like 'max_allowed_packet'" )
+      result = execute( "SELECT @@max_allowed_packet" )
       # original Mysql gem responds to #fetch_row while Mysql2 responds to #first
-      val = result.respond_to?(:fetch_row) ? result.fetch_row[1] : result.first[1]
+      val = result.respond_to?(:fetch_row) ? result.fetch_row[0] : result.first[0]
       val.to_i
     end
   end

data/lib/activerecord-import/adapters/postgresql_adapter.rb

@@ -1,5 +1,6 @@
 module ActiveRecord::Import::PostgreSQLAdapter
   include ActiveRecord::Import::ImportSupport
+  include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
 
   MIN_VERSION_FOR_UPSERT = 90_500
 
@@ -27,7 +28,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
       else
         select_values( sql2insert, *args )
       end
-      query_cache.clear if query_cache_enabled
+      clear_query_cache if query_cache_enabled
     end
 
     if options[:returning].blank?

data/lib/activerecord-import/adapters/sqlite3_adapter.rb

@@ -1,5 +1,6 @@
 module ActiveRecord::Import::SQLite3Adapter
   include ActiveRecord::Import::ImportSupport
+  include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
 
   MIN_VERSION_FOR_IMPORT = "3.7.11".freeze
   MIN_VERSION_FOR_UPSERT = "3.24.0".freeze

data/lib/activerecord-import/import.rb

@@ -11,6 +11,12 @@ module ActiveRecord::Import #:nodoc:
     end
   end
 
+  module OnDuplicateKeyUpdateSupport #:nodoc:
+    def supports_on_duplicate_key_update? #:nodoc:
+      true
+    end
+  end
+
   class MissingColumnError < StandardError
     def initialize(name, index)
       super "Missing column for value <#{name}> at index #{index}"
@@ -241,8 +247,9 @@ end
 
 module ActiveRecord::Import::Connection
   def establish_connection(args = nil)
-    super(args)
+    conn = super(args)
     ActiveRecord::Import.load_from_connection_pool connection_pool
+    conn
   end
 end
 
@@ -540,7 +547,7 @@ class ActiveRecord::Base
     alias import! bulk_import! unless ActiveRecord::Base.respond_to? :import!
 
     def import_helper( *args )
-      options = { validate: true, timestamps: true }
+      options = { validate: true, timestamps: true, track_validation_failures: false }
       options.merge!( args.pop ) if args.last.is_a? Hash
       # making sure that current model's primary key is used
       options[:primary_key] = primary_key
@@ -575,7 +582,7 @@ class ActiveRecord::Base
           if respond_to?(:timestamp_attributes_for_update, true)
             send(:timestamp_attributes_for_update).map(&:to_sym)
           else
-            new.send(:timestamp_attributes_for_update_in_model)
+            allocate.send(:timestamp_attributes_for_update_in_model)
           end
         end
 
@@ -696,14 +703,18 @@ class ActiveRecord::Base
             # keep track of the instance and the position it is currently at. if this fails
             # validation we'll use the index to remove it from the array_of_attributes
             arr.each_with_index do |hsh, i|
-              model = new
-              hsh.each_pair { |k, v| model[k] = v }
+              # utilize block initializer syntax to prevent failure when 'mass_assignment_sanitizer = :strict'
+              model = new do |m|
+                hsh.each_pair { |k, v| m[k] = v }
+              end
+
               next if validator.valid_model?(model)
               raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
+
               array_of_attributes[i] = nil
               failure = model.dup
               failure.errors.send(:initialize_dup, model.errors)
-              failed_instances << failure
+              failed_instances << (options[:track_validation_failures] ? [i, failure] : failure )
             end
             array_of_attributes.compact!
           end
@@ -794,17 +805,29 @@ class ActiveRecord::Base
       if supports_import?
         # generate the sql
         post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
+        import_size = values_sql.size
+
+        batch_size = options[:batch_size] || import_size
+        run_proc = options[:batch_size].to_i.positive? && options[:batch_progress].respond_to?( :call )
+        progress_proc = options[:batch_progress]
+        current_batch = 0
+        batches = (import_size / batch_size.to_f).ceil
 
-        batch_size = options[:batch_size] || values_sql.size
         values_sql.each_slice(batch_size) do |batch_values|
+          batch_started_at = Time.now.to_i
+
           # perform the inserts
           result = connection.insert_many( [insert_sql, post_sql_statements].flatten,
             batch_values,
             options,
-            "#{model_name} Create Many Without Validations Or Callbacks" )
+            "#{model_name} Create Many" )
+
           number_inserted += result.num_inserts
           ids += result.ids
           results += result.results
+          current_batch += 1
+
+          progress_proc.call(import_size, batches, current_batch, Time.now.to_i - batch_started_at) if run_proc
         end
       else
         transaction(requires_new: true) do
@@ -835,6 +858,19 @@ class ActiveRecord::Base
         end
       end
 
+      deserialize_value = lambda do |column, value|
+        column = columns_hash[column]
+        return value unless column
+        if respond_to?(:type_caster)
+          type = type_for_attribute(column.name)
+          type.deserialize(value)
+        elsif column.respond_to?(:type_cast_from_database)
+          column.type_cast_from_database(value)
+        else
+          value
+        end
+      end
+
       if models.size == import_result.results.size
         columns = Array(options[:returning])
         single_column = "#{columns.first}=" if columns.size == 1
@@ -842,10 +878,12 @@ class ActiveRecord::Base
           model = models[index]
 
           if single_column
-            model.send(single_column, result)
+            val = deserialize_value.call(columns.first, result)
+            model.send(single_column, val)
           else
             columns.each_with_index do |column, col_index|
-              model.send("#{column}=", result[col_index])
+              val = deserialize_value.call(column, result[col_index])
+              model.send("#{column}=", val)
             end
           end
         end
@@ -866,10 +904,12 @@ class ActiveRecord::Base
 
     # Sync belongs_to association ids with foreign key field
     def load_association_ids(model)
+      changed_columns = model.changed
       association_reflections = model.class.reflect_on_all_associations(:belongs_to)
       association_reflections.each do |association_reflection|
         column_name = association_reflection.foreign_key
         next if association_reflection.options[:polymorphic]
+        next if changed_columns.include?(column_name)
         association = model.association(association_reflection.name)
         association = association.target
         next if association.blank? || model.public_send(column_name).present?
@@ -949,7 +989,7 @@ class ActiveRecord::Base
           elsif column
             if respond_to?(:type_caster)                                         # Rails 5.0 and higher
               type = type_for_attribute(column.name)
-              val = type.type == :boolean ? type.cast(val) : type.serialize(val)
+              val = !type.respond_to?(:subtype) && type.type == :boolean ? type.cast(val) : type.serialize(val)
               connection_memo.quote(val)
             elsif column.respond_to?(:type_cast_from_user)                       # Rails 4.2
               connection_memo.quote(column.type_cast_from_user(val), column)
@@ -977,7 +1017,7 @@ class ActiveRecord::Base
         timestamp_columns[:create] = timestamp_attributes_for_create_in_model
         timestamp_columns[:update] = timestamp_attributes_for_update_in_model
       else
-        instance = new
+        instance = allocate
         timestamp_columns[:create] = instance.send(:timestamp_attributes_for_create_in_model)
         timestamp_columns[:update] = instance.send(:timestamp_attributes_for_update_in_model)
       end

data/lib/activerecord-import/value_sets_parser.rb

@@ -1,3 +1,5 @@
+require 'active_support/core_ext/array'
+
 module ActiveRecord::Import
   class ValueSetTooLargeError < StandardError
     attr_reader :size

data/lib/activerecord-import/version.rb

@@ -1,5 +1,5 @@
 module ActiveRecord
   module Import
-    VERSION = "1.0.4".freeze
+    VERSION = "1.1.0".freeze
   end
 end

data/test/import_test.rb

@@ -252,6 +252,16 @@ describe "#import" do
         end
       end
 
+      it "should index the failed instances by their poistion in the set if `track_failures` is true" do
+        index_offset = valid_values.length
+        results = Topic.import columns, valid_values + invalid_values, validate: true, track_validation_failures: true
+        assert_equal invalid_values.size, results.failed_instances.size
+        invalid_values.each_with_index do |value_set, index|
+          assert_equal index + index_offset, results.failed_instances[index].first
+          assert_equal value_set.first, results.failed_instances[index].last.title
+        end
+      end
+
       it "should set ids in valid models if adapter supports setting primary key of imported objects" do
         if ActiveRecord::Base.supports_setting_primary_key_of_imported_objects?
           Topic.import (invalid_models + valid_models), validate: true
@@ -395,6 +405,15 @@ describe "#import" do
         assert_equal 3, result.num_inserts if Topic.supports_import?
       end
     end
+
+    it "should accept and call an optional callable to run after each batch" do
+      lambda_called = 0
+
+      my_proc = ->(_row_count, _batches, _batch, _duration) { lambda_called += 1 }
+      Topic.import Build(10, :topics), batch_size: 4, batch_progress: my_proc
+
+      assert_equal 3, lambda_called
+    end
   end
 
   context "with :synchronize option" do
@@ -900,4 +919,33 @@ describe "#import" do
       end
     end
   end
+  describe "importing model with after_initialize callback" do
+    let(:columns) { %w(name size) }
+    let(:valid_values) { [%w("Deer", "Small"), %w("Monkey", "Medium")] }
+    let(:invalid_values) do
+      [
+        { name: "giraffe", size: "Large" },
+        { size: "Medium" } # name is missing
+      ]
+    end
+    context "with validation checks turned off" do
+      it "should import valid data" do
+        Animal.import(columns, valid_values, validate: false)
+        assert_equal 2, Animal.count
+      end
+      it "should raise ArgumentError" do
+        assert_raise(ArgumentError) { Animal.import(invalid_values, validate: false) }
+      end
+    end
+
+    context "with validation checks turned on" do
+      it "should import valid data" do
+        Animal.import(columns, valid_values, validate: true)
+        assert_equal 2, Animal.count
+      end
+      it "should raise ArgumentError" do
+        assert_raise(ArgumentError) { Animal.import(invalid_values, validate: true) }
+      end
+    end
+  end
 end

data/test/models/animal.rb

@@ -0,0 +1,6 @@
+class Animal < ActiveRecord::Base
+  after_initialize :validate_name_presence, if: :new_record?
+  def validate_name_presence
+    raise ArgumentError if name.nil?
+  end
+end

data/test/schema/postgresql_schema.rb

@@ -16,6 +16,7 @@ ActiveRecord::Schema.define do
 
   create_table :vendors, id: :uuid, force: :cascade do |t|
     t.string :name, null: true
+    t.text :hours
     t.text :preferences
 
     if t.respond_to?(:json)

data/test/support/postgresql/import_examples.rb

@@ -127,6 +127,15 @@ def should_support_postgresql_import_functionality
         end
       end
 
+      context "when a returning column is a serialized attribute" do
+        let(:vendor) { Vendor.new(hours: { monday: '8-5' }) }
+        let(:result) { Vendor.import([vendor], returning: %w(hours)) }
+
+        it "creates records" do
+          assert_difference("Vendor.count", +1) { result }
+        end
+      end
+
       context "when primary key and returning overlap" do
         let(:result) { Book.import(books, returning: %w(id title)) }
 

data/test/support/shared_examples/recursive_import.rb

@@ -138,6 +138,15 @@ def should_support_recursive_import
       books.each do |book|
         assert_equal book.topic_id, second_new_topic.id
       end
+
+      books.each { |book| book.topic_id = nil }
+      assert_no_difference "Book.count", books.size do
+        Book.import books, validate: false, on_duplicate_key_update: [:topic_id]
+      end
+
+      books.each do |book|
+        assert_equal book.topic_id, nil
+      end
     end
 
     unless ENV["SKIP_COMPOSITE_PK"]

data/test/test_helper.rb

@@ -48,7 +48,15 @@ adapter = ENV["ARE_DB"] || "sqlite3"
 FileUtils.mkdir_p 'log'
 ActiveRecord::Base.logger = Logger.new("log/test.log")
 ActiveRecord::Base.logger.level = Logger::DEBUG
-ActiveRecord::Base.configurations["test"] = YAML.load_file(test_dir.join("database.yml"))[adapter]
+
+if ENV['AR_VERSION'].to_f >= 6.0
+  yaml_config = YAML.load_file(test_dir.join("database.yml"))[adapter]
+  config = ActiveRecord::DatabaseConfigurations::HashConfig.new("test", adapter, yaml_config)
+  ActiveRecord::Base.configurations.configurations << config
+else
+  ActiveRecord::Base.configurations["test"] = YAML.load_file(test_dir.join("database.yml"))[adapter]
+end
+
 ActiveRecord::Base.default_timezone = :utc
 
 require "activerecord-import"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-import
 version: !ruby/object:Gem::Version
-  version: 1.0.4
+  version: 1.1.0
 platform: ruby
 authors:
 - Zach Dennis
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-12-24 00:00:00.000000000 Z
+date: 2021-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -67,7 +67,7 @@ files:
 - benchmarks/models/test_innodb.rb
 - benchmarks/models/test_memory.rb
 - benchmarks/models/test_myisam.rb
-- benchmarks/schema/mysql_schema.rb
+- benchmarks/schema/mysql2_schema.rb
 - gemfiles/3.2.gemfile
 - gemfiles/4.0.gemfile
 - gemfiles/4.1.gemfile
@@ -121,6 +121,7 @@ files:
 - test/makara_postgis/import_test.rb
 - test/models/account.rb
 - test/models/alarm.rb
+- test/models/animal.rb
 - test/models/bike_maker.rb
 - test/models/book.rb
 - test/models/car.rb
@@ -166,11 +167,11 @@ files:
 - test/travis/database.yml
 - test/value_sets_bytes_parser_test.rb
 - test/value_sets_records_parser_test.rb
-homepage: http://github.com/zdennis/activerecord-import
+homepage: https://github.com/zdennis/activerecord-import
 licenses:
-- Ruby
+- MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -178,15 +179,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.2
+      version: 2.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
-signing_key: 
+rubygems_version: 3.0.8
+signing_key:
 specification_version: 4
 summary: Bulk insert extension for ActiveRecord
 test_files:
@@ -211,6 +212,7 @@ test_files:
 - test/makara_postgis/import_test.rb
 - test/models/account.rb
 - test/models/alarm.rb
+- test/models/animal.rb
 - test/models/bike_maker.rb
 - test/models/book.rb
 - test/models/car.rb