activerecord-import 1.0.6 → 1.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bedb8323d872ae8ca6daebc691f82fedc918a3f2c01e7e354a8534854c06d80e
-  data.tar.gz: 2df3dfaef596b93d208854557700167a4a8e0e9bb151ff224e4a6255e374086f
+  metadata.gz: fdf26ba513a4d3365d24cf9b2bf1fc814db9918d06ccdb91c5f32dec026fb7b7
+  data.tar.gz: 35ba31f95ef4b585cbe8ca4b0819d8ddc1ba50b12db6a70d3be2352669fd9289
 SHA512:
-  metadata.gz: c97ba946d5453394831959f384868812ca59833d1c694feccc19d68967d38e3501305fca11fcd39dec32ddc8c88d26982ea2f087d590e34a19da88ff7def1c9c
-  data.tar.gz: 0fcdc64e20ac0ee42584ab97f060e116f2542a353f0dc815a616adbe3810d6a343f8b021c0204adde9eb484aac5befc4e0b4699b1bbdfbddb2d486f8418d1839
+  metadata.gz: 3ef741e92b76c9705c63145b58351e82813eb71b003993a73cff5db2d0779d6dc14b48d4dc43a354ad8ba6a24a6acd2aa54bdf167775dee8e0404db4905c03f2
+  data.tar.gz: f8634e6ae1e9345ca9b1d68bca8fe4e67c04b1ae51101fdf97608a609d1fc349737d424ce23f809548d4e6a42931eb38d246d6be354acf49282fcaf20d827fa0

data/.gitignore

@@ -24,6 +24,7 @@ pkg
 log/*.log
 test.db
 test/database.yml
+benchmarks/log/
 
 .ruby-*
 .bundle/

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## Changes in 1.0.7
+
+* Use @@max_allowed_packet session variable instead of querying SHOW VARIABLES. Thanks to @diclophis via \#706.
+* Add option :track_validation_failures. When this is set to true, failed_instances will be an array of arrays, with each inner array having the form [:index_in_dataset, :object_with_errors]. Thanks to @rorymckinley  via \#684.
+
+### Fixes
+
+* Prevent mass-assignment errors in Rails strict mode. Thanks to @diclophis via \##709.
+
 ## Changes in 1.0.6
 
 ### Fixes
@@ -5,7 +14,7 @@
 * Handle after_initialize callbacks. Thanks to @AhMohsen46 via \#691 and
   \#692.
 * Fix regression introduced in 1.0.4. Explicity allow adapters to
-  support on duplicate key update. Thanks to @dsobiera, @jkowens via \#698.
+  support on duplicate key update. Thanks to @dsobiera, @jkowens via \#696.
 
 ## Changes in 1.0.5
 
@@ -283,7 +292,7 @@
   Thanks to @jkowens via \#301.
 * Allow for custom timestamp columns. Thanks to @mojidabckuu, @jkowens
   via \#401.
- 
+
 ### Fixes
 
 * Fix ActiveRecord 5 issue coercing boolean values when serializing
@@ -295,7 +304,7 @@
 
 * Fix issue where PostgreSQL cannot recognize columns if names
   include mixed case characters. Thanks to @hugobgranja via \#379.
-* Fix an issue for ActiveRecord 5 where serialized fields with 
+* Fix an issue for ActiveRecord 5 where serialized fields with
   default values were not being typecast. Thanks to @whistlerbrk,
   @jkowens via \#386.
 * Add option :force_single_insert for MySQL to make sure a single

data/README.markdown

@@ -250,20 +250,21 @@ Book.import books, recursive: true
 
 ### Options
 
-Key                     | Options               | Default            | Description
------------------------ | --------------------- | ------------------ | -----------
-:validate               | `true`/`false`        | `true`             | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
-:validate_uniqueness    | `true`/`false`        | `false`            | Whether or not to run uniqueness validations, has potential pitfalls, use with caution (requires `>= v0.27.0`).
-:validate_with_context  | `Symbol`              |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones. 
-:on_duplicate_key_ignore| `true`/`false`        | `false`            | Allows skipping records with duplicate keys. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-ignore) for more details.
-:ignore                 | `true`/`false`        | `false`            | Alias for :on_duplicate_key_ignore.
-:on_duplicate_key_update| :all, `Array`, `Hash` | N/A                | Allows upsert logic to be used. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-update) for more details.
-:synchronize            | `Array`               | N/A                | An array of ActiveRecord instances. This synchronizes existing instances in memory with updates from the import.
-:timestamps             | `true`/`false`        | `true`             | Enables/disables timestamps on imported records.
-:recursive              | `true`/`false`        | `false`            | Imports has_many/has_one associations (PostgreSQL only).
-:batch_size             | `Integer`             | total # of records | Max number of records to insert per import
-:raise_error            | `true`/`false`        | `false`            | Raises an exception at the first invalid record. This means there will not be a result object returned. The `import!` method is a shortcut for this.
-:all_or_none            | `true`/`false`        | `false`            | Will not import any records if there is a record with validation errors.
+Key                       | Options               | Default            | Description
+------------------------- | --------------------- | ------------------ | -----------
+:validate                 | `true`/`false`        | `true`             | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
+:validate_uniqueness      | `true`/`false`        | `false`            | Whether or not to run uniqueness validations, has potential pitfalls, use with caution (requires `>= v0.27.0`).
+:validate_with_context    | `Symbol`              |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones. 
+:track_validation_failures| `true`/`false`        | `false`            | When this is set to true, `failed_instances` will be an array of arrays, with each inner array having the form `[:index_in_dataset, :object_with_errors]` 
+:on_duplicate_key_ignore  | `true`/`false`        | `false`            | Allows skipping records with duplicate keys. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-ignore) for more details.
+:ignore                   | `true`/`false`        | `false`            | Alias for :on_duplicate_key_ignore.
+:on_duplicate_key_update  | :all, `Array`, `Hash` | N/A                | Allows upsert logic to be used. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-update) for more details.
+:synchronize              | `Array`               | N/A                | An array of ActiveRecord instances. This synchronizes existing instances in memory with updates from the import.
+:timestamps               | `true`/`false`        | `true`             | Enables/disables timestamps on imported records.
+:recursive                | `true`/`false`        | `false`            | Imports has_many/has_one associations (PostgreSQL only).
+:batch_size               | `Integer`             | total # of records | Max number of records to insert per import
+:raise_error              | `true`/`false`        | `false`            | Raises an exception at the first invalid record. This means there will not be a result object returned. The `import!` method is a shortcut for this.
+:all_or_none              | `true`/`false`        | `false`            | Will not import any records if there is a record with validation errors.
 
 #### Duplicate Key Ignore
 

data/activerecord-import.gemspec

@@ -6,7 +6,7 @@ Gem::Specification.new do |gem|
   gem.email         = ["zach.dennis@gmail.com"]
   gem.summary       = "Bulk insert extension for ActiveRecord"
   gem.description   = "A library for bulk inserting data using ActiveRecord."
-  gem.homepage      = "http://github.com/zdennis/activerecord-import"
+  gem.homepage      = "https://github.com/zdennis/activerecord-import"
   gem.license       = "MIT"
 
   gem.files         = `git ls-files`.split($\)

data/lib/activerecord-import/adapters/mysql_adapter.rb

@@ -56,9 +56,9 @@ module ActiveRecord::Import::MysqlAdapter
   # in a single packet
   def max_allowed_packet # :nodoc:
     @max_allowed_packet ||= begin
-      result = execute( "SHOW VARIABLES like 'max_allowed_packet'" )
+      result = execute( "SELECT @@max_allowed_packet" )
       # original Mysql gem responds to #fetch_row while Mysql2 responds to #first
-      val = result.respond_to?(:fetch_row) ? result.fetch_row[1] : result.first[1]
+      val = result.respond_to?(:fetch_row) ? result.fetch_row[0] : result.first[0]
       val.to_i
     end
   end

data/lib/activerecord-import/import.rb

@@ -547,7 +547,7 @@ class ActiveRecord::Base
     alias import! bulk_import! unless ActiveRecord::Base.respond_to? :import!
 
     def import_helper( *args )
-      options = { validate: true, timestamps: true }
+      options = { validate: true, timestamps: true, track_validation_failures: false }
       options.merge!( args.pop ) if args.last.is_a? Hash
       # making sure that current model's primary key is used
       options[:primary_key] = primary_key
@@ -703,13 +703,18 @@ class ActiveRecord::Base
             # keep track of the instance and the position it is currently at. if this fails
             # validation we'll use the index to remove it from the array_of_attributes
             arr.each_with_index do |hsh, i|
-              model = new(hsh)
+              # utilize block initializer syntax to prevent failure when 'mass_assignment_sanitizer = :strict'
+              model = new do |m|
+                hsh.each_pair { |k, v| m[k] = v }
+              end
+
               next if validator.valid_model?(model)
               raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
+
               array_of_attributes[i] = nil
               failure = model.dup
               failure.errors.send(:initialize_dup, model.errors)
-              failed_instances << failure
+              failed_instances << (options[:track_validation_failures] ? [i, failure] : failure )
             end
             array_of_attributes.compact!
           end
@@ -807,7 +812,7 @@ class ActiveRecord::Base
           result = connection.insert_many( [insert_sql, post_sql_statements].flatten,
             batch_values,
             options,
-            "#{model_name} Create Many Without Validations Or Callbacks" )
+            "#{model_name} Create Many" )
           number_inserted += result.num_inserts
           ids += result.ids
           results += result.results

data/lib/activerecord-import/version.rb

@@ -1,5 +1,5 @@
 module ActiveRecord
   module Import
-    VERSION = "1.0.6".freeze
+    VERSION = "1.0.7".freeze
   end
 end

data/test/import_test.rb

@@ -252,6 +252,16 @@ describe "#import" do
         end
       end
 
+      it "should index the failed instances by their poistion in the set if `track_failures` is true" do
+        index_offset = valid_values.length
+        results = Topic.import columns, valid_values + invalid_values, validate: true, track_validation_failures: true
+        assert_equal invalid_values.size, results.failed_instances.size
+        invalid_values.each_with_index do |value_set, index|
+          assert_equal index + index_offset, results.failed_instances[index].first
+          assert_equal value_set.first, results.failed_instances[index].last.title
+        end
+      end
+
       it "should set ids in valid models if adapter supports setting primary key of imported objects" do
         if ActiveRecord::Base.supports_setting_primary_key_of_imported_objects?
           Topic.import (invalid_models + valid_models), validate: true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-import
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 1.0.7
 platform: ruby
 authors:
 - Zach Dennis
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-01 00:00:00.000000000 Z
+date: 2020-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -67,7 +67,7 @@ files:
 - benchmarks/models/test_innodb.rb
 - benchmarks/models/test_memory.rb
 - benchmarks/models/test_myisam.rb
-- benchmarks/schema/mysql_schema.rb
+- benchmarks/schema/mysql2_schema.rb
 - gemfiles/3.2.gemfile
 - gemfiles/4.0.gemfile
 - gemfiles/4.1.gemfile
@@ -167,7 +167,7 @@ files:
 - test/travis/database.yml
 - test/value_sets_bytes_parser_test.rb
 - test/value_sets_records_parser_test.rb
-homepage: http://github.com/zdennis/activerecord-import
+homepage: https://github.com/zdennis/activerecord-import
 licenses:
 - MIT
 metadata: {}
@@ -186,7 +186,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.0.8
 signing_key:
 specification_version: 4
 summary: Bulk insert extension for ActiveRecord