activerecord-import 0.17.2 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (52) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +1 -0
  3. data/.travis.yml +40 -23
  4. data/CHANGELOG.md +315 -1
  5. data/Gemfile +23 -13
  6. data/LICENSE +21 -56
  7. data/README.markdown +564 -33
  8. data/Rakefile +2 -1
  9. data/activerecord-import.gemspec +3 -3
  10. data/benchmarks/lib/cli_parser.rb +2 -1
  11. data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +0 -0
  12. data/gemfiles/5.1.gemfile +2 -0
  13. data/gemfiles/5.2.gemfile +2 -0
  14. data/gemfiles/6.0.gemfile +2 -0
  15. data/gemfiles/6.1.gemfile +1 -0
  16. data/lib/activerecord-import.rb +2 -15
  17. data/lib/activerecord-import/adapters/abstract_adapter.rb +9 -3
  18. data/lib/activerecord-import/adapters/mysql_adapter.rb +17 -11
  19. data/lib/activerecord-import/adapters/postgresql_adapter.rb +68 -20
  20. data/lib/activerecord-import/adapters/sqlite3_adapter.rb +128 -9
  21. data/lib/activerecord-import/base.rb +12 -7
  22. data/lib/activerecord-import/import.rb +514 -166
  23. data/lib/activerecord-import/synchronize.rb +2 -2
  24. data/lib/activerecord-import/value_sets_parser.rb +16 -0
  25. data/lib/activerecord-import/version.rb +1 -1
  26. data/test/adapters/makara_postgis.rb +1 -0
  27. data/test/import_test.rb +274 -23
  28. data/test/makara_postgis/import_test.rb +8 -0
  29. data/test/models/account.rb +3 -0
  30. data/test/models/animal.rb +6 -0
  31. data/test/models/bike_maker.rb +7 -0
  32. data/test/models/tag.rb +1 -1
  33. data/test/models/topic.rb +14 -0
  34. data/test/models/user.rb +3 -0
  35. data/test/models/user_token.rb +4 -0
  36. data/test/schema/generic_schema.rb +30 -8
  37. data/test/schema/mysql2_schema.rb +19 -0
  38. data/test/schema/postgresql_schema.rb +18 -0
  39. data/test/schema/sqlite3_schema.rb +13 -0
  40. data/test/support/factories.rb +9 -8
  41. data/test/support/generate.rb +6 -6
  42. data/test/support/mysql/import_examples.rb +14 -2
  43. data/test/support/postgresql/import_examples.rb +220 -1
  44. data/test/support/shared_examples/on_duplicate_key_ignore.rb +15 -9
  45. data/test/support/shared_examples/on_duplicate_key_update.rb +271 -8
  46. data/test/support/shared_examples/recursive_import.rb +91 -21
  47. data/test/support/sqlite3/import_examples.rb +189 -25
  48. data/test/synchronize_test.rb +8 -0
  49. data/test/test_helper.rb +24 -3
  50. data/test/value_sets_bytes_parser_test.rb +13 -2
  51. metadata +32 -13
  52. data/test/schema/mysql_schema.rb +0 -16
@@ -11,24 +11,29 @@ module ActiveRecord::Import
11
11
  when 'mysql2spatial' then 'mysql2'
12
12
  when 'spatialite' then 'sqlite3'
13
13
  when 'postgresql_makara' then 'postgresql'
14
+ when 'makara_postgis' then 'postgresql'
14
15
  when 'postgis' then 'postgresql'
16
+ when 'cockroachdb' then 'postgresql'
15
17
  else adapter
16
18
  end
17
19
  end
18
20
 
19
21
  # Loads the import functionality for a specific database adapter
20
22
  def self.require_adapter(adapter)
21
- require File.join(ADAPTER_PATH, "/abstract_adapter")
22
- begin
23
- require File.join(ADAPTER_PATH, "/#{base_adapter(adapter)}_adapter")
24
- rescue LoadError
25
- # fallback
26
- end
23
+ require File.join(ADAPTER_PATH, "/#{base_adapter(adapter)}_adapter")
24
+ rescue LoadError
25
+ # fallback
27
26
  end
28
27
 
29
28
  # Loads the import functionality for the passed in ActiveRecord connection
30
29
  def self.load_from_connection_pool(connection_pool)
31
- require_adapter connection_pool.spec.config[:adapter]
30
+ adapter =
31
+ if connection_pool.respond_to?(:db_config) # ActiveRecord >= 6.1
32
+ connection_pool.db_config.adapter
33
+ else
34
+ connection_pool.spec.config[:adapter]
35
+ end
36
+ require_adapter adapter
32
37
  end
33
38
  end
34
39
 
@@ -3,7 +3,7 @@ require "ostruct"
3
3
  module ActiveRecord::Import::ConnectionAdapters; end
4
4
 
5
5
  module ActiveRecord::Import #:nodoc:
6
- Result = Struct.new(:failed_instances, :num_inserts, :ids)
6
+ Result = Struct.new(:failed_instances, :num_inserts, :ids, :results)
7
7
 
8
8
  module ImportSupport #:nodoc:
9
9
  def supports_import? #:nodoc:
@@ -22,16 +22,110 @@ module ActiveRecord::Import #:nodoc:
22
22
  super "Missing column for value <#{name}> at index #{index}"
23
23
  end
24
24
  end
25
+
26
+ class Validator
27
+ def initialize(klass, options = {})
28
+ @options = options
29
+ @validator_class = klass
30
+ init_validations(klass)
31
+ end
32
+
33
+ def init_validations(klass)
34
+ @validate_callbacks = klass._validate_callbacks.dup
35
+
36
+ @validate_callbacks.each_with_index do |callback, i|
37
+ filter = callback.raw_filter
38
+ next unless filter.class.name =~ /Validations::PresenceValidator/ ||
39
+ (!@options[:validate_uniqueness] &&
40
+ filter.is_a?(ActiveRecord::Validations::UniquenessValidator))
41
+
42
+ callback = callback.dup
43
+ filter = filter.dup
44
+ attrs = filter.instance_variable_get(:@attributes).dup
45
+
46
+ if filter.is_a?(ActiveRecord::Validations::UniquenessValidator)
47
+ attrs = []
48
+ else
49
+ associations = klass.reflect_on_all_associations(:belongs_to)
50
+ associations.each do |assoc|
51
+ if (index = attrs.index(assoc.name))
52
+ key = assoc.foreign_key.to_sym
53
+ attrs[index] = key unless attrs.include?(key)
54
+ end
55
+ end
56
+ end
57
+
58
+ filter.instance_variable_set(:@attributes, attrs)
59
+
60
+ if @validate_callbacks.respond_to?(:chain, true)
61
+ @validate_callbacks.send(:chain).tap do |chain|
62
+ callback.instance_variable_set(:@filter, filter)
63
+ chain[i] = callback
64
+ end
65
+ else
66
+ callback.raw_filter = filter
67
+ callback.filter = callback.send(:_compile_filter, filter)
68
+ @validate_callbacks[i] = callback
69
+ end
70
+ end
71
+ end
72
+
73
+ def valid_model?(model)
74
+ init_validations(model.class) unless model.class == @validator_class
75
+
76
+ validation_context = @options[:validate_with_context]
77
+ validation_context ||= (model.new_record? ? :create : :update)
78
+ current_context = model.send(:validation_context)
79
+
80
+ begin
81
+ model.send(:validation_context=, validation_context)
82
+ model.errors.clear
83
+
84
+ model.run_callbacks(:validation) do
85
+ if defined?(ActiveSupport::Callbacks::Filters::Environment) # ActiveRecord >= 4.1
86
+ runner = @validate_callbacks.compile
87
+ env = ActiveSupport::Callbacks::Filters::Environment.new(model, false, nil)
88
+ if runner.respond_to?(:call) # ActiveRecord < 5.1
89
+ runner.call(env)
90
+ else # ActiveRecord 5.1
91
+ # Note that this is a gross simplification of ActiveSupport::Callbacks#run_callbacks.
92
+ # It's technically possible for there to exist an "around" callback in the
93
+ # :validate chain, but this would be an aberration, since Rails doesn't define
94
+ # "around_validate". Still, rather than silently ignoring such callbacks, we
95
+ # explicitly raise a RuntimeError, since activerecord-import was asked to perform
96
+ # validations and it's unable to do so.
97
+ #
98
+ # The alternative here would be to copy-and-paste the bulk of the
99
+ # ActiveSupport::Callbacks#run_callbacks method, which is undesirable if there's
100
+ # no real-world use case for it.
101
+ raise "The :validate callback chain contains an 'around' callback, which is unsupported" unless runner.final?
102
+ runner.invoke_before(env)
103
+ runner.invoke_after(env)
104
+ end
105
+ elsif @validate_callbacks.method(:compile).arity == 0 # ActiveRecord = 4.0
106
+ model.instance_eval @validate_callbacks.compile
107
+ else # ActiveRecord 3.x
108
+ model.instance_eval @validate_callbacks.compile(nil, model)
109
+ end
110
+ end
111
+
112
+ model.errors.empty?
113
+ ensure
114
+ model.send(:validation_context=, current_context)
115
+ end
116
+ end
117
+ end
25
118
  end
26
119
 
27
120
  class ActiveRecord::Associations::CollectionProxy
28
- def import(*args, &block)
29
- @association.import(*args, &block)
121
+ def bulk_import(*args, &block)
122
+ @association.bulk_import(*args, &block)
30
123
  end
124
+ alias import bulk_import unless respond_to? :import
31
125
  end
32
126
 
33
127
  class ActiveRecord::Associations::CollectionAssociation
34
- def import(*args, &block)
128
+ def bulk_import(*args, &block)
35
129
  unless owner.persisted?
36
130
  raise ActiveRecord::RecordNotSaved, "You cannot call import unless the parent is saved"
37
131
  end
@@ -40,16 +134,21 @@ class ActiveRecord::Associations::CollectionAssociation
40
134
 
41
135
  model_klass = reflection.klass
42
136
  symbolized_foreign_key = reflection.foreign_key.to_sym
43
- symbolized_column_names = model_klass.column_names.map(&:to_sym)
44
137
 
45
- owner_primary_key = owner.class.primary_key
138
+ symbolized_column_names = if model_klass.connection.respond_to?(:supports_virtual_columns?) && model_klass.connection.supports_virtual_columns?
139
+ model_klass.columns.reject(&:virtual?).map { |c| c.name.to_sym }
140
+ else
141
+ model_klass.column_names.map(&:to_sym)
142
+ end
143
+
144
+ owner_primary_key = reflection.active_record_primary_key.to_sym
46
145
  owner_primary_key_value = owner.send(owner_primary_key)
47
146
 
48
147
  # assume array of model objects
49
148
  if args.last.is_a?( Array ) && args.last.first.is_a?(ActiveRecord::Base)
50
149
  if args.length == 2
51
150
  models = args.last
52
- column_names = args.first
151
+ column_names = args.first.dup
53
152
  else
54
153
  models = args.first
55
154
  column_names = symbolized_column_names
@@ -64,7 +163,46 @@ class ActiveRecord::Associations::CollectionAssociation
64
163
  m.public_send "#{reflection.type}=", owner.class.name if reflection.type
65
164
  end
66
165
 
67
- return model_klass.import column_names, models, options
166
+ return model_klass.bulk_import column_names, models, options
167
+
168
+ # supports array of hash objects
169
+ elsif args.last.is_a?( Array ) && args.last.first.is_a?(Hash)
170
+ if args.length == 2
171
+ array_of_hashes = args.last
172
+ column_names = args.first.dup
173
+ allow_extra_hash_keys = true
174
+ else
175
+ array_of_hashes = args.first
176
+ column_names = array_of_hashes.first.keys
177
+ allow_extra_hash_keys = false
178
+ end
179
+
180
+ symbolized_column_names = column_names.map(&:to_sym)
181
+ unless symbolized_column_names.include?(symbolized_foreign_key)
182
+ column_names << symbolized_foreign_key
183
+ end
184
+
185
+ if reflection.type && !symbolized_column_names.include?(reflection.type.to_sym)
186
+ column_names << reflection.type.to_sym
187
+ end
188
+
189
+ array_of_attributes = array_of_hashes.map do |h|
190
+ error_message = model_klass.send(:validate_hash_import, h, symbolized_column_names, allow_extra_hash_keys)
191
+
192
+ raise ArgumentError, error_message if error_message
193
+
194
+ column_names.map do |key|
195
+ if key == symbolized_foreign_key
196
+ owner_primary_key_value
197
+ elsif reflection.type && key == reflection.type.to_sym
198
+ owner.class.name
199
+ else
200
+ h[key]
201
+ end
202
+ end
203
+ end
204
+
205
+ return model_klass.bulk_import column_names, array_of_attributes, options
68
206
 
69
207
  # supports empty array
70
208
  elsif args.last.is_a?( Array ) && args.last.empty?
@@ -73,7 +211,12 @@ class ActiveRecord::Associations::CollectionAssociation
73
211
  # supports 2-element array and array
74
212
  elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
75
213
  column_names, array_of_attributes = args
76
- symbolized_column_names = column_names.map(&:to_s)
214
+
215
+ # dup the passed args so we don't modify unintentionally
216
+ column_names = column_names.dup
217
+ array_of_attributes = array_of_attributes.map(&:dup)
218
+
219
+ symbolized_column_names = column_names.map(&:to_sym)
77
220
 
78
221
  if symbolized_column_names.include?(symbolized_foreign_key)
79
222
  index = symbolized_column_names.index(symbolized_foreign_key)
@@ -84,31 +227,35 @@ class ActiveRecord::Associations::CollectionAssociation
84
227
  end
85
228
 
86
229
  if reflection.type
87
- column_names << reflection.type
88
- array_of_attributes.each { |attrs| attrs << owner.class.name }
230
+ symbolized_type = reflection.type.to_sym
231
+ if symbolized_column_names.include?(symbolized_type)
232
+ index = symbolized_column_names.index(symbolized_type)
233
+ array_of_attributes.each { |attrs| attrs[index] = owner.class.name }
234
+ else
235
+ column_names << symbolized_type
236
+ array_of_attributes.each { |attrs| attrs << owner.class.name }
237
+ end
89
238
  end
90
239
 
91
- return model_klass.import column_names, array_of_attributes, options
240
+ return model_klass.bulk_import column_names, array_of_attributes, options
92
241
  else
93
242
  raise ArgumentError, "Invalid arguments!"
94
243
  end
95
244
  end
245
+ alias import bulk_import unless respond_to? :import
246
+ end
247
+
248
+ module ActiveRecord::Import::Connection
249
+ def establish_connection(args = nil)
250
+ conn = super(args)
251
+ ActiveRecord::Import.load_from_connection_pool connection_pool
252
+ conn
253
+ end
96
254
  end
97
255
 
98
256
  class ActiveRecord::Base
99
257
  class << self
100
- # use tz as set in ActiveRecord::Base
101
- tproc = lambda do
102
- ActiveRecord::Base.default_timezone == :utc ? Time.now.utc : Time.now
103
- end
104
-
105
- AREXT_RAILS_COLUMNS = {
106
- create: { "created_on" => tproc,
107
- "created_at" => tproc },
108
- update: { "updated_on" => tproc,
109
- "updated_at" => tproc }
110
- }.freeze
111
- AREXT_RAILS_COLUMN_NAMES = AREXT_RAILS_COLUMNS[:create].keys + AREXT_RAILS_COLUMNS[:update].keys
258
+ prepend ActiveRecord::Import::Connection
112
259
 
113
260
  # Returns true if the current database connection adapter
114
261
  # supports import functionality, otherwise returns false.
@@ -120,14 +267,14 @@ class ActiveRecord::Base
120
267
  # supports on duplicate key update functionality, otherwise
121
268
  # returns false.
122
269
  def supports_on_duplicate_key_update?
123
- connection.supports_on_duplicate_key_update?
270
+ connection.respond_to?(:supports_on_duplicate_key_update?) && connection.supports_on_duplicate_key_update?
124
271
  end
125
272
 
126
273
  # returns true if the current database connection adapter
127
274
  # supports setting the primary key of bulk imported models, otherwise
128
275
  # returns false
129
- def support_setting_primary_key_of_imported_objects?
130
- connection.respond_to?(:support_setting_primary_key_of_imported_objects?) && connection.support_setting_primary_key_of_imported_objects?
276
+ def supports_setting_primary_key_of_imported_objects?
277
+ connection.respond_to?(:supports_setting_primary_key_of_imported_objects?) && connection.supports_setting_primary_key_of_imported_objects?
131
278
  end
132
279
 
133
280
  # Imports a collection of values to the database.
@@ -173,16 +320,21 @@ class ActiveRecord::Base
173
320
  #
174
321
  # == Options
175
322
  # * +validate+ - true|false, tells import whether or not to use
176
- # ActiveRecord validations. Validations are enforced by default.
323
+ # ActiveRecord validations. Validations are enforced by default.
324
+ # It skips the uniqueness validation for performance reasons.
325
+ # You can find more details here:
326
+ # https://github.com/zdennis/activerecord-import/issues/228
177
327
  # * +ignore+ - true|false, an alias for on_duplicate_key_ignore.
178
328
  # * +on_duplicate_key_ignore+ - true|false, tells import to discard
179
- # records that contain duplicate keys. For Postgres 9.5+ it adds
180
- # ON CONFLICT DO NOTHING, for MySQL it uses INSERT IGNORE, and for
181
- # SQLite it uses INSERT OR IGNORE. Cannot be enabled on a
182
- # recursive import.
183
- # * +on_duplicate_key_update+ - an Array or Hash, tells import to
184
- # use MySQL's ON DUPLICATE KEY UPDATE or Postgres 9.5+ ON CONFLICT
185
- # DO UPDATE ability. See On Duplicate Key Update below.
329
+ # records that contain duplicate keys. For Postgres 9.5+ it adds
330
+ # ON CONFLICT DO NOTHING, for MySQL it uses INSERT IGNORE, and for
331
+ # SQLite it uses INSERT OR IGNORE. Cannot be enabled on a
332
+ # recursive import. For database adapters that normally support
333
+ # setting primary keys on imported objects, this option prevents
334
+ # that from occurring.
335
+ # * +on_duplicate_key_update+ - :all, an Array, or Hash, tells import to
336
+ # use MySQL's ON DUPLICATE KEY UPDATE or Postgres/SQLite ON CONFLICT
337
+ # DO UPDATE ability. See On Duplicate Key Update below.
186
338
  # * +synchronize+ - an array of ActiveRecord instances for the model
187
339
  # that you are currently importing data into. This synchronizes
188
340
  # existing model instances in memory with updates from the import.
@@ -204,6 +356,9 @@ class ActiveRecord::Base
204
356
  # BlogPost.import posts
205
357
  #
206
358
  # # Example using array_of_hash_objects
359
+ # # NOTE: column_names will be determined by using the keys of the first hash in the array. If later hashes in the
360
+ # # array have different keys an exception will be raised. If you have hashes to import with different sets of keys
361
+ # # we recommend grouping these into batches before importing.
207
362
  # values = [ {author_name: 'zdennis', title: 'test post'} ], [ {author_name: 'jdoe', title: 'another test post'} ] ]
208
363
  # BlogPost.import values
209
364
  #
@@ -237,7 +392,15 @@ class ActiveRecord::Base
237
392
  #
238
393
  # == On Duplicate Key Update (MySQL)
239
394
  #
240
- # The :on_duplicate_key_update option can be either an Array or a Hash.
395
+ # The :on_duplicate_key_update option can be either :all, an Array, or a Hash.
396
+ #
397
+ # ==== Using :all
398
+ #
399
+ # The :on_duplicate_key_update option can be set to :all. All columns
400
+ # other than the primary key are updated. If a list of column names is
401
+ # supplied, only those columns will be updated. Below is an example:
402
+ #
403
+ # BlogPost.import columns, values, on_duplicate_key_update: :all
241
404
  #
242
405
  # ==== Using an Array
243
406
  #
@@ -256,11 +419,19 @@ class ActiveRecord::Base
256
419
  #
257
420
  # BlogPost.import columns, attributes, on_duplicate_key_update: { title: :title }
258
421
  #
259
- # == On Duplicate Key Update (Postgres 9.5+)
422
+ # == On Duplicate Key Update (Postgres 9.5+ and SQLite 3.24+)
260
423
  #
261
- # The :on_duplicate_key_update option can be an Array or a Hash with up to
424
+ # The :on_duplicate_key_update option can be :all, an Array, or a Hash with up to
262
425
  # three attributes, :conflict_target (and optionally :index_predicate) or
263
- # :constraint_name, and :columns.
426
+ # :constraint_name (Postgres), and :columns.
427
+ #
428
+ # ==== Using :all
429
+ #
430
+ # The :on_duplicate_key_update option can be set to :all. All columns
431
+ # other than the primary key are updated. If a list of column names is
432
+ # supplied, only those columns will be updated. Below is an example:
433
+ #
434
+ # BlogPost.import columns, values, on_duplicate_key_update: :all
264
435
  #
265
436
  # ==== Using an Array
266
437
  #
@@ -280,7 +451,7 @@ class ActiveRecord::Base
280
451
  # conflicting constraint to be explicitly specified. Using this option
281
452
  # allows you to specify a constraint other than the primary key.
282
453
  #
283
- # ====== :conflict_target
454
+ # ===== :conflict_target
284
455
  #
285
456
  # The :conflict_target attribute specifies the columns that make up the
286
457
  # conflicting unique constraint and can be a single column or an array of
@@ -290,7 +461,7 @@ class ActiveRecord::Base
290
461
  #
291
462
  # BlogPost.import columns, values, on_duplicate_key_update: { conflict_target: [ :author_id, :slug ], columns: [ :date_modified ] }
292
463
  #
293
- # ====== :index_predicate
464
+ # ===== :index_predicate
294
465
  #
295
466
  # The :index_predicate attribute optionally specifies a WHERE condition
296
467
  # on :conflict_target, which is required for matching against partial
@@ -299,7 +470,7 @@ class ActiveRecord::Base
299
470
  #
300
471
  # BlogPost.import columns, values, on_duplicate_key_update: { conflict_target: [ :author_id, :slug ], index_predicate: 'status <> 0', columns: [ :date_modified ] }
301
472
  #
302
- # ====== :constraint_name
473
+ # ===== :constraint_name
303
474
  #
304
475
  # The :constraint_name attribute explicitly identifies the conflicting
305
476
  # unique index by name. Postgres documentation discourages using this method
@@ -307,11 +478,28 @@ class ActiveRecord::Base
307
478
  #
308
479
  # BlogPost.import columns, values, on_duplicate_key_update: { constraint_name: :blog_posts_pkey, columns: [ :date_modified ] }
309
480
  #
310
- # ====== :columns
481
+ # ===== :condition
482
+ #
483
+ # The :condition attribute optionally specifies a WHERE condition
484
+ # on :conflict_action. Only rows for which this expression returns true will be updated.
485
+ # Note that it's evaluated last, after a conflict has been identified as a candidate to update.
486
+ # Below is an example:
487
+ #
488
+ # BlogPost.import columns, values, on_duplicate_key_update: { conflict_target: [ :author_id ], condition: "blog_posts.title NOT LIKE '%sample%'", columns: [ :author_name ] }
489
+ #
490
+ # ===== :columns
491
+ #
492
+ # The :columns attribute can be either :all, an Array, or a Hash.
311
493
  #
312
- # The :columns attribute can be either an Array or a Hash.
494
+ # ===== Using :all
495
+ #
496
+ # The :columns attribute can be :all. All columns other than the primary key will be updated.
497
+ # If a list of column names is supplied, only those columns will be updated.
498
+ # Below is an example:
313
499
  #
314
- # ======== Using an Array
500
+ # BlogPost.import columns, values, on_duplicate_key_update: { conflict_target: :slug, columns: :all }
501
+ #
502
+ # ===== Using an Array
315
503
  #
316
504
  # The :columns attribute can be an array of column names. The column names
317
505
  # are the only fields that are updated if a duplicate record is found.
@@ -319,7 +507,7 @@ class ActiveRecord::Base
319
507
  #
320
508
  # BlogPost.import columns, values, on_duplicate_key_update: { conflict_target: :slug, columns: [ :date_modified, :content, :author ] }
321
509
  #
322
- # ======== Using a Hash
510
+ # ===== Using a Hash
323
511
  #
324
512
  # The :columns option can be a hash of column names to model attribute name
325
513
  # mappings. This gives you finer grained control over what fields are updated
@@ -332,7 +520,8 @@ class ActiveRecord::Base
332
520
  # * failed_instances - an array of objects that fails validation and were not committed to the database. An empty array if no validation is performed.
333
521
  # * num_inserts - the number of insert statements it took to import the data
334
522
  # * ids - the primary keys of the imported ids if the adapter supports it, otherwise an empty array.
335
- def import(*args)
523
+ # * results - import results if the adapter supports it, otherwise an empty array.
524
+ def bulk_import(*args)
336
525
  if args.first.is_a?( Array ) && args.first.first.is_a?(ActiveRecord::Base)
337
526
  options = {}
338
527
  options.merge!( args.pop ) if args.last.is_a?(Hash)
@@ -343,31 +532,29 @@ class ActiveRecord::Base
343
532
  import_helper(*args)
344
533
  end
345
534
  end
535
+ alias import bulk_import unless ActiveRecord::Base.respond_to? :import
346
536
 
347
537
  # Imports a collection of values if all values are valid. Import fails at the
348
538
  # first encountered validation error and raises ActiveRecord::RecordInvalid
349
539
  # with the failed instance.
350
- def import!(*args)
540
+ def bulk_import!(*args)
351
541
  options = args.last.is_a?( Hash ) ? args.pop : {}
352
542
  options[:validate] = true
353
543
  options[:raise_error] = true
354
544
 
355
- import(*args, options)
545
+ bulk_import(*args, options)
356
546
  end
547
+ alias import! bulk_import! unless ActiveRecord::Base.respond_to? :import!
357
548
 
358
549
  def import_helper( *args )
359
- options = { validate: true, timestamps: true }
550
+ options = { validate: true, timestamps: true, track_validation_failures: false }
360
551
  options.merge!( args.pop ) if args.last.is_a? Hash
361
552
  # making sure that current model's primary key is used
362
553
  options[:primary_key] = primary_key
554
+ options[:locking_column] = locking_column if attribute_names.include?(locking_column)
363
555
 
364
- # Don't modify incoming arguments
365
- if options[:on_duplicate_key_update] && options[:on_duplicate_key_update].duplicable?
366
- options[:on_duplicate_key_update] = options[:on_duplicate_key_update].dup
367
- end
368
-
369
- is_validating = options[:validate]
370
- is_validating = true unless options[:validate_with_context].nil?
556
+ is_validating = options[:validate_with_context].present? ? true : options[:validate]
557
+ validator = ActiveRecord::Import::Validator.new(self, options)
371
558
 
372
559
  # assume array of model objects
373
560
  if args.last.is_a?( Array ) && args.last.first.is_a?(ActiveRecord::Base)
@@ -376,23 +563,48 @@ class ActiveRecord::Base
376
563
  column_names = args.first.dup
377
564
  else
378
565
  models = args.first
379
- column_names = self.column_names.dup
566
+ column_names = if connection.respond_to?(:supports_virtual_columns?) && connection.supports_virtual_columns?
567
+ columns.reject(&:virtual?).map(&:name)
568
+ else
569
+ self.column_names.dup
570
+ end
380
571
  end
381
572
 
382
- if models.first.id.nil? && column_names.include?(primary_key) && columns_hash[primary_key].type == :uuid
383
- column_names.delete(primary_key)
573
+ if models.first.id.nil?
574
+ Array(primary_key).each do |c|
575
+ if column_names.include?(c) && columns_hash[c].type == :uuid
576
+ column_names.delete(c)
577
+ end
578
+ end
384
579
  end
385
580
 
386
- stored_attrs = respond_to?(:stored_attributes) ? stored_attributes : {}
387
- default_values = column_defaults
581
+ update_attrs = if record_timestamps && options[:timestamps]
582
+ if respond_to?(:timestamp_attributes_for_update, true)
583
+ send(:timestamp_attributes_for_update).map(&:to_sym)
584
+ else
585
+ allocate.send(:timestamp_attributes_for_update_in_model)
586
+ end
587
+ end
388
588
 
389
- array_of_attributes = models.map do |model|
390
- column_names.map do |name|
391
- is_stored_attr = stored_attrs.any? && stored_attrs.key?(name.to_sym)
392
- if is_stored_attr || default_values[name].is_a?(Hash)
393
- model.read_attribute(name.to_s)
589
+ array_of_attributes = []
590
+
591
+ models.each do |model|
592
+ if supports_setting_primary_key_of_imported_objects?
593
+ load_association_ids(model)
594
+ end
595
+
596
+ if is_validating && !validator.valid_model?(model)
597
+ raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
598
+ next
599
+ end
600
+
601
+ array_of_attributes << column_names.map do |name|
602
+ if model.persisted? &&
603
+ update_attrs && update_attrs.include?(name.to_sym) &&
604
+ !model.send("#{name}_changed?")
605
+ nil
394
606
  else
395
- model.read_attribute_before_type_cast(name.to_s)
607
+ model.read_attribute(name.to_s)
396
608
  end
397
609
  end
398
610
  end
@@ -401,19 +613,25 @@ class ActiveRecord::Base
401
613
  if args.length == 2
402
614
  array_of_hashes = args.last
403
615
  column_names = args.first.dup
616
+ allow_extra_hash_keys = true
404
617
  else
405
618
  array_of_hashes = args.first
406
619
  column_names = array_of_hashes.first.keys
620
+ allow_extra_hash_keys = false
407
621
  end
408
622
 
409
623
  array_of_attributes = array_of_hashes.map do |h|
624
+ error_message = validate_hash_import(h, column_names, allow_extra_hash_keys)
625
+
626
+ raise ArgumentError, error_message if error_message
627
+
410
628
  column_names.map do |key|
411
629
  h[key]
412
630
  end
413
631
  end
414
632
  # supports empty array
415
633
  elsif args.last.is_a?( Array ) && args.last.empty?
416
- return ActiveRecord::Import::Result.new([], 0, [])
634
+ return ActiveRecord::Import::Result.new([], 0, [], [])
417
635
  # supports 2-element array and array
418
636
  elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
419
637
 
@@ -438,47 +656,82 @@ class ActiveRecord::Base
438
656
 
439
657
  if !symbolized_primary_key.to_set.subset?(symbolized_column_names.to_set) && connection.prefetch_primary_key? && sequence_name
440
658
  column_count = column_names.size
441
- column_names.concat(primary_key).uniq!
659
+ column_names.concat(Array(primary_key)).uniq!
442
660
  columns_added = column_names.size - column_count
443
661
  new_fields = Array.new(columns_added)
444
662
  array_of_attributes.each { |a| a.concat(new_fields) }
445
663
  end
446
664
 
665
+ # Don't modify incoming arguments
666
+ on_duplicate_key_update = options[:on_duplicate_key_update]
667
+ if on_duplicate_key_update
668
+ updatable_columns = symbolized_column_names.reject { |c| symbolized_primary_key.include? c }
669
+ options[:on_duplicate_key_update] = if on_duplicate_key_update.is_a?(Hash)
670
+ on_duplicate_key_update.each_with_object({}) do |(k, v), duped_options|
671
+ duped_options[k] = if k == :columns && v == :all
672
+ updatable_columns
673
+ elsif v.duplicable?
674
+ v.dup
675
+ else
676
+ v
677
+ end
678
+ end
679
+ elsif on_duplicate_key_update == :all
680
+ updatable_columns
681
+ elsif on_duplicate_key_update.duplicable?
682
+ on_duplicate_key_update.dup
683
+ else
684
+ on_duplicate_key_update
685
+ end
686
+ end
687
+
447
688
  timestamps = {}
448
689
 
449
690
  # record timestamps unless disabled in ActiveRecord::Base
450
- if record_timestamps && options.delete( :timestamps )
691
+ if record_timestamps && options[:timestamps]
451
692
  timestamps = add_special_rails_stamps column_names, array_of_attributes, options
452
693
  end
453
694
 
454
695
  return_obj = if is_validating
455
- if models
456
- import_with_validations( column_names, array_of_attributes, options ) do |failed|
457
- models.each_with_index do |model, i|
458
- model = model.dup if options[:recursive]
459
- next if model.valid?(options[:validate_with_context])
696
+ import_with_validations( column_names, array_of_attributes, options ) do |failed_instances|
697
+ if models
698
+ models.each { |m| failed_instances << m if m.errors.any? }
699
+ else
700
+ # create instances for each of our column/value sets
701
+ arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
702
+
703
+ # keep track of the instance and the position it is currently at. if this fails
704
+ # validation we'll use the index to remove it from the array_of_attributes
705
+ arr.each_with_index do |hsh, i|
706
+ # utilize block initializer syntax to prevent failure when 'mass_assignment_sanitizer = :strict'
707
+ model = new do |m|
708
+ hsh.each_pair { |k, v| m[k] = v }
709
+ end
710
+
711
+ next if validator.valid_model?(model)
460
712
  raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
713
+
461
714
  array_of_attributes[i] = nil
462
- failed << model
715
+ failure = model.dup
716
+ failure.errors.send(:initialize_dup, model.errors)
717
+ failed_instances << (options[:track_validation_failures] ? [i, failure] : failure )
463
718
  end
719
+ array_of_attributes.compact!
464
720
  end
465
- else
466
- import_with_validations( column_names, array_of_attributes, options )
467
721
  end
468
722
  else
469
- (num_inserts, ids) = import_without_validations_or_callbacks( column_names, array_of_attributes, options )
470
- ActiveRecord::Import::Result.new([], num_inserts, ids)
723
+ import_without_validations_or_callbacks( column_names, array_of_attributes, options )
471
724
  end
472
725
 
473
726
  if options[:synchronize]
474
- sync_keys = options[:synchronize_keys] || [primary_key]
727
+ sync_keys = options[:synchronize_keys] || Array(primary_key)
475
728
  synchronize( options[:synchronize], sync_keys)
476
729
  end
477
730
  return_obj.num_inserts = 0 if return_obj.num_inserts.nil?
478
731
 
479
732
  # if we have ids, then set the id on the models and mark the models as clean.
480
- if models && support_setting_primary_key_of_imported_objects?
481
- set_attributes_and_mark_clean(models, return_obj, timestamps)
733
+ if models && supports_setting_primary_key_of_imported_objects?
734
+ set_attributes_and_mark_clean(models, return_obj, timestamps, options)
482
735
 
483
736
  # if there are auto-save associations on the models we imported that are new, import them as well
484
737
  import_associations(models, options.dup) if options[:recursive]
@@ -487,10 +740,6 @@ class ActiveRecord::Base
487
740
  return_obj
488
741
  end
489
742
 
490
- # TODO import_from_table needs to be implemented.
491
- def import_from_table( options ) # :nodoc:
492
- end
493
-
494
743
  # Imports the passed in +column_names+ and +array_of_attributes+
495
744
  # given the passed in +options+ Hash with validations. Returns an
496
745
  # object with the methods +failed_instances+ and +num_inserts+.
@@ -501,34 +750,14 @@ class ActiveRecord::Base
501
750
  def import_with_validations( column_names, array_of_attributes, options = {} )
502
751
  failed_instances = []
503
752
 
504
- if block_given?
505
- yield failed_instances
506
- else
507
- # create instances for each of our column/value sets
508
- arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
509
-
510
- # keep track of the instance and the position it is currently at. if this fails
511
- # validation we'll use the index to remove it from the array_of_attributes
512
- model = new
513
- arr.each_with_index do |hsh, i|
514
- hsh.each_pair { |k, v| model[k] = v }
515
- next if model.valid?(options[:validate_with_context])
516
- raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
517
- array_of_attributes[i] = nil
518
- failure = model.dup
519
- failure.errors.send(:initialize_dup, model.errors)
520
- failed_instances << failure
521
- end
522
- end
753
+ yield failed_instances if block_given?
523
754
 
524
- array_of_attributes.compact!
525
-
526
- num_inserts, ids = if array_of_attributes.empty? || options[:all_or_none] && failed_instances.any?
527
- [0, []]
755
+ result = if options[:all_or_none] && failed_instances.any?
756
+ ActiveRecord::Import::Result.new([], 0, [], [])
528
757
  else
529
758
  import_without_validations_or_callbacks( column_names, array_of_attributes, options )
530
759
  end
531
- ActiveRecord::Import::Result.new(failed_instances, num_inserts, ids)
760
+ ActiveRecord::Import::Result.new(failed_instances, result.num_inserts, result.ids, result.results)
532
761
  end
533
762
 
534
763
  # Imports the passed in +column_names+ and +array_of_attributes+
@@ -538,6 +767,8 @@ class ActiveRecord::Base
538
767
  # information on +column_names+, +array_of_attributes_ and
539
768
  # +options+.
540
769
  def import_without_validations_or_callbacks( column_names, array_of_attributes, options = {} )
770
+ return ActiveRecord::Import::Result.new([], 0, [], []) if array_of_attributes.empty?
771
+
541
772
  column_names = column_names.map(&:to_sym)
542
773
  scope_columns, scope_values = scope_attributes.to_a.transpose
543
774
 
@@ -547,7 +778,7 @@ class ActiveRecord::Base
547
778
  next if column_names.include?(name_as_sym)
548
779
 
549
780
  is_sti = (name_as_sym == inheritance_column.to_sym && self < base_class)
550
- value = value.first if is_sti
781
+ value = Array(value).first if is_sti
551
782
 
552
783
  column_names << name_as_sym
553
784
  array_of_attributes.each { |attrs| attrs << value }
@@ -570,19 +801,33 @@ class ActiveRecord::Base
570
801
 
571
802
  number_inserted = 0
572
803
  ids = []
804
+ results = []
573
805
  if supports_import?
574
806
  # generate the sql
575
807
  post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
808
+ import_size = values_sql.size
809
+
810
+ batch_size = options[:batch_size] || import_size
811
+ run_proc = options[:batch_size].to_i.positive? && options[:batch_progress].respond_to?( :call )
812
+ progress_proc = options[:batch_progress]
813
+ current_batch = 0
814
+ batches = (import_size / batch_size.to_f).ceil
576
815
 
577
- batch_size = options[:batch_size] || values_sql.size
578
816
  values_sql.each_slice(batch_size) do |batch_values|
817
+ batch_started_at = Time.now.to_i
818
+
579
819
  # perform the inserts
580
820
  result = connection.insert_many( [insert_sql, post_sql_statements].flatten,
581
821
  batch_values,
582
822
  options,
583
- "#{self.class.name} Create Many Without Validations Or Callbacks" )
584
- number_inserted += result[0]
585
- ids += result[1]
823
+ "#{model_name} Create Many" )
824
+
825
+ number_inserted += result.num_inserts
826
+ ids += result.ids
827
+ results += result.results
828
+ current_batch += 1
829
+
830
+ progress_proc.call(import_size, batches, current_batch, Time.now.to_i - batch_started_at) if run_proc
586
831
  end
587
832
  else
588
833
  transaction(requires_new: true) do
@@ -592,27 +837,85 @@ class ActiveRecord::Base
592
837
  end
593
838
  end
594
839
  end
595
- [number_inserted, ids]
840
+ ActiveRecord::Import::Result.new([], number_inserted, ids, results)
596
841
  end
597
842
 
598
843
  private
599
844
 
600
- def set_attributes_and_mark_clean(models, import_result, timestamps)
845
+ def set_attributes_and_mark_clean(models, import_result, timestamps, options)
601
846
  return if models.nil?
602
847
  models -= import_result.failed_instances
603
- import_result.ids.each_with_index do |id, index|
604
- model = models[index]
605
- model.id = id
606
- if model.respond_to?(:clear_changes_information) # Rails 4.0 and higher
848
+
849
+ # if ids were returned for all models we know all were updated
850
+ if models.size == import_result.ids.size
851
+ import_result.ids.each_with_index do |id, index|
852
+ model = models[index]
853
+ model.id = id
854
+
855
+ timestamps.each do |attr, value|
856
+ model.send(attr + "=", value)
857
+ end
858
+ end
859
+ end
860
+
861
+ deserialize_value = lambda do |column, value|
862
+ column = columns_hash[column]
863
+ return value unless column
864
+ if respond_to?(:type_caster)
865
+ type = type_for_attribute(column.name)
866
+ type.deserialize(value)
867
+ elsif column.respond_to?(:type_cast_from_database)
868
+ column.type_cast_from_database(value)
869
+ else
870
+ value
871
+ end
872
+ end
873
+
874
+ if models.size == import_result.results.size
875
+ columns = Array(options[:returning])
876
+ single_column = "#{columns.first}=" if columns.size == 1
877
+ import_result.results.each_with_index do |result, index|
878
+ model = models[index]
879
+
880
+ if single_column
881
+ val = deserialize_value.call(columns.first, result)
882
+ model.send(single_column, val)
883
+ else
884
+ columns.each_with_index do |column, col_index|
885
+ val = deserialize_value.call(column, result[col_index])
886
+ model.send("#{column}=", val)
887
+ end
888
+ end
889
+ end
890
+ end
891
+
892
+ models.each do |model|
893
+ if model.respond_to?(:changes_applied) # Rails 4.1.8 and higher
894
+ model.changes_internally_applied if model.respond_to?(:changes_internally_applied) # legacy behavior for Rails 5.1
895
+ model.changes_applied
896
+ elsif model.respond_to?(:clear_changes_information) # Rails 4.0 and higher
607
897
  model.clear_changes_information
608
898
  else # Rails 3.2
609
899
  model.instance_variable_get(:@changed_attributes).clear
610
900
  end
611
901
  model.instance_variable_set(:@new_record, false)
902
+ end
903
+ end
612
904
 
613
- timestamps.each do |attr, value|
614
- model.send(attr + "=", value)
615
- end
905
+ # Sync belongs_to association ids with foreign key field
906
+ def load_association_ids(model)
907
+ changed_columns = model.changed
908
+ association_reflections = model.class.reflect_on_all_associations(:belongs_to)
909
+ association_reflections.each do |association_reflection|
910
+ column_name = association_reflection.foreign_key
911
+ next if association_reflection.options[:polymorphic]
912
+ next if changed_columns.include?(column_name)
913
+ association = model.association(association_reflection.name)
914
+ association = association.target
915
+ next if association.blank? || model.public_send(column_name).present?
916
+
917
+ association_primary_key = association_reflection.association_primary_key
918
+ model.public_send("#{column_name}=", association.send(association_primary_key))
616
919
  end
617
920
  end
618
921
 
@@ -625,12 +928,13 @@ class ActiveRecord::Base
625
928
  associated_objects_by_class = {}
626
929
  models.each { |model| find_associated_objects_for_import(associated_objects_by_class, model) }
627
930
 
628
- # :on_duplicate_key_update not supported for associations
931
+ # :on_duplicate_key_update and :returning not supported for associations
629
932
  options.delete(:on_duplicate_key_update)
933
+ options.delete(:returning)
630
934
 
631
935
  associated_objects_by_class.each_value do |associations|
632
936
  associations.each_value do |associated_records|
633
- associated_records.first.class.import(associated_records, options) unless associated_records.empty?
937
+ associated_records.first.class.bulk_import(associated_records, options) unless associated_records.empty?
634
938
  end
635
939
  end
636
940
  end
@@ -639,6 +943,7 @@ class ActiveRecord::Base
639
943
  # of class => objects to import.
640
944
  def find_associated_objects_for_import(associated_objects_by_class, model)
641
945
  associated_objects_by_class[model.class.name] ||= {}
946
+ return associated_objects_by_class unless model.id
642
947
 
643
948
  association_reflections =
644
949
  model.class.reflect_on_all_associations(:has_one) +
@@ -668,29 +973,36 @@ class ActiveRecord::Base
668
973
  # Returns SQL the VALUES for an INSERT statement given the passed in +columns+
669
974
  # and +array_of_attributes+.
670
975
  def values_sql_for_columns_and_attributes(columns, array_of_attributes) # :nodoc:
671
- # connection and type_caster get called a *lot* in this high intensity loop.
672
- # Reuse the same ones w/in the loop, otherwise they would keep being re-retreived (= lots of time for large imports)
976
+ # connection gets called a *lot* in this high intensity loop.
977
+ # Reuse the same one w/in the loop, otherwise it would keep being re-retreived (= lots of time for large imports)
673
978
  connection_memo = connection
674
- type_caster_memo = type_caster if respond_to?(:type_caster)
675
979
 
676
980
  array_of_attributes.map do |arr|
677
981
  my_values = arr.each_with_index.map do |val, j|
678
982
  column = columns[j]
679
983
 
680
984
  # be sure to query sequence_name *last*, only if cheaper tests fail, because it's costly
681
- if val.nil? && column.name == primary_key && !sequence_name.blank?
985
+ if val.nil? && Array(primary_key).first == column.name && !sequence_name.blank?
682
986
  connection_memo.next_value_for_sequence(sequence_name)
987
+ elsif val.respond_to?(:to_sql)
988
+ "(#{val.to_sql})"
683
989
  elsif column
684
- if defined?(type_caster_memo) && type_caster_memo.respond_to?(:type_cast_for_database) # Rails 5.0 and higher
685
- connection_memo.quote(type_caster_memo.type_cast_for_database(column.name, val))
686
- elsif column.respond_to?(:type_cast_from_user) # Rails 4.2 and higher
990
+ if respond_to?(:type_caster) # Rails 5.0 and higher
991
+ type = type_for_attribute(column.name)
992
+ val = !type.respond_to?(:subtype) && type.type == :boolean ? type.cast(val) : type.serialize(val)
993
+ connection_memo.quote(val)
994
+ elsif column.respond_to?(:type_cast_from_user) # Rails 4.2
687
995
  connection_memo.quote(column.type_cast_from_user(val), column)
688
- else # Rails 3.2, 4.0 and 4.1
996
+ else # Rails 3.2, 4.0 and 4.1
689
997
  if serialized_attributes.include?(column.name)
690
998
  val = serialized_attributes[column.name].dump(val)
691
999
  end
692
- connection_memo.quote(column.type_cast(val), column)
1000
+ # Fixes #443 to support binary (i.e. bytea) columns on PG
1001
+ val = column.type_cast(val) unless column.type && column.type.to_sym == :binary
1002
+ connection_memo.quote(val, column)
693
1003
  end
1004
+ else
1005
+ raise ArgumentError, "Number of values (#{arr.length}) exceeds number of columns (#{columns.length})"
694
1006
  end
695
1007
  end
696
1008
  "(#{my_values.join(',')})"
@@ -698,39 +1010,38 @@ class ActiveRecord::Base
698
1010
  end
699
1011
 
700
1012
  def add_special_rails_stamps( column_names, array_of_attributes, options )
701
- timestamps = {}
1013
+ timestamp_columns = {}
1014
+ timestamps = {}
702
1015
 
703
- AREXT_RAILS_COLUMNS[:create].each_pair do |key, blk|
704
- next unless self.column_names.include?(key)
705
- value = blk.call
706
- timestamps[key] = value
707
-
708
- index = column_names.index(key) || column_names.index(key.to_sym)
709
- if index
710
- # replace every instance of the array of attributes with our value
711
- array_of_attributes.each { |arr| arr[index] = value if arr[index].nil? }
712
- else
713
- column_names << key
714
- array_of_attributes.each { |arr| arr << value }
715
- end
1016
+ if respond_to?(:all_timestamp_attributes_in_model, true) # Rails 5.1 and higher
1017
+ timestamp_columns[:create] = timestamp_attributes_for_create_in_model
1018
+ timestamp_columns[:update] = timestamp_attributes_for_update_in_model
1019
+ else
1020
+ instance = allocate
1021
+ timestamp_columns[:create] = instance.send(:timestamp_attributes_for_create_in_model)
1022
+ timestamp_columns[:update] = instance.send(:timestamp_attributes_for_update_in_model)
716
1023
  end
717
1024
 
718
- AREXT_RAILS_COLUMNS[:update].each_pair do |key, blk|
719
- next unless self.column_names.include?(key)
720
- value = blk.call
721
- timestamps[key] = value
722
-
723
- index = column_names.index(key) || column_names.index(key.to_sym)
724
- if index
725
- # replace every instance of the array of attributes with our value
726
- array_of_attributes.each { |arr| arr[index] = value }
727
- else
728
- column_names << key
729
- array_of_attributes.each { |arr| arr << value }
730
- end
1025
+ # use tz as set in ActiveRecord::Base
1026
+ timestamp = ActiveRecord::Base.default_timezone == :utc ? Time.now.utc : Time.now
1027
+
1028
+ [:create, :update].each do |action|
1029
+ timestamp_columns[action].each do |column|
1030
+ column = column.to_s
1031
+ timestamps[column] = timestamp
1032
+
1033
+ index = column_names.index(column) || column_names.index(column.to_sym)
1034
+ if index
1035
+ # replace every instance of the array of attributes with our value
1036
+ array_of_attributes.each { |arr| arr[index] = timestamp if arr[index].nil? }
1037
+ else
1038
+ column_names << column
1039
+ array_of_attributes.each { |arr| arr << timestamp }
1040
+ end
731
1041
 
732
- if supports_on_duplicate_key_update?
733
- connection.add_column_for_on_duplicate_key_update(key, options)
1042
+ if supports_on_duplicate_key_update? && action == :update
1043
+ connection.add_column_for_on_duplicate_key_update(column, options)
1044
+ end
734
1045
  end
735
1046
  end
736
1047
 
@@ -741,5 +1052,42 @@ class ActiveRecord::Base
741
1052
  def validations_array_for_column_names_and_attributes( column_names, array_of_attributes ) # :nodoc:
742
1053
  array_of_attributes.map { |values| Hash[column_names.zip(values)] }
743
1054
  end
1055
+
1056
+ # Checks that the imported hash has the required_keys, optionally also checks that the hash has
1057
+ # no keys beyond those required when `allow_extra_keys` is false.
1058
+ # returns `nil` if validation passes, or an error message if it fails
1059
+ def validate_hash_import(hash, required_keys, allow_extra_keys) # :nodoc:
1060
+ extra_keys = allow_extra_keys ? [] : hash.keys - required_keys
1061
+ missing_keys = required_keys - hash.keys
1062
+
1063
+ return nil if extra_keys.empty? && missing_keys.empty?
1064
+
1065
+ if allow_extra_keys
1066
+ <<-EOS
1067
+ Hash key mismatch.
1068
+
1069
+ When importing an array of hashes with provided columns_names, each hash must contain keys for all column_names.
1070
+
1071
+ Required keys: #{required_keys}
1072
+ Missing keys: #{missing_keys}
1073
+
1074
+ Hash: #{hash}
1075
+ EOS
1076
+ else
1077
+ <<-EOS
1078
+ Hash key mismatch.
1079
+
1080
+ When importing an array of hashes, all hashes must have the same keys.
1081
+ If you have records that are missing some values, we recommend you either set default values
1082
+ for the missing keys or group these records into batches by key set before importing.
1083
+
1084
+ Required keys: #{required_keys}
1085
+ Extra keys: #{extra_keys}
1086
+ Missing keys: #{missing_keys}
1087
+
1088
+ Hash: #{hash}
1089
+ EOS
1090
+ end
1091
+ end
744
1092
  end
745
1093
  end