activerecord-import 0.23.0 → 1.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/workflows/test.yaml +107 -0
- data/.gitignore +1 -0
- data/CHANGELOG.md +214 -4
- data/Gemfile +11 -9
- data/LICENSE +21 -56
- data/README.markdown +574 -22
- data/Rakefile +2 -1
- data/activerecord-import.gemspec +4 -4
- data/benchmarks/benchmark.rb +5 -1
- data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +0 -0
- data/gemfiles/5.0.gemfile +1 -0
- data/gemfiles/5.1.gemfile +1 -0
- data/gemfiles/5.2.gemfile +2 -2
- data/gemfiles/6.0.gemfile +2 -0
- data/gemfiles/6.1.gemfile +2 -0
- data/gemfiles/7.0.gemfile +1 -0
- data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +4 -4
- data/lib/activerecord-import/adapters/abstract_adapter.rb +7 -1
- data/lib/activerecord-import/adapters/mysql_adapter.rb +8 -11
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +14 -16
- data/lib/activerecord-import/adapters/sqlite3_adapter.rb +125 -8
- data/lib/activerecord-import/base.rb +9 -1
- data/lib/activerecord-import/import.rb +269 -123
- data/lib/activerecord-import/synchronize.rb +2 -2
- data/lib/activerecord-import/value_sets_parser.rb +2 -0
- data/lib/activerecord-import/version.rb +1 -1
- data/lib/activerecord-import.rb +1 -0
- data/test/adapters/makara_postgis.rb +1 -0
- data/test/{travis → github}/database.yml +3 -1
- data/test/import_test.rb +138 -8
- data/test/makara_postgis/import_test.rb +8 -0
- data/test/models/animal.rb +6 -0
- data/test/models/card.rb +3 -0
- data/test/models/customer.rb +6 -0
- data/test/models/deck.rb +6 -0
- data/test/models/order.rb +6 -0
- data/test/models/playing_card.rb +2 -0
- data/test/models/user.rb +3 -1
- data/test/models/user_token.rb +4 -0
- data/test/schema/generic_schema.rb +30 -0
- data/test/schema/mysql2_schema.rb +19 -0
- data/test/schema/postgresql_schema.rb +16 -0
- data/test/schema/sqlite3_schema.rb +13 -0
- data/test/support/factories.rb +8 -8
- data/test/support/generate.rb +6 -6
- data/test/support/mysql/import_examples.rb +12 -0
- data/test/support/postgresql/import_examples.rb +100 -2
- data/test/support/shared_examples/on_duplicate_key_update.rb +54 -0
- data/test/support/shared_examples/recursive_import.rb +74 -4
- data/test/support/sqlite3/import_examples.rb +189 -25
- data/test/test_helper.rb +28 -3
- metadata +37 -18
- data/.travis.yml +0 -62
- data/gemfiles/3.2.gemfile +0 -2
- data/gemfiles/4.0.gemfile +0 -2
- data/gemfiles/4.1.gemfile +0 -2
- data/test/schema/mysql_schema.rb +0 -16
@@ -24,27 +24,66 @@ module ActiveRecord::Import #:nodoc:
|
|
24
24
|
end
|
25
25
|
|
26
26
|
class Validator
|
27
|
-
def initialize(options = {})
|
27
|
+
def initialize(klass, options = {})
|
28
28
|
@options = options
|
29
|
+
@validator_class = klass
|
30
|
+
init_validations(klass)
|
31
|
+
end
|
32
|
+
|
33
|
+
def init_validations(klass)
|
34
|
+
@validate_callbacks = klass._validate_callbacks.dup
|
35
|
+
|
36
|
+
@validate_callbacks.each_with_index do |callback, i|
|
37
|
+
filter = callback.respond_to?(:raw_filter) ? callback.raw_filter : callback.filter
|
38
|
+
next unless filter.class.name =~ /Validations::PresenceValidator/ ||
|
39
|
+
(!@options[:validate_uniqueness] &&
|
40
|
+
filter.is_a?(ActiveRecord::Validations::UniquenessValidator))
|
41
|
+
|
42
|
+
callback = callback.dup
|
43
|
+
filter = filter.dup
|
44
|
+
attrs = filter.instance_variable_get(:@attributes).dup
|
45
|
+
|
46
|
+
if filter.is_a?(ActiveRecord::Validations::UniquenessValidator)
|
47
|
+
attrs = []
|
48
|
+
else
|
49
|
+
associations = klass.reflect_on_all_associations(:belongs_to)
|
50
|
+
associations.each do |assoc|
|
51
|
+
if (index = attrs.index(assoc.name))
|
52
|
+
key = assoc.foreign_key.is_a?(Array) ? assoc.foreign_key.map(&:to_sym) : assoc.foreign_key.to_sym
|
53
|
+
attrs[index] = key unless attrs.include?(key)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
filter.instance_variable_set(:@attributes, attrs)
|
59
|
+
|
60
|
+
if @validate_callbacks.respond_to?(:chain, true)
|
61
|
+
@validate_callbacks.send(:chain).tap do |chain|
|
62
|
+
callback.instance_variable_set(:@filter, filter)
|
63
|
+
chain[i] = callback
|
64
|
+
end
|
65
|
+
else
|
66
|
+
callback.raw_filter = filter
|
67
|
+
callback.filter = callback.send(:_compile_filter, filter)
|
68
|
+
@validate_callbacks[i] = callback
|
69
|
+
end
|
70
|
+
end
|
29
71
|
end
|
30
72
|
|
31
73
|
def valid_model?(model)
|
74
|
+
init_validations(model.class) unless model.class == @validator_class
|
75
|
+
|
32
76
|
validation_context = @options[:validate_with_context]
|
33
77
|
validation_context ||= (model.new_record? ? :create : :update)
|
34
|
-
|
35
78
|
current_context = model.send(:validation_context)
|
79
|
+
|
36
80
|
begin
|
37
81
|
model.send(:validation_context=, validation_context)
|
38
82
|
model.errors.clear
|
39
83
|
|
40
|
-
validate_callbacks = model._validate_callbacks.dup
|
41
|
-
model._validate_callbacks.each do |callback|
|
42
|
-
validate_callbacks.delete(callback) if callback.raw_filter.is_a? ActiveRecord::Validations::UniquenessValidator
|
43
|
-
end
|
44
|
-
|
45
84
|
model.run_callbacks(:validation) do
|
46
85
|
if defined?(ActiveSupport::Callbacks::Filters::Environment) # ActiveRecord >= 4.1
|
47
|
-
runner = validate_callbacks.compile
|
86
|
+
runner = @validate_callbacks.compile
|
48
87
|
env = ActiveSupport::Callbacks::Filters::Environment.new(model, false, nil)
|
49
88
|
if runner.respond_to?(:call) # ActiveRecord < 5.1
|
50
89
|
runner.call(env)
|
@@ -63,10 +102,10 @@ module ActiveRecord::Import #:nodoc:
|
|
63
102
|
runner.invoke_before(env)
|
64
103
|
runner.invoke_after(env)
|
65
104
|
end
|
66
|
-
elsif validate_callbacks.method(:compile).arity == 0 # ActiveRecord = 4.0
|
67
|
-
model.instance_eval validate_callbacks.compile
|
105
|
+
elsif @validate_callbacks.method(:compile).arity == 0 # ActiveRecord = 4.0
|
106
|
+
model.instance_eval @validate_callbacks.compile
|
68
107
|
else # ActiveRecord 3.x
|
69
|
-
model.instance_eval validate_callbacks.compile(nil, model)
|
108
|
+
model.instance_eval @validate_callbacks.compile(nil, model)
|
70
109
|
end
|
71
110
|
end
|
72
111
|
|
@@ -95,9 +134,14 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
95
134
|
|
96
135
|
model_klass = reflection.klass
|
97
136
|
symbolized_foreign_key = reflection.foreign_key.to_sym
|
98
|
-
symbolized_column_names = model_klass.column_names.map(&:to_sym)
|
99
137
|
|
100
|
-
|
138
|
+
symbolized_column_names = if model_klass.connection.respond_to?(:supports_virtual_columns?) && model_klass.connection.supports_virtual_columns?
|
139
|
+
model_klass.columns.reject(&:virtual?).map { |c| c.name.to_sym }
|
140
|
+
else
|
141
|
+
model_klass.column_names.map(&:to_sym)
|
142
|
+
end
|
143
|
+
|
144
|
+
owner_primary_key = reflection.active_record_primary_key.to_sym
|
101
145
|
owner_primary_key_value = owner.send(owner_primary_key)
|
102
146
|
|
103
147
|
# assume array of model objects
|
@@ -201,16 +245,17 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
201
245
|
alias import bulk_import unless respond_to? :import
|
202
246
|
end
|
203
247
|
|
248
|
+
module ActiveRecord::Import::Connection
|
249
|
+
def establish_connection(args = nil)
|
250
|
+
conn = super(args)
|
251
|
+
ActiveRecord::Import.load_from_connection_pool connection_pool
|
252
|
+
conn
|
253
|
+
end
|
254
|
+
end
|
255
|
+
|
204
256
|
class ActiveRecord::Base
|
205
257
|
class << self
|
206
|
-
|
207
|
-
conn = establish_connection_without_activerecord_import(*args)
|
208
|
-
ActiveRecord::Import.load_from_connection_pool connection_pool
|
209
|
-
conn
|
210
|
-
end
|
211
|
-
|
212
|
-
alias establish_connection_without_activerecord_import establish_connection
|
213
|
-
alias establish_connection establish_connection_with_activerecord_import
|
258
|
+
prepend ActiveRecord::Import::Connection
|
214
259
|
|
215
260
|
# Returns true if the current database connection adapter
|
216
261
|
# supports import functionality, otherwise returns false.
|
@@ -276,6 +321,9 @@ class ActiveRecord::Base
|
|
276
321
|
# == Options
|
277
322
|
# * +validate+ - true|false, tells import whether or not to use
|
278
323
|
# ActiveRecord validations. Validations are enforced by default.
|
324
|
+
# It skips the uniqueness validation for performance reasons.
|
325
|
+
# You can find more details here:
|
326
|
+
# https://github.com/zdennis/activerecord-import/issues/228
|
279
327
|
# * +ignore+ - true|false, an alias for on_duplicate_key_ignore.
|
280
328
|
# * +on_duplicate_key_ignore+ - true|false, tells import to discard
|
281
329
|
# records that contain duplicate keys. For Postgres 9.5+ it adds
|
@@ -284,8 +332,8 @@ class ActiveRecord::Base
|
|
284
332
|
# recursive import. For database adapters that normally support
|
285
333
|
# setting primary keys on imported objects, this option prevents
|
286
334
|
# that from occurring.
|
287
|
-
# * +on_duplicate_key_update+ - an Array or Hash, tells import to
|
288
|
-
# use MySQL's ON DUPLICATE KEY UPDATE or Postgres
|
335
|
+
# * +on_duplicate_key_update+ - :all, an Array, or Hash, tells import to
|
336
|
+
# use MySQL's ON DUPLICATE KEY UPDATE or Postgres/SQLite ON CONFLICT
|
289
337
|
# DO UPDATE ability. See On Duplicate Key Update below.
|
290
338
|
# * +synchronize+ - an array of ActiveRecord instances for the model
|
291
339
|
# that you are currently importing data into. This synchronizes
|
@@ -344,7 +392,15 @@ class ActiveRecord::Base
|
|
344
392
|
#
|
345
393
|
# == On Duplicate Key Update (MySQL)
|
346
394
|
#
|
347
|
-
# The :on_duplicate_key_update option can be either an Array or a Hash.
|
395
|
+
# The :on_duplicate_key_update option can be either :all, an Array, or a Hash.
|
396
|
+
#
|
397
|
+
# ==== Using :all
|
398
|
+
#
|
399
|
+
# The :on_duplicate_key_update option can be set to :all. All columns
|
400
|
+
# other than the primary key are updated. If a list of column names is
|
401
|
+
# supplied, only those columns will be updated. Below is an example:
|
402
|
+
#
|
403
|
+
# BlogPost.import columns, values, on_duplicate_key_update: :all
|
348
404
|
#
|
349
405
|
# ==== Using an Array
|
350
406
|
#
|
@@ -363,11 +419,19 @@ class ActiveRecord::Base
|
|
363
419
|
#
|
364
420
|
# BlogPost.import columns, attributes, on_duplicate_key_update: { title: :title }
|
365
421
|
#
|
366
|
-
# == On Duplicate Key Update (Postgres 9.5+)
|
422
|
+
# == On Duplicate Key Update (Postgres 9.5+ and SQLite 3.24+)
|
367
423
|
#
|
368
|
-
# The :on_duplicate_key_update option can be an Array or a Hash with up to
|
424
|
+
# The :on_duplicate_key_update option can be :all, an Array, or a Hash with up to
|
369
425
|
# three attributes, :conflict_target (and optionally :index_predicate) or
|
370
|
-
# :constraint_name, and :columns.
|
426
|
+
# :constraint_name (Postgres), and :columns.
|
427
|
+
#
|
428
|
+
# ==== Using :all
|
429
|
+
#
|
430
|
+
# The :on_duplicate_key_update option can be set to :all. All columns
|
431
|
+
# other than the primary key are updated. If a list of column names is
|
432
|
+
# supplied, only those columns will be updated. Below is an example:
|
433
|
+
#
|
434
|
+
# BlogPost.import columns, values, on_duplicate_key_update: :all
|
371
435
|
#
|
372
436
|
# ==== Using an Array
|
373
437
|
#
|
@@ -425,7 +489,15 @@ class ActiveRecord::Base
|
|
425
489
|
#
|
426
490
|
# ===== :columns
|
427
491
|
#
|
428
|
-
# The :columns attribute can be either an Array or a Hash.
|
492
|
+
# The :columns attribute can be either :all, an Array, or a Hash.
|
493
|
+
#
|
494
|
+
# ===== Using :all
|
495
|
+
#
|
496
|
+
# The :columns attribute can be :all. All columns other than the primary key will be updated.
|
497
|
+
# If a list of column names is supplied, only those columns will be updated.
|
498
|
+
# Below is an example:
|
499
|
+
#
|
500
|
+
# BlogPost.import columns, values, on_duplicate_key_update: { conflict_target: :slug, columns: :all }
|
429
501
|
#
|
430
502
|
# ===== Using an Array
|
431
503
|
#
|
@@ -460,7 +532,7 @@ class ActiveRecord::Base
|
|
460
532
|
import_helper(*args)
|
461
533
|
end
|
462
534
|
end
|
463
|
-
alias import bulk_import unless respond_to? :import
|
535
|
+
alias import bulk_import unless ActiveRecord::Base.respond_to? :import
|
464
536
|
|
465
537
|
# Imports a collection of values if all values are valid. Import fails at the
|
466
538
|
# first encountered validation error and raises ActiveRecord::RecordInvalid
|
@@ -472,27 +544,17 @@ class ActiveRecord::Base
|
|
472
544
|
|
473
545
|
bulk_import(*args, options)
|
474
546
|
end
|
475
|
-
alias import! bulk_import! unless respond_to? :import!
|
547
|
+
alias import! bulk_import! unless ActiveRecord::Base.respond_to? :import!
|
476
548
|
|
477
549
|
def import_helper( *args )
|
478
|
-
options = { validate: true, timestamps: true }
|
550
|
+
options = { validate: true, timestamps: true, track_validation_failures: false }
|
479
551
|
options.merge!( args.pop ) if args.last.is_a? Hash
|
480
552
|
# making sure that current model's primary key is used
|
481
553
|
options[:primary_key] = primary_key
|
482
554
|
options[:locking_column] = locking_column if attribute_names.include?(locking_column)
|
483
555
|
|
484
|
-
|
485
|
-
|
486
|
-
if on_duplicate_key_update && on_duplicate_key_update.duplicable?
|
487
|
-
options[:on_duplicate_key_update] = if on_duplicate_key_update.is_a?(Hash)
|
488
|
-
on_duplicate_key_update.each { |k, v| on_duplicate_key_update[k] = v.dup if v.duplicable? }
|
489
|
-
else
|
490
|
-
on_duplicate_key_update.dup
|
491
|
-
end
|
492
|
-
end
|
493
|
-
|
494
|
-
is_validating = options[:validate]
|
495
|
-
is_validating = true unless options[:validate_with_context].nil?
|
556
|
+
is_validating = options[:validate_with_context].present? ? true : options[:validate]
|
557
|
+
validator = ActiveRecord::Import::Validator.new(self, options)
|
496
558
|
|
497
559
|
# assume array of model objects
|
498
560
|
if args.last.is_a?( Array ) && args.last.first.is_a?(ActiveRecord::Base)
|
@@ -501,34 +563,48 @@ class ActiveRecord::Base
|
|
501
563
|
column_names = args.first.dup
|
502
564
|
else
|
503
565
|
models = args.first
|
504
|
-
column_names =
|
566
|
+
column_names = if connection.respond_to?(:supports_virtual_columns?) && connection.supports_virtual_columns?
|
567
|
+
columns.reject(&:virtual?).map(&:name)
|
568
|
+
else
|
569
|
+
self.column_names.dup
|
570
|
+
end
|
505
571
|
end
|
506
572
|
|
507
|
-
if models.first.id.nil?
|
508
|
-
|
573
|
+
if models.first.id.nil?
|
574
|
+
Array(primary_key).each do |c|
|
575
|
+
if column_names.include?(c) && columns_hash[c].type == :uuid
|
576
|
+
column_names.delete(c)
|
577
|
+
end
|
578
|
+
end
|
509
579
|
end
|
510
580
|
|
511
|
-
|
512
|
-
|
513
|
-
|
514
|
-
|
515
|
-
|
516
|
-
|
517
|
-
serialized_attributes
|
581
|
+
update_attrs = if record_timestamps && options[:timestamps]
|
582
|
+
if respond_to?(:timestamp_attributes_for_update, true)
|
583
|
+
send(:timestamp_attributes_for_update).map(&:to_sym)
|
584
|
+
else
|
585
|
+
allocate.send(:timestamp_attributes_for_update_in_model)
|
586
|
+
end
|
518
587
|
end
|
519
588
|
|
520
|
-
array_of_attributes =
|
589
|
+
array_of_attributes = []
|
590
|
+
|
591
|
+
models.each do |model|
|
521
592
|
if supports_setting_primary_key_of_imported_objects?
|
522
593
|
load_association_ids(model)
|
523
594
|
end
|
524
595
|
|
525
|
-
|
526
|
-
|
527
|
-
|
528
|
-
|
529
|
-
|
596
|
+
if is_validating && !validator.valid_model?(model)
|
597
|
+
raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
|
598
|
+
next
|
599
|
+
end
|
600
|
+
|
601
|
+
array_of_attributes << column_names.map do |name|
|
602
|
+
if model.persisted? &&
|
603
|
+
update_attrs && update_attrs.include?(name.to_sym) &&
|
604
|
+
!model.send("#{name}_changed?")
|
605
|
+
nil
|
530
606
|
else
|
531
|
-
model.
|
607
|
+
model.read_attribute(name.to_s)
|
532
608
|
end
|
533
609
|
end
|
534
610
|
end
|
@@ -555,7 +631,7 @@ class ActiveRecord::Base
|
|
555
631
|
end
|
556
632
|
# supports empty array
|
557
633
|
elsif args.last.is_a?( Array ) && args.last.empty?
|
558
|
-
return ActiveRecord::Import::Result.new([], 0, [])
|
634
|
+
return ActiveRecord::Import::Result.new([], 0, [], [])
|
559
635
|
# supports 2-element array and array
|
560
636
|
elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
|
561
637
|
|
@@ -586,32 +662,69 @@ class ActiveRecord::Base
|
|
586
662
|
array_of_attributes.each { |a| a.concat(new_fields) }
|
587
663
|
end
|
588
664
|
|
665
|
+
# Don't modify incoming arguments
|
666
|
+
on_duplicate_key_update = options[:on_duplicate_key_update]
|
667
|
+
if on_duplicate_key_update
|
668
|
+
updatable_columns = symbolized_column_names.reject { |c| symbolized_primary_key.include? c }
|
669
|
+
options[:on_duplicate_key_update] = if on_duplicate_key_update.is_a?(Hash)
|
670
|
+
on_duplicate_key_update.each_with_object({}) do |(k, v), duped_options|
|
671
|
+
duped_options[k] = if k == :columns && v == :all
|
672
|
+
updatable_columns
|
673
|
+
elsif v.duplicable?
|
674
|
+
v.dup
|
675
|
+
else
|
676
|
+
v
|
677
|
+
end
|
678
|
+
end
|
679
|
+
elsif on_duplicate_key_update == :all
|
680
|
+
updatable_columns
|
681
|
+
elsif on_duplicate_key_update.duplicable?
|
682
|
+
on_duplicate_key_update.dup
|
683
|
+
else
|
684
|
+
on_duplicate_key_update
|
685
|
+
end
|
686
|
+
end
|
687
|
+
|
589
688
|
timestamps = {}
|
590
689
|
|
591
690
|
# record timestamps unless disabled in ActiveRecord::Base
|
592
|
-
if record_timestamps && options
|
691
|
+
if record_timestamps && options[:timestamps]
|
593
692
|
timestamps = add_special_rails_stamps column_names, array_of_attributes, options
|
594
693
|
end
|
595
694
|
|
596
695
|
return_obj = if is_validating
|
597
|
-
|
598
|
-
|
599
|
-
models.
|
600
|
-
|
696
|
+
import_with_validations( column_names, array_of_attributes, options ) do |failed_instances|
|
697
|
+
if models
|
698
|
+
models.each { |m| failed_instances << m if m.errors.any? }
|
699
|
+
else
|
700
|
+
# create instances for each of our column/value sets
|
701
|
+
arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
|
702
|
+
|
703
|
+
# keep track of the instance and the position it is currently at. if this fails
|
704
|
+
# validation we'll use the index to remove it from the array_of_attributes
|
705
|
+
arr.each_with_index do |hsh, i|
|
706
|
+
# utilize block initializer syntax to prevent failure when 'mass_assignment_sanitizer = :strict'
|
707
|
+
model = new do |m|
|
708
|
+
hsh.each_pair { |k, v| m[k] = v }
|
709
|
+
end
|
710
|
+
|
711
|
+
next if validator.valid_model?(model)
|
601
712
|
raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
|
713
|
+
|
602
714
|
array_of_attributes[i] = nil
|
603
|
-
|
715
|
+
failure = model.dup
|
716
|
+
failure.errors.send(:initialize_dup, model.errors)
|
717
|
+
failed_instances << (options[:track_validation_failures] ? [i, failure] : failure )
|
604
718
|
end
|
719
|
+
array_of_attributes.compact!
|
605
720
|
end
|
606
|
-
else
|
607
|
-
import_with_validations( column_names, array_of_attributes, options )
|
608
721
|
end
|
609
722
|
else
|
610
723
|
import_without_validations_or_callbacks( column_names, array_of_attributes, options )
|
611
724
|
end
|
612
725
|
|
613
726
|
if options[:synchronize]
|
614
|
-
sync_keys = options[:synchronize_keys] ||
|
727
|
+
sync_keys = options[:synchronize_keys] || Array(primary_key)
|
615
728
|
synchronize( options[:synchronize], sync_keys)
|
616
729
|
end
|
617
730
|
return_obj.num_inserts = 0 if return_obj.num_inserts.nil?
|
@@ -621,7 +734,10 @@ class ActiveRecord::Base
|
|
621
734
|
set_attributes_and_mark_clean(models, return_obj, timestamps, options)
|
622
735
|
|
623
736
|
# if there are auto-save associations on the models we imported that are new, import them as well
|
624
|
-
|
737
|
+
if options[:recursive]
|
738
|
+
options[:on_duplicate_key_update] = on_duplicate_key_update unless on_duplicate_key_update.nil?
|
739
|
+
import_associations(models, options.dup.merge(validate: false))
|
740
|
+
end
|
625
741
|
end
|
626
742
|
|
627
743
|
return_obj
|
@@ -637,29 +753,7 @@ class ActiveRecord::Base
|
|
637
753
|
def import_with_validations( column_names, array_of_attributes, options = {} )
|
638
754
|
failed_instances = []
|
639
755
|
|
640
|
-
|
641
|
-
|
642
|
-
if block_given?
|
643
|
-
yield validator, failed_instances
|
644
|
-
else
|
645
|
-
# create instances for each of our column/value sets
|
646
|
-
arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
|
647
|
-
|
648
|
-
# keep track of the instance and the position it is currently at. if this fails
|
649
|
-
# validation we'll use the index to remove it from the array_of_attributes
|
650
|
-
arr.each_with_index do |hsh, i|
|
651
|
-
model = new
|
652
|
-
hsh.each_pair { |k, v| model[k] = v }
|
653
|
-
next if validator.valid_model? model
|
654
|
-
raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
|
655
|
-
array_of_attributes[i] = nil
|
656
|
-
failure = model.dup
|
657
|
-
failure.errors.send(:initialize_dup, model.errors)
|
658
|
-
failed_instances << failure
|
659
|
-
end
|
660
|
-
end
|
661
|
-
|
662
|
-
array_of_attributes.compact!
|
756
|
+
yield failed_instances if block_given?
|
663
757
|
|
664
758
|
result = if options[:all_or_none] && failed_instances.any?
|
665
759
|
ActiveRecord::Import::Result.new([], 0, [], [])
|
@@ -684,21 +778,22 @@ class ActiveRecord::Base
|
|
684
778
|
unless scope_columns.blank?
|
685
779
|
scope_columns.zip(scope_values).each do |name, value|
|
686
780
|
name_as_sym = name.to_sym
|
687
|
-
next if column_names.include?(name_as_sym)
|
688
|
-
|
689
|
-
is_sti = (name_as_sym == inheritance_column.to_sym && self < base_class)
|
690
|
-
value = Array(value).first if is_sti
|
691
|
-
|
781
|
+
next if column_names.include?(name_as_sym) || name_as_sym == inheritance_column.to_sym
|
692
782
|
column_names << name_as_sym
|
693
783
|
array_of_attributes.each { |attrs| attrs << value }
|
694
784
|
end
|
695
785
|
end
|
696
786
|
|
787
|
+
if finder_needs_type_condition?
|
788
|
+
unless column_names.include?(inheritance_column.to_sym)
|
789
|
+
column_names << inheritance_column.to_sym
|
790
|
+
array_of_attributes.each { |attrs| attrs << sti_name }
|
791
|
+
end
|
792
|
+
end
|
793
|
+
|
697
794
|
columns = column_names.each_with_index.map do |name, i|
|
698
795
|
column = columns_hash[name.to_s]
|
699
|
-
|
700
796
|
raise ActiveRecord::Import::MissingColumnError.new(name.to_s, i) if column.nil?
|
701
|
-
|
702
797
|
column
|
703
798
|
end
|
704
799
|
|
@@ -714,17 +809,29 @@ class ActiveRecord::Base
|
|
714
809
|
if supports_import?
|
715
810
|
# generate the sql
|
716
811
|
post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
|
812
|
+
import_size = values_sql.size
|
813
|
+
|
814
|
+
batch_size = options[:batch_size] || import_size
|
815
|
+
run_proc = options[:batch_size].to_i.positive? && options[:batch_progress].respond_to?( :call )
|
816
|
+
progress_proc = options[:batch_progress]
|
817
|
+
current_batch = 0
|
818
|
+
batches = (import_size / batch_size.to_f).ceil
|
717
819
|
|
718
|
-
batch_size = options[:batch_size] || values_sql.size
|
719
820
|
values_sql.each_slice(batch_size) do |batch_values|
|
821
|
+
batch_started_at = Time.now.to_i
|
822
|
+
|
720
823
|
# perform the inserts
|
721
824
|
result = connection.insert_many( [insert_sql, post_sql_statements].flatten,
|
722
825
|
batch_values,
|
723
826
|
options,
|
724
|
-
"#{model_name} Create Many
|
827
|
+
"#{model_name} Create Many" )
|
828
|
+
|
725
829
|
number_inserted += result.num_inserts
|
726
830
|
ids += result.ids
|
727
831
|
results += result.results
|
832
|
+
current_batch += 1
|
833
|
+
|
834
|
+
progress_proc.call(import_size, batches, current_batch, Time.now.to_i - batch_started_at) if run_proc
|
728
835
|
end
|
729
836
|
else
|
730
837
|
transaction(requires_new: true) do
|
@@ -750,11 +857,24 @@ class ActiveRecord::Base
|
|
750
857
|
model.id = id
|
751
858
|
|
752
859
|
timestamps.each do |attr, value|
|
753
|
-
model.send(attr + "=", value)
|
860
|
+
model.send(attr + "=", value) if model.send(attr).nil?
|
754
861
|
end
|
755
862
|
end
|
756
863
|
end
|
757
864
|
|
865
|
+
deserialize_value = lambda do |column, value|
|
866
|
+
column = columns_hash[column]
|
867
|
+
return value unless column
|
868
|
+
if respond_to?(:type_caster)
|
869
|
+
type = type_for_attribute(column.name)
|
870
|
+
type.deserialize(value)
|
871
|
+
elsif column.respond_to?(:type_cast_from_database)
|
872
|
+
column.type_cast_from_database(value)
|
873
|
+
else
|
874
|
+
value
|
875
|
+
end
|
876
|
+
end
|
877
|
+
|
758
878
|
if models.size == import_result.results.size
|
759
879
|
columns = Array(options[:returning])
|
760
880
|
single_column = "#{columns.first}=" if columns.size == 1
|
@@ -762,17 +882,22 @@ class ActiveRecord::Base
|
|
762
882
|
model = models[index]
|
763
883
|
|
764
884
|
if single_column
|
765
|
-
|
885
|
+
val = deserialize_value.call(columns.first, result)
|
886
|
+
model.send(single_column, val)
|
766
887
|
else
|
767
888
|
columns.each_with_index do |column, col_index|
|
768
|
-
|
889
|
+
val = deserialize_value.call(column, result[col_index])
|
890
|
+
model.send("#{column}=", val)
|
769
891
|
end
|
770
892
|
end
|
771
893
|
end
|
772
894
|
end
|
773
895
|
|
774
896
|
models.each do |model|
|
775
|
-
if model.respond_to?(:
|
897
|
+
if model.respond_to?(:changes_applied) # Rails 4.1.8 and higher
|
898
|
+
model.changes_internally_applied if model.respond_to?(:changes_internally_applied) # legacy behavior for Rails 5.1
|
899
|
+
model.changes_applied
|
900
|
+
elsif model.respond_to?(:clear_changes_information) # Rails 4.0 and higher
|
776
901
|
model.clear_changes_information
|
777
902
|
else # Rails 3.2
|
778
903
|
model.instance_variable_get(:@changed_attributes).clear
|
@@ -783,14 +908,21 @@ class ActiveRecord::Base
|
|
783
908
|
|
784
909
|
# Sync belongs_to association ids with foreign key field
|
785
910
|
def load_association_ids(model)
|
911
|
+
changed_columns = model.changed
|
786
912
|
association_reflections = model.class.reflect_on_all_associations(:belongs_to)
|
787
913
|
association_reflections.each do |association_reflection|
|
788
914
|
next if association_reflection.options[:polymorphic]
|
789
|
-
|
790
|
-
|
791
|
-
|
792
|
-
|
793
|
-
|
915
|
+
|
916
|
+
column_names = Array(association_reflection.foreign_key).map(&:to_s)
|
917
|
+
column_names.each_with_index do |column_name, column_index|
|
918
|
+
next if changed_columns.include?(column_name)
|
919
|
+
|
920
|
+
association = model.association(association_reflection.name)
|
921
|
+
association = association.target
|
922
|
+
next if association.blank? || model.public_send(column_name).present?
|
923
|
+
|
924
|
+
association_primary_key = Array(association_reflection.association_primary_key)[column_index]
|
925
|
+
model.public_send("#{column_name}=", association.send(association_primary_key))
|
794
926
|
end
|
795
927
|
end
|
796
928
|
end
|
@@ -804,8 +936,10 @@ class ActiveRecord::Base
|
|
804
936
|
associated_objects_by_class = {}
|
805
937
|
models.each { |model| find_associated_objects_for_import(associated_objects_by_class, model) }
|
806
938
|
|
807
|
-
# :on_duplicate_key_update
|
808
|
-
options.delete(:on_duplicate_key_update)
|
939
|
+
# :on_duplicate_key_update only supported for all fields
|
940
|
+
options.delete(:on_duplicate_key_update) unless options[:on_duplicate_key_update] == :all
|
941
|
+
# :returning not supported for associations
|
942
|
+
options.delete(:returning)
|
809
943
|
|
810
944
|
associated_objects_by_class.each_value do |associations|
|
811
945
|
associations.each_value do |associated_records|
|
@@ -836,8 +970,13 @@ class ActiveRecord::Base
|
|
836
970
|
changed_objects.each do |child|
|
837
971
|
child.public_send("#{association_reflection.foreign_key}=", model.id)
|
838
972
|
# For polymorphic associations
|
973
|
+
association_name = if model.class.respond_to?(:polymorphic_name)
|
974
|
+
model.class.polymorphic_name
|
975
|
+
else
|
976
|
+
model.class.base_class
|
977
|
+
end
|
839
978
|
association_reflection.type.try do |type|
|
840
|
-
child.public_send("#{type}=",
|
979
|
+
child.public_send("#{type}=", association_name)
|
841
980
|
end
|
842
981
|
end
|
843
982
|
associated_objects_by_class[model.class.name][association_reflection.name].concat changed_objects
|
@@ -857,14 +996,14 @@ class ActiveRecord::Base
|
|
857
996
|
column = columns[j]
|
858
997
|
|
859
998
|
# be sure to query sequence_name *last*, only if cheaper tests fail, because it's costly
|
860
|
-
if val.nil? &&
|
999
|
+
if val.nil? && Array(primary_key).first == column.name && !sequence_name.blank?
|
861
1000
|
connection_memo.next_value_for_sequence(sequence_name)
|
862
1001
|
elsif val.respond_to?(:to_sql)
|
863
1002
|
"(#{val.to_sql})"
|
864
1003
|
elsif column
|
865
1004
|
if respond_to?(:type_caster) # Rails 5.0 and higher
|
866
1005
|
type = type_for_attribute(column.name)
|
867
|
-
val = type.type == :boolean ? type.cast(val) : type.serialize(val)
|
1006
|
+
val = !type.respond_to?(:subtype) && type.type == :boolean ? type.cast(val) : type.serialize(val)
|
868
1007
|
connection_memo.quote(val)
|
869
1008
|
elsif column.respond_to?(:type_cast_from_user) # Rails 4.2
|
870
1009
|
connection_memo.quote(column.type_cast_from_user(val), column)
|
@@ -873,9 +1012,11 @@ class ActiveRecord::Base
|
|
873
1012
|
val = serialized_attributes[column.name].dump(val)
|
874
1013
|
end
|
875
1014
|
# Fixes #443 to support binary (i.e. bytea) columns on PG
|
876
|
-
val = column.type_cast(val) unless column.type.to_sym == :binary
|
1015
|
+
val = column.type_cast(val) unless column.type && column.type.to_sym == :binary
|
877
1016
|
connection_memo.quote(val, column)
|
878
1017
|
end
|
1018
|
+
else
|
1019
|
+
raise ArgumentError, "Number of values (#{arr.length}) exceeds number of columns (#{columns.length})"
|
879
1020
|
end
|
880
1021
|
end
|
881
1022
|
"(#{my_values.join(',')})"
|
@@ -890,13 +1031,18 @@ class ActiveRecord::Base
|
|
890
1031
|
timestamp_columns[:create] = timestamp_attributes_for_create_in_model
|
891
1032
|
timestamp_columns[:update] = timestamp_attributes_for_update_in_model
|
892
1033
|
else
|
893
|
-
instance =
|
1034
|
+
instance = allocate
|
894
1035
|
timestamp_columns[:create] = instance.send(:timestamp_attributes_for_create_in_model)
|
895
1036
|
timestamp_columns[:update] = instance.send(:timestamp_attributes_for_update_in_model)
|
896
1037
|
end
|
897
1038
|
|
898
1039
|
# use tz as set in ActiveRecord::Base
|
899
|
-
|
1040
|
+
default_timezone = if ActiveRecord.respond_to?(:default_timezone)
|
1041
|
+
ActiveRecord.default_timezone
|
1042
|
+
else
|
1043
|
+
ActiveRecord::Base.default_timezone
|
1044
|
+
end
|
1045
|
+
timestamp = default_timezone == :utc ? Time.now.utc : Time.now
|
900
1046
|
|
901
1047
|
[:create, :update].each do |action|
|
902
1048
|
timestamp_columns[action].each do |column|
|
@@ -906,7 +1052,7 @@ class ActiveRecord::Base
|
|
906
1052
|
index = column_names.index(column) || column_names.index(column.to_sym)
|
907
1053
|
if index
|
908
1054
|
# replace every instance of the array of attributes with our value
|
909
|
-
array_of_attributes.each { |arr| arr[index] = timestamp if arr[index].nil?
|
1055
|
+
array_of_attributes.each { |arr| arr[index] = timestamp if arr[index].nil? }
|
910
1056
|
else
|
911
1057
|
column_names << column
|
912
1058
|
array_of_attributes.each { |arr| arr << timestamp }
|
@@ -941,7 +1087,7 @@ Hash key mismatch.
|
|
941
1087
|
|
942
1088
|
When importing an array of hashes with provided columns_names, each hash must contain keys for all column_names.
|
943
1089
|
|
944
|
-
Required keys: #{
|
1090
|
+
Required keys: #{required_keys}
|
945
1091
|
Missing keys: #{missing_keys}
|
946
1092
|
|
947
1093
|
Hash: #{hash}
|
@@ -954,7 +1100,7 @@ When importing an array of hashes, all hashes must have the same keys.
|
|
954
1100
|
If you have records that are missing some values, we recommend you either set default values
|
955
1101
|
for the missing keys or group these records into batches by key set before importing.
|
956
1102
|
|
957
|
-
Required keys: #{
|
1103
|
+
Required keys: #{required_keys}
|
958
1104
|
Extra keys: #{extra_keys}
|
959
1105
|
Missing keys: #{missing_keys}
|
960
1106
|
|