activerecord-import 0.23.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/test.yaml +107 -0
- data/.gitignore +1 -0
- data/CHANGELOG.md +214 -4
- data/Gemfile +11 -9
- data/LICENSE +21 -56
- data/README.markdown +574 -22
- data/Rakefile +2 -1
- data/activerecord-import.gemspec +4 -4
- data/benchmarks/benchmark.rb +5 -1
- data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +0 -0
- data/gemfiles/5.0.gemfile +1 -0
- data/gemfiles/5.1.gemfile +1 -0
- data/gemfiles/5.2.gemfile +2 -2
- data/gemfiles/6.0.gemfile +2 -0
- data/gemfiles/6.1.gemfile +2 -0
- data/gemfiles/7.0.gemfile +1 -0
- data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +4 -4
- data/lib/activerecord-import/adapters/abstract_adapter.rb +7 -1
- data/lib/activerecord-import/adapters/mysql_adapter.rb +8 -11
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +14 -16
- data/lib/activerecord-import/adapters/sqlite3_adapter.rb +125 -8
- data/lib/activerecord-import/base.rb +9 -1
- data/lib/activerecord-import/import.rb +269 -123
- data/lib/activerecord-import/synchronize.rb +2 -2
- data/lib/activerecord-import/value_sets_parser.rb +2 -0
- data/lib/activerecord-import/version.rb +1 -1
- data/lib/activerecord-import.rb +1 -0
- data/test/adapters/makara_postgis.rb +1 -0
- data/test/{travis → github}/database.yml +3 -1
- data/test/import_test.rb +138 -8
- data/test/makara_postgis/import_test.rb +8 -0
- data/test/models/animal.rb +6 -0
- data/test/models/card.rb +3 -0
- data/test/models/customer.rb +6 -0
- data/test/models/deck.rb +6 -0
- data/test/models/order.rb +6 -0
- data/test/models/playing_card.rb +2 -0
- data/test/models/user.rb +3 -1
- data/test/models/user_token.rb +4 -0
- data/test/schema/generic_schema.rb +30 -0
- data/test/schema/mysql2_schema.rb +19 -0
- data/test/schema/postgresql_schema.rb +16 -0
- data/test/schema/sqlite3_schema.rb +13 -0
- data/test/support/factories.rb +8 -8
- data/test/support/generate.rb +6 -6
- data/test/support/mysql/import_examples.rb +12 -0
- data/test/support/postgresql/import_examples.rb +100 -2
- data/test/support/shared_examples/on_duplicate_key_update.rb +54 -0
- data/test/support/shared_examples/recursive_import.rb +74 -4
- data/test/support/sqlite3/import_examples.rb +189 -25
- data/test/test_helper.rb +28 -3
- metadata +37 -18
- data/.travis.yml +0 -62
- data/gemfiles/3.2.gemfile +0 -2
- data/gemfiles/4.0.gemfile +0 -2
- data/gemfiles/4.1.gemfile +0 -2
- data/test/schema/mysql_schema.rb +0 -16
@@ -24,27 +24,66 @@ module ActiveRecord::Import #:nodoc:
|
|
24
24
|
end
|
25
25
|
|
26
26
|
class Validator
|
27
|
-
def initialize(options = {})
|
27
|
+
def initialize(klass, options = {})
|
28
28
|
@options = options
|
29
|
+
@validator_class = klass
|
30
|
+
init_validations(klass)
|
31
|
+
end
|
32
|
+
|
33
|
+
def init_validations(klass)
|
34
|
+
@validate_callbacks = klass._validate_callbacks.dup
|
35
|
+
|
36
|
+
@validate_callbacks.each_with_index do |callback, i|
|
37
|
+
filter = callback.respond_to?(:raw_filter) ? callback.raw_filter : callback.filter
|
38
|
+
next unless filter.class.name =~ /Validations::PresenceValidator/ ||
|
39
|
+
(!@options[:validate_uniqueness] &&
|
40
|
+
filter.is_a?(ActiveRecord::Validations::UniquenessValidator))
|
41
|
+
|
42
|
+
callback = callback.dup
|
43
|
+
filter = filter.dup
|
44
|
+
attrs = filter.instance_variable_get(:@attributes).dup
|
45
|
+
|
46
|
+
if filter.is_a?(ActiveRecord::Validations::UniquenessValidator)
|
47
|
+
attrs = []
|
48
|
+
else
|
49
|
+
associations = klass.reflect_on_all_associations(:belongs_to)
|
50
|
+
associations.each do |assoc|
|
51
|
+
if (index = attrs.index(assoc.name))
|
52
|
+
key = assoc.foreign_key.is_a?(Array) ? assoc.foreign_key.map(&:to_sym) : assoc.foreign_key.to_sym
|
53
|
+
attrs[index] = key unless attrs.include?(key)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
filter.instance_variable_set(:@attributes, attrs)
|
59
|
+
|
60
|
+
if @validate_callbacks.respond_to?(:chain, true)
|
61
|
+
@validate_callbacks.send(:chain).tap do |chain|
|
62
|
+
callback.instance_variable_set(:@filter, filter)
|
63
|
+
chain[i] = callback
|
64
|
+
end
|
65
|
+
else
|
66
|
+
callback.raw_filter = filter
|
67
|
+
callback.filter = callback.send(:_compile_filter, filter)
|
68
|
+
@validate_callbacks[i] = callback
|
69
|
+
end
|
70
|
+
end
|
29
71
|
end
|
30
72
|
|
31
73
|
def valid_model?(model)
|
74
|
+
init_validations(model.class) unless model.class == @validator_class
|
75
|
+
|
32
76
|
validation_context = @options[:validate_with_context]
|
33
77
|
validation_context ||= (model.new_record? ? :create : :update)
|
34
|
-
|
35
78
|
current_context = model.send(:validation_context)
|
79
|
+
|
36
80
|
begin
|
37
81
|
model.send(:validation_context=, validation_context)
|
38
82
|
model.errors.clear
|
39
83
|
|
40
|
-
validate_callbacks = model._validate_callbacks.dup
|
41
|
-
model._validate_callbacks.each do |callback|
|
42
|
-
validate_callbacks.delete(callback) if callback.raw_filter.is_a? ActiveRecord::Validations::UniquenessValidator
|
43
|
-
end
|
44
|
-
|
45
84
|
model.run_callbacks(:validation) do
|
46
85
|
if defined?(ActiveSupport::Callbacks::Filters::Environment) # ActiveRecord >= 4.1
|
47
|
-
runner = validate_callbacks.compile
|
86
|
+
runner = @validate_callbacks.compile
|
48
87
|
env = ActiveSupport::Callbacks::Filters::Environment.new(model, false, nil)
|
49
88
|
if runner.respond_to?(:call) # ActiveRecord < 5.1
|
50
89
|
runner.call(env)
|
@@ -63,10 +102,10 @@ module ActiveRecord::Import #:nodoc:
|
|
63
102
|
runner.invoke_before(env)
|
64
103
|
runner.invoke_after(env)
|
65
104
|
end
|
66
|
-
elsif validate_callbacks.method(:compile).arity == 0 # ActiveRecord = 4.0
|
67
|
-
model.instance_eval validate_callbacks.compile
|
105
|
+
elsif @validate_callbacks.method(:compile).arity == 0 # ActiveRecord = 4.0
|
106
|
+
model.instance_eval @validate_callbacks.compile
|
68
107
|
else # ActiveRecord 3.x
|
69
|
-
model.instance_eval validate_callbacks.compile(nil, model)
|
108
|
+
model.instance_eval @validate_callbacks.compile(nil, model)
|
70
109
|
end
|
71
110
|
end
|
72
111
|
|
@@ -95,9 +134,14 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
95
134
|
|
96
135
|
model_klass = reflection.klass
|
97
136
|
symbolized_foreign_key = reflection.foreign_key.to_sym
|
98
|
-
symbolized_column_names = model_klass.column_names.map(&:to_sym)
|
99
137
|
|
100
|
-
|
138
|
+
symbolized_column_names = if model_klass.connection.respond_to?(:supports_virtual_columns?) && model_klass.connection.supports_virtual_columns?
|
139
|
+
model_klass.columns.reject(&:virtual?).map { |c| c.name.to_sym }
|
140
|
+
else
|
141
|
+
model_klass.column_names.map(&:to_sym)
|
142
|
+
end
|
143
|
+
|
144
|
+
owner_primary_key = reflection.active_record_primary_key.to_sym
|
101
145
|
owner_primary_key_value = owner.send(owner_primary_key)
|
102
146
|
|
103
147
|
# assume array of model objects
|
@@ -201,16 +245,17 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
201
245
|
alias import bulk_import unless respond_to? :import
|
202
246
|
end
|
203
247
|
|
248
|
+
module ActiveRecord::Import::Connection
|
249
|
+
def establish_connection(args = nil)
|
250
|
+
conn = super(args)
|
251
|
+
ActiveRecord::Import.load_from_connection_pool connection_pool
|
252
|
+
conn
|
253
|
+
end
|
254
|
+
end
|
255
|
+
|
204
256
|
class ActiveRecord::Base
|
205
257
|
class << self
|
206
|
-
|
207
|
-
conn = establish_connection_without_activerecord_import(*args)
|
208
|
-
ActiveRecord::Import.load_from_connection_pool connection_pool
|
209
|
-
conn
|
210
|
-
end
|
211
|
-
|
212
|
-
alias establish_connection_without_activerecord_import establish_connection
|
213
|
-
alias establish_connection establish_connection_with_activerecord_import
|
258
|
+
prepend ActiveRecord::Import::Connection
|
214
259
|
|
215
260
|
# Returns true if the current database connection adapter
|
216
261
|
# supports import functionality, otherwise returns false.
|
@@ -276,6 +321,9 @@ class ActiveRecord::Base
|
|
276
321
|
# == Options
|
277
322
|
# * +validate+ - true|false, tells import whether or not to use
|
278
323
|
# ActiveRecord validations. Validations are enforced by default.
|
324
|
+
# It skips the uniqueness validation for performance reasons.
|
325
|
+
# You can find more details here:
|
326
|
+
# https://github.com/zdennis/activerecord-import/issues/228
|
279
327
|
# * +ignore+ - true|false, an alias for on_duplicate_key_ignore.
|
280
328
|
# * +on_duplicate_key_ignore+ - true|false, tells import to discard
|
281
329
|
# records that contain duplicate keys. For Postgres 9.5+ it adds
|
@@ -284,8 +332,8 @@ class ActiveRecord::Base
|
|
284
332
|
# recursive import. For database adapters that normally support
|
285
333
|
# setting primary keys on imported objects, this option prevents
|
286
334
|
# that from occurring.
|
287
|
-
# * +on_duplicate_key_update+ - an Array or Hash, tells import to
|
288
|
-
# use MySQL's ON DUPLICATE KEY UPDATE or Postgres
|
335
|
+
# * +on_duplicate_key_update+ - :all, an Array, or Hash, tells import to
|
336
|
+
# use MySQL's ON DUPLICATE KEY UPDATE or Postgres/SQLite ON CONFLICT
|
289
337
|
# DO UPDATE ability. See On Duplicate Key Update below.
|
290
338
|
# * +synchronize+ - an array of ActiveRecord instances for the model
|
291
339
|
# that you are currently importing data into. This synchronizes
|
@@ -344,7 +392,15 @@ class ActiveRecord::Base
|
|
344
392
|
#
|
345
393
|
# == On Duplicate Key Update (MySQL)
|
346
394
|
#
|
347
|
-
# The :on_duplicate_key_update option can be either an Array or a Hash.
|
395
|
+
# The :on_duplicate_key_update option can be either :all, an Array, or a Hash.
|
396
|
+
#
|
397
|
+
# ==== Using :all
|
398
|
+
#
|
399
|
+
# The :on_duplicate_key_update option can be set to :all. All columns
|
400
|
+
# other than the primary key are updated. If a list of column names is
|
401
|
+
# supplied, only those columns will be updated. Below is an example:
|
402
|
+
#
|
403
|
+
# BlogPost.import columns, values, on_duplicate_key_update: :all
|
348
404
|
#
|
349
405
|
# ==== Using an Array
|
350
406
|
#
|
@@ -363,11 +419,19 @@ class ActiveRecord::Base
|
|
363
419
|
#
|
364
420
|
# BlogPost.import columns, attributes, on_duplicate_key_update: { title: :title }
|
365
421
|
#
|
366
|
-
# == On Duplicate Key Update (Postgres 9.5+)
|
422
|
+
# == On Duplicate Key Update (Postgres 9.5+ and SQLite 3.24+)
|
367
423
|
#
|
368
|
-
# The :on_duplicate_key_update option can be an Array or a Hash with up to
|
424
|
+
# The :on_duplicate_key_update option can be :all, an Array, or a Hash with up to
|
369
425
|
# three attributes, :conflict_target (and optionally :index_predicate) or
|
370
|
-
# :constraint_name, and :columns.
|
426
|
+
# :constraint_name (Postgres), and :columns.
|
427
|
+
#
|
428
|
+
# ==== Using :all
|
429
|
+
#
|
430
|
+
# The :on_duplicate_key_update option can be set to :all. All columns
|
431
|
+
# other than the primary key are updated. If a list of column names is
|
432
|
+
# supplied, only those columns will be updated. Below is an example:
|
433
|
+
#
|
434
|
+
# BlogPost.import columns, values, on_duplicate_key_update: :all
|
371
435
|
#
|
372
436
|
# ==== Using an Array
|
373
437
|
#
|
@@ -425,7 +489,15 @@ class ActiveRecord::Base
|
|
425
489
|
#
|
426
490
|
# ===== :columns
|
427
491
|
#
|
428
|
-
# The :columns attribute can be either an Array or a Hash.
|
492
|
+
# The :columns attribute can be either :all, an Array, or a Hash.
|
493
|
+
#
|
494
|
+
# ===== Using :all
|
495
|
+
#
|
496
|
+
# The :columns attribute can be :all. All columns other than the primary key will be updated.
|
497
|
+
# If a list of column names is supplied, only those columns will be updated.
|
498
|
+
# Below is an example:
|
499
|
+
#
|
500
|
+
# BlogPost.import columns, values, on_duplicate_key_update: { conflict_target: :slug, columns: :all }
|
429
501
|
#
|
430
502
|
# ===== Using an Array
|
431
503
|
#
|
@@ -460,7 +532,7 @@ class ActiveRecord::Base
|
|
460
532
|
import_helper(*args)
|
461
533
|
end
|
462
534
|
end
|
463
|
-
alias import bulk_import unless respond_to? :import
|
535
|
+
alias import bulk_import unless ActiveRecord::Base.respond_to? :import
|
464
536
|
|
465
537
|
# Imports a collection of values if all values are valid. Import fails at the
|
466
538
|
# first encountered validation error and raises ActiveRecord::RecordInvalid
|
@@ -472,27 +544,17 @@ class ActiveRecord::Base
|
|
472
544
|
|
473
545
|
bulk_import(*args, options)
|
474
546
|
end
|
475
|
-
alias import! bulk_import! unless respond_to? :import!
|
547
|
+
alias import! bulk_import! unless ActiveRecord::Base.respond_to? :import!
|
476
548
|
|
477
549
|
def import_helper( *args )
|
478
|
-
options = { validate: true, timestamps: true }
|
550
|
+
options = { validate: true, timestamps: true, track_validation_failures: false }
|
479
551
|
options.merge!( args.pop ) if args.last.is_a? Hash
|
480
552
|
# making sure that current model's primary key is used
|
481
553
|
options[:primary_key] = primary_key
|
482
554
|
options[:locking_column] = locking_column if attribute_names.include?(locking_column)
|
483
555
|
|
484
|
-
|
485
|
-
|
486
|
-
if on_duplicate_key_update && on_duplicate_key_update.duplicable?
|
487
|
-
options[:on_duplicate_key_update] = if on_duplicate_key_update.is_a?(Hash)
|
488
|
-
on_duplicate_key_update.each { |k, v| on_duplicate_key_update[k] = v.dup if v.duplicable? }
|
489
|
-
else
|
490
|
-
on_duplicate_key_update.dup
|
491
|
-
end
|
492
|
-
end
|
493
|
-
|
494
|
-
is_validating = options[:validate]
|
495
|
-
is_validating = true unless options[:validate_with_context].nil?
|
556
|
+
is_validating = options[:validate_with_context].present? ? true : options[:validate]
|
557
|
+
validator = ActiveRecord::Import::Validator.new(self, options)
|
496
558
|
|
497
559
|
# assume array of model objects
|
498
560
|
if args.last.is_a?( Array ) && args.last.first.is_a?(ActiveRecord::Base)
|
@@ -501,34 +563,48 @@ class ActiveRecord::Base
|
|
501
563
|
column_names = args.first.dup
|
502
564
|
else
|
503
565
|
models = args.first
|
504
|
-
column_names =
|
566
|
+
column_names = if connection.respond_to?(:supports_virtual_columns?) && connection.supports_virtual_columns?
|
567
|
+
columns.reject(&:virtual?).map(&:name)
|
568
|
+
else
|
569
|
+
self.column_names.dup
|
570
|
+
end
|
505
571
|
end
|
506
572
|
|
507
|
-
if models.first.id.nil?
|
508
|
-
|
573
|
+
if models.first.id.nil?
|
574
|
+
Array(primary_key).each do |c|
|
575
|
+
if column_names.include?(c) && columns_hash[c].type == :uuid
|
576
|
+
column_names.delete(c)
|
577
|
+
end
|
578
|
+
end
|
509
579
|
end
|
510
580
|
|
511
|
-
|
512
|
-
|
513
|
-
|
514
|
-
|
515
|
-
|
516
|
-
|
517
|
-
serialized_attributes
|
581
|
+
update_attrs = if record_timestamps && options[:timestamps]
|
582
|
+
if respond_to?(:timestamp_attributes_for_update, true)
|
583
|
+
send(:timestamp_attributes_for_update).map(&:to_sym)
|
584
|
+
else
|
585
|
+
allocate.send(:timestamp_attributes_for_update_in_model)
|
586
|
+
end
|
518
587
|
end
|
519
588
|
|
520
|
-
array_of_attributes =
|
589
|
+
array_of_attributes = []
|
590
|
+
|
591
|
+
models.each do |model|
|
521
592
|
if supports_setting_primary_key_of_imported_objects?
|
522
593
|
load_association_ids(model)
|
523
594
|
end
|
524
595
|
|
525
|
-
|
526
|
-
|
527
|
-
|
528
|
-
|
529
|
-
|
596
|
+
if is_validating && !validator.valid_model?(model)
|
597
|
+
raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
|
598
|
+
next
|
599
|
+
end
|
600
|
+
|
601
|
+
array_of_attributes << column_names.map do |name|
|
602
|
+
if model.persisted? &&
|
603
|
+
update_attrs && update_attrs.include?(name.to_sym) &&
|
604
|
+
!model.send("#{name}_changed?")
|
605
|
+
nil
|
530
606
|
else
|
531
|
-
model.
|
607
|
+
model.read_attribute(name.to_s)
|
532
608
|
end
|
533
609
|
end
|
534
610
|
end
|
@@ -555,7 +631,7 @@ class ActiveRecord::Base
|
|
555
631
|
end
|
556
632
|
# supports empty array
|
557
633
|
elsif args.last.is_a?( Array ) && args.last.empty?
|
558
|
-
return ActiveRecord::Import::Result.new([], 0, [])
|
634
|
+
return ActiveRecord::Import::Result.new([], 0, [], [])
|
559
635
|
# supports 2-element array and array
|
560
636
|
elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
|
561
637
|
|
@@ -586,32 +662,69 @@ class ActiveRecord::Base
|
|
586
662
|
array_of_attributes.each { |a| a.concat(new_fields) }
|
587
663
|
end
|
588
664
|
|
665
|
+
# Don't modify incoming arguments
|
666
|
+
on_duplicate_key_update = options[:on_duplicate_key_update]
|
667
|
+
if on_duplicate_key_update
|
668
|
+
updatable_columns = symbolized_column_names.reject { |c| symbolized_primary_key.include? c }
|
669
|
+
options[:on_duplicate_key_update] = if on_duplicate_key_update.is_a?(Hash)
|
670
|
+
on_duplicate_key_update.each_with_object({}) do |(k, v), duped_options|
|
671
|
+
duped_options[k] = if k == :columns && v == :all
|
672
|
+
updatable_columns
|
673
|
+
elsif v.duplicable?
|
674
|
+
v.dup
|
675
|
+
else
|
676
|
+
v
|
677
|
+
end
|
678
|
+
end
|
679
|
+
elsif on_duplicate_key_update == :all
|
680
|
+
updatable_columns
|
681
|
+
elsif on_duplicate_key_update.duplicable?
|
682
|
+
on_duplicate_key_update.dup
|
683
|
+
else
|
684
|
+
on_duplicate_key_update
|
685
|
+
end
|
686
|
+
end
|
687
|
+
|
589
688
|
timestamps = {}
|
590
689
|
|
591
690
|
# record timestamps unless disabled in ActiveRecord::Base
|
592
|
-
if record_timestamps && options
|
691
|
+
if record_timestamps && options[:timestamps]
|
593
692
|
timestamps = add_special_rails_stamps column_names, array_of_attributes, options
|
594
693
|
end
|
595
694
|
|
596
695
|
return_obj = if is_validating
|
597
|
-
|
598
|
-
|
599
|
-
models.
|
600
|
-
|
696
|
+
import_with_validations( column_names, array_of_attributes, options ) do |failed_instances|
|
697
|
+
if models
|
698
|
+
models.each { |m| failed_instances << m if m.errors.any? }
|
699
|
+
else
|
700
|
+
# create instances for each of our column/value sets
|
701
|
+
arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
|
702
|
+
|
703
|
+
# keep track of the instance and the position it is currently at. if this fails
|
704
|
+
# validation we'll use the index to remove it from the array_of_attributes
|
705
|
+
arr.each_with_index do |hsh, i|
|
706
|
+
# utilize block initializer syntax to prevent failure when 'mass_assignment_sanitizer = :strict'
|
707
|
+
model = new do |m|
|
708
|
+
hsh.each_pair { |k, v| m[k] = v }
|
709
|
+
end
|
710
|
+
|
711
|
+
next if validator.valid_model?(model)
|
601
712
|
raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
|
713
|
+
|
602
714
|
array_of_attributes[i] = nil
|
603
|
-
|
715
|
+
failure = model.dup
|
716
|
+
failure.errors.send(:initialize_dup, model.errors)
|
717
|
+
failed_instances << (options[:track_validation_failures] ? [i, failure] : failure )
|
604
718
|
end
|
719
|
+
array_of_attributes.compact!
|
605
720
|
end
|
606
|
-
else
|
607
|
-
import_with_validations( column_names, array_of_attributes, options )
|
608
721
|
end
|
609
722
|
else
|
610
723
|
import_without_validations_or_callbacks( column_names, array_of_attributes, options )
|
611
724
|
end
|
612
725
|
|
613
726
|
if options[:synchronize]
|
614
|
-
sync_keys = options[:synchronize_keys] ||
|
727
|
+
sync_keys = options[:synchronize_keys] || Array(primary_key)
|
615
728
|
synchronize( options[:synchronize], sync_keys)
|
616
729
|
end
|
617
730
|
return_obj.num_inserts = 0 if return_obj.num_inserts.nil?
|
@@ -621,7 +734,10 @@ class ActiveRecord::Base
|
|
621
734
|
set_attributes_and_mark_clean(models, return_obj, timestamps, options)
|
622
735
|
|
623
736
|
# if there are auto-save associations on the models we imported that are new, import them as well
|
624
|
-
|
737
|
+
if options[:recursive]
|
738
|
+
options[:on_duplicate_key_update] = on_duplicate_key_update unless on_duplicate_key_update.nil?
|
739
|
+
import_associations(models, options.dup.merge(validate: false))
|
740
|
+
end
|
625
741
|
end
|
626
742
|
|
627
743
|
return_obj
|
@@ -637,29 +753,7 @@ class ActiveRecord::Base
|
|
637
753
|
def import_with_validations( column_names, array_of_attributes, options = {} )
|
638
754
|
failed_instances = []
|
639
755
|
|
640
|
-
|
641
|
-
|
642
|
-
if block_given?
|
643
|
-
yield validator, failed_instances
|
644
|
-
else
|
645
|
-
# create instances for each of our column/value sets
|
646
|
-
arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
|
647
|
-
|
648
|
-
# keep track of the instance and the position it is currently at. if this fails
|
649
|
-
# validation we'll use the index to remove it from the array_of_attributes
|
650
|
-
arr.each_with_index do |hsh, i|
|
651
|
-
model = new
|
652
|
-
hsh.each_pair { |k, v| model[k] = v }
|
653
|
-
next if validator.valid_model? model
|
654
|
-
raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
|
655
|
-
array_of_attributes[i] = nil
|
656
|
-
failure = model.dup
|
657
|
-
failure.errors.send(:initialize_dup, model.errors)
|
658
|
-
failed_instances << failure
|
659
|
-
end
|
660
|
-
end
|
661
|
-
|
662
|
-
array_of_attributes.compact!
|
756
|
+
yield failed_instances if block_given?
|
663
757
|
|
664
758
|
result = if options[:all_or_none] && failed_instances.any?
|
665
759
|
ActiveRecord::Import::Result.new([], 0, [], [])
|
@@ -684,21 +778,22 @@ class ActiveRecord::Base
|
|
684
778
|
unless scope_columns.blank?
|
685
779
|
scope_columns.zip(scope_values).each do |name, value|
|
686
780
|
name_as_sym = name.to_sym
|
687
|
-
next if column_names.include?(name_as_sym)
|
688
|
-
|
689
|
-
is_sti = (name_as_sym == inheritance_column.to_sym && self < base_class)
|
690
|
-
value = Array(value).first if is_sti
|
691
|
-
|
781
|
+
next if column_names.include?(name_as_sym) || name_as_sym == inheritance_column.to_sym
|
692
782
|
column_names << name_as_sym
|
693
783
|
array_of_attributes.each { |attrs| attrs << value }
|
694
784
|
end
|
695
785
|
end
|
696
786
|
|
787
|
+
if finder_needs_type_condition?
|
788
|
+
unless column_names.include?(inheritance_column.to_sym)
|
789
|
+
column_names << inheritance_column.to_sym
|
790
|
+
array_of_attributes.each { |attrs| attrs << sti_name }
|
791
|
+
end
|
792
|
+
end
|
793
|
+
|
697
794
|
columns = column_names.each_with_index.map do |name, i|
|
698
795
|
column = columns_hash[name.to_s]
|
699
|
-
|
700
796
|
raise ActiveRecord::Import::MissingColumnError.new(name.to_s, i) if column.nil?
|
701
|
-
|
702
797
|
column
|
703
798
|
end
|
704
799
|
|
@@ -714,17 +809,29 @@ class ActiveRecord::Base
|
|
714
809
|
if supports_import?
|
715
810
|
# generate the sql
|
716
811
|
post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
|
812
|
+
import_size = values_sql.size
|
813
|
+
|
814
|
+
batch_size = options[:batch_size] || import_size
|
815
|
+
run_proc = options[:batch_size].to_i.positive? && options[:batch_progress].respond_to?( :call )
|
816
|
+
progress_proc = options[:batch_progress]
|
817
|
+
current_batch = 0
|
818
|
+
batches = (import_size / batch_size.to_f).ceil
|
717
819
|
|
718
|
-
batch_size = options[:batch_size] || values_sql.size
|
719
820
|
values_sql.each_slice(batch_size) do |batch_values|
|
821
|
+
batch_started_at = Time.now.to_i
|
822
|
+
|
720
823
|
# perform the inserts
|
721
824
|
result = connection.insert_many( [insert_sql, post_sql_statements].flatten,
|
722
825
|
batch_values,
|
723
826
|
options,
|
724
|
-
"#{model_name} Create Many
|
827
|
+
"#{model_name} Create Many" )
|
828
|
+
|
725
829
|
number_inserted += result.num_inserts
|
726
830
|
ids += result.ids
|
727
831
|
results += result.results
|
832
|
+
current_batch += 1
|
833
|
+
|
834
|
+
progress_proc.call(import_size, batches, current_batch, Time.now.to_i - batch_started_at) if run_proc
|
728
835
|
end
|
729
836
|
else
|
730
837
|
transaction(requires_new: true) do
|
@@ -750,11 +857,24 @@ class ActiveRecord::Base
|
|
750
857
|
model.id = id
|
751
858
|
|
752
859
|
timestamps.each do |attr, value|
|
753
|
-
model.send(attr + "=", value)
|
860
|
+
model.send(attr + "=", value) if model.send(attr).nil?
|
754
861
|
end
|
755
862
|
end
|
756
863
|
end
|
757
864
|
|
865
|
+
deserialize_value = lambda do |column, value|
|
866
|
+
column = columns_hash[column]
|
867
|
+
return value unless column
|
868
|
+
if respond_to?(:type_caster)
|
869
|
+
type = type_for_attribute(column.name)
|
870
|
+
type.deserialize(value)
|
871
|
+
elsif column.respond_to?(:type_cast_from_database)
|
872
|
+
column.type_cast_from_database(value)
|
873
|
+
else
|
874
|
+
value
|
875
|
+
end
|
876
|
+
end
|
877
|
+
|
758
878
|
if models.size == import_result.results.size
|
759
879
|
columns = Array(options[:returning])
|
760
880
|
single_column = "#{columns.first}=" if columns.size == 1
|
@@ -762,17 +882,22 @@ class ActiveRecord::Base
|
|
762
882
|
model = models[index]
|
763
883
|
|
764
884
|
if single_column
|
765
|
-
|
885
|
+
val = deserialize_value.call(columns.first, result)
|
886
|
+
model.send(single_column, val)
|
766
887
|
else
|
767
888
|
columns.each_with_index do |column, col_index|
|
768
|
-
|
889
|
+
val = deserialize_value.call(column, result[col_index])
|
890
|
+
model.send("#{column}=", val)
|
769
891
|
end
|
770
892
|
end
|
771
893
|
end
|
772
894
|
end
|
773
895
|
|
774
896
|
models.each do |model|
|
775
|
-
if model.respond_to?(:
|
897
|
+
if model.respond_to?(:changes_applied) # Rails 4.1.8 and higher
|
898
|
+
model.changes_internally_applied if model.respond_to?(:changes_internally_applied) # legacy behavior for Rails 5.1
|
899
|
+
model.changes_applied
|
900
|
+
elsif model.respond_to?(:clear_changes_information) # Rails 4.0 and higher
|
776
901
|
model.clear_changes_information
|
777
902
|
else # Rails 3.2
|
778
903
|
model.instance_variable_get(:@changed_attributes).clear
|
@@ -783,14 +908,21 @@ class ActiveRecord::Base
|
|
783
908
|
|
784
909
|
# Sync belongs_to association ids with foreign key field
|
785
910
|
def load_association_ids(model)
|
911
|
+
changed_columns = model.changed
|
786
912
|
association_reflections = model.class.reflect_on_all_associations(:belongs_to)
|
787
913
|
association_reflections.each do |association_reflection|
|
788
914
|
next if association_reflection.options[:polymorphic]
|
789
|
-
|
790
|
-
|
791
|
-
|
792
|
-
|
793
|
-
|
915
|
+
|
916
|
+
column_names = Array(association_reflection.foreign_key).map(&:to_s)
|
917
|
+
column_names.each_with_index do |column_name, column_index|
|
918
|
+
next if changed_columns.include?(column_name)
|
919
|
+
|
920
|
+
association = model.association(association_reflection.name)
|
921
|
+
association = association.target
|
922
|
+
next if association.blank? || model.public_send(column_name).present?
|
923
|
+
|
924
|
+
association_primary_key = Array(association_reflection.association_primary_key)[column_index]
|
925
|
+
model.public_send("#{column_name}=", association.send(association_primary_key))
|
794
926
|
end
|
795
927
|
end
|
796
928
|
end
|
@@ -804,8 +936,10 @@ class ActiveRecord::Base
|
|
804
936
|
associated_objects_by_class = {}
|
805
937
|
models.each { |model| find_associated_objects_for_import(associated_objects_by_class, model) }
|
806
938
|
|
807
|
-
# :on_duplicate_key_update
|
808
|
-
options.delete(:on_duplicate_key_update)
|
939
|
+
# :on_duplicate_key_update only supported for all fields
|
940
|
+
options.delete(:on_duplicate_key_update) unless options[:on_duplicate_key_update] == :all
|
941
|
+
# :returning not supported for associations
|
942
|
+
options.delete(:returning)
|
809
943
|
|
810
944
|
associated_objects_by_class.each_value do |associations|
|
811
945
|
associations.each_value do |associated_records|
|
@@ -836,8 +970,13 @@ class ActiveRecord::Base
|
|
836
970
|
changed_objects.each do |child|
|
837
971
|
child.public_send("#{association_reflection.foreign_key}=", model.id)
|
838
972
|
# For polymorphic associations
|
973
|
+
association_name = if model.class.respond_to?(:polymorphic_name)
|
974
|
+
model.class.polymorphic_name
|
975
|
+
else
|
976
|
+
model.class.base_class
|
977
|
+
end
|
839
978
|
association_reflection.type.try do |type|
|
840
|
-
child.public_send("#{type}=",
|
979
|
+
child.public_send("#{type}=", association_name)
|
841
980
|
end
|
842
981
|
end
|
843
982
|
associated_objects_by_class[model.class.name][association_reflection.name].concat changed_objects
|
@@ -857,14 +996,14 @@ class ActiveRecord::Base
|
|
857
996
|
column = columns[j]
|
858
997
|
|
859
998
|
# be sure to query sequence_name *last*, only if cheaper tests fail, because it's costly
|
860
|
-
if val.nil? &&
|
999
|
+
if val.nil? && Array(primary_key).first == column.name && !sequence_name.blank?
|
861
1000
|
connection_memo.next_value_for_sequence(sequence_name)
|
862
1001
|
elsif val.respond_to?(:to_sql)
|
863
1002
|
"(#{val.to_sql})"
|
864
1003
|
elsif column
|
865
1004
|
if respond_to?(:type_caster) # Rails 5.0 and higher
|
866
1005
|
type = type_for_attribute(column.name)
|
867
|
-
val = type.type == :boolean ? type.cast(val) : type.serialize(val)
|
1006
|
+
val = !type.respond_to?(:subtype) && type.type == :boolean ? type.cast(val) : type.serialize(val)
|
868
1007
|
connection_memo.quote(val)
|
869
1008
|
elsif column.respond_to?(:type_cast_from_user) # Rails 4.2
|
870
1009
|
connection_memo.quote(column.type_cast_from_user(val), column)
|
@@ -873,9 +1012,11 @@ class ActiveRecord::Base
|
|
873
1012
|
val = serialized_attributes[column.name].dump(val)
|
874
1013
|
end
|
875
1014
|
# Fixes #443 to support binary (i.e. bytea) columns on PG
|
876
|
-
val = column.type_cast(val) unless column.type.to_sym == :binary
|
1015
|
+
val = column.type_cast(val) unless column.type && column.type.to_sym == :binary
|
877
1016
|
connection_memo.quote(val, column)
|
878
1017
|
end
|
1018
|
+
else
|
1019
|
+
raise ArgumentError, "Number of values (#{arr.length}) exceeds number of columns (#{columns.length})"
|
879
1020
|
end
|
880
1021
|
end
|
881
1022
|
"(#{my_values.join(',')})"
|
@@ -890,13 +1031,18 @@ class ActiveRecord::Base
|
|
890
1031
|
timestamp_columns[:create] = timestamp_attributes_for_create_in_model
|
891
1032
|
timestamp_columns[:update] = timestamp_attributes_for_update_in_model
|
892
1033
|
else
|
893
|
-
instance =
|
1034
|
+
instance = allocate
|
894
1035
|
timestamp_columns[:create] = instance.send(:timestamp_attributes_for_create_in_model)
|
895
1036
|
timestamp_columns[:update] = instance.send(:timestamp_attributes_for_update_in_model)
|
896
1037
|
end
|
897
1038
|
|
898
1039
|
# use tz as set in ActiveRecord::Base
|
899
|
-
|
1040
|
+
default_timezone = if ActiveRecord.respond_to?(:default_timezone)
|
1041
|
+
ActiveRecord.default_timezone
|
1042
|
+
else
|
1043
|
+
ActiveRecord::Base.default_timezone
|
1044
|
+
end
|
1045
|
+
timestamp = default_timezone == :utc ? Time.now.utc : Time.now
|
900
1046
|
|
901
1047
|
[:create, :update].each do |action|
|
902
1048
|
timestamp_columns[action].each do |column|
|
@@ -906,7 +1052,7 @@ class ActiveRecord::Base
|
|
906
1052
|
index = column_names.index(column) || column_names.index(column.to_sym)
|
907
1053
|
if index
|
908
1054
|
# replace every instance of the array of attributes with our value
|
909
|
-
array_of_attributes.each { |arr| arr[index] = timestamp if arr[index].nil?
|
1055
|
+
array_of_attributes.each { |arr| arr[index] = timestamp if arr[index].nil? }
|
910
1056
|
else
|
911
1057
|
column_names << column
|
912
1058
|
array_of_attributes.each { |arr| arr << timestamp }
|
@@ -941,7 +1087,7 @@ Hash key mismatch.
|
|
941
1087
|
|
942
1088
|
When importing an array of hashes with provided columns_names, each hash must contain keys for all column_names.
|
943
1089
|
|
944
|
-
Required keys: #{
|
1090
|
+
Required keys: #{required_keys}
|
945
1091
|
Missing keys: #{missing_keys}
|
946
1092
|
|
947
1093
|
Hash: #{hash}
|
@@ -954,7 +1100,7 @@ When importing an array of hashes, all hashes must have the same keys.
|
|
954
1100
|
If you have records that are missing some values, we recommend you either set default values
|
955
1101
|
for the missing keys or group these records into batches by key set before importing.
|
956
1102
|
|
957
|
-
Required keys: #{
|
1103
|
+
Required keys: #{required_keys}
|
958
1104
|
Extra keys: #{extra_keys}
|
959
1105
|
Missing keys: #{missing_keys}
|
960
1106
|
|