activerecord-import 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +33 -0
- data/README.markdown +9 -9
- data/lib/activerecord-import/active_record/adapters/mysql2_proxy_adapter.rb +8 -0
- data/lib/activerecord-import/active_record/adapters/postgresql_proxy_adapter.rb +8 -0
- data/lib/activerecord-import/active_record/adapters/sqlite3_proxy_adapter.rb +8 -0
- data/lib/activerecord-import/active_record/adapters/trilogy_proxy_adapter.rb +8 -0
- data/lib/activerecord-import/adapters/active_record_proxy_adapter.rb +19 -0
- data/lib/activerecord-import/adapters/mysql2_proxy_adapter.rb +9 -0
- data/lib/activerecord-import/adapters/mysql_adapter.rb +10 -3
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +1 -1
- data/lib/activerecord-import/adapters/postgresql_proxy_adapter.rb +9 -0
- data/lib/activerecord-import/adapters/sqlite3_proxy_adapter.rb +9 -0
- data/lib/activerecord-import/adapters/trilogy_proxy_adapter.rb +9 -0
- data/lib/activerecord-import/base.rb +0 -2
- data/lib/activerecord-import/import.rb +243 -207
- data/lib/activerecord-import/version.rb +1 -1
- metadata +13 -208
- data/.github/workflows/test.yaml +0 -165
- data/.gitignore +0 -37
- data/.rubocop.yml +0 -118
- data/.rubocop_todo.yml +0 -30
- data/Brewfile +0 -5
- data/Dockerfile +0 -23
- data/Gemfile +0 -63
- data/Rakefile +0 -75
- data/activerecord-import.gemspec +0 -28
- data/benchmarks/README +0 -32
- data/benchmarks/benchmark.rb +0 -72
- data/benchmarks/lib/base.rb +0 -143
- data/benchmarks/lib/cli_parser.rb +0 -111
- data/benchmarks/lib/float.rb +0 -17
- data/benchmarks/lib/mysql2_benchmark.rb +0 -21
- data/benchmarks/lib/output_to_csv.rb +0 -21
- data/benchmarks/lib/output_to_html.rb +0 -66
- data/benchmarks/models/test_innodb.rb +0 -5
- data/benchmarks/models/test_memory.rb +0 -5
- data/benchmarks/models/test_myisam.rb +0 -5
- data/benchmarks/schema/mysql2_schema.rb +0 -18
- data/docker-compose.yml +0 -34
- data/gemfiles/5.2.gemfile +0 -4
- data/gemfiles/6.0.gemfile +0 -4
- data/gemfiles/6.1.gemfile +0 -4
- data/gemfiles/7.0.gemfile +0 -4
- data/gemfiles/7.1.gemfile +0 -3
- data/gemfiles/7.2.gemfile +0 -3
- data/gemfiles/8.0.gemfile +0 -3
- data/test/adapters/janus_mysql2.rb +0 -3
- data/test/adapters/janus_trilogy.rb +0 -11
- data/test/adapters/jdbcmysql.rb +0 -3
- data/test/adapters/jdbcpostgresql.rb +0 -3
- data/test/adapters/jdbcsqlite3.rb +0 -3
- data/test/adapters/makara_postgis.rb +0 -3
- data/test/adapters/mysql2.rb +0 -3
- data/test/adapters/mysql2_makara.rb +0 -3
- data/test/adapters/mysql2_proxy.rb +0 -3
- data/test/adapters/mysql2spatial.rb +0 -3
- data/test/adapters/postgis.rb +0 -3
- data/test/adapters/postgresql.rb +0 -3
- data/test/adapters/postgresql_makara.rb +0 -3
- data/test/adapters/postgresql_proxy.rb +0 -3
- data/test/adapters/seamless_database_pool.rb +0 -3
- data/test/adapters/spatialite.rb +0 -3
- data/test/adapters/sqlite3.rb +0 -3
- data/test/adapters/trilogy.rb +0 -11
- data/test/database.yml.sample +0 -73
- data/test/github/database.yml +0 -85
- data/test/import_test.rb +0 -970
- data/test/janus_mysql2/import_test.rb +0 -8
- data/test/janus_trilogy/import_test.rb +0 -7
- data/test/jdbcmysql/import_test.rb +0 -7
- data/test/jdbcpostgresql/import_test.rb +0 -6
- data/test/jdbcsqlite3/import_test.rb +0 -6
- data/test/makara_postgis/import_test.rb +0 -10
- data/test/models/account.rb +0 -5
- data/test/models/alarm.rb +0 -4
- data/test/models/animal.rb +0 -8
- data/test/models/author.rb +0 -9
- data/test/models/bike_maker.rb +0 -10
- data/test/models/book.rb +0 -18
- data/test/models/car.rb +0 -5
- data/test/models/card.rb +0 -5
- data/test/models/chapter.rb +0 -6
- data/test/models/composite_book.rb +0 -19
- data/test/models/composite_chapter.rb +0 -12
- data/test/models/customer.rb +0 -18
- data/test/models/deck.rb +0 -8
- data/test/models/dictionary.rb +0 -6
- data/test/models/discount.rb +0 -5
- data/test/models/end_note.rb +0 -6
- data/test/models/group.rb +0 -5
- data/test/models/order.rb +0 -17
- data/test/models/playing_card.rb +0 -4
- data/test/models/promotion.rb +0 -5
- data/test/models/question.rb +0 -5
- data/test/models/rule.rb +0 -5
- data/test/models/tag.rb +0 -12
- data/test/models/tag_alias.rb +0 -11
- data/test/models/topic.rb +0 -31
- data/test/models/user.rb +0 -5
- data/test/models/user_token.rb +0 -6
- data/test/models/vendor.rb +0 -9
- data/test/models/widget.rb +0 -33
- data/test/mysql2/import_test.rb +0 -7
- data/test/mysql2_makara/import_test.rb +0 -8
- data/test/mysql2_proxy/import_test.rb +0 -6
- data/test/mysqlspatial2/import_test.rb +0 -8
- data/test/postgis/import_test.rb +0 -10
- data/test/postgresql/import_test.rb +0 -6
- data/test/postgresql_proxy/import_test.rb +0 -6
- data/test/schema/generic_schema.rb +0 -230
- data/test/schema/jdbcpostgresql_schema.rb +0 -3
- data/test/schema/mysql2_schema.rb +0 -21
- data/test/schema/postgis_schema.rb +0 -3
- data/test/schema/postgresql_schema.rb +0 -94
- data/test/schema/sqlite3_schema.rb +0 -15
- data/test/schema/version.rb +0 -12
- data/test/sqlite3/import_test.rb +0 -6
- data/test/support/active_support/test_case_extensions.rb +0 -73
- data/test/support/assertions.rb +0 -75
- data/test/support/factories.rb +0 -66
- data/test/support/generate.rb +0 -31
- data/test/support/mysql/import_examples.rb +0 -97
- data/test/support/postgresql/import_examples.rb +0 -642
- data/test/support/shared_examples/on_duplicate_key_ignore.rb +0 -45
- data/test/support/shared_examples/on_duplicate_key_update.rb +0 -437
- data/test/support/shared_examples/recursive_import.rb +0 -352
- data/test/support/sqlite3/import_examples.rb +0 -232
- data/test/synchronize_test.rb +0 -43
- data/test/test_helper.rb +0 -91
- data/test/trilogy/import_test.rb +0 -7
- data/test/value_sets_bytes_parser_test.rb +0 -106
- data/test/value_sets_records_parser_test.rb +0 -34
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
module ActiveRecord::Import::ConnectionAdapters; end
|
|
4
4
|
|
|
5
5
|
module ActiveRecord::Import # :nodoc:
|
|
6
|
-
Result = Struct.new(:failed_instances, :num_inserts, :ids, :results)
|
|
6
|
+
Result = Struct.new(:failed_instances, :num_inserts, :ids, :results, :affected_rows)
|
|
7
7
|
|
|
8
8
|
module ImportSupport # :nodoc:
|
|
9
9
|
def supports_import? # :nodoc:
|
|
@@ -79,7 +79,7 @@ module ActiveRecord::Import # :nodoc:
|
|
|
79
79
|
current_context = model.send(:validation_context)
|
|
80
80
|
|
|
81
81
|
begin
|
|
82
|
-
model
|
|
82
|
+
set_validation_context(model, validation_context)
|
|
83
83
|
model.errors.clear
|
|
84
84
|
|
|
85
85
|
model.run_callbacks(:validation) do
|
|
@@ -117,7 +117,22 @@ module ActiveRecord::Import # :nodoc:
|
|
|
117
117
|
|
|
118
118
|
model.errors.empty?
|
|
119
119
|
ensure
|
|
120
|
-
model
|
|
120
|
+
set_validation_context(model, current_context)
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
private
|
|
125
|
+
|
|
126
|
+
# ActiveModel's private +validation_context=+ writer was removed in Rails
|
|
127
|
+
# 8.2 (rails/rails@0f9d1270834a6407a59637650bf910d8ae826169) in favor of a
|
|
128
|
+
# validation context object reachable through the private
|
|
129
|
+
# +context_for_validation+ reader. Support both so that validated imports
|
|
130
|
+
# keep working across all supported ActiveRecord versions.
|
|
131
|
+
def set_validation_context(model, context)
|
|
132
|
+
if model.respond_to?(:validation_context=, true)
|
|
133
|
+
model.send(:validation_context=, context)
|
|
134
|
+
else
|
|
135
|
+
model.send(:context_for_validation).context = context
|
|
121
136
|
end
|
|
122
137
|
end
|
|
123
138
|
end
|
|
@@ -141,7 +156,7 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
|
141
156
|
model_klass = reflection.klass
|
|
142
157
|
symbolized_foreign_key = reflection.foreign_key.to_sym
|
|
143
158
|
|
|
144
|
-
symbolized_column_names = if model_klass.
|
|
159
|
+
symbolized_column_names = if model_klass.connection_pool.with_connection { |conn| conn.respond_to?(:supports_virtual_columns?) && conn.supports_virtual_columns? }
|
|
145
160
|
model_klass.columns.reject(&:virtual?).map { |c| c.name.to_sym }
|
|
146
161
|
else
|
|
147
162
|
model_klass.column_names.map(&:to_sym)
|
|
@@ -251,36 +266,56 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
|
251
266
|
alias import bulk_import unless respond_to? :import
|
|
252
267
|
end
|
|
253
268
|
|
|
254
|
-
module ActiveRecord::Import::
|
|
255
|
-
def establish_connection(args
|
|
256
|
-
|
|
257
|
-
ActiveRecord::Import.load_from_connection_pool
|
|
258
|
-
|
|
269
|
+
module ActiveRecord::Import::ConnectionHandler
|
|
270
|
+
def establish_connection(*args, **kwargs, &block)
|
|
271
|
+
pool = super(*args, **kwargs, &block)
|
|
272
|
+
ActiveRecord::Import.load_from_connection_pool pool
|
|
273
|
+
pool
|
|
259
274
|
end
|
|
260
275
|
end
|
|
261
276
|
|
|
277
|
+
class ActiveRecord::ConnectionAdapters::ConnectionHandler
|
|
278
|
+
prepend ActiveRecord::Import::ConnectionHandler
|
|
279
|
+
end
|
|
280
|
+
|
|
262
281
|
class ActiveRecord::Base
|
|
263
282
|
class << self
|
|
264
|
-
prepend ActiveRecord::Import::Connection
|
|
265
|
-
|
|
266
283
|
# Returns true if the current database connection adapter
|
|
267
284
|
# supports import functionality, otherwise returns false.
|
|
268
|
-
def supports_import?(
|
|
269
|
-
|
|
285
|
+
def supports_import?(conn = nil)
|
|
286
|
+
if conn
|
|
287
|
+
conn.respond_to?(:supports_import?) && conn.supports_import?
|
|
288
|
+
else
|
|
289
|
+
connection_pool.with_connection do |conn|
|
|
290
|
+
conn.respond_to?(:supports_import?) && conn.supports_import?
|
|
291
|
+
end
|
|
292
|
+
end
|
|
270
293
|
end
|
|
271
294
|
|
|
272
295
|
# Returns true if the current database connection adapter
|
|
273
296
|
# supports on duplicate key update functionality, otherwise
|
|
274
297
|
# returns false.
|
|
275
|
-
def supports_on_duplicate_key_update?
|
|
276
|
-
|
|
298
|
+
def supports_on_duplicate_key_update?(conn = nil)
|
|
299
|
+
if conn
|
|
300
|
+
conn.respond_to?(:supports_on_duplicate_key_update?) && conn.supports_on_duplicate_key_update?
|
|
301
|
+
else
|
|
302
|
+
connection_pool.with_connection do |conn|
|
|
303
|
+
conn.respond_to?(:supports_on_duplicate_key_update?) && conn.supports_on_duplicate_key_update?
|
|
304
|
+
end
|
|
305
|
+
end
|
|
277
306
|
end
|
|
278
307
|
|
|
279
308
|
# returns true if the current database connection adapter
|
|
280
309
|
# supports setting the primary key of bulk imported models, otherwise
|
|
281
310
|
# returns false
|
|
282
|
-
def supports_setting_primary_key_of_imported_objects?
|
|
283
|
-
|
|
311
|
+
def supports_setting_primary_key_of_imported_objects?(conn = nil)
|
|
312
|
+
if conn
|
|
313
|
+
conn.respond_to?(:supports_setting_primary_key_of_imported_objects?) && conn.supports_setting_primary_key_of_imported_objects?
|
|
314
|
+
else
|
|
315
|
+
connection_pool.with_connection do |conn|
|
|
316
|
+
conn.respond_to?(:supports_setting_primary_key_of_imported_objects?) && conn.supports_setting_primary_key_of_imported_objects?
|
|
317
|
+
end
|
|
318
|
+
end
|
|
284
319
|
end
|
|
285
320
|
|
|
286
321
|
# Imports a collection of values to the database.
|
|
@@ -553,204 +588,206 @@ class ActiveRecord::Base
|
|
|
553
588
|
alias import! bulk_import! unless ActiveRecord::Base.respond_to? :import!
|
|
554
589
|
|
|
555
590
|
def import_helper( *args )
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
if args.
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
models = args.first
|
|
572
|
-
column_names = if connection.respond_to?(:supports_virtual_columns?) && connection.supports_virtual_columns?
|
|
573
|
-
columns.reject(&:virtual?).map(&:name)
|
|
591
|
+
connection_pool.with_connection do |conn|
|
|
592
|
+
options = { model: self, validate: true, timestamps: true, track_validation_failures: false }
|
|
593
|
+
options.merge!( args.pop ) if args.last.is_a? Hash
|
|
594
|
+
# making sure that current model's primary key is used
|
|
595
|
+
options[:primary_key] = primary_key
|
|
596
|
+
options[:locking_column] = locking_column if locking_enabled?
|
|
597
|
+
|
|
598
|
+
is_validating = options[:validate_with_context].present? || options[:validate]
|
|
599
|
+
validator = ActiveRecord::Import::Validator.new(self, options)
|
|
600
|
+
|
|
601
|
+
# assume array of model objects
|
|
602
|
+
if args.last.is_a?( Array ) && args.last.first.is_a?(ActiveRecord::Base)
|
|
603
|
+
if args.length == 2
|
|
604
|
+
models = args.last
|
|
605
|
+
column_names = args.first.dup
|
|
574
606
|
else
|
|
575
|
-
|
|
607
|
+
models = args.first
|
|
608
|
+
column_names = if conn.try(:supports_virtual_columns?)
|
|
609
|
+
columns.reject(&:virtual?).map(&:name)
|
|
610
|
+
else
|
|
611
|
+
self.column_names.dup
|
|
612
|
+
end
|
|
576
613
|
end
|
|
577
|
-
end
|
|
578
614
|
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
615
|
+
if models.first.id.nil?
|
|
616
|
+
Array(primary_key).each do |c|
|
|
617
|
+
if column_names.include?(c) && schema_columns_hash(conn)[c].type == :uuid
|
|
618
|
+
column_names.delete(c)
|
|
619
|
+
end
|
|
583
620
|
end
|
|
584
621
|
end
|
|
585
|
-
end
|
|
586
622
|
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
623
|
+
update_attrs = if record_timestamps && options[:timestamps]
|
|
624
|
+
if respond_to?(:timestamp_attributes_for_update, true)
|
|
625
|
+
send(:timestamp_attributes_for_update).map(&:to_sym)
|
|
626
|
+
else
|
|
627
|
+
allocate.send(:timestamp_attributes_for_update_in_model)
|
|
628
|
+
end
|
|
592
629
|
end
|
|
593
|
-
end
|
|
594
630
|
|
|
595
|
-
|
|
631
|
+
array_of_attributes = []
|
|
596
632
|
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
633
|
+
models.each do |model|
|
|
634
|
+
if supports_setting_primary_key_of_imported_objects?(conn)
|
|
635
|
+
load_association_ids(model)
|
|
636
|
+
end
|
|
601
637
|
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
638
|
+
if is_validating && !validator.valid_model?(model)
|
|
639
|
+
raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
|
|
640
|
+
next
|
|
641
|
+
end
|
|
606
642
|
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
643
|
+
array_of_attributes << column_names.map do |name|
|
|
644
|
+
if model.persisted? &&
|
|
645
|
+
update_attrs && update_attrs.include?(name.to_sym) &&
|
|
646
|
+
!model.send("#{name}_changed?")
|
|
647
|
+
nil
|
|
648
|
+
else
|
|
649
|
+
model.read_attribute(name.to_s)
|
|
650
|
+
end
|
|
614
651
|
end
|
|
615
652
|
end
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
end
|
|
653
|
+
# supports array of hash objects
|
|
654
|
+
elsif args.last.is_a?( Array ) && args.last.first.is_a?(Hash)
|
|
655
|
+
if args.length == 2
|
|
656
|
+
array_of_hashes = args.last
|
|
657
|
+
column_names = args.first.dup
|
|
658
|
+
allow_extra_hash_keys = true
|
|
659
|
+
else
|
|
660
|
+
array_of_hashes = args.first
|
|
661
|
+
column_names = array_of_hashes.first.keys
|
|
662
|
+
allow_extra_hash_keys = false
|
|
663
|
+
end
|
|
628
664
|
|
|
629
|
-
|
|
630
|
-
|
|
665
|
+
array_of_attributes = array_of_hashes.map do |h|
|
|
666
|
+
error_message = validate_hash_import(h, column_names, allow_extra_hash_keys)
|
|
631
667
|
|
|
632
|
-
|
|
668
|
+
raise ArgumentError, error_message if error_message
|
|
633
669
|
|
|
634
|
-
|
|
635
|
-
|
|
670
|
+
column_names.map do |key|
|
|
671
|
+
h[key]
|
|
672
|
+
end
|
|
673
|
+
end
|
|
674
|
+
# supports empty array
|
|
675
|
+
elsif args.last.is_a?( Array ) && args.last.empty?
|
|
676
|
+
return ActiveRecord::Import::Result.new([], 0, [], [])
|
|
677
|
+
# supports 2-element array and array
|
|
678
|
+
elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
|
|
679
|
+
|
|
680
|
+
unless args.last.first.is_a?(Array)
|
|
681
|
+
raise ArgumentError, "Last argument should be a two dimensional array '[[]]'. First element in array was a #{args.last.first.class}"
|
|
636
682
|
end
|
|
637
|
-
end
|
|
638
|
-
# supports empty array
|
|
639
|
-
elsif args.last.is_a?( Array ) && args.last.empty?
|
|
640
|
-
return ActiveRecord::Import::Result.new([], 0, [], [])
|
|
641
|
-
# supports 2-element array and array
|
|
642
|
-
elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
|
|
643
|
-
|
|
644
|
-
unless args.last.first.is_a?(Array)
|
|
645
|
-
raise ArgumentError, "Last argument should be a two dimensional array '[[]]'. First element in array was a #{args.last.first.class}"
|
|
646
|
-
end
|
|
647
683
|
|
|
648
|
-
|
|
684
|
+
column_names, array_of_attributes = args
|
|
649
685
|
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
686
|
+
# dup the passed args so we don't modify unintentionally
|
|
687
|
+
column_names = column_names.dup
|
|
688
|
+
array_of_attributes = array_of_attributes.map(&:dup)
|
|
689
|
+
else
|
|
690
|
+
raise ArgumentError, "Invalid arguments!"
|
|
691
|
+
end
|
|
656
692
|
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
693
|
+
# Force the primary key col into the insert if it's not
|
|
694
|
+
# on the list and we are using a sequence and stuff a nil
|
|
695
|
+
# value for it into each row so the sequencer will fire later
|
|
696
|
+
symbolized_column_names = Array(column_names).map(&:to_sym)
|
|
697
|
+
symbolized_primary_key = Array(primary_key).map(&:to_sym)
|
|
698
|
+
|
|
699
|
+
if !symbolized_primary_key.to_set.subset?(symbolized_column_names.to_set) && conn.prefetch_primary_key? && sequence_name
|
|
700
|
+
column_count = column_names.size
|
|
701
|
+
column_names.concat(Array(primary_key)).uniq!
|
|
702
|
+
columns_added = column_names.size - column_count
|
|
703
|
+
new_fields = Array.new(columns_added)
|
|
704
|
+
array_of_attributes.each { |a| a.concat(new_fields) }
|
|
705
|
+
end
|
|
670
706
|
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
707
|
+
# Don't modify incoming arguments
|
|
708
|
+
on_duplicate_key_update = options[:on_duplicate_key_update]
|
|
709
|
+
if on_duplicate_key_update
|
|
710
|
+
updatable_columns = symbolized_column_names.reject { |c| symbolized_primary_key.include? c }
|
|
711
|
+
options[:on_duplicate_key_update] = if on_duplicate_key_update.is_a?(Hash)
|
|
712
|
+
on_duplicate_key_update.each_with_object({}) do |(k, v), duped_options|
|
|
713
|
+
duped_options[k] = if k == :columns && v == :all
|
|
714
|
+
updatable_columns
|
|
715
|
+
elsif v.duplicable?
|
|
716
|
+
v.dup
|
|
717
|
+
else
|
|
718
|
+
v
|
|
719
|
+
end
|
|
683
720
|
end
|
|
721
|
+
elsif on_duplicate_key_update == :all
|
|
722
|
+
updatable_columns
|
|
723
|
+
elsif on_duplicate_key_update.duplicable?
|
|
724
|
+
on_duplicate_key_update.dup
|
|
725
|
+
else
|
|
726
|
+
on_duplicate_key_update
|
|
684
727
|
end
|
|
685
|
-
elsif on_duplicate_key_update == :all
|
|
686
|
-
updatable_columns
|
|
687
|
-
elsif on_duplicate_key_update.duplicable?
|
|
688
|
-
on_duplicate_key_update.dup
|
|
689
|
-
else
|
|
690
|
-
on_duplicate_key_update
|
|
691
728
|
end
|
|
692
|
-
end
|
|
693
729
|
|
|
694
|
-
|
|
730
|
+
timestamps = {}
|
|
695
731
|
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
732
|
+
# record timestamps unless disabled in ActiveRecord::Base
|
|
733
|
+
if record_timestamps && options[:timestamps]
|
|
734
|
+
timestamps = add_special_rails_stamps conn, column_names, array_of_attributes, options
|
|
735
|
+
end
|
|
700
736
|
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
737
|
+
return_obj = if is_validating
|
|
738
|
+
import_with_validations( conn, column_names, array_of_attributes, options ) do |failed_instances|
|
|
739
|
+
if models
|
|
740
|
+
models.each_with_index do |m, i|
|
|
741
|
+
next unless m.errors.any?
|
|
706
742
|
|
|
707
|
-
|
|
708
|
-
end
|
|
709
|
-
else
|
|
710
|
-
# create instances for each of our column/value sets
|
|
711
|
-
arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
|
|
712
|
-
|
|
713
|
-
# keep track of the instance and the position it is currently at. if this fails
|
|
714
|
-
# validation we'll use the index to remove it from the array_of_attributes
|
|
715
|
-
arr.each_with_index do |hsh, i|
|
|
716
|
-
# utilize block initializer syntax to prevent failure when 'mass_assignment_sanitizer = :strict'
|
|
717
|
-
model = new do |m|
|
|
718
|
-
hsh.each_pair { |k, v| m[k] = v }
|
|
743
|
+
failed_instances << (options[:track_validation_failures] ? [i, m] : m)
|
|
719
744
|
end
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
745
|
+
else
|
|
746
|
+
# create instances for each of our column/value sets
|
|
747
|
+
arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
|
|
748
|
+
|
|
749
|
+
# keep track of the instance and the position it is currently at. if this fails
|
|
750
|
+
# validation we'll use the index to remove it from the array_of_attributes
|
|
751
|
+
arr.each_with_index do |hsh, i|
|
|
752
|
+
# utilize block initializer syntax to prevent failure when 'mass_assignment_sanitizer = :strict'
|
|
753
|
+
model = new do |m|
|
|
754
|
+
hsh.each_pair { |k, v| m[k] = v }
|
|
755
|
+
end
|
|
756
|
+
|
|
757
|
+
next if validator.valid_model?(model)
|
|
758
|
+
raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
|
|
759
|
+
|
|
760
|
+
array_of_attributes[i] = nil
|
|
761
|
+
failure = model.dup
|
|
762
|
+
failure.errors.send(:initialize_dup, model.errors)
|
|
763
|
+
failed_instances << (options[:track_validation_failures] ? [i, failure] : failure )
|
|
764
|
+
end
|
|
765
|
+
array_of_attributes.compact!
|
|
728
766
|
end
|
|
729
|
-
array_of_attributes.compact!
|
|
730
767
|
end
|
|
768
|
+
else
|
|
769
|
+
import_without_validations_or_callbacks( conn, column_names, array_of_attributes, options )
|
|
731
770
|
end
|
|
732
|
-
else
|
|
733
|
-
import_without_validations_or_callbacks( column_names, array_of_attributes, options )
|
|
734
|
-
end
|
|
735
771
|
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
772
|
+
if options[:synchronize]
|
|
773
|
+
sync_keys = options[:synchronize_keys] || Array(primary_key)
|
|
774
|
+
synchronize( options[:synchronize], sync_keys)
|
|
775
|
+
end
|
|
776
|
+
return_obj.num_inserts = 0 if return_obj.num_inserts.nil?
|
|
741
777
|
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
778
|
+
# if we have ids, then set the id on the models and mark the models as clean.
|
|
779
|
+
if models && supports_setting_primary_key_of_imported_objects?(conn)
|
|
780
|
+
set_attributes_and_mark_clean(conn, models, return_obj, timestamps, options)
|
|
745
781
|
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
782
|
+
# if there are auto-save associations on the models we imported that are new, import them as well
|
|
783
|
+
if options[:recursive]
|
|
784
|
+
options[:on_duplicate_key_update] = on_duplicate_key_update unless on_duplicate_key_update.nil?
|
|
785
|
+
import_associations(models, options.dup.merge(validate: false))
|
|
786
|
+
end
|
|
750
787
|
end
|
|
751
|
-
end
|
|
752
788
|
|
|
753
|
-
|
|
789
|
+
return_obj
|
|
790
|
+
end
|
|
754
791
|
end
|
|
755
792
|
|
|
756
793
|
# Imports the passed in +column_names+ and +array_of_attributes+
|
|
@@ -760,7 +797,7 @@ class ActiveRecord::Base
|
|
|
760
797
|
# +num_inserts+ is the number of inserts it took to import the data. See
|
|
761
798
|
# ActiveRecord::Base.import for more information on
|
|
762
799
|
# +column_names+, +array_of_attributes+ and +options+.
|
|
763
|
-
def import_with_validations( column_names, array_of_attributes, options = {} )
|
|
800
|
+
def import_with_validations( conn, column_names, array_of_attributes, options = {} )
|
|
764
801
|
failed_instances = []
|
|
765
802
|
|
|
766
803
|
yield failed_instances if block_given?
|
|
@@ -768,9 +805,9 @@ class ActiveRecord::Base
|
|
|
768
805
|
result = if options[:all_or_none] && failed_instances.any?
|
|
769
806
|
ActiveRecord::Import::Result.new([], 0, [], [])
|
|
770
807
|
else
|
|
771
|
-
import_without_validations_or_callbacks( column_names, array_of_attributes, options )
|
|
808
|
+
import_without_validations_or_callbacks( conn, column_names, array_of_attributes, options )
|
|
772
809
|
end
|
|
773
|
-
ActiveRecord::Import::Result.new(failed_instances, result.num_inserts, result.ids, result.results)
|
|
810
|
+
ActiveRecord::Import::Result.new(failed_instances, result.num_inserts, result.ids, result.results, result.affected_rows)
|
|
774
811
|
end
|
|
775
812
|
|
|
776
813
|
# Imports the passed in +column_names+ and +array_of_attributes+
|
|
@@ -779,7 +816,7 @@ class ActiveRecord::Base
|
|
|
779
816
|
# validations or callbacks. See ActiveRecord::Base.import for more
|
|
780
817
|
# information on +column_names+, +array_of_attributes_ and
|
|
781
818
|
# +options+.
|
|
782
|
-
def import_without_validations_or_callbacks( column_names, array_of_attributes, options = {} )
|
|
819
|
+
def import_without_validations_or_callbacks( conn, column_names, array_of_attributes, options = {} )
|
|
783
820
|
return ActiveRecord::Import::Result.new([], 0, [], []) if array_of_attributes.empty?
|
|
784
821
|
|
|
785
822
|
column_names = column_names.map do |name|
|
|
@@ -803,23 +840,25 @@ class ActiveRecord::Base
|
|
|
803
840
|
end
|
|
804
841
|
|
|
805
842
|
columns = column_names.each_with_index.map do |name, i|
|
|
806
|
-
column = schema_columns_hash[name.to_s]
|
|
843
|
+
column = schema_columns_hash(conn)[name.to_s]
|
|
807
844
|
raise ActiveRecord::Import::MissingColumnError.new(name.to_s, i) if column.nil?
|
|
808
845
|
column
|
|
809
846
|
end
|
|
810
847
|
|
|
811
|
-
columns_sql = "(#{column_names.map { |name|
|
|
812
|
-
pre_sql_statements =
|
|
848
|
+
columns_sql = "(#{column_names.map { |name| conn.quote_column_name(name) }.join(',')})"
|
|
849
|
+
pre_sql_statements = conn.pre_sql_statements( options )
|
|
813
850
|
insert_sql = ['INSERT', pre_sql_statements, "INTO #{quoted_table_name} #{columns_sql} VALUES "]
|
|
814
851
|
insert_sql = insert_sql.flatten.join(' ')
|
|
815
|
-
values_sql = values_sql_for_columns_and_attributes(columns, array_of_attributes)
|
|
852
|
+
values_sql = values_sql_for_columns_and_attributes(conn, columns, array_of_attributes)
|
|
816
853
|
|
|
817
854
|
number_inserted = 0
|
|
818
855
|
ids = []
|
|
819
856
|
results = []
|
|
820
|
-
|
|
857
|
+
affected_rows = 0
|
|
858
|
+
|
|
859
|
+
if supports_import?(conn)
|
|
821
860
|
# generate the sql
|
|
822
|
-
post_sql_statements =
|
|
861
|
+
post_sql_statements = conn.post_sql_statements( quoted_table_name, options )
|
|
823
862
|
import_size = values_sql.size
|
|
824
863
|
|
|
825
864
|
batch_size = options[:batch_size] || import_size
|
|
@@ -832,7 +871,7 @@ class ActiveRecord::Base
|
|
|
832
871
|
batch_started_at = Time.now.to_i
|
|
833
872
|
|
|
834
873
|
# perform the inserts
|
|
835
|
-
result =
|
|
874
|
+
result = conn.insert_many( [insert_sql, post_sql_statements].flatten,
|
|
836
875
|
batch_values,
|
|
837
876
|
options,
|
|
838
877
|
"#{model_name} Create Many" )
|
|
@@ -840,6 +879,7 @@ class ActiveRecord::Base
|
|
|
840
879
|
number_inserted += result.num_inserts
|
|
841
880
|
ids += result.ids
|
|
842
881
|
results += result.results
|
|
882
|
+
affected_rows = result.affected_rows.nil? ? nil : (affected_rows || 0) + result.affected_rows
|
|
843
883
|
current_batch += 1
|
|
844
884
|
|
|
845
885
|
progress_proc.call(import_size, batches, current_batch, Time.now.to_i - batch_started_at) if run_proc
|
|
@@ -847,12 +887,12 @@ class ActiveRecord::Base
|
|
|
847
887
|
else
|
|
848
888
|
transaction(requires_new: true) do
|
|
849
889
|
values_sql.each do |values|
|
|
850
|
-
ids <<
|
|
890
|
+
ids << conn.insert(insert_sql + values)
|
|
851
891
|
number_inserted += 1
|
|
852
892
|
end
|
|
853
893
|
end
|
|
854
894
|
end
|
|
855
|
-
ActiveRecord::Import::Result.new([], number_inserted, ids, results)
|
|
895
|
+
ActiveRecord::Import::Result.new([], number_inserted, ids, results, affected_rows)
|
|
856
896
|
end
|
|
857
897
|
|
|
858
898
|
private
|
|
@@ -865,7 +905,7 @@ class ActiveRecord::Base
|
|
|
865
905
|
)
|
|
866
906
|
end
|
|
867
907
|
|
|
868
|
-
def set_attributes_and_mark_clean(models, import_result, timestamps, options)
|
|
908
|
+
def set_attributes_and_mark_clean(conn, models, import_result, timestamps, options)
|
|
869
909
|
return if models.nil?
|
|
870
910
|
models -= import_result.failed_instances
|
|
871
911
|
|
|
@@ -882,7 +922,7 @@ class ActiveRecord::Base
|
|
|
882
922
|
end
|
|
883
923
|
|
|
884
924
|
deserialize_value = lambda do |column, value|
|
|
885
|
-
column = schema_columns_hash[column]
|
|
925
|
+
column = schema_columns_hash(conn)[column]
|
|
886
926
|
return value unless column
|
|
887
927
|
if respond_to?(:type_caster)
|
|
888
928
|
type = type_for_attribute(column.name)
|
|
@@ -980,9 +1020,9 @@ class ActiveRecord::Base
|
|
|
980
1020
|
end
|
|
981
1021
|
end
|
|
982
1022
|
|
|
983
|
-
def schema_columns_hash
|
|
1023
|
+
def schema_columns_hash(conn)
|
|
984
1024
|
if respond_to?(:ignored_columns) && ignored_columns.any?
|
|
985
|
-
|
|
1025
|
+
conn.schema_cache.columns_hash(table_name)
|
|
986
1026
|
else
|
|
987
1027
|
columns_hash
|
|
988
1028
|
end
|
|
@@ -1029,34 +1069,30 @@ class ActiveRecord::Base
|
|
|
1029
1069
|
|
|
1030
1070
|
# Returns SQL the VALUES for an INSERT statement given the passed in +columns+
|
|
1031
1071
|
# and +array_of_attributes+.
|
|
1032
|
-
def values_sql_for_columns_and_attributes(columns, array_of_attributes) # :nodoc:
|
|
1033
|
-
# connection gets called a *lot* in this high intensity loop.
|
|
1034
|
-
# Reuse the same one w/in the loop, otherwise it would keep being re-retreived (= lots of time for large imports)
|
|
1035
|
-
connection_memo = connection
|
|
1036
|
-
|
|
1072
|
+
def values_sql_for_columns_and_attributes(conn, columns, array_of_attributes) # :nodoc:
|
|
1037
1073
|
array_of_attributes.map do |arr|
|
|
1038
1074
|
my_values = arr.each_with_index.map do |val, j|
|
|
1039
1075
|
column = columns[j]
|
|
1040
1076
|
|
|
1041
1077
|
# be sure to query sequence_name *last*, only if cheaper tests fail, because it's costly
|
|
1042
1078
|
if val.nil? && Array(primary_key).first == column.name && !sequence_name.blank?
|
|
1043
|
-
|
|
1079
|
+
conn.next_value_for_sequence(sequence_name)
|
|
1044
1080
|
elsif val.respond_to?(:to_sql)
|
|
1045
1081
|
"(#{val.to_sql})"
|
|
1046
1082
|
elsif column
|
|
1047
1083
|
if respond_to?(:type_caster) # Rails 5.0 and higher
|
|
1048
1084
|
type = type_for_attribute(column.name)
|
|
1049
1085
|
val = !type.respond_to?(:subtype) && type.type == :boolean ? type.cast(val) : type.serialize(val)
|
|
1050
|
-
|
|
1086
|
+
conn.quote(val)
|
|
1051
1087
|
elsif column.respond_to?(:type_cast_from_user) # Rails 4.2
|
|
1052
|
-
|
|
1088
|
+
conn.quote(column.type_cast_from_user(val), column)
|
|
1053
1089
|
else # Rails 3.2, 4.0 and 4.1
|
|
1054
1090
|
if serialized_attributes.include?(column.name)
|
|
1055
1091
|
val = serialized_attributes[column.name].dump(val)
|
|
1056
1092
|
end
|
|
1057
1093
|
# Fixes #443 to support binary (i.e. bytea) columns on PG
|
|
1058
1094
|
val = column.type_cast(val) unless column.type && column.type.to_sym == :binary
|
|
1059
|
-
|
|
1095
|
+
conn.quote(val, column)
|
|
1060
1096
|
end
|
|
1061
1097
|
else
|
|
1062
1098
|
raise ArgumentError, "Number of values (#{arr.length}) exceeds number of columns (#{columns.length})"
|
|
@@ -1066,7 +1102,7 @@ class ActiveRecord::Base
|
|
|
1066
1102
|
end
|
|
1067
1103
|
end
|
|
1068
1104
|
|
|
1069
|
-
def add_special_rails_stamps( column_names, array_of_attributes, options )
|
|
1105
|
+
def add_special_rails_stamps( conn, column_names, array_of_attributes, options )
|
|
1070
1106
|
timestamp_columns = {}
|
|
1071
1107
|
timestamps = {}
|
|
1072
1108
|
|
|
@@ -1101,8 +1137,8 @@ class ActiveRecord::Base
|
|
|
1101
1137
|
array_of_attributes.each { |arr| arr << timestamp }
|
|
1102
1138
|
end
|
|
1103
1139
|
|
|
1104
|
-
if supports_on_duplicate_key_update? && action == :update
|
|
1105
|
-
|
|
1140
|
+
if supports_on_duplicate_key_update?(conn) && action == :update
|
|
1141
|
+
conn.add_column_for_on_duplicate_key_update(column, options)
|
|
1106
1142
|
end
|
|
1107
1143
|
end
|
|
1108
1144
|
end
|