activerecord-import 0.14.1 → 1.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/workflows/test.yaml +107 -0
- data/.gitignore +1 -0
- data/.rubocop.yml +74 -8
- data/Brewfile +3 -1
- data/CHANGELOG.md +448 -2
- data/Gemfile +26 -19
- data/LICENSE +21 -56
- data/README.markdown +568 -32
- data/Rakefile +5 -1
- data/activerecord-import.gemspec +8 -7
- data/benchmarks/README +2 -2
- data/benchmarks/benchmark.rb +8 -1
- data/benchmarks/lib/base.rb +2 -0
- data/benchmarks/lib/cli_parser.rb +5 -2
- data/benchmarks/lib/float.rb +2 -0
- data/benchmarks/lib/mysql2_benchmark.rb +2 -0
- data/benchmarks/lib/output_to_csv.rb +2 -0
- data/benchmarks/lib/output_to_html.rb +4 -2
- data/benchmarks/models/test_innodb.rb +2 -0
- data/benchmarks/models/test_memory.rb +2 -0
- data/benchmarks/models/test_myisam.rb +2 -0
- data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +2 -0
- data/gemfiles/4.2.gemfile +4 -3
- data/gemfiles/5.0.gemfile +4 -3
- data/gemfiles/5.1.gemfile +4 -0
- data/gemfiles/5.2.gemfile +4 -0
- data/gemfiles/6.0.gemfile +4 -0
- data/gemfiles/6.1.gemfile +4 -0
- data/gemfiles/7.0.gemfile +4 -0
- data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +6 -4
- data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +8 -0
- data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/abstract_adapter.rb +12 -16
- data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql_adapter.rb +35 -18
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +107 -28
- data/lib/activerecord-import/adapters/sqlite3_adapter.rb +141 -18
- data/lib/activerecord-import/base.rb +15 -8
- data/lib/activerecord-import/import.rb +642 -178
- data/lib/activerecord-import/mysql2.rb +2 -0
- data/lib/activerecord-import/postgresql.rb +2 -0
- data/lib/activerecord-import/sqlite3.rb +2 -0
- data/lib/activerecord-import/synchronize.rb +5 -3
- data/lib/activerecord-import/value_sets_parser.rb +29 -3
- data/lib/activerecord-import/version.rb +3 -1
- data/lib/activerecord-import.rb +5 -16
- data/test/adapters/jdbcmysql.rb +2 -0
- data/test/adapters/jdbcpostgresql.rb +2 -0
- data/test/adapters/jdbcsqlite3.rb +3 -0
- data/test/adapters/makara_postgis.rb +3 -0
- data/test/adapters/mysql2.rb +2 -0
- data/test/adapters/mysql2_makara.rb +2 -0
- data/test/adapters/mysql2spatial.rb +2 -0
- data/test/adapters/postgis.rb +2 -0
- data/test/adapters/postgresql.rb +2 -0
- data/test/adapters/postgresql_makara.rb +2 -0
- data/test/adapters/seamless_database_pool.rb +2 -0
- data/test/adapters/spatialite.rb +2 -0
- data/test/adapters/sqlite3.rb +2 -0
- data/test/{travis → github}/database.yml +7 -1
- data/test/import_test.rb +498 -32
- data/test/jdbcmysql/import_test.rb +2 -1
- data/test/jdbcpostgresql/import_test.rb +2 -1
- data/test/jdbcsqlite3/import_test.rb +6 -0
- data/test/makara_postgis/import_test.rb +10 -0
- data/test/models/account.rb +5 -0
- data/test/models/alarm.rb +4 -0
- data/test/models/animal.rb +8 -0
- data/test/models/bike_maker.rb +9 -0
- data/test/models/book.rb +4 -0
- data/test/models/car.rb +5 -0
- data/test/models/card.rb +5 -0
- data/test/models/chapter.rb +2 -0
- data/test/models/customer.rb +8 -0
- data/test/models/deck.rb +8 -0
- data/test/models/dictionary.rb +6 -0
- data/test/models/discount.rb +2 -0
- data/test/models/end_note.rb +2 -0
- data/test/models/group.rb +2 -0
- data/test/models/order.rb +8 -0
- data/test/models/playing_card.rb +4 -0
- data/test/models/promotion.rb +2 -0
- data/test/models/question.rb +2 -0
- data/test/models/rule.rb +2 -0
- data/test/models/tag.rb +7 -0
- data/test/models/tag_alias.rb +5 -0
- data/test/models/topic.rb +16 -0
- data/test/models/user.rb +5 -0
- data/test/models/user_token.rb +6 -0
- data/test/models/vendor.rb +9 -0
- data/test/models/widget.rb +18 -0
- data/test/mysql2/import_test.rb +2 -0
- data/test/mysql2_makara/import_test.rb +2 -0
- data/test/mysqlspatial2/import_test.rb +2 -0
- data/test/postgis/import_test.rb +6 -0
- data/test/postgresql/import_test.rb +2 -4
- data/test/schema/generic_schema.rb +88 -3
- data/test/schema/jdbcpostgresql_schema.rb +3 -0
- data/test/schema/mysql2_schema.rb +21 -0
- data/test/schema/postgis_schema.rb +3 -0
- data/test/schema/postgresql_schema.rb +63 -0
- data/test/schema/sqlite3_schema.rb +15 -0
- data/test/schema/version.rb +2 -0
- data/test/sqlite3/import_test.rb +4 -50
- data/test/support/active_support/test_case_extensions.rb +8 -1
- data/test/support/assertions.rb +2 -0
- data/test/support/factories.rb +17 -8
- data/test/support/generate.rb +10 -8
- data/test/support/mysql/import_examples.rb +17 -3
- data/test/support/postgresql/import_examples.rb +442 -9
- data/test/support/shared_examples/on_duplicate_key_ignore.rb +45 -0
- data/test/support/shared_examples/on_duplicate_key_update.rb +278 -1
- data/test/support/shared_examples/recursive_import.rb +137 -12
- data/test/support/sqlite3/import_examples.rb +232 -0
- data/test/synchronize_test.rb +10 -0
- data/test/test_helper.rb +44 -3
- data/test/value_sets_bytes_parser_test.rb +15 -2
- data/test/value_sets_records_parser_test.rb +2 -0
- metadata +74 -22
- data/.travis.yml +0 -52
- data/gemfiles/3.2.gemfile +0 -3
- data/gemfiles/4.0.gemfile +0 -3
- data/gemfiles/4.1.gemfile +0 -3
- data/test/schema/mysql_schema.rb +0 -16
@@ -1,9 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "ostruct"
|
2
4
|
|
3
5
|
module ActiveRecord::Import::ConnectionAdapters; end
|
4
6
|
|
5
7
|
module ActiveRecord::Import #:nodoc:
|
6
|
-
Result = Struct.new(:failed_instances, :num_inserts, :ids)
|
8
|
+
Result = Struct.new(:failed_instances, :num_inserts, :ids, :results)
|
7
9
|
|
8
10
|
module ImportSupport #:nodoc:
|
9
11
|
def supports_import? #:nodoc:
|
@@ -22,16 +24,110 @@ module ActiveRecord::Import #:nodoc:
|
|
22
24
|
super "Missing column for value <#{name}> at index #{index}"
|
23
25
|
end
|
24
26
|
end
|
27
|
+
|
28
|
+
class Validator
|
29
|
+
def initialize(klass, options = {})
|
30
|
+
@options = options
|
31
|
+
@validator_class = klass
|
32
|
+
init_validations(klass)
|
33
|
+
end
|
34
|
+
|
35
|
+
def init_validations(klass)
|
36
|
+
@validate_callbacks = klass._validate_callbacks.dup
|
37
|
+
|
38
|
+
@validate_callbacks.each_with_index do |callback, i|
|
39
|
+
filter = callback.respond_to?(:raw_filter) ? callback.raw_filter : callback.filter
|
40
|
+
next unless filter.class.name =~ /Validations::PresenceValidator/ ||
|
41
|
+
(!@options[:validate_uniqueness] &&
|
42
|
+
filter.is_a?(ActiveRecord::Validations::UniquenessValidator))
|
43
|
+
|
44
|
+
callback = callback.dup
|
45
|
+
filter = filter.dup
|
46
|
+
attrs = filter.instance_variable_get(:@attributes).dup
|
47
|
+
|
48
|
+
if filter.is_a?(ActiveRecord::Validations::UniquenessValidator)
|
49
|
+
attrs = []
|
50
|
+
else
|
51
|
+
associations = klass.reflect_on_all_associations(:belongs_to)
|
52
|
+
associations.each do |assoc|
|
53
|
+
if (index = attrs.index(assoc.name))
|
54
|
+
key = assoc.foreign_key.is_a?(Array) ? assoc.foreign_key.map(&:to_sym) : assoc.foreign_key.to_sym
|
55
|
+
attrs[index] = key unless attrs.include?(key)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
filter.instance_variable_set(:@attributes, attrs.flatten)
|
61
|
+
|
62
|
+
if @validate_callbacks.respond_to?(:chain, true)
|
63
|
+
@validate_callbacks.send(:chain).tap do |chain|
|
64
|
+
callback.instance_variable_set(:@filter, filter)
|
65
|
+
chain[i] = callback
|
66
|
+
end
|
67
|
+
else
|
68
|
+
callback.raw_filter = filter
|
69
|
+
callback.filter = callback.send(:_compile_filter, filter)
|
70
|
+
@validate_callbacks[i] = callback
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def valid_model?(model)
|
76
|
+
init_validations(model.class) unless model.class == @validator_class
|
77
|
+
|
78
|
+
validation_context = @options[:validate_with_context]
|
79
|
+
validation_context ||= (model.new_record? ? :create : :update)
|
80
|
+
current_context = model.send(:validation_context)
|
81
|
+
|
82
|
+
begin
|
83
|
+
model.send(:validation_context=, validation_context)
|
84
|
+
model.errors.clear
|
85
|
+
|
86
|
+
model.run_callbacks(:validation) do
|
87
|
+
if defined?(ActiveSupport::Callbacks::Filters::Environment) # ActiveRecord >= 4.1
|
88
|
+
runner = @validate_callbacks.compile
|
89
|
+
env = ActiveSupport::Callbacks::Filters::Environment.new(model, false, nil)
|
90
|
+
if runner.respond_to?(:call) # ActiveRecord < 5.1
|
91
|
+
runner.call(env)
|
92
|
+
else # ActiveRecord 5.1
|
93
|
+
# Note that this is a gross simplification of ActiveSupport::Callbacks#run_callbacks.
|
94
|
+
# It's technically possible for there to exist an "around" callback in the
|
95
|
+
# :validate chain, but this would be an aberration, since Rails doesn't define
|
96
|
+
# "around_validate". Still, rather than silently ignoring such callbacks, we
|
97
|
+
# explicitly raise a RuntimeError, since activerecord-import was asked to perform
|
98
|
+
# validations and it's unable to do so.
|
99
|
+
#
|
100
|
+
# The alternative here would be to copy-and-paste the bulk of the
|
101
|
+
# ActiveSupport::Callbacks#run_callbacks method, which is undesirable if there's
|
102
|
+
# no real-world use case for it.
|
103
|
+
raise "The :validate callback chain contains an 'around' callback, which is unsupported" unless runner.final?
|
104
|
+
runner.invoke_before(env)
|
105
|
+
runner.invoke_after(env)
|
106
|
+
end
|
107
|
+
elsif @validate_callbacks.method(:compile).arity == 0 # ActiveRecord = 4.0
|
108
|
+
model.instance_eval @validate_callbacks.compile
|
109
|
+
else # ActiveRecord 3.x
|
110
|
+
model.instance_eval @validate_callbacks.compile(nil, model)
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
model.errors.empty?
|
115
|
+
ensure
|
116
|
+
model.send(:validation_context=, current_context)
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
25
120
|
end
|
26
121
|
|
27
122
|
class ActiveRecord::Associations::CollectionProxy
|
28
|
-
def
|
29
|
-
@association.
|
123
|
+
def bulk_import(*args, &block)
|
124
|
+
@association.bulk_import(*args, &block)
|
30
125
|
end
|
126
|
+
alias import bulk_import unless respond_to? :import
|
31
127
|
end
|
32
128
|
|
33
129
|
class ActiveRecord::Associations::CollectionAssociation
|
34
|
-
def
|
130
|
+
def bulk_import(*args, &block)
|
35
131
|
unless owner.persisted?
|
36
132
|
raise ActiveRecord::RecordNotSaved, "You cannot call import unless the parent is saved"
|
37
133
|
end
|
@@ -40,16 +136,21 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
40
136
|
|
41
137
|
model_klass = reflection.klass
|
42
138
|
symbolized_foreign_key = reflection.foreign_key.to_sym
|
43
|
-
symbolized_column_names = model_klass.column_names.map(&:to_sym)
|
44
139
|
|
45
|
-
|
140
|
+
symbolized_column_names = if model_klass.connection.respond_to?(:supports_virtual_columns?) && model_klass.connection.supports_virtual_columns?
|
141
|
+
model_klass.columns.reject(&:virtual?).map { |c| c.name.to_sym }
|
142
|
+
else
|
143
|
+
model_klass.column_names.map(&:to_sym)
|
144
|
+
end
|
145
|
+
|
146
|
+
owner_primary_key = reflection.active_record_primary_key.to_sym
|
46
147
|
owner_primary_key_value = owner.send(owner_primary_key)
|
47
148
|
|
48
149
|
# assume array of model objects
|
49
150
|
if args.last.is_a?( Array ) && args.last.first.is_a?(ActiveRecord::Base)
|
50
151
|
if args.length == 2
|
51
152
|
models = args.last
|
52
|
-
column_names = args.first
|
153
|
+
column_names = args.first.dup
|
53
154
|
else
|
54
155
|
models = args.first
|
55
156
|
column_names = symbolized_column_names
|
@@ -64,7 +165,46 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
64
165
|
m.public_send "#{reflection.type}=", owner.class.name if reflection.type
|
65
166
|
end
|
66
167
|
|
67
|
-
return model_klass.
|
168
|
+
return model_klass.bulk_import column_names, models, options
|
169
|
+
|
170
|
+
# supports array of hash objects
|
171
|
+
elsif args.last.is_a?( Array ) && args.last.first.is_a?(Hash)
|
172
|
+
if args.length == 2
|
173
|
+
array_of_hashes = args.last
|
174
|
+
column_names = args.first.dup
|
175
|
+
allow_extra_hash_keys = true
|
176
|
+
else
|
177
|
+
array_of_hashes = args.first
|
178
|
+
column_names = array_of_hashes.first.keys
|
179
|
+
allow_extra_hash_keys = false
|
180
|
+
end
|
181
|
+
|
182
|
+
symbolized_column_names = column_names.map(&:to_sym)
|
183
|
+
unless symbolized_column_names.include?(symbolized_foreign_key)
|
184
|
+
column_names << symbolized_foreign_key
|
185
|
+
end
|
186
|
+
|
187
|
+
if reflection.type && !symbolized_column_names.include?(reflection.type.to_sym)
|
188
|
+
column_names << reflection.type.to_sym
|
189
|
+
end
|
190
|
+
|
191
|
+
array_of_attributes = array_of_hashes.map do |h|
|
192
|
+
error_message = model_klass.send(:validate_hash_import, h, symbolized_column_names, allow_extra_hash_keys)
|
193
|
+
|
194
|
+
raise ArgumentError, error_message if error_message
|
195
|
+
|
196
|
+
column_names.map do |key|
|
197
|
+
if key == symbolized_foreign_key
|
198
|
+
owner_primary_key_value
|
199
|
+
elsif reflection.type && key == reflection.type.to_sym
|
200
|
+
owner.class.name
|
201
|
+
else
|
202
|
+
h[key]
|
203
|
+
end
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
return model_klass.bulk_import column_names, array_of_attributes, options
|
68
208
|
|
69
209
|
# supports empty array
|
70
210
|
elsif args.last.is_a?( Array ) && args.last.empty?
|
@@ -73,7 +213,12 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
73
213
|
# supports 2-element array and array
|
74
214
|
elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
|
75
215
|
column_names, array_of_attributes = args
|
76
|
-
|
216
|
+
|
217
|
+
# dup the passed args so we don't modify unintentionally
|
218
|
+
column_names = column_names.dup
|
219
|
+
array_of_attributes = array_of_attributes.map(&:dup)
|
220
|
+
|
221
|
+
symbolized_column_names = column_names.map(&:to_sym)
|
77
222
|
|
78
223
|
if symbolized_column_names.include?(symbolized_foreign_key)
|
79
224
|
index = symbolized_column_names.index(symbolized_foreign_key)
|
@@ -84,31 +229,35 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
84
229
|
end
|
85
230
|
|
86
231
|
if reflection.type
|
87
|
-
|
88
|
-
|
232
|
+
symbolized_type = reflection.type.to_sym
|
233
|
+
if symbolized_column_names.include?(symbolized_type)
|
234
|
+
index = symbolized_column_names.index(symbolized_type)
|
235
|
+
array_of_attributes.each { |attrs| attrs[index] = owner.class.name }
|
236
|
+
else
|
237
|
+
column_names << symbolized_type
|
238
|
+
array_of_attributes.each { |attrs| attrs << owner.class.name }
|
239
|
+
end
|
89
240
|
end
|
90
241
|
|
91
|
-
return model_klass.
|
242
|
+
return model_klass.bulk_import column_names, array_of_attributes, options
|
92
243
|
else
|
93
244
|
raise ArgumentError, "Invalid arguments!"
|
94
245
|
end
|
95
246
|
end
|
247
|
+
alias import bulk_import unless respond_to? :import
|
248
|
+
end
|
249
|
+
|
250
|
+
module ActiveRecord::Import::Connection
|
251
|
+
def establish_connection(args = nil)
|
252
|
+
conn = super(args)
|
253
|
+
ActiveRecord::Import.load_from_connection_pool connection_pool
|
254
|
+
conn
|
255
|
+
end
|
96
256
|
end
|
97
257
|
|
98
258
|
class ActiveRecord::Base
|
99
259
|
class << self
|
100
|
-
|
101
|
-
tproc = lambda do
|
102
|
-
ActiveRecord::Base.default_timezone == :utc ? Time.now.utc : Time.now
|
103
|
-
end
|
104
|
-
|
105
|
-
AREXT_RAILS_COLUMNS = {
|
106
|
-
create: { "created_on" => tproc,
|
107
|
-
"created_at" => tproc },
|
108
|
-
update: { "updated_on" => tproc,
|
109
|
-
"updated_at" => tproc }
|
110
|
-
}.freeze
|
111
|
-
AREXT_RAILS_COLUMN_NAMES = AREXT_RAILS_COLUMNS[:create].keys + AREXT_RAILS_COLUMNS[:update].keys
|
260
|
+
prepend ActiveRecord::Import::Connection
|
112
261
|
|
113
262
|
# Returns true if the current database connection adapter
|
114
263
|
# supports import functionality, otherwise returns false.
|
@@ -120,14 +269,14 @@ class ActiveRecord::Base
|
|
120
269
|
# supports on duplicate key update functionality, otherwise
|
121
270
|
# returns false.
|
122
271
|
def supports_on_duplicate_key_update?
|
123
|
-
connection.supports_on_duplicate_key_update?
|
272
|
+
connection.respond_to?(:supports_on_duplicate_key_update?) && connection.supports_on_duplicate_key_update?
|
124
273
|
end
|
125
274
|
|
126
275
|
# returns true if the current database connection adapter
|
127
276
|
# supports setting the primary key of bulk imported models, otherwise
|
128
277
|
# returns false
|
129
|
-
def
|
130
|
-
connection.respond_to?(:
|
278
|
+
def supports_setting_primary_key_of_imported_objects?
|
279
|
+
connection.respond_to?(:supports_setting_primary_key_of_imported_objects?) && connection.supports_setting_primary_key_of_imported_objects?
|
131
280
|
end
|
132
281
|
|
133
282
|
# Imports a collection of values to the database.
|
@@ -144,6 +293,9 @@ class ActiveRecord::Base
|
|
144
293
|
#
|
145
294
|
# == Usage
|
146
295
|
# Model.import array_of_models
|
296
|
+
# Model.import column_names, array_of_models
|
297
|
+
# Model.import array_of_hash_objects
|
298
|
+
# Model.import column_names, array_of_hash_objects
|
147
299
|
# Model.import column_names, array_of_values
|
148
300
|
# Model.import column_names, array_of_values, options
|
149
301
|
#
|
@@ -170,15 +322,21 @@ class ActiveRecord::Base
|
|
170
322
|
#
|
171
323
|
# == Options
|
172
324
|
# * +validate+ - true|false, tells import whether or not to use
|
173
|
-
#
|
174
|
-
#
|
175
|
-
#
|
176
|
-
#
|
177
|
-
#
|
178
|
-
#
|
179
|
-
#
|
180
|
-
#
|
181
|
-
#
|
325
|
+
# ActiveRecord validations. Validations are enforced by default.
|
326
|
+
# It skips the uniqueness validation for performance reasons.
|
327
|
+
# You can find more details here:
|
328
|
+
# https://github.com/zdennis/activerecord-import/issues/228
|
329
|
+
# * +ignore+ - true|false, an alias for on_duplicate_key_ignore.
|
330
|
+
# * +on_duplicate_key_ignore+ - true|false, tells import to discard
|
331
|
+
# records that contain duplicate keys. For Postgres 9.5+ it adds
|
332
|
+
# ON CONFLICT DO NOTHING, for MySQL it uses INSERT IGNORE, and for
|
333
|
+
# SQLite it uses INSERT OR IGNORE. Cannot be enabled on a
|
334
|
+
# recursive import. For database adapters that normally support
|
335
|
+
# setting primary keys on imported objects, this option prevents
|
336
|
+
# that from occurring.
|
337
|
+
# * +on_duplicate_key_update+ - :all, an Array, or Hash, tells import to
|
338
|
+
# use MySQL's ON DUPLICATE KEY UPDATE or Postgres/SQLite ON CONFLICT
|
339
|
+
# DO UPDATE ability. See On Duplicate Key Update below.
|
182
340
|
# * +synchronize+ - an array of ActiveRecord instances for the model
|
183
341
|
# that you are currently importing data into. This synchronizes
|
184
342
|
# existing model instances in memory with updates from the import.
|
@@ -186,7 +344,7 @@ class ActiveRecord::Base
|
|
186
344
|
# (if false) even if record timestamps is disabled in ActiveRecord::Base
|
187
345
|
# * +recursive+ - true|false, tells import to import all has_many/has_one
|
188
346
|
# associations if the adapter supports setting the primary keys of the
|
189
|
-
# newly imported objects.
|
347
|
+
# newly imported objects. PostgreSQL only.
|
190
348
|
# * +batch_size+ - an integer value to specify the max number of records to
|
191
349
|
# include per insert. Defaults to the total number of records to import.
|
192
350
|
#
|
@@ -199,6 +357,18 @@ class ActiveRecord::Base
|
|
199
357
|
# BlogPost.new author_name: 'Zach Dennis', title: 'AREXT3' ]
|
200
358
|
# BlogPost.import posts
|
201
359
|
#
|
360
|
+
# # Example using array_of_hash_objects
|
361
|
+
# # NOTE: column_names will be determined by using the keys of the first hash in the array. If later hashes in the
|
362
|
+
# # array have different keys an exception will be raised. If you have hashes to import with different sets of keys
|
363
|
+
# # we recommend grouping these into batches before importing.
|
364
|
+
# values = [ {author_name: 'zdennis', title: 'test post'} ], [ {author_name: 'jdoe', title: 'another test post'} ] ]
|
365
|
+
# BlogPost.import values
|
366
|
+
#
|
367
|
+
# # Example using column_names and array_of_hash_objects
|
368
|
+
# columns = [ :author_name, :title ]
|
369
|
+
# values = [ {author_name: 'zdennis', title: 'test post'} ], [ {author_name: 'jdoe', title: 'another test post'} ] ]
|
370
|
+
# BlogPost.import columns, values
|
371
|
+
#
|
202
372
|
# # Example using column_names and array_of_values
|
203
373
|
# columns = [ :author_name, :title ]
|
204
374
|
# values = [ [ 'zdennis', 'test post' ], [ 'jdoe', 'another test post' ] ]
|
@@ -224,7 +394,15 @@ class ActiveRecord::Base
|
|
224
394
|
#
|
225
395
|
# == On Duplicate Key Update (MySQL)
|
226
396
|
#
|
227
|
-
# The :on_duplicate_key_update option can be either an Array or a Hash.
|
397
|
+
# The :on_duplicate_key_update option can be either :all, an Array, or a Hash.
|
398
|
+
#
|
399
|
+
# ==== Using :all
|
400
|
+
#
|
401
|
+
# The :on_duplicate_key_update option can be set to :all. All columns
|
402
|
+
# other than the primary key are updated. If a list of column names is
|
403
|
+
# supplied, only those columns will be updated. Below is an example:
|
404
|
+
#
|
405
|
+
# BlogPost.import columns, values, on_duplicate_key_update: :all
|
228
406
|
#
|
229
407
|
# ==== Using an Array
|
230
408
|
#
|
@@ -243,10 +421,19 @@ class ActiveRecord::Base
|
|
243
421
|
#
|
244
422
|
# BlogPost.import columns, attributes, on_duplicate_key_update: { title: :title }
|
245
423
|
#
|
246
|
-
# == On Duplicate Key Update (Postgres 9.5+)
|
424
|
+
# == On Duplicate Key Update (Postgres 9.5+ and SQLite 3.24+)
|
425
|
+
#
|
426
|
+
# The :on_duplicate_key_update option can be :all, an Array, or a Hash with up to
|
427
|
+
# three attributes, :conflict_target (and optionally :index_predicate) or
|
428
|
+
# :constraint_name (Postgres), and :columns.
|
429
|
+
#
|
430
|
+
# ==== Using :all
|
247
431
|
#
|
248
|
-
# The :on_duplicate_key_update option can be
|
249
|
-
#
|
432
|
+
# The :on_duplicate_key_update option can be set to :all. All columns
|
433
|
+
# other than the primary key are updated. If a list of column names is
|
434
|
+
# supplied, only those columns will be updated. Below is an example:
|
435
|
+
#
|
436
|
+
# BlogPost.import columns, values, on_duplicate_key_update: :all
|
250
437
|
#
|
251
438
|
# ==== Using an Array
|
252
439
|
#
|
@@ -260,12 +447,13 @@ class ActiveRecord::Base
|
|
260
447
|
#
|
261
448
|
# ==== Using a Hash
|
262
449
|
#
|
263
|
-
# The :
|
264
|
-
# :conflict_target
|
265
|
-
#
|
266
|
-
#
|
450
|
+
# The :on_duplicate_key_update option can be a hash with up to three
|
451
|
+
# attributes, :conflict_target (and optionally :index_predicate) or
|
452
|
+
# :constraint_name, and :columns. Unlike MySQL, Postgres requires the
|
453
|
+
# conflicting constraint to be explicitly specified. Using this option
|
454
|
+
# allows you to specify a constraint other than the primary key.
|
267
455
|
#
|
268
|
-
#
|
456
|
+
# ===== :conflict_target
|
269
457
|
#
|
270
458
|
# The :conflict_target attribute specifies the columns that make up the
|
271
459
|
# conflicting unique constraint and can be a single column or an array of
|
@@ -273,9 +461,18 @@ class ActiveRecord::Base
|
|
273
461
|
# but it is the preferred method of identifying a constraint. It will
|
274
462
|
# default to the primary key. Below is an example:
|
275
463
|
#
|
276
|
-
# BlogPost.import columns, values, on_duplicate_key_update: { conflict_target: [:author_id, :slug], columns: [ :date_modified ] }
|
464
|
+
# BlogPost.import columns, values, on_duplicate_key_update: { conflict_target: [ :author_id, :slug ], columns: [ :date_modified ] }
|
465
|
+
#
|
466
|
+
# ===== :index_predicate
|
467
|
+
#
|
468
|
+
# The :index_predicate attribute optionally specifies a WHERE condition
|
469
|
+
# on :conflict_target, which is required for matching against partial
|
470
|
+
# indexes. This attribute is ignored if :constraint_name is included.
|
471
|
+
# Below is an example:
|
472
|
+
#
|
473
|
+
# BlogPost.import columns, values, on_duplicate_key_update: { conflict_target: [ :author_id, :slug ], index_predicate: 'status <> 0', columns: [ :date_modified ] }
|
277
474
|
#
|
278
|
-
#
|
475
|
+
# ===== :constraint_name
|
279
476
|
#
|
280
477
|
# The :constraint_name attribute explicitly identifies the conflicting
|
281
478
|
# unique index by name. Postgres documentation discourages using this method
|
@@ -283,11 +480,28 @@ class ActiveRecord::Base
|
|
283
480
|
#
|
284
481
|
# BlogPost.import columns, values, on_duplicate_key_update: { constraint_name: :blog_posts_pkey, columns: [ :date_modified ] }
|
285
482
|
#
|
286
|
-
#
|
483
|
+
# ===== :condition
|
484
|
+
#
|
485
|
+
# The :condition attribute optionally specifies a WHERE condition
|
486
|
+
# on :conflict_action. Only rows for which this expression returns true will be updated.
|
487
|
+
# Note that it's evaluated last, after a conflict has been identified as a candidate to update.
|
488
|
+
# Below is an example:
|
489
|
+
#
|
490
|
+
# BlogPost.import columns, values, on_duplicate_key_update: { conflict_target: [ :author_id ], condition: "blog_posts.title NOT LIKE '%sample%'", columns: [ :author_name ] }
|
491
|
+
#
|
492
|
+
# ===== :columns
|
287
493
|
#
|
288
|
-
# The :columns attribute can be either an Array or a Hash.
|
494
|
+
# The :columns attribute can be either :all, an Array, or a Hash.
|
289
495
|
#
|
290
|
-
#
|
496
|
+
# ===== Using :all
|
497
|
+
#
|
498
|
+
# The :columns attribute can be :all. All columns other than the primary key will be updated.
|
499
|
+
# If a list of column names is supplied, only those columns will be updated.
|
500
|
+
# Below is an example:
|
501
|
+
#
|
502
|
+
# BlogPost.import columns, values, on_duplicate_key_update: { conflict_target: :slug, columns: :all }
|
503
|
+
#
|
504
|
+
# ===== Using an Array
|
291
505
|
#
|
292
506
|
# The :columns attribute can be an array of column names. The column names
|
293
507
|
# are the only fields that are updated if a duplicate record is found.
|
@@ -295,7 +509,7 @@ class ActiveRecord::Base
|
|
295
509
|
#
|
296
510
|
# BlogPost.import columns, values, on_duplicate_key_update: { conflict_target: :slug, columns: [ :date_modified, :content, :author ] }
|
297
511
|
#
|
298
|
-
#
|
512
|
+
# ===== Using a Hash
|
299
513
|
#
|
300
514
|
# The :columns option can be a hash of column names to model attribute name
|
301
515
|
# mappings. This gives you finer grained control over what fields are updated
|
@@ -307,8 +521,9 @@ class ActiveRecord::Base
|
|
307
521
|
# This returns an object which responds to +failed_instances+ and +num_inserts+.
|
308
522
|
# * failed_instances - an array of objects that fails validation and were not committed to the database. An empty array if no validation is performed.
|
309
523
|
# * num_inserts - the number of insert statements it took to import the data
|
310
|
-
# * ids - the primary keys of the imported ids
|
311
|
-
|
524
|
+
# * ids - the primary keys of the imported ids if the adapter supports it, otherwise an empty array.
|
525
|
+
# * results - import results if the adapter supports it, otherwise an empty array.
|
526
|
+
def bulk_import(*args)
|
312
527
|
if args.first.is_a?( Array ) && args.first.first.is_a?(ActiveRecord::Base)
|
313
528
|
options = {}
|
314
529
|
options.merge!( args.pop ) if args.last.is_a?(Hash)
|
@@ -319,115 +534,217 @@ class ActiveRecord::Base
|
|
319
534
|
import_helper(*args)
|
320
535
|
end
|
321
536
|
end
|
537
|
+
alias import bulk_import unless ActiveRecord::Base.respond_to? :import
|
322
538
|
|
323
539
|
# Imports a collection of values if all values are valid. Import fails at the
|
324
540
|
# first encountered validation error and raises ActiveRecord::RecordInvalid
|
325
541
|
# with the failed instance.
|
326
|
-
def
|
542
|
+
def bulk_import!(*args)
|
327
543
|
options = args.last.is_a?( Hash ) ? args.pop : {}
|
328
544
|
options[:validate] = true
|
329
545
|
options[:raise_error] = true
|
330
546
|
|
331
|
-
|
547
|
+
bulk_import(*args, options)
|
332
548
|
end
|
549
|
+
alias import! bulk_import! unless ActiveRecord::Base.respond_to? :import!
|
333
550
|
|
334
551
|
def import_helper( *args )
|
335
|
-
options = { validate: true, timestamps: true,
|
552
|
+
options = { model: self, validate: true, timestamps: true, track_validation_failures: false }
|
336
553
|
options.merge!( args.pop ) if args.last.is_a? Hash
|
554
|
+
# making sure that current model's primary key is used
|
555
|
+
options[:primary_key] = primary_key
|
556
|
+
options[:locking_column] = locking_column if attribute_names.include?(locking_column)
|
337
557
|
|
338
|
-
|
339
|
-
|
340
|
-
options[:on_duplicate_key_update] = options[:on_duplicate_key_update].dup
|
341
|
-
end
|
342
|
-
|
343
|
-
is_validating = options[:validate]
|
344
|
-
is_validating = true unless options[:validate_with_context].nil?
|
558
|
+
is_validating = options[:validate_with_context].present? ? true : options[:validate]
|
559
|
+
validator = ActiveRecord::Import::Validator.new(self, options)
|
345
560
|
|
346
561
|
# assume array of model objects
|
347
562
|
if args.last.is_a?( Array ) && args.last.first.is_a?(ActiveRecord::Base)
|
348
563
|
if args.length == 2
|
349
564
|
models = args.last
|
350
|
-
column_names = args.first
|
565
|
+
column_names = args.first.dup
|
351
566
|
else
|
352
567
|
models = args.first
|
353
|
-
column_names =
|
568
|
+
column_names = if connection.respond_to?(:supports_virtual_columns?) && connection.supports_virtual_columns?
|
569
|
+
columns.reject(&:virtual?).map(&:name)
|
570
|
+
else
|
571
|
+
self.column_names.dup
|
572
|
+
end
|
573
|
+
end
|
574
|
+
|
575
|
+
if models.first.id.nil?
|
576
|
+
Array(primary_key).each do |c|
|
577
|
+
if column_names.include?(c) && columns_hash[c].type == :uuid
|
578
|
+
column_names.delete(c)
|
579
|
+
end
|
580
|
+
end
|
354
581
|
end
|
355
582
|
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
583
|
+
update_attrs = if record_timestamps && options[:timestamps]
|
584
|
+
if respond_to?(:timestamp_attributes_for_update, true)
|
585
|
+
send(:timestamp_attributes_for_update).map(&:to_sym)
|
586
|
+
else
|
587
|
+
allocate.send(:timestamp_attributes_for_update_in_model)
|
588
|
+
end
|
589
|
+
end
|
590
|
+
|
591
|
+
array_of_attributes = []
|
592
|
+
|
593
|
+
models.each do |model|
|
594
|
+
if supports_setting_primary_key_of_imported_objects?
|
595
|
+
load_association_ids(model)
|
596
|
+
end
|
597
|
+
|
598
|
+
if is_validating && !validator.valid_model?(model)
|
599
|
+
raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
|
600
|
+
next
|
601
|
+
end
|
602
|
+
|
603
|
+
array_of_attributes << column_names.map do |name|
|
604
|
+
if model.persisted? &&
|
605
|
+
update_attrs && update_attrs.include?(name.to_sym) &&
|
606
|
+
!model.send("#{name}_changed?")
|
607
|
+
nil
|
608
|
+
else
|
609
|
+
model.read_attribute(name.to_s)
|
610
|
+
end
|
611
|
+
end
|
612
|
+
end
|
613
|
+
# supports array of hash objects
|
614
|
+
elsif args.last.is_a?( Array ) && args.last.first.is_a?(Hash)
|
615
|
+
if args.length == 2
|
616
|
+
array_of_hashes = args.last
|
617
|
+
column_names = args.first.dup
|
618
|
+
allow_extra_hash_keys = true
|
619
|
+
else
|
620
|
+
array_of_hashes = args.first
|
621
|
+
column_names = array_of_hashes.first.keys
|
622
|
+
allow_extra_hash_keys = false
|
623
|
+
end
|
624
|
+
|
625
|
+
array_of_attributes = array_of_hashes.map do |h|
|
626
|
+
error_message = validate_hash_import(h, column_names, allow_extra_hash_keys)
|
627
|
+
|
628
|
+
raise ArgumentError, error_message if error_message
|
629
|
+
|
630
|
+
column_names.map do |key|
|
631
|
+
h[key]
|
361
632
|
end
|
362
|
-
# end
|
363
633
|
end
|
364
634
|
# supports empty array
|
365
635
|
elsif args.last.is_a?( Array ) && args.last.empty?
|
366
|
-
return ActiveRecord::Import::Result.new([], 0, [])
|
636
|
+
return ActiveRecord::Import::Result.new([], 0, [], [])
|
367
637
|
# supports 2-element array and array
|
368
638
|
elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
|
639
|
+
|
640
|
+
unless args.last.first.is_a?(Array)
|
641
|
+
raise ArgumentError, "Last argument should be a two dimensional array '[[]]'. First element in array was a #{args.last.first.class}"
|
642
|
+
end
|
643
|
+
|
369
644
|
column_names, array_of_attributes = args
|
645
|
+
|
646
|
+
# dup the passed args so we don't modify unintentionally
|
647
|
+
column_names = column_names.dup
|
648
|
+
array_of_attributes = array_of_attributes.map(&:dup)
|
370
649
|
else
|
371
650
|
raise ArgumentError, "Invalid arguments!"
|
372
651
|
end
|
373
652
|
|
374
|
-
# dup the passed in array so we don't modify it unintentionally
|
375
|
-
column_names = column_names.dup
|
376
|
-
array_of_attributes = array_of_attributes.dup
|
377
|
-
|
378
653
|
# Force the primary key col into the insert if it's not
|
379
654
|
# on the list and we are using a sequence and stuff a nil
|
380
655
|
# value for it into each row so the sequencer will fire later
|
381
|
-
|
382
|
-
|
383
|
-
|
656
|
+
symbolized_column_names = Array(column_names).map(&:to_sym)
|
657
|
+
symbolized_primary_key = Array(primary_key).map(&:to_sym)
|
658
|
+
|
659
|
+
if !symbolized_primary_key.to_set.subset?(symbolized_column_names.to_set) && connection.prefetch_primary_key? && sequence_name
|
660
|
+
column_count = column_names.size
|
661
|
+
column_names.concat(Array(primary_key)).uniq!
|
662
|
+
columns_added = column_names.size - column_count
|
663
|
+
new_fields = Array.new(columns_added)
|
664
|
+
array_of_attributes.each { |a| a.concat(new_fields) }
|
384
665
|
end
|
385
666
|
|
667
|
+
# Don't modify incoming arguments
|
668
|
+
on_duplicate_key_update = options[:on_duplicate_key_update]
|
669
|
+
if on_duplicate_key_update
|
670
|
+
updatable_columns = symbolized_column_names.reject { |c| symbolized_primary_key.include? c }
|
671
|
+
options[:on_duplicate_key_update] = if on_duplicate_key_update.is_a?(Hash)
|
672
|
+
on_duplicate_key_update.each_with_object({}) do |(k, v), duped_options|
|
673
|
+
duped_options[k] = if k == :columns && v == :all
|
674
|
+
updatable_columns
|
675
|
+
elsif v.duplicable?
|
676
|
+
v.dup
|
677
|
+
else
|
678
|
+
v
|
679
|
+
end
|
680
|
+
end
|
681
|
+
elsif on_duplicate_key_update == :all
|
682
|
+
updatable_columns
|
683
|
+
elsif on_duplicate_key_update.duplicable?
|
684
|
+
on_duplicate_key_update.dup
|
685
|
+
else
|
686
|
+
on_duplicate_key_update
|
687
|
+
end
|
688
|
+
end
|
689
|
+
|
690
|
+
timestamps = {}
|
691
|
+
|
386
692
|
# record timestamps unless disabled in ActiveRecord::Base
|
387
|
-
if record_timestamps && options
|
388
|
-
add_special_rails_stamps column_names, array_of_attributes, options
|
693
|
+
if record_timestamps && options[:timestamps]
|
694
|
+
timestamps = add_special_rails_stamps column_names, array_of_attributes, options
|
389
695
|
end
|
390
696
|
|
391
697
|
return_obj = if is_validating
|
392
|
-
|
393
|
-
|
394
|
-
models.
|
395
|
-
|
396
|
-
|
397
|
-
|
698
|
+
import_with_validations( column_names, array_of_attributes, options ) do |failed_instances|
|
699
|
+
if models
|
700
|
+
models.each { |m| failed_instances << m if m.errors.any? }
|
701
|
+
else
|
702
|
+
# create instances for each of our column/value sets
|
703
|
+
arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
|
704
|
+
|
705
|
+
# keep track of the instance and the position it is currently at. if this fails
|
706
|
+
# validation we'll use the index to remove it from the array_of_attributes
|
707
|
+
arr.each_with_index do |hsh, i|
|
708
|
+
# utilize block initializer syntax to prevent failure when 'mass_assignment_sanitizer = :strict'
|
709
|
+
model = new do |m|
|
710
|
+
hsh.each_pair { |k, v| m[k] = v }
|
711
|
+
end
|
712
|
+
|
713
|
+
next if validator.valid_model?(model)
|
714
|
+
raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
|
715
|
+
|
398
716
|
array_of_attributes[i] = nil
|
399
|
-
|
717
|
+
failure = model.dup
|
718
|
+
failure.errors.send(:initialize_dup, model.errors)
|
719
|
+
failed_instances << (options[:track_validation_failures] ? [i, failure] : failure )
|
400
720
|
end
|
721
|
+
array_of_attributes.compact!
|
401
722
|
end
|
402
|
-
else
|
403
|
-
import_with_validations( column_names, array_of_attributes, options )
|
404
723
|
end
|
405
724
|
else
|
406
|
-
|
407
|
-
ActiveRecord::Import::Result.new([], num_inserts, ids)
|
725
|
+
import_without_validations_or_callbacks( column_names, array_of_attributes, options )
|
408
726
|
end
|
409
727
|
|
410
728
|
if options[:synchronize]
|
411
|
-
sync_keys = options[:synchronize_keys] ||
|
729
|
+
sync_keys = options[:synchronize_keys] || Array(primary_key)
|
412
730
|
synchronize( options[:synchronize], sync_keys)
|
413
731
|
end
|
414
732
|
return_obj.num_inserts = 0 if return_obj.num_inserts.nil?
|
415
733
|
|
416
734
|
# if we have ids, then set the id on the models and mark the models as clean.
|
417
|
-
if
|
418
|
-
|
735
|
+
if models && supports_setting_primary_key_of_imported_objects?
|
736
|
+
set_attributes_and_mark_clean(models, return_obj, timestamps, options)
|
419
737
|
|
420
738
|
# if there are auto-save associations on the models we imported that are new, import them as well
|
421
|
-
|
739
|
+
if options[:recursive]
|
740
|
+
options[:on_duplicate_key_update] = on_duplicate_key_update unless on_duplicate_key_update.nil?
|
741
|
+
import_associations(models, options.dup.merge(validate: false))
|
742
|
+
end
|
422
743
|
end
|
423
744
|
|
424
745
|
return_obj
|
425
746
|
end
|
426
747
|
|
427
|
-
# TODO import_from_table needs to be implemented.
|
428
|
-
def import_from_table( options ) # :nodoc:
|
429
|
-
end
|
430
|
-
|
431
748
|
# Imports the passed in +column_names+ and +array_of_attributes+
|
432
749
|
# given the passed in +options+ Hash with validations. Returns an
|
433
750
|
# object with the methods +failed_instances+ and +num_inserts+.
|
@@ -438,32 +755,14 @@ class ActiveRecord::Base
|
|
438
755
|
def import_with_validations( column_names, array_of_attributes, options = {} )
|
439
756
|
failed_instances = []
|
440
757
|
|
441
|
-
if block_given?
|
442
|
-
yield failed_instances
|
443
|
-
else
|
444
|
-
# create instances for each of our column/value sets
|
445
|
-
arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
|
446
|
-
|
447
|
-
# keep track of the instance and the position it is currently at. if this fails
|
448
|
-
# validation we'll use the index to remove it from the array_of_attributes
|
449
|
-
model = new
|
450
|
-
arr.each_with_index do |hsh, i|
|
451
|
-
hsh.each_pair { |k, v| model[k] = v }
|
452
|
-
next if model.valid?(options[:validate_with_context])
|
453
|
-
raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
|
454
|
-
array_of_attributes[i] = nil
|
455
|
-
failed_instances << model.dup
|
456
|
-
end
|
457
|
-
end
|
458
|
-
|
459
|
-
array_of_attributes.compact!
|
758
|
+
yield failed_instances if block_given?
|
460
759
|
|
461
|
-
|
462
|
-
[0, []]
|
760
|
+
result = if options[:all_or_none] && failed_instances.any?
|
761
|
+
ActiveRecord::Import::Result.new([], 0, [], [])
|
463
762
|
else
|
464
763
|
import_without_validations_or_callbacks( column_names, array_of_attributes, options )
|
465
764
|
end
|
466
|
-
ActiveRecord::Import::Result.new(failed_instances, num_inserts, ids)
|
765
|
+
ActiveRecord::Import::Result.new(failed_instances, result.num_inserts, result.ids, result.results)
|
467
766
|
end
|
468
767
|
|
469
768
|
# Imports the passed in +column_names+ and +array_of_attributes+
|
@@ -473,66 +772,143 @@ class ActiveRecord::Base
|
|
473
772
|
# information on +column_names+, +array_of_attributes_ and
|
474
773
|
# +options+.
|
475
774
|
def import_without_validations_or_callbacks( column_names, array_of_attributes, options = {} )
|
775
|
+
return ActiveRecord::Import::Result.new([], 0, [], []) if array_of_attributes.empty?
|
776
|
+
|
476
777
|
column_names = column_names.map(&:to_sym)
|
477
778
|
scope_columns, scope_values = scope_attributes.to_a.transpose
|
478
779
|
|
479
780
|
unless scope_columns.blank?
|
480
781
|
scope_columns.zip(scope_values).each do |name, value|
|
481
782
|
name_as_sym = name.to_sym
|
482
|
-
next if column_names.include?(name_as_sym)
|
483
|
-
|
484
|
-
is_sti = (name_as_sym == inheritance_column.to_sym && self < base_class)
|
485
|
-
value = value.first if is_sti
|
486
|
-
|
783
|
+
next if column_names.include?(name_as_sym) || name_as_sym == inheritance_column.to_sym
|
487
784
|
column_names << name_as_sym
|
488
785
|
array_of_attributes.each { |attrs| attrs << value }
|
489
786
|
end
|
490
787
|
end
|
491
788
|
|
789
|
+
if finder_needs_type_condition?
|
790
|
+
unless column_names.include?(inheritance_column.to_sym)
|
791
|
+
column_names << inheritance_column.to_sym
|
792
|
+
array_of_attributes.each { |attrs| attrs << sti_name }
|
793
|
+
end
|
794
|
+
end
|
795
|
+
|
492
796
|
columns = column_names.each_with_index.map do |name, i|
|
493
797
|
column = columns_hash[name.to_s]
|
494
|
-
|
495
798
|
raise ActiveRecord::Import::MissingColumnError.new(name.to_s, i) if column.nil?
|
496
|
-
|
497
799
|
column
|
498
800
|
end
|
499
801
|
|
500
802
|
columns_sql = "(#{column_names.map { |name| connection.quote_column_name(name) }.join(',')})"
|
501
|
-
|
803
|
+
pre_sql_statements = connection.pre_sql_statements( options )
|
804
|
+
insert_sql = ['INSERT', pre_sql_statements, "INTO #{quoted_table_name} #{columns_sql} VALUES "]
|
805
|
+
insert_sql = insert_sql.flatten.join(' ')
|
502
806
|
values_sql = values_sql_for_columns_and_attributes(columns, array_of_attributes)
|
503
807
|
|
504
808
|
number_inserted = 0
|
505
809
|
ids = []
|
810
|
+
results = []
|
506
811
|
if supports_import?
|
507
812
|
# generate the sql
|
508
813
|
post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
|
814
|
+
import_size = values_sql.size
|
815
|
+
|
816
|
+
batch_size = options[:batch_size] || import_size
|
817
|
+
run_proc = options[:batch_size].to_i.positive? && options[:batch_progress].respond_to?( :call )
|
818
|
+
progress_proc = options[:batch_progress]
|
819
|
+
current_batch = 0
|
820
|
+
batches = (import_size / batch_size.to_f).ceil
|
509
821
|
|
510
|
-
batch_size = options[:batch_size] || values_sql.size
|
511
822
|
values_sql.each_slice(batch_size) do |batch_values|
|
823
|
+
batch_started_at = Time.now.to_i
|
824
|
+
|
512
825
|
# perform the inserts
|
513
826
|
result = connection.insert_many( [insert_sql, post_sql_statements].flatten,
|
514
827
|
batch_values,
|
515
|
-
|
516
|
-
|
517
|
-
|
828
|
+
options,
|
829
|
+
"#{model_name} Create Many" )
|
830
|
+
|
831
|
+
number_inserted += result.num_inserts
|
832
|
+
ids += result.ids
|
833
|
+
results += result.results
|
834
|
+
current_batch += 1
|
835
|
+
|
836
|
+
progress_proc.call(import_size, batches, current_batch, Time.now.to_i - batch_started_at) if run_proc
|
518
837
|
end
|
519
838
|
else
|
520
|
-
|
521
|
-
|
522
|
-
|
839
|
+
transaction(requires_new: true) do
|
840
|
+
values_sql.each do |values|
|
841
|
+
ids << connection.insert(insert_sql + values)
|
842
|
+
number_inserted += 1
|
843
|
+
end
|
523
844
|
end
|
524
845
|
end
|
525
|
-
[number_inserted, ids
|
846
|
+
ActiveRecord::Import::Result.new([], number_inserted, ids, results)
|
526
847
|
end
|
527
848
|
|
528
849
|
private
|
529
850
|
|
530
|
-
def
|
851
|
+
def set_attributes_and_mark_clean(models, import_result, timestamps, options)
|
531
852
|
return if models.nil?
|
532
|
-
import_result.
|
533
|
-
|
534
|
-
|
535
|
-
|
853
|
+
models -= import_result.failed_instances
|
854
|
+
|
855
|
+
# if ids were returned for all models we know all were updated
|
856
|
+
if models.size == import_result.ids.size
|
857
|
+
import_result.ids.each_with_index do |id, index|
|
858
|
+
model = models[index]
|
859
|
+
model.id = id
|
860
|
+
|
861
|
+
timestamps.each do |attr, value|
|
862
|
+
model.send(attr + "=", value) if model.send(attr).nil?
|
863
|
+
end
|
864
|
+
end
|
865
|
+
end
|
866
|
+
|
867
|
+
deserialize_value = lambda do |column, value|
|
868
|
+
column = columns_hash[column]
|
869
|
+
return value unless column
|
870
|
+
if respond_to?(:type_caster)
|
871
|
+
type = type_for_attribute(column.name)
|
872
|
+
type.deserialize(value)
|
873
|
+
elsif column.respond_to?(:type_cast_from_database)
|
874
|
+
column.type_cast_from_database(value)
|
875
|
+
else
|
876
|
+
value
|
877
|
+
end
|
878
|
+
end
|
879
|
+
|
880
|
+
set_value = lambda do |model, column, value|
|
881
|
+
val = deserialize_value.call(column, value)
|
882
|
+
if model.attribute_names.include?(column)
|
883
|
+
model.send("#{column}=", val)
|
884
|
+
else
|
885
|
+
attributes = attributes_builder.build_from_database(model.attributes.merge(column => val))
|
886
|
+
model.instance_variable_set(:@attributes, attributes)
|
887
|
+
end
|
888
|
+
end
|
889
|
+
|
890
|
+
columns = Array(options[:returning_columns])
|
891
|
+
results = Array(import_result.results)
|
892
|
+
if models.size == results.size
|
893
|
+
single_column = columns.first if columns.size == 1
|
894
|
+
results.each_with_index do |result, index|
|
895
|
+
model = models[index]
|
896
|
+
|
897
|
+
if single_column
|
898
|
+
set_value.call(model, single_column, result)
|
899
|
+
else
|
900
|
+
columns.each_with_index do |column, col_index|
|
901
|
+
set_value.call(model, column, result[col_index])
|
902
|
+
end
|
903
|
+
end
|
904
|
+
end
|
905
|
+
end
|
906
|
+
|
907
|
+
models.each do |model|
|
908
|
+
if model.respond_to?(:changes_applied) # Rails 4.1.8 and higher
|
909
|
+
model.changes_internally_applied if model.respond_to?(:changes_internally_applied) # legacy behavior for Rails 5.1
|
910
|
+
model.changes_applied
|
911
|
+
elsif model.respond_to?(:clear_changes_information) # Rails 4.0 and higher
|
536
912
|
model.clear_changes_information
|
537
913
|
else # Rails 3.2
|
538
914
|
model.instance_variable_get(:@changed_attributes).clear
|
@@ -541,20 +917,44 @@ class ActiveRecord::Base
|
|
541
917
|
end
|
542
918
|
end
|
543
919
|
|
920
|
+
# Sync belongs_to association ids with foreign key field
|
921
|
+
def load_association_ids(model)
|
922
|
+
changed_columns = model.changed
|
923
|
+
association_reflections = model.class.reflect_on_all_associations(:belongs_to)
|
924
|
+
association_reflections.each do |association_reflection|
|
925
|
+
next if association_reflection.options[:polymorphic]
|
926
|
+
|
927
|
+
column_names = Array(association_reflection.foreign_key).map(&:to_s)
|
928
|
+
column_names.each_with_index do |column_name, column_index|
|
929
|
+
next if changed_columns.include?(column_name)
|
930
|
+
|
931
|
+
association = model.association(association_reflection.name)
|
932
|
+
association = association.target
|
933
|
+
next if association.blank? || model.public_send(column_name).present?
|
934
|
+
|
935
|
+
association_primary_key = Array(association_reflection.association_primary_key)[column_index]
|
936
|
+
model.public_send("#{column_name}=", association.send(association_primary_key))
|
937
|
+
end
|
938
|
+
end
|
939
|
+
end
|
940
|
+
|
544
941
|
def import_associations(models, options)
|
545
942
|
# now, for all the dirty associations, collect them into a new set of models, then recurse.
|
546
943
|
# notes:
|
547
944
|
# does not handle associations that reference themselves
|
548
945
|
# should probably take a hash to associations to follow.
|
946
|
+
return if models.nil?
|
549
947
|
associated_objects_by_class = {}
|
550
948
|
models.each { |model| find_associated_objects_for_import(associated_objects_by_class, model) }
|
551
949
|
|
552
|
-
# :on_duplicate_key_update
|
553
|
-
options.delete(:on_duplicate_key_update)
|
950
|
+
# :on_duplicate_key_update only supported for all fields
|
951
|
+
options.delete(:on_duplicate_key_update) unless options[:on_duplicate_key_update] == :all
|
952
|
+
# :returning not supported for associations
|
953
|
+
options.delete(:returning)
|
554
954
|
|
555
955
|
associated_objects_by_class.each_value do |associations|
|
556
956
|
associations.each_value do |associated_records|
|
557
|
-
associated_records.first.class.
|
957
|
+
associated_records.first.class.bulk_import(associated_records, options) unless associated_records.empty?
|
558
958
|
end
|
559
959
|
end
|
560
960
|
end
|
@@ -563,6 +963,7 @@ class ActiveRecord::Base
|
|
563
963
|
# of class => objects to import.
|
564
964
|
def find_associated_objects_for_import(associated_objects_by_class, model)
|
565
965
|
associated_objects_by_class[model.class.name] ||= {}
|
966
|
+
return associated_objects_by_class unless model.id
|
566
967
|
|
567
968
|
association_reflections =
|
568
969
|
model.class.reflect_on_all_associations(:has_one) +
|
@@ -580,8 +981,13 @@ class ActiveRecord::Base
|
|
580
981
|
changed_objects.each do |child|
|
581
982
|
child.public_send("#{association_reflection.foreign_key}=", model.id)
|
582
983
|
# For polymorphic associations
|
984
|
+
association_name = if model.class.respond_to?(:polymorphic_name)
|
985
|
+
model.class.polymorphic_name
|
986
|
+
else
|
987
|
+
model.class.base_class
|
988
|
+
end
|
583
989
|
association_reflection.type.try do |type|
|
584
|
-
child.public_send("#{type}=",
|
990
|
+
child.public_send("#{type}=", association_name)
|
585
991
|
end
|
586
992
|
end
|
587
993
|
associated_objects_by_class[model.class.name][association_reflection.name].concat changed_objects
|
@@ -595,24 +1001,33 @@ class ActiveRecord::Base
|
|
595
1001
|
# connection gets called a *lot* in this high intensity loop.
|
596
1002
|
# Reuse the same one w/in the loop, otherwise it would keep being re-retreived (= lots of time for large imports)
|
597
1003
|
connection_memo = connection
|
1004
|
+
|
598
1005
|
array_of_attributes.map do |arr|
|
599
1006
|
my_values = arr.each_with_index.map do |val, j|
|
600
1007
|
column = columns[j]
|
601
1008
|
|
602
1009
|
# be sure to query sequence_name *last*, only if cheaper tests fail, because it's costly
|
603
|
-
if val.nil? &&
|
1010
|
+
if val.nil? && Array(primary_key).first == column.name && !sequence_name.blank?
|
604
1011
|
connection_memo.next_value_for_sequence(sequence_name)
|
1012
|
+
elsif val.respond_to?(:to_sql)
|
1013
|
+
"(#{val.to_sql})"
|
605
1014
|
elsif column
|
606
|
-
if respond_to?(:type_caster)
|
607
|
-
|
608
|
-
|
1015
|
+
if respond_to?(:type_caster) # Rails 5.0 and higher
|
1016
|
+
type = type_for_attribute(column.name)
|
1017
|
+
val = !type.respond_to?(:subtype) && type.type == :boolean ? type.cast(val) : type.serialize(val)
|
1018
|
+
connection_memo.quote(val)
|
1019
|
+
elsif column.respond_to?(:type_cast_from_user) # Rails 4.2
|
609
1020
|
connection_memo.quote(column.type_cast_from_user(val), column)
|
610
|
-
else
|
1021
|
+
else # Rails 3.2, 4.0 and 4.1
|
611
1022
|
if serialized_attributes.include?(column.name)
|
612
1023
|
val = serialized_attributes[column.name].dump(val)
|
613
1024
|
end
|
614
|
-
|
1025
|
+
# Fixes #443 to support binary (i.e. bytea) columns on PG
|
1026
|
+
val = column.type_cast(val) unless column.type && column.type.to_sym == :binary
|
1027
|
+
connection_memo.quote(val, column)
|
615
1028
|
end
|
1029
|
+
else
|
1030
|
+
raise ArgumentError, "Number of values (#{arr.length}) exceeds number of columns (#{columns.length})"
|
616
1031
|
end
|
617
1032
|
end
|
618
1033
|
"(#{my_values.join(',')})"
|
@@ -620,40 +1035,89 @@ class ActiveRecord::Base
|
|
620
1035
|
end
|
621
1036
|
|
622
1037
|
def add_special_rails_stamps( column_names, array_of_attributes, options )
|
623
|
-
|
624
|
-
|
625
|
-
|
626
|
-
|
627
|
-
|
628
|
-
|
629
|
-
|
630
|
-
|
631
|
-
|
632
|
-
|
633
|
-
end
|
1038
|
+
timestamp_columns = {}
|
1039
|
+
timestamps = {}
|
1040
|
+
|
1041
|
+
if respond_to?(:all_timestamp_attributes_in_model, true) # Rails 5.1 and higher
|
1042
|
+
timestamp_columns[:create] = timestamp_attributes_for_create_in_model
|
1043
|
+
timestamp_columns[:update] = timestamp_attributes_for_update_in_model
|
1044
|
+
else
|
1045
|
+
instance = allocate
|
1046
|
+
timestamp_columns[:create] = instance.send(:timestamp_attributes_for_create_in_model)
|
1047
|
+
timestamp_columns[:update] = instance.send(:timestamp_attributes_for_update_in_model)
|
634
1048
|
end
|
635
1049
|
|
636
|
-
|
637
|
-
|
638
|
-
|
639
|
-
|
640
|
-
|
641
|
-
|
642
|
-
|
643
|
-
|
644
|
-
|
645
|
-
|
646
|
-
|
1050
|
+
# use tz as set in ActiveRecord::Base
|
1051
|
+
default_timezone = if ActiveRecord.respond_to?(:default_timezone)
|
1052
|
+
ActiveRecord.default_timezone
|
1053
|
+
else
|
1054
|
+
ActiveRecord::Base.default_timezone
|
1055
|
+
end
|
1056
|
+
timestamp = default_timezone == :utc ? Time.now.utc : Time.now
|
1057
|
+
|
1058
|
+
[:create, :update].each do |action|
|
1059
|
+
timestamp_columns[action].each do |column|
|
1060
|
+
column = column.to_s
|
1061
|
+
timestamps[column] = timestamp
|
1062
|
+
|
1063
|
+
index = column_names.index(column) || column_names.index(column.to_sym)
|
1064
|
+
if index
|
1065
|
+
# replace every instance of the array of attributes with our value
|
1066
|
+
array_of_attributes.each { |arr| arr[index] = timestamp if arr[index].nil? }
|
1067
|
+
else
|
1068
|
+
column_names << column
|
1069
|
+
array_of_attributes.each { |arr| arr << timestamp }
|
1070
|
+
end
|
647
1071
|
|
648
|
-
|
649
|
-
|
1072
|
+
if supports_on_duplicate_key_update? && action == :update
|
1073
|
+
connection.add_column_for_on_duplicate_key_update(column, options)
|
1074
|
+
end
|
650
1075
|
end
|
651
1076
|
end
|
1077
|
+
|
1078
|
+
timestamps
|
652
1079
|
end
|
653
1080
|
|
654
1081
|
# Returns an Array of Hashes for the passed in +column_names+ and +array_of_attributes+.
|
655
1082
|
def validations_array_for_column_names_and_attributes( column_names, array_of_attributes ) # :nodoc:
|
656
1083
|
array_of_attributes.map { |values| Hash[column_names.zip(values)] }
|
657
1084
|
end
|
1085
|
+
|
1086
|
+
# Checks that the imported hash has the required_keys, optionally also checks that the hash has
|
1087
|
+
# no keys beyond those required when `allow_extra_keys` is false.
|
1088
|
+
# returns `nil` if validation passes, or an error message if it fails
|
1089
|
+
def validate_hash_import(hash, required_keys, allow_extra_keys) # :nodoc:
|
1090
|
+
extra_keys = allow_extra_keys ? [] : hash.keys - required_keys
|
1091
|
+
missing_keys = required_keys - hash.keys
|
1092
|
+
|
1093
|
+
return nil if extra_keys.empty? && missing_keys.empty?
|
1094
|
+
|
1095
|
+
if allow_extra_keys
|
1096
|
+
<<-EOS
|
1097
|
+
Hash key mismatch.
|
1098
|
+
|
1099
|
+
When importing an array of hashes with provided columns_names, each hash must contain keys for all column_names.
|
1100
|
+
|
1101
|
+
Required keys: #{required_keys}
|
1102
|
+
Missing keys: #{missing_keys}
|
1103
|
+
|
1104
|
+
Hash: #{hash}
|
1105
|
+
EOS
|
1106
|
+
else
|
1107
|
+
<<-EOS
|
1108
|
+
Hash key mismatch.
|
1109
|
+
|
1110
|
+
When importing an array of hashes, all hashes must have the same keys.
|
1111
|
+
If you have records that are missing some values, we recommend you either set default values
|
1112
|
+
for the missing keys or group these records into batches by key set before importing.
|
1113
|
+
|
1114
|
+
Required keys: #{required_keys}
|
1115
|
+
Extra keys: #{extra_keys}
|
1116
|
+
Missing keys: #{missing_keys}
|
1117
|
+
|
1118
|
+
Hash: #{hash}
|
1119
|
+
EOS
|
1120
|
+
end
|
1121
|
+
end
|
658
1122
|
end
|
659
1123
|
end
|