activerecord-import 1.0.2 → 1.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/test.yaml +151 -0
- data/.gitignore +5 -0
- data/.rubocop.yml +74 -8
- data/.rubocop_todo.yml +10 -16
- data/Brewfile +3 -1
- data/CHANGELOG.md +138 -3
- data/Dockerfile +23 -0
- data/Gemfile +24 -14
- data/LICENSE +21 -56
- data/README.markdown +108 -60
- data/Rakefile +3 -0
- data/activerecord-import.gemspec +6 -5
- data/benchmarks/benchmark.rb +10 -4
- data/benchmarks/lib/base.rb +4 -2
- data/benchmarks/lib/cli_parser.rb +4 -2
- data/benchmarks/lib/float.rb +2 -0
- data/benchmarks/lib/mysql2_benchmark.rb +2 -0
- data/benchmarks/lib/output_to_csv.rb +2 -0
- data/benchmarks/lib/output_to_html.rb +4 -2
- data/benchmarks/models/test_innodb.rb +2 -0
- data/benchmarks/models/test_memory.rb +2 -0
- data/benchmarks/models/test_myisam.rb +2 -0
- data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +2 -0
- data/docker-compose.yml +34 -0
- data/gemfiles/4.2.gemfile +2 -0
- data/gemfiles/5.0.gemfile +2 -0
- data/gemfiles/5.1.gemfile +2 -0
- data/gemfiles/5.2.gemfile +2 -0
- data/gemfiles/6.0.gemfile +4 -1
- data/gemfiles/6.1.gemfile +4 -1
- data/gemfiles/7.0.gemfile +4 -0
- data/gemfiles/7.1.gemfile +3 -0
- data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +6 -4
- data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/trilogy_adapter.rb +8 -0
- data/lib/activerecord-import/adapters/abstract_adapter.rb +14 -5
- data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql_adapter.rb +33 -25
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +69 -56
- data/lib/activerecord-import/adapters/sqlite3_adapter.rb +39 -39
- data/lib/activerecord-import/adapters/trilogy_adapter.rb +7 -0
- data/lib/activerecord-import/base.rb +10 -2
- data/lib/activerecord-import/import.rb +162 -65
- data/lib/activerecord-import/mysql2.rb +2 -0
- data/lib/activerecord-import/postgresql.rb +2 -0
- data/lib/activerecord-import/sqlite3.rb +2 -0
- data/lib/activerecord-import/synchronize.rb +3 -1
- data/lib/activerecord-import/value_sets_parser.rb +5 -0
- data/lib/activerecord-import/version.rb +3 -1
- data/lib/activerecord-import.rb +2 -1
- data/test/adapters/jdbcmysql.rb +2 -0
- data/test/adapters/jdbcpostgresql.rb +2 -0
- data/test/adapters/jdbcsqlite3.rb +2 -0
- data/test/adapters/makara_postgis.rb +2 -0
- data/test/adapters/mysql2.rb +2 -0
- data/test/adapters/mysql2_makara.rb +2 -0
- data/test/adapters/mysql2spatial.rb +2 -0
- data/test/adapters/postgis.rb +2 -0
- data/test/adapters/postgresql.rb +2 -0
- data/test/adapters/postgresql_makara.rb +2 -0
- data/test/adapters/seamless_database_pool.rb +2 -0
- data/test/adapters/spatialite.rb +2 -0
- data/test/adapters/sqlite3.rb +2 -0
- data/test/adapters/trilogy.rb +9 -0
- data/test/database.yml.sample +7 -0
- data/test/{travis → github}/database.yml +7 -1
- data/test/import_test.rb +93 -2
- data/test/jdbcmysql/import_test.rb +5 -3
- data/test/jdbcpostgresql/import_test.rb +4 -2
- data/test/jdbcsqlite3/import_test.rb +4 -2
- data/test/makara_postgis/import_test.rb +4 -2
- data/test/models/account.rb +2 -0
- data/test/models/alarm.rb +2 -0
- data/test/models/animal.rb +8 -0
- data/test/models/author.rb +7 -0
- data/test/models/bike_maker.rb +3 -0
- data/test/models/book.rb +7 -2
- data/test/models/car.rb +2 -0
- data/test/models/card.rb +5 -0
- data/test/models/chapter.rb +2 -0
- data/test/models/composite_book.rb +19 -0
- data/test/models/composite_chapter.rb +9 -0
- data/test/models/customer.rb +18 -0
- data/test/models/deck.rb +8 -0
- data/test/models/dictionary.rb +2 -0
- data/test/models/discount.rb +2 -0
- data/test/models/end_note.rb +2 -0
- data/test/models/group.rb +2 -0
- data/test/models/order.rb +17 -0
- data/test/models/playing_card.rb +4 -0
- data/test/models/promotion.rb +2 -0
- data/test/models/question.rb +2 -0
- data/test/models/rule.rb +2 -0
- data/test/models/tag.rb +9 -1
- data/test/models/tag_alias.rb +11 -0
- data/test/models/topic.rb +7 -0
- data/test/models/user.rb +2 -0
- data/test/models/user_token.rb +2 -0
- data/test/models/vendor.rb +2 -0
- data/test/models/widget.rb +2 -0
- data/test/mysql2/import_test.rb +5 -3
- data/test/mysql2_makara/import_test.rb +5 -3
- data/test/mysqlspatial2/import_test.rb +5 -3
- data/test/postgis/import_test.rb +4 -2
- data/test/postgresql/import_test.rb +4 -2
- data/test/schema/generic_schema.rb +37 -1
- data/test/schema/jdbcpostgresql_schema.rb +3 -1
- data/test/schema/mysql2_schema.rb +2 -0
- data/test/schema/postgis_schema.rb +3 -1
- data/test/schema/postgresql_schema.rb +47 -0
- data/test/schema/sqlite3_schema.rb +2 -0
- data/test/schema/version.rb +2 -0
- data/test/sqlite3/import_test.rb +4 -2
- data/test/support/active_support/test_case_extensions.rb +2 -0
- data/test/support/assertions.rb +2 -0
- data/test/support/factories.rb +2 -0
- data/test/support/generate.rb +4 -2
- data/test/support/mysql/import_examples.rb +2 -1
- data/test/support/postgresql/import_examples.rb +108 -2
- data/test/support/shared_examples/on_duplicate_key_ignore.rb +2 -0
- data/test/support/shared_examples/on_duplicate_key_update.rb +78 -9
- data/test/support/shared_examples/recursive_import.rb +98 -1
- data/test/support/sqlite3/import_examples.rb +2 -1
- data/test/synchronize_test.rb +2 -0
- data/test/test_helper.rb +33 -6
- data/test/trilogy/import_test.rb +7 -0
- data/test/value_sets_bytes_parser_test.rb +3 -1
- data/test/value_sets_records_parser_test.rb +3 -1
- metadata +42 -16
- data/.travis.yml +0 -70
- data/gemfiles/3.2.gemfile +0 -2
- data/gemfiles/4.0.gemfile +0 -2
- data/gemfiles/4.1.gemfile +0 -2
@@ -1,9 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module ActiveRecord::Import::SQLite3Adapter
|
2
4
|
include ActiveRecord::Import::ImportSupport
|
3
5
|
include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
|
4
6
|
|
5
|
-
MIN_VERSION_FOR_IMPORT = "3.7.11"
|
6
|
-
MIN_VERSION_FOR_UPSERT = "3.24.0"
|
7
|
+
MIN_VERSION_FOR_IMPORT = "3.7.11"
|
8
|
+
MIN_VERSION_FOR_UPSERT = "3.24.0"
|
7
9
|
SQLITE_LIMIT_COMPOUND_SELECT = 500
|
8
10
|
|
9
11
|
# Override our conformance to ActiveRecord::Import::ImportSupport interface
|
@@ -22,10 +24,11 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
22
24
|
def insert_many( sql, values, _options = {}, *args ) # :nodoc:
|
23
25
|
number_of_inserts = 0
|
24
26
|
|
25
|
-
base_sql, post_sql =
|
26
|
-
|
27
|
-
|
28
|
-
|
27
|
+
base_sql, post_sql = case sql
|
28
|
+
when String
|
29
|
+
[sql, '']
|
30
|
+
when Array
|
31
|
+
[sql.shift, sql.join( ' ' )]
|
29
32
|
end
|
30
33
|
|
31
34
|
value_sets = ::ActiveRecord::Import::ValueSetsRecordsParser.parse(values,
|
@@ -54,11 +57,9 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
54
57
|
def post_sql_statements( table_name, options ) # :nodoc:
|
55
58
|
sql = []
|
56
59
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
sql << sql_for_on_duplicate_key_ignore( options[:on_duplicate_key_ignore] )
|
61
|
-
end
|
60
|
+
# Options :recursive and :on_duplicate_key_ignore are mutually exclusive
|
61
|
+
if supports_on_duplicate_key_update? && ((options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:on_duplicate_key_update])
|
62
|
+
sql << sql_for_on_duplicate_key_ignore( options[:on_duplicate_key_ignore] )
|
62
63
|
end
|
63
64
|
|
64
65
|
sql + super
|
@@ -71,13 +72,14 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
71
72
|
# Add a column to be updated on duplicate key update
|
72
73
|
def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
|
73
74
|
arg = options[:on_duplicate_key_update]
|
74
|
-
|
75
|
+
case arg
|
76
|
+
when Hash
|
75
77
|
columns = arg.fetch( :columns ) { arg[:columns] = [] }
|
76
78
|
case columns
|
77
79
|
when Array then columns << column.to_sym unless columns.include?( column.to_sym )
|
78
80
|
when Hash then columns[column.to_sym] = column.to_sym
|
79
81
|
end
|
80
|
-
|
82
|
+
when Array
|
81
83
|
arg << column.to_sym unless arg.include?( column.to_sym )
|
82
84
|
end
|
83
85
|
end
|
@@ -92,12 +94,12 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
92
94
|
|
93
95
|
# Returns a generated ON CONFLICT DO UPDATE statement given the passed
|
94
96
|
# in +args+.
|
95
|
-
def sql_for_on_duplicate_key_update(
|
96
|
-
arg, primary_key, locking_column = args
|
97
|
+
def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
|
98
|
+
arg, model, primary_key, locking_column = args
|
97
99
|
arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
|
98
100
|
return unless arg.is_a?( Hash )
|
99
101
|
|
100
|
-
sql = ' ON CONFLICT '
|
102
|
+
sql = ' ON CONFLICT '.dup
|
101
103
|
conflict_target = sql_for_conflict_target( arg )
|
102
104
|
|
103
105
|
columns = arg.fetch( :columns, [] )
|
@@ -112,11 +114,12 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
112
114
|
end
|
113
115
|
|
114
116
|
sql << "#{conflict_target}DO UPDATE SET "
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
117
|
+
case columns
|
118
|
+
when Array
|
119
|
+
sql << sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, columns )
|
120
|
+
when Hash
|
121
|
+
sql << sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, columns )
|
122
|
+
when String
|
120
123
|
sql << columns
|
121
124
|
else
|
122
125
|
raise ArgumentError, 'Expected :columns to be an Array or Hash'
|
@@ -127,22 +130,27 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
127
130
|
sql
|
128
131
|
end
|
129
132
|
|
130
|
-
def sql_for_on_duplicate_key_update_as_array( locking_column, arr ) # :nodoc:
|
133
|
+
def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
|
131
134
|
results = arr.map do |column|
|
132
|
-
|
135
|
+
original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
|
136
|
+
qc = quote_column_name( original_column_name )
|
133
137
|
"#{qc}=EXCLUDED.#{qc}"
|
134
138
|
end
|
135
|
-
increment_locking_column!(results, locking_column)
|
139
|
+
increment_locking_column!(table_name, results, locking_column)
|
136
140
|
results.join( ',' )
|
137
141
|
end
|
138
142
|
|
139
|
-
def sql_for_on_duplicate_key_update_as_hash( locking_column, hsh ) # :nodoc:
|
143
|
+
def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
|
140
144
|
results = hsh.map do |column1, column2|
|
141
|
-
|
142
|
-
|
145
|
+
original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
|
146
|
+
qc1 = quote_column_name( original_column1_name )
|
147
|
+
|
148
|
+
original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
|
149
|
+
qc2 = quote_column_name( original_column2_name )
|
150
|
+
|
143
151
|
"#{qc1}=EXCLUDED.#{qc2}"
|
144
152
|
end
|
145
|
-
increment_locking_column!(results, locking_column)
|
153
|
+
increment_locking_column!(table_name, results, locking_column)
|
146
154
|
results.join( ',' )
|
147
155
|
end
|
148
156
|
|
@@ -150,9 +158,9 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
150
158
|
conflict_target = args[:conflict_target]
|
151
159
|
index_predicate = args[:index_predicate]
|
152
160
|
if conflict_target.present?
|
153
|
-
|
154
|
-
|
155
|
-
|
161
|
+
sql = "(#{Array( conflict_target ).reject( &:blank? ).join( ', ' )}) "
|
162
|
+
sql += "WHERE #{index_predicate} " if index_predicate
|
163
|
+
sql
|
156
164
|
end
|
157
165
|
end
|
158
166
|
|
@@ -166,14 +174,6 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
166
174
|
exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key')
|
167
175
|
end
|
168
176
|
|
169
|
-
def increment_locking_column!(results, locking_column)
|
170
|
-
if locking_column.present?
|
171
|
-
results << "\"#{locking_column}\"=EXCLUDED.\"#{locking_column}\"+1"
|
172
|
-
end
|
173
|
-
end
|
174
|
-
|
175
|
-
private
|
176
|
-
|
177
177
|
def database_version
|
178
178
|
defined?(sqlite_version) ? sqlite_version : super
|
179
179
|
end
|
@@ -1,9 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "pathname"
|
2
4
|
require "active_record"
|
3
5
|
require "active_record/version"
|
4
6
|
|
5
7
|
module ActiveRecord::Import
|
6
|
-
ADAPTER_PATH = "activerecord-import/active_record/adapters"
|
8
|
+
ADAPTER_PATH = "activerecord-import/active_record/adapters"
|
7
9
|
|
8
10
|
def self.base_adapter(adapter)
|
9
11
|
case adapter
|
@@ -27,7 +29,13 @@ module ActiveRecord::Import
|
|
27
29
|
|
28
30
|
# Loads the import functionality for the passed in ActiveRecord connection
|
29
31
|
def self.load_from_connection_pool(connection_pool)
|
30
|
-
|
32
|
+
adapter =
|
33
|
+
if connection_pool.respond_to?(:db_config) # ActiveRecord >= 6.1
|
34
|
+
connection_pool.db_config.adapter
|
35
|
+
else
|
36
|
+
connection_pool.spec.config[:adapter]
|
37
|
+
end
|
38
|
+
require_adapter adapter
|
31
39
|
end
|
32
40
|
end
|
33
41
|
|
@@ -1,18 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "ostruct"
|
2
4
|
|
3
5
|
module ActiveRecord::Import::ConnectionAdapters; end
|
4
6
|
|
5
|
-
module ActiveRecord::Import
|
7
|
+
module ActiveRecord::Import # :nodoc:
|
6
8
|
Result = Struct.new(:failed_instances, :num_inserts, :ids, :results)
|
7
9
|
|
8
|
-
module ImportSupport
|
9
|
-
def supports_import?
|
10
|
+
module ImportSupport # :nodoc:
|
11
|
+
def supports_import? # :nodoc:
|
10
12
|
true
|
11
13
|
end
|
12
14
|
end
|
13
15
|
|
14
|
-
module OnDuplicateKeyUpdateSupport
|
15
|
-
def supports_on_duplicate_key_update?
|
16
|
+
module OnDuplicateKeyUpdateSupport # :nodoc:
|
17
|
+
def supports_on_duplicate_key_update? # :nodoc:
|
16
18
|
true
|
17
19
|
end
|
18
20
|
end
|
@@ -34,7 +36,7 @@ module ActiveRecord::Import #:nodoc:
|
|
34
36
|
@validate_callbacks = klass._validate_callbacks.dup
|
35
37
|
|
36
38
|
@validate_callbacks.each_with_index do |callback, i|
|
37
|
-
filter = callback.raw_filter
|
39
|
+
filter = callback.respond_to?(:raw_filter) ? callback.raw_filter : callback.filter
|
38
40
|
next unless filter.class.name =~ /Validations::PresenceValidator/ ||
|
39
41
|
(!@options[:validate_uniqueness] &&
|
40
42
|
filter.is_a?(ActiveRecord::Validations::UniquenessValidator))
|
@@ -49,13 +51,13 @@ module ActiveRecord::Import #:nodoc:
|
|
49
51
|
associations = klass.reflect_on_all_associations(:belongs_to)
|
50
52
|
associations.each do |assoc|
|
51
53
|
if (index = attrs.index(assoc.name))
|
52
|
-
key = assoc.foreign_key.to_sym
|
54
|
+
key = assoc.foreign_key.is_a?(Array) ? assoc.foreign_key.map(&:to_sym) : assoc.foreign_key.to_sym
|
53
55
|
attrs[index] = key unless attrs.include?(key)
|
54
56
|
end
|
55
57
|
end
|
56
58
|
end
|
57
59
|
|
58
|
-
filter.instance_variable_set(:@attributes, attrs)
|
60
|
+
filter.instance_variable_set(:@attributes, attrs.flatten)
|
59
61
|
|
60
62
|
if @validate_callbacks.respond_to?(:chain, true)
|
61
63
|
@validate_callbacks.send(:chain).tap do |chain|
|
@@ -71,7 +73,7 @@ module ActiveRecord::Import #:nodoc:
|
|
71
73
|
end
|
72
74
|
|
73
75
|
def valid_model?(model)
|
74
|
-
init_validations(model.class) unless model.
|
76
|
+
init_validations(model.class) unless model.instance_of?(@validator_class)
|
75
77
|
|
76
78
|
validation_context = @options[:validate_with_context]
|
77
79
|
validation_context ||= (model.new_record? ? :create : :update)
|
@@ -83,7 +85,11 @@ module ActiveRecord::Import #:nodoc:
|
|
83
85
|
|
84
86
|
model.run_callbacks(:validation) do
|
85
87
|
if defined?(ActiveSupport::Callbacks::Filters::Environment) # ActiveRecord >= 4.1
|
86
|
-
runner = @validate_callbacks.compile
|
88
|
+
runner = if @validate_callbacks.method(:compile).arity == 0
|
89
|
+
@validate_callbacks.compile
|
90
|
+
else # ActiveRecord >= 7.1
|
91
|
+
@validate_callbacks.compile(nil)
|
92
|
+
end
|
87
93
|
env = ActiveSupport::Callbacks::Filters::Environment.new(model, false, nil)
|
88
94
|
if runner.respond_to?(:call) # ActiveRecord < 5.1
|
89
95
|
runner.call(env)
|
@@ -163,7 +169,7 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
163
169
|
m.public_send "#{reflection.type}=", owner.class.name if reflection.type
|
164
170
|
end
|
165
171
|
|
166
|
-
|
172
|
+
model_klass.bulk_import column_names, models, options
|
167
173
|
|
168
174
|
# supports array of hash objects
|
169
175
|
elsif args.last.is_a?( Array ) && args.last.first.is_a?(Hash)
|
@@ -202,11 +208,11 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
202
208
|
end
|
203
209
|
end
|
204
210
|
|
205
|
-
|
211
|
+
model_klass.bulk_import column_names, array_of_attributes, options
|
206
212
|
|
207
213
|
# supports empty array
|
208
214
|
elsif args.last.is_a?( Array ) && args.last.empty?
|
209
|
-
|
215
|
+
ActiveRecord::Import::Result.new([], 0, [])
|
210
216
|
|
211
217
|
# supports 2-element array and array
|
212
218
|
elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
|
@@ -237,7 +243,7 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
237
243
|
end
|
238
244
|
end
|
239
245
|
|
240
|
-
|
246
|
+
model_klass.bulk_import column_names, array_of_attributes, options
|
241
247
|
else
|
242
248
|
raise ArgumentError, "Invalid arguments!"
|
243
249
|
end
|
@@ -245,16 +251,17 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
245
251
|
alias import bulk_import unless respond_to? :import
|
246
252
|
end
|
247
253
|
|
254
|
+
module ActiveRecord::Import::Connection
|
255
|
+
def establish_connection(args = nil)
|
256
|
+
conn = super(args)
|
257
|
+
ActiveRecord::Import.load_from_connection_pool connection_pool
|
258
|
+
conn
|
259
|
+
end
|
260
|
+
end
|
261
|
+
|
248
262
|
class ActiveRecord::Base
|
249
263
|
class << self
|
250
|
-
|
251
|
-
conn = establish_connection_without_activerecord_import(*args)
|
252
|
-
ActiveRecord::Import.load_from_connection_pool connection_pool
|
253
|
-
conn
|
254
|
-
end
|
255
|
-
|
256
|
-
alias establish_connection_without_activerecord_import establish_connection
|
257
|
-
alias establish_connection establish_connection_with_activerecord_import
|
264
|
+
prepend ActiveRecord::Import::Connection
|
258
265
|
|
259
266
|
# Returns true if the current database connection adapter
|
260
267
|
# supports import functionality, otherwise returns false.
|
@@ -546,11 +553,11 @@ class ActiveRecord::Base
|
|
546
553
|
alias import! bulk_import! unless ActiveRecord::Base.respond_to? :import!
|
547
554
|
|
548
555
|
def import_helper( *args )
|
549
|
-
options = { validate: true, timestamps: true }
|
556
|
+
options = { model: self, validate: true, timestamps: true, track_validation_failures: false }
|
550
557
|
options.merge!( args.pop ) if args.last.is_a? Hash
|
551
558
|
# making sure that current model's primary key is used
|
552
559
|
options[:primary_key] = primary_key
|
553
|
-
options[:locking_column] = locking_column if
|
560
|
+
options[:locking_column] = locking_column if locking_enabled?
|
554
561
|
|
555
562
|
is_validating = options[:validate_with_context].present? ? true : options[:validate]
|
556
563
|
validator = ActiveRecord::Import::Validator.new(self, options)
|
@@ -571,7 +578,7 @@ class ActiveRecord::Base
|
|
571
578
|
|
572
579
|
if models.first.id.nil?
|
573
580
|
Array(primary_key).each do |c|
|
574
|
-
if column_names.include?(c) &&
|
581
|
+
if column_names.include?(c) && schema_columns_hash[c].type == :uuid
|
575
582
|
column_names.delete(c)
|
576
583
|
end
|
577
584
|
end
|
@@ -581,7 +588,7 @@ class ActiveRecord::Base
|
|
581
588
|
if respond_to?(:timestamp_attributes_for_update, true)
|
582
589
|
send(:timestamp_attributes_for_update).map(&:to_sym)
|
583
590
|
else
|
584
|
-
|
591
|
+
allocate.send(:timestamp_attributes_for_update_in_model)
|
585
592
|
end
|
586
593
|
end
|
587
594
|
|
@@ -630,7 +637,7 @@ class ActiveRecord::Base
|
|
630
637
|
end
|
631
638
|
# supports empty array
|
632
639
|
elsif args.last.is_a?( Array ) && args.last.empty?
|
633
|
-
return ActiveRecord::Import::Result.new([], 0, [])
|
640
|
+
return ActiveRecord::Import::Result.new([], 0, [], [])
|
634
641
|
# supports 2-element array and array
|
635
642
|
elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
|
636
643
|
|
@@ -694,7 +701,11 @@ class ActiveRecord::Base
|
|
694
701
|
return_obj = if is_validating
|
695
702
|
import_with_validations( column_names, array_of_attributes, options ) do |failed_instances|
|
696
703
|
if models
|
697
|
-
models.
|
704
|
+
models.each_with_index do |m, i|
|
705
|
+
next unless m.errors.any?
|
706
|
+
|
707
|
+
failed_instances << (options[:track_validation_failures] ? [i, m] : m)
|
708
|
+
end
|
698
709
|
else
|
699
710
|
# create instances for each of our column/value sets
|
700
711
|
arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
|
@@ -702,14 +713,18 @@ class ActiveRecord::Base
|
|
702
713
|
# keep track of the instance and the position it is currently at. if this fails
|
703
714
|
# validation we'll use the index to remove it from the array_of_attributes
|
704
715
|
arr.each_with_index do |hsh, i|
|
705
|
-
|
706
|
-
|
716
|
+
# utilize block initializer syntax to prevent failure when 'mass_assignment_sanitizer = :strict'
|
717
|
+
model = new do |m|
|
718
|
+
hsh.each_pair { |k, v| m[k] = v }
|
719
|
+
end
|
720
|
+
|
707
721
|
next if validator.valid_model?(model)
|
708
722
|
raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
|
723
|
+
|
709
724
|
array_of_attributes[i] = nil
|
710
725
|
failure = model.dup
|
711
726
|
failure.errors.send(:initialize_dup, model.errors)
|
712
|
-
failed_instances << failure
|
727
|
+
failed_instances << (options[:track_validation_failures] ? [i, failure] : failure )
|
713
728
|
end
|
714
729
|
array_of_attributes.compact!
|
715
730
|
end
|
@@ -729,7 +744,10 @@ class ActiveRecord::Base
|
|
729
744
|
set_attributes_and_mark_clean(models, return_obj, timestamps, options)
|
730
745
|
|
731
746
|
# if there are auto-save associations on the models we imported that are new, import them as well
|
732
|
-
|
747
|
+
if options[:recursive]
|
748
|
+
options[:on_duplicate_key_update] = on_duplicate_key_update unless on_duplicate_key_update.nil?
|
749
|
+
import_associations(models, options.dup.merge(validate: false))
|
750
|
+
end
|
733
751
|
end
|
734
752
|
|
735
753
|
return_obj
|
@@ -764,27 +782,29 @@ class ActiveRecord::Base
|
|
764
782
|
def import_without_validations_or_callbacks( column_names, array_of_attributes, options = {} )
|
765
783
|
return ActiveRecord::Import::Result.new([], 0, [], []) if array_of_attributes.empty?
|
766
784
|
|
767
|
-
column_names = column_names.map
|
785
|
+
column_names = column_names.map do |name|
|
786
|
+
original_name = attribute_alias?(name) ? attribute_alias(name) : name
|
787
|
+
original_name.to_sym
|
788
|
+
end
|
768
789
|
scope_columns, scope_values = scope_attributes.to_a.transpose
|
769
790
|
|
770
791
|
unless scope_columns.blank?
|
771
792
|
scope_columns.zip(scope_values).each do |name, value|
|
772
793
|
name_as_sym = name.to_sym
|
773
|
-
next if column_names.include?(name_as_sym)
|
774
|
-
|
775
|
-
is_sti = (name_as_sym == inheritance_column.to_sym && self < base_class)
|
776
|
-
value = Array(value).first if is_sti
|
777
|
-
|
794
|
+
next if column_names.include?(name_as_sym) || name_as_sym == inheritance_column.to_sym
|
778
795
|
column_names << name_as_sym
|
779
796
|
array_of_attributes.each { |attrs| attrs << value }
|
780
797
|
end
|
781
798
|
end
|
782
799
|
|
783
|
-
|
784
|
-
|
800
|
+
if finder_needs_type_condition? && !column_names.include?(inheritance_column.to_sym)
|
801
|
+
column_names << inheritance_column.to_sym
|
802
|
+
array_of_attributes.each { |attrs| attrs << sti_name }
|
803
|
+
end
|
785
804
|
|
805
|
+
columns = column_names.each_with_index.map do |name, i|
|
806
|
+
column = schema_columns_hash[name.to_s]
|
786
807
|
raise ActiveRecord::Import::MissingColumnError.new(name.to_s, i) if column.nil?
|
787
|
-
|
788
808
|
column
|
789
809
|
end
|
790
810
|
|
@@ -800,17 +820,29 @@ class ActiveRecord::Base
|
|
800
820
|
if supports_import?
|
801
821
|
# generate the sql
|
802
822
|
post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
|
823
|
+
import_size = values_sql.size
|
824
|
+
|
825
|
+
batch_size = options[:batch_size] || import_size
|
826
|
+
run_proc = options[:batch_size].to_i.positive? && options[:batch_progress].respond_to?( :call )
|
827
|
+
progress_proc = options[:batch_progress]
|
828
|
+
current_batch = 0
|
829
|
+
batches = (import_size / batch_size.to_f).ceil
|
803
830
|
|
804
|
-
batch_size = options[:batch_size] || values_sql.size
|
805
831
|
values_sql.each_slice(batch_size) do |batch_values|
|
832
|
+
batch_started_at = Time.now.to_i
|
833
|
+
|
806
834
|
# perform the inserts
|
807
835
|
result = connection.insert_many( [insert_sql, post_sql_statements].flatten,
|
808
836
|
batch_values,
|
809
837
|
options,
|
810
|
-
"#{model_name} Create Many
|
838
|
+
"#{model_name} Create Many" )
|
839
|
+
|
811
840
|
number_inserted += result.num_inserts
|
812
841
|
ids += result.ids
|
813
842
|
results += result.results
|
843
|
+
current_batch += 1
|
844
|
+
|
845
|
+
progress_proc.call(import_size, batches, current_batch, Time.now.to_i - batch_started_at) if run_proc
|
814
846
|
end
|
815
847
|
else
|
816
848
|
transaction(requires_new: true) do
|
@@ -825,6 +857,15 @@ class ActiveRecord::Base
|
|
825
857
|
|
826
858
|
private
|
827
859
|
|
860
|
+
def associated_options(options, associated_class)
|
861
|
+
return options unless options.key?(:recursive_on_duplicate_key_update)
|
862
|
+
|
863
|
+
table_name = associated_class.arel_table.name.to_sym
|
864
|
+
options.merge(
|
865
|
+
on_duplicate_key_update: options[:recursive_on_duplicate_key_update][table_name]
|
866
|
+
)
|
867
|
+
end
|
868
|
+
|
828
869
|
def set_attributes_and_mark_clean(models, import_result, timestamps, options)
|
829
870
|
return if models.nil?
|
830
871
|
models -= import_result.failed_instances
|
@@ -836,22 +877,46 @@ class ActiveRecord::Base
|
|
836
877
|
model.id = id
|
837
878
|
|
838
879
|
timestamps.each do |attr, value|
|
839
|
-
model.send(attr
|
880
|
+
model.send("#{attr}=", value) if model.send(attr).nil?
|
840
881
|
end
|
841
882
|
end
|
842
883
|
end
|
843
884
|
|
844
|
-
|
845
|
-
|
846
|
-
|
847
|
-
|
885
|
+
deserialize_value = lambda do |column, value|
|
886
|
+
column = schema_columns_hash[column]
|
887
|
+
return value unless column
|
888
|
+
if respond_to?(:type_caster)
|
889
|
+
type = type_for_attribute(column.name)
|
890
|
+
type.deserialize(value)
|
891
|
+
elsif column.respond_to?(:type_cast_from_database)
|
892
|
+
column.type_cast_from_database(value)
|
893
|
+
else
|
894
|
+
value
|
895
|
+
end
|
896
|
+
end
|
897
|
+
|
898
|
+
set_value = lambda do |model, column, value|
|
899
|
+
val = deserialize_value.call(column, value)
|
900
|
+
if model.attribute_names.include?(column)
|
901
|
+
model.send("#{column}=", val)
|
902
|
+
else
|
903
|
+
attributes = attributes_builder.build_from_database(model.attributes.merge(column => val))
|
904
|
+
model.instance_variable_set(:@attributes, attributes)
|
905
|
+
end
|
906
|
+
end
|
907
|
+
|
908
|
+
columns = Array(options[:returning_columns])
|
909
|
+
results = Array(import_result.results)
|
910
|
+
if models.size == results.size
|
911
|
+
single_column = columns.first if columns.size == 1
|
912
|
+
results.each_with_index do |result, index|
|
848
913
|
model = models[index]
|
849
914
|
|
850
915
|
if single_column
|
851
|
-
|
916
|
+
set_value.call(model, single_column, result)
|
852
917
|
else
|
853
918
|
columns.each_with_index do |column, col_index|
|
854
|
-
|
919
|
+
set_value.call(model, column, result[col_index])
|
855
920
|
end
|
856
921
|
end
|
857
922
|
end
|
@@ -872,16 +937,22 @@ class ActiveRecord::Base
|
|
872
937
|
|
873
938
|
# Sync belongs_to association ids with foreign key field
|
874
939
|
def load_association_ids(model)
|
940
|
+
changed_columns = model.changed
|
875
941
|
association_reflections = model.class.reflect_on_all_associations(:belongs_to)
|
876
942
|
association_reflections.each do |association_reflection|
|
877
|
-
column_name = association_reflection.foreign_key
|
878
943
|
next if association_reflection.options[:polymorphic]
|
879
|
-
association = model.association(association_reflection.name)
|
880
|
-
association = association.target
|
881
|
-
next if association.blank? || model.public_send(column_name).present?
|
882
944
|
|
883
|
-
|
884
|
-
|
945
|
+
column_names = Array(association_reflection.foreign_key).map(&:to_s)
|
946
|
+
column_names.each_with_index do |column_name, column_index|
|
947
|
+
next if changed_columns.include?(column_name)
|
948
|
+
|
949
|
+
association = model.association(association_reflection.name)
|
950
|
+
association = association.target
|
951
|
+
next if association.blank? || model.public_send(column_name).present?
|
952
|
+
|
953
|
+
association_primary_key = Array(association_reflection.association_primary_key.tr("[]:", "").split(", "))[column_index]
|
954
|
+
model.public_send("#{column_name}=", association.send(association_primary_key))
|
955
|
+
end
|
885
956
|
end
|
886
957
|
end
|
887
958
|
|
@@ -894,17 +965,30 @@ class ActiveRecord::Base
|
|
894
965
|
associated_objects_by_class = {}
|
895
966
|
models.each { |model| find_associated_objects_for_import(associated_objects_by_class, model) }
|
896
967
|
|
897
|
-
# :on_duplicate_key_update
|
898
|
-
options.delete(:on_duplicate_key_update)
|
968
|
+
# :on_duplicate_key_update only supported for all fields
|
969
|
+
options.delete(:on_duplicate_key_update) unless options[:on_duplicate_key_update] == :all
|
970
|
+
# :returning not supported for associations
|
899
971
|
options.delete(:returning)
|
900
972
|
|
901
973
|
associated_objects_by_class.each_value do |associations|
|
902
974
|
associations.each_value do |associated_records|
|
903
|
-
|
975
|
+
next if associated_records.empty?
|
976
|
+
|
977
|
+
associated_class = associated_records.first.class
|
978
|
+
associated_class.bulk_import(associated_records,
|
979
|
+
associated_options(options, associated_class))
|
904
980
|
end
|
905
981
|
end
|
906
982
|
end
|
907
983
|
|
984
|
+
def schema_columns_hash
|
985
|
+
if respond_to?(:ignored_columns) && ignored_columns.any?
|
986
|
+
connection.schema_cache.columns_hash(table_name)
|
987
|
+
else
|
988
|
+
columns_hash
|
989
|
+
end
|
990
|
+
end
|
991
|
+
|
908
992
|
# We are eventually going to call Class.import <objects> so we build up a hash
|
909
993
|
# of class => objects to import.
|
910
994
|
def find_associated_objects_for_import(associated_objects_by_class, model)
|
@@ -925,10 +1009,18 @@ class ActiveRecord::Base
|
|
925
1009
|
|
926
1010
|
changed_objects = association.select { |a| a.new_record? || a.changed? }
|
927
1011
|
changed_objects.each do |child|
|
928
|
-
|
1012
|
+
Array(association_reflection.inverse_of&.foreign_key || association_reflection.foreign_key).each_with_index do |column, index|
|
1013
|
+
child.public_send("#{column}=", Array(model.id)[index])
|
1014
|
+
end
|
1015
|
+
|
929
1016
|
# For polymorphic associations
|
1017
|
+
association_name = if model.class.respond_to?(:polymorphic_name)
|
1018
|
+
model.class.polymorphic_name
|
1019
|
+
else
|
1020
|
+
model.class.base_class
|
1021
|
+
end
|
930
1022
|
association_reflection.type.try do |type|
|
931
|
-
child.public_send("#{type}=",
|
1023
|
+
child.public_send("#{type}=", association_name)
|
932
1024
|
end
|
933
1025
|
end
|
934
1026
|
associated_objects_by_class[model.class.name][association_reflection.name].concat changed_objects
|
@@ -955,7 +1047,7 @@ class ActiveRecord::Base
|
|
955
1047
|
elsif column
|
956
1048
|
if respond_to?(:type_caster) # Rails 5.0 and higher
|
957
1049
|
type = type_for_attribute(column.name)
|
958
|
-
val = type.type == :boolean ? type.cast(val) : type.serialize(val)
|
1050
|
+
val = !type.respond_to?(:subtype) && type.type == :boolean ? type.cast(val) : type.serialize(val)
|
959
1051
|
connection_memo.quote(val)
|
960
1052
|
elsif column.respond_to?(:type_cast_from_user) # Rails 4.2
|
961
1053
|
connection_memo.quote(column.type_cast_from_user(val), column)
|
@@ -964,7 +1056,7 @@ class ActiveRecord::Base
|
|
964
1056
|
val = serialized_attributes[column.name].dump(val)
|
965
1057
|
end
|
966
1058
|
# Fixes #443 to support binary (i.e. bytea) columns on PG
|
967
|
-
val = column.type_cast(val) unless column.type.to_sym == :binary
|
1059
|
+
val = column.type_cast(val) unless column.type && column.type.to_sym == :binary
|
968
1060
|
connection_memo.quote(val, column)
|
969
1061
|
end
|
970
1062
|
else
|
@@ -983,13 +1075,18 @@ class ActiveRecord::Base
|
|
983
1075
|
timestamp_columns[:create] = timestamp_attributes_for_create_in_model
|
984
1076
|
timestamp_columns[:update] = timestamp_attributes_for_update_in_model
|
985
1077
|
else
|
986
|
-
instance =
|
1078
|
+
instance = allocate
|
987
1079
|
timestamp_columns[:create] = instance.send(:timestamp_attributes_for_create_in_model)
|
988
1080
|
timestamp_columns[:update] = instance.send(:timestamp_attributes_for_update_in_model)
|
989
1081
|
end
|
990
1082
|
|
991
1083
|
# use tz as set in ActiveRecord::Base
|
992
|
-
|
1084
|
+
default_timezone = if ActiveRecord.respond_to?(:default_timezone)
|
1085
|
+
ActiveRecord.default_timezone
|
1086
|
+
else
|
1087
|
+
ActiveRecord::Base.default_timezone
|
1088
|
+
end
|
1089
|
+
timestamp = default_timezone == :utc ? Time.now.utc : Time.now
|
993
1090
|
|
994
1091
|
[:create, :update].each do |action|
|
995
1092
|
timestamp_columns[action].each do |column|
|