activerecord-import 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yaml +7 -1
- data/.rubocop.yml +74 -8
- data/.rubocop_todo.yml +6 -16
- data/Brewfile +3 -1
- data/CHANGELOG.md +18 -0
- data/Gemfile +5 -3
- data/README.markdown +8 -6
- data/Rakefile +2 -0
- data/activerecord-import.gemspec +2 -1
- data/benchmarks/benchmark.rb +5 -3
- data/benchmarks/lib/base.rb +4 -2
- data/benchmarks/lib/cli_parser.rb +4 -2
- data/benchmarks/lib/float.rb +2 -0
- data/benchmarks/lib/mysql2_benchmark.rb +2 -0
- data/benchmarks/lib/output_to_csv.rb +2 -0
- data/benchmarks/lib/output_to_html.rb +4 -2
- data/benchmarks/models/test_innodb.rb +2 -0
- data/benchmarks/models/test_memory.rb +2 -0
- data/benchmarks/models/test_myisam.rb +2 -0
- data/benchmarks/schema/mysql2_schema.rb +2 -0
- data/gemfiles/4.2.gemfile +2 -0
- data/gemfiles/5.0.gemfile +2 -0
- data/gemfiles/5.1.gemfile +2 -0
- data/gemfiles/5.2.gemfile +2 -0
- data/gemfiles/6.0.gemfile +2 -0
- data/gemfiles/6.1.gemfile +2 -0
- data/gemfiles/7.0.gemfile +3 -0
- data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/abstract_adapter.rb +8 -5
- data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql_adapter.rb +26 -18
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +63 -42
- data/lib/activerecord-import/adapters/sqlite3_adapter.rb +33 -25
- data/lib/activerecord-import/base.rb +3 -1
- data/lib/activerecord-import/import.rb +60 -32
- data/lib/activerecord-import/mysql2.rb +2 -0
- data/lib/activerecord-import/postgresql.rb +2 -0
- data/lib/activerecord-import/sqlite3.rb +2 -0
- data/lib/activerecord-import/synchronize.rb +2 -0
- data/lib/activerecord-import/value_sets_parser.rb +3 -0
- data/lib/activerecord-import/version.rb +3 -1
- data/lib/activerecord-import.rb +3 -1
- data/test/adapters/jdbcmysql.rb +2 -0
- data/test/adapters/jdbcpostgresql.rb +2 -0
- data/test/adapters/jdbcsqlite3.rb +2 -0
- data/test/adapters/makara_postgis.rb +2 -0
- data/test/adapters/mysql2.rb +2 -0
- data/test/adapters/mysql2_makara.rb +2 -0
- data/test/adapters/mysql2spatial.rb +2 -0
- data/test/adapters/postgis.rb +2 -0
- data/test/adapters/postgresql.rb +2 -0
- data/test/adapters/postgresql_makara.rb +2 -0
- data/test/adapters/seamless_database_pool.rb +2 -0
- data/test/adapters/spatialite.rb +2 -0
- data/test/adapters/sqlite3.rb +2 -0
- data/test/import_test.rb +21 -0
- data/test/jdbcmysql/import_test.rb +5 -3
- data/test/jdbcpostgresql/import_test.rb +4 -2
- data/test/jdbcsqlite3/import_test.rb +4 -2
- data/test/makara_postgis/import_test.rb +4 -2
- data/test/models/account.rb +2 -0
- data/test/models/alarm.rb +2 -0
- data/test/models/animal.rb +2 -0
- data/test/models/bike_maker.rb +3 -0
- data/test/models/book.rb +2 -0
- data/test/models/car.rb +2 -0
- data/test/models/card.rb +2 -0
- data/test/models/chapter.rb +2 -0
- data/test/models/customer.rb +2 -0
- data/test/models/deck.rb +2 -0
- data/test/models/dictionary.rb +2 -0
- data/test/models/discount.rb +2 -0
- data/test/models/end_note.rb +2 -0
- data/test/models/group.rb +2 -0
- data/test/models/order.rb +2 -0
- data/test/models/playing_card.rb +2 -0
- data/test/models/promotion.rb +2 -0
- data/test/models/question.rb +2 -0
- data/test/models/rule.rb +2 -0
- data/test/models/tag.rb +3 -0
- data/test/models/tag_alias.rb +5 -0
- data/test/models/topic.rb +7 -0
- data/test/models/user.rb +2 -0
- data/test/models/user_token.rb +2 -0
- data/test/models/vendor.rb +2 -0
- data/test/models/widget.rb +2 -0
- data/test/mysql2/import_test.rb +5 -3
- data/test/mysql2_makara/import_test.rb +5 -3
- data/test/mysqlspatial2/import_test.rb +5 -3
- data/test/postgis/import_test.rb +4 -2
- data/test/postgresql/import_test.rb +4 -2
- data/test/schema/generic_schema.rb +9 -0
- data/test/schema/jdbcpostgresql_schema.rb +3 -1
- data/test/schema/mysql2_schema.rb +2 -0
- data/test/schema/postgis_schema.rb +3 -1
- data/test/schema/postgresql_schema.rb +2 -0
- data/test/schema/sqlite3_schema.rb +2 -0
- data/test/schema/version.rb +2 -0
- data/test/sqlite3/import_test.rb +4 -2
- data/test/support/active_support/test_case_extensions.rb +2 -0
- data/test/support/assertions.rb +2 -0
- data/test/support/factories.rb +2 -0
- data/test/support/generate.rb +4 -2
- data/test/support/mysql/import_examples.rb +2 -1
- data/test/support/postgresql/import_examples.rb +40 -1
- data/test/support/shared_examples/on_duplicate_key_ignore.rb +2 -0
- data/test/support/shared_examples/on_duplicate_key_update.rb +41 -10
- data/test/support/shared_examples/recursive_import.rb +2 -0
- data/test/support/sqlite3/import_examples.rb +2 -1
- data/test/synchronize_test.rb +2 -0
- data/test/test_helper.rb +11 -3
- data/test/value_sets_bytes_parser_test.rb +3 -1
- data/test/value_sets_records_parser_test.rb +3 -1
- metadata +5 -3
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module ActiveRecord::Import::MysqlAdapter
|
|
2
4
|
include ActiveRecord::Import::ImportSupport
|
|
3
5
|
include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
|
|
@@ -11,13 +13,14 @@ module ActiveRecord::Import::MysqlAdapter
|
|
|
11
13
|
# the number of inserts default
|
|
12
14
|
number_of_inserts = 0
|
|
13
15
|
|
|
14
|
-
base_sql, post_sql =
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
16
|
+
base_sql, post_sql = case sql
|
|
17
|
+
when String
|
|
18
|
+
[sql, '']
|
|
19
|
+
when Array
|
|
20
|
+
[sql.shift, sql.join( ' ' )]
|
|
18
21
|
end
|
|
19
22
|
|
|
20
|
-
sql_size = QUERY_OVERHEAD + base_sql.
|
|
23
|
+
sql_size = QUERY_OVERHEAD + base_sql.bytesize + post_sql.bytesize
|
|
21
24
|
|
|
22
25
|
# the number of bytes the requested insert statement values will take up
|
|
23
26
|
values_in_bytes = values.sum(&:bytesize)
|
|
@@ -31,7 +34,7 @@ module ActiveRecord::Import::MysqlAdapter
|
|
|
31
34
|
max = max_allowed_packet
|
|
32
35
|
|
|
33
36
|
# if we can insert it all as one statement
|
|
34
|
-
if
|
|
37
|
+
if max == NO_MAX_PACKET || total_bytes <= max || options[:force_single_insert]
|
|
35
38
|
number_of_inserts += 1
|
|
36
39
|
sql2insert = base_sql + values.join( ',' ) + post_sql
|
|
37
40
|
insert( sql2insert, *args )
|
|
@@ -83,13 +86,13 @@ module ActiveRecord::Import::MysqlAdapter
|
|
|
83
86
|
# in +args+.
|
|
84
87
|
def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
|
|
85
88
|
sql = ' ON DUPLICATE KEY UPDATE '.dup
|
|
86
|
-
arg = args
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arg )
|
|
90
|
-
|
|
91
|
-
sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, arg )
|
|
92
|
-
|
|
89
|
+
arg, model, _primary_key, locking_column = args
|
|
90
|
+
case arg
|
|
91
|
+
when Array
|
|
92
|
+
sql << sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arg )
|
|
93
|
+
when Hash
|
|
94
|
+
sql << sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, arg )
|
|
95
|
+
when String
|
|
93
96
|
sql << arg
|
|
94
97
|
else
|
|
95
98
|
raise ArgumentError, "Expected Array or Hash"
|
|
@@ -97,19 +100,24 @@ module ActiveRecord::Import::MysqlAdapter
|
|
|
97
100
|
sql
|
|
98
101
|
end
|
|
99
102
|
|
|
100
|
-
def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
|
|
103
|
+
def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
|
|
101
104
|
results = arr.map do |column|
|
|
102
|
-
|
|
105
|
+
original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
|
|
106
|
+
qc = quote_column_name( original_column_name )
|
|
103
107
|
"#{table_name}.#{qc}=VALUES(#{qc})"
|
|
104
108
|
end
|
|
105
109
|
increment_locking_column!(table_name, results, locking_column)
|
|
106
110
|
results.join( ',' )
|
|
107
111
|
end
|
|
108
112
|
|
|
109
|
-
def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
|
|
113
|
+
def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
|
|
110
114
|
results = hsh.map do |column1, column2|
|
|
111
|
-
|
|
112
|
-
|
|
115
|
+
original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
|
|
116
|
+
qc1 = quote_column_name( original_column1_name )
|
|
117
|
+
|
|
118
|
+
original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
|
|
119
|
+
qc2 = quote_column_name( original_column2_name )
|
|
120
|
+
|
|
113
121
|
"#{table_name}.#{qc1}=VALUES( #{qc2} )"
|
|
114
122
|
end
|
|
115
123
|
increment_locking_column!(table_name, results, locking_column)
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module ActiveRecord::Import::PostgreSQLAdapter
|
|
2
4
|
include ActiveRecord::Import::ImportSupport
|
|
3
5
|
include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
|
|
@@ -6,59 +8,66 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
|
6
8
|
|
|
7
9
|
def insert_many( sql, values, options = {}, *args ) # :nodoc:
|
|
8
10
|
number_of_inserts = 1
|
|
9
|
-
returned_values =
|
|
11
|
+
returned_values = {}
|
|
10
12
|
ids = []
|
|
11
13
|
results = []
|
|
12
14
|
|
|
13
|
-
base_sql, post_sql =
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
base_sql, post_sql = case sql
|
|
16
|
+
when String
|
|
17
|
+
[sql, '']
|
|
18
|
+
when Array
|
|
19
|
+
[sql.shift, sql.join( ' ' )]
|
|
17
20
|
end
|
|
18
21
|
|
|
19
22
|
sql2insert = base_sql + values.join( ',' ) + post_sql
|
|
20
23
|
|
|
21
|
-
|
|
22
|
-
if
|
|
24
|
+
selections = returning_selections(options)
|
|
25
|
+
if selections.blank? || (options[:no_returning] && !options[:recursive])
|
|
23
26
|
insert( sql2insert, *args )
|
|
24
27
|
else
|
|
25
|
-
returned_values = if
|
|
28
|
+
returned_values = if selections.size > 1
|
|
26
29
|
# Select composite columns
|
|
27
|
-
|
|
30
|
+
db_result = select_all( sql2insert, *args )
|
|
31
|
+
{ values: db_result.rows, columns: db_result.columns }
|
|
28
32
|
else
|
|
29
|
-
select_values( sql2insert, *args )
|
|
33
|
+
{ values: select_values( sql2insert, *args ) }
|
|
30
34
|
end
|
|
31
35
|
clear_query_cache if query_cache_enabled
|
|
32
36
|
end
|
|
33
37
|
|
|
34
38
|
if options[:returning].blank?
|
|
35
|
-
ids = returned_values
|
|
39
|
+
ids = Array(returned_values[:values])
|
|
36
40
|
elsif options[:primary_key].blank?
|
|
37
|
-
|
|
41
|
+
options[:returning_columns] ||= returned_values[:columns]
|
|
42
|
+
results = Array(returned_values[:values])
|
|
38
43
|
else
|
|
39
44
|
# split primary key and returning columns
|
|
40
|
-
ids, results = split_ids_and_results(returned_values,
|
|
45
|
+
ids, results, options[:returning_columns] = split_ids_and_results(returned_values, options)
|
|
41
46
|
end
|
|
42
47
|
|
|
43
48
|
ActiveRecord::Import::Result.new([], number_of_inserts, ids, results)
|
|
44
49
|
end
|
|
45
50
|
|
|
46
|
-
def split_ids_and_results(
|
|
51
|
+
def split_ids_and_results( selections, options )
|
|
47
52
|
ids = []
|
|
48
|
-
|
|
53
|
+
returning_values = []
|
|
54
|
+
|
|
55
|
+
columns = Array(selections[:columns])
|
|
56
|
+
values = Array(selections[:values])
|
|
49
57
|
id_indexes = Array(options[:primary_key]).map { |key| columns.index(key) }
|
|
50
|
-
|
|
58
|
+
returning_columns = columns.reject.with_index { |_, index| id_indexes.include?(index) }
|
|
59
|
+
returning_indexes = returning_columns.map { |column| columns.index(column) }
|
|
51
60
|
|
|
52
61
|
values.each do |value|
|
|
53
62
|
value_array = Array(value)
|
|
54
|
-
ids << id_indexes.map { |
|
|
55
|
-
|
|
63
|
+
ids << id_indexes.map { |index| value_array[index] }
|
|
64
|
+
returning_values << returning_indexes.map { |index| value_array[index] }
|
|
56
65
|
end
|
|
57
66
|
|
|
58
67
|
ids.map!(&:first) if id_indexes.size == 1
|
|
59
|
-
|
|
68
|
+
returning_values.map!(&:first) if returning_columns.size == 1
|
|
60
69
|
|
|
61
|
-
[ids,
|
|
70
|
+
[ids, returning_values, returning_columns]
|
|
62
71
|
end
|
|
63
72
|
|
|
64
73
|
def next_value_for_sequence(sequence_name)
|
|
@@ -79,31 +88,37 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
|
79
88
|
|
|
80
89
|
sql += super(table_name, options)
|
|
81
90
|
|
|
82
|
-
|
|
83
|
-
unless
|
|
84
|
-
sql << " RETURNING
|
|
91
|
+
selections = returning_selections(options)
|
|
92
|
+
unless selections.blank? || (options[:no_returning] && !options[:recursive])
|
|
93
|
+
sql << " RETURNING #{selections.join(', ')}"
|
|
85
94
|
end
|
|
86
95
|
|
|
87
96
|
sql
|
|
88
97
|
end
|
|
89
98
|
|
|
90
|
-
def
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
99
|
+
def returning_selections(options)
|
|
100
|
+
selections = []
|
|
101
|
+
column_names = Array(options[:model].column_names)
|
|
102
|
+
|
|
103
|
+
selections += Array(options[:primary_key]) if options[:primary_key].present?
|
|
104
|
+
selections += Array(options[:returning]) if options[:returning].present?
|
|
105
|
+
|
|
106
|
+
selections.map do |selection|
|
|
107
|
+
column_names.include?(selection.to_s) ? "\"#{selection}\"" : selection
|
|
108
|
+
end
|
|
95
109
|
end
|
|
96
110
|
|
|
97
111
|
# Add a column to be updated on duplicate key update
|
|
98
112
|
def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
|
|
99
113
|
arg = options[:on_duplicate_key_update]
|
|
100
|
-
|
|
114
|
+
case arg
|
|
115
|
+
when Hash
|
|
101
116
|
columns = arg.fetch( :columns ) { arg[:columns] = [] }
|
|
102
117
|
case columns
|
|
103
118
|
when Array then columns << column.to_sym unless columns.include?( column.to_sym )
|
|
104
119
|
when Hash then columns[column.to_sym] = column.to_sym
|
|
105
120
|
end
|
|
106
|
-
|
|
121
|
+
when Array
|
|
107
122
|
arg << column.to_sym unless arg.include?( column.to_sym )
|
|
108
123
|
end
|
|
109
124
|
end
|
|
@@ -119,7 +134,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
|
119
134
|
# Returns a generated ON CONFLICT DO UPDATE statement given the passed
|
|
120
135
|
# in +args+.
|
|
121
136
|
def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
|
|
122
|
-
arg, primary_key, locking_column = args
|
|
137
|
+
arg, model, primary_key, locking_column = args
|
|
123
138
|
arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
|
|
124
139
|
return unless arg.is_a?( Hash )
|
|
125
140
|
|
|
@@ -138,11 +153,12 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
|
138
153
|
end
|
|
139
154
|
|
|
140
155
|
sql << "#{conflict_target}DO UPDATE SET "
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
156
|
+
case columns
|
|
157
|
+
when Array
|
|
158
|
+
sql << sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, columns )
|
|
159
|
+
when Hash
|
|
160
|
+
sql << sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, columns )
|
|
161
|
+
when String
|
|
146
162
|
sql << columns
|
|
147
163
|
else
|
|
148
164
|
raise ArgumentError, 'Expected :columns to be an Array or Hash'
|
|
@@ -153,19 +169,24 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
|
153
169
|
sql
|
|
154
170
|
end
|
|
155
171
|
|
|
156
|
-
def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
|
|
172
|
+
def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
|
|
157
173
|
results = arr.map do |column|
|
|
158
|
-
|
|
174
|
+
original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
|
|
175
|
+
qc = quote_column_name( original_column_name )
|
|
159
176
|
"#{qc}=EXCLUDED.#{qc}"
|
|
160
177
|
end
|
|
161
178
|
increment_locking_column!(table_name, results, locking_column)
|
|
162
179
|
results.join( ',' )
|
|
163
180
|
end
|
|
164
181
|
|
|
165
|
-
def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
|
|
182
|
+
def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
|
|
166
183
|
results = hsh.map do |column1, column2|
|
|
167
|
-
|
|
168
|
-
|
|
184
|
+
original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
|
|
185
|
+
qc1 = quote_column_name( original_column1_name )
|
|
186
|
+
|
|
187
|
+
original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
|
|
188
|
+
qc2 = quote_column_name( original_column2_name )
|
|
189
|
+
|
|
169
190
|
"#{qc1}=EXCLUDED.#{qc2}"
|
|
170
191
|
end
|
|
171
192
|
increment_locking_column!(table_name, results, locking_column)
|
|
@@ -179,7 +200,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
|
179
200
|
if constraint_name.present?
|
|
180
201
|
"ON CONSTRAINT #{constraint_name} "
|
|
181
202
|
elsif conflict_target.present?
|
|
182
|
-
sql =
|
|
203
|
+
sql = "(#{Array( conflict_target ).reject( &:blank? ).join( ', ' )}) "
|
|
183
204
|
sql += "WHERE #{index_predicate} " if index_predicate
|
|
184
205
|
sql
|
|
185
206
|
end
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module ActiveRecord::Import::SQLite3Adapter
|
|
2
4
|
include ActiveRecord::Import::ImportSupport
|
|
3
5
|
include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
|
|
4
6
|
|
|
5
|
-
MIN_VERSION_FOR_IMPORT = "3.7.11"
|
|
6
|
-
MIN_VERSION_FOR_UPSERT = "3.24.0"
|
|
7
|
+
MIN_VERSION_FOR_IMPORT = "3.7.11"
|
|
8
|
+
MIN_VERSION_FOR_UPSERT = "3.24.0"
|
|
7
9
|
SQLITE_LIMIT_COMPOUND_SELECT = 500
|
|
8
10
|
|
|
9
11
|
# Override our conformance to ActiveRecord::Import::ImportSupport interface
|
|
@@ -22,10 +24,11 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
22
24
|
def insert_many( sql, values, _options = {}, *args ) # :nodoc:
|
|
23
25
|
number_of_inserts = 0
|
|
24
26
|
|
|
25
|
-
base_sql, post_sql =
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
27
|
+
base_sql, post_sql = case sql
|
|
28
|
+
when String
|
|
29
|
+
[sql, '']
|
|
30
|
+
when Array
|
|
31
|
+
[sql.shift, sql.join( ' ' )]
|
|
29
32
|
end
|
|
30
33
|
|
|
31
34
|
value_sets = ::ActiveRecord::Import::ValueSetsRecordsParser.parse(values,
|
|
@@ -54,11 +57,9 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
54
57
|
def post_sql_statements( table_name, options ) # :nodoc:
|
|
55
58
|
sql = []
|
|
56
59
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
sql << sql_for_on_duplicate_key_ignore( options[:on_duplicate_key_ignore] )
|
|
61
|
-
end
|
|
60
|
+
# Options :recursive and :on_duplicate_key_ignore are mutually exclusive
|
|
61
|
+
if supports_on_duplicate_key_update? && ((options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:on_duplicate_key_update])
|
|
62
|
+
sql << sql_for_on_duplicate_key_ignore( options[:on_duplicate_key_ignore] )
|
|
62
63
|
end
|
|
63
64
|
|
|
64
65
|
sql + super
|
|
@@ -71,13 +72,14 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
71
72
|
# Add a column to be updated on duplicate key update
|
|
72
73
|
def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
|
|
73
74
|
arg = options[:on_duplicate_key_update]
|
|
74
|
-
|
|
75
|
+
case arg
|
|
76
|
+
when Hash
|
|
75
77
|
columns = arg.fetch( :columns ) { arg[:columns] = [] }
|
|
76
78
|
case columns
|
|
77
79
|
when Array then columns << column.to_sym unless columns.include?( column.to_sym )
|
|
78
80
|
when Hash then columns[column.to_sym] = column.to_sym
|
|
79
81
|
end
|
|
80
|
-
|
|
82
|
+
when Array
|
|
81
83
|
arg << column.to_sym unless arg.include?( column.to_sym )
|
|
82
84
|
end
|
|
83
85
|
end
|
|
@@ -93,7 +95,7 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
93
95
|
# Returns a generated ON CONFLICT DO UPDATE statement given the passed
|
|
94
96
|
# in +args+.
|
|
95
97
|
def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
|
|
96
|
-
arg, primary_key, locking_column = args
|
|
98
|
+
arg, model, primary_key, locking_column = args
|
|
97
99
|
arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
|
|
98
100
|
return unless arg.is_a?( Hash )
|
|
99
101
|
|
|
@@ -112,11 +114,12 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
112
114
|
end
|
|
113
115
|
|
|
114
116
|
sql << "#{conflict_target}DO UPDATE SET "
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
117
|
+
case columns
|
|
118
|
+
when Array
|
|
119
|
+
sql << sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, columns )
|
|
120
|
+
when Hash
|
|
121
|
+
sql << sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, columns )
|
|
122
|
+
when String
|
|
120
123
|
sql << columns
|
|
121
124
|
else
|
|
122
125
|
raise ArgumentError, 'Expected :columns to be an Array or Hash'
|
|
@@ -127,19 +130,24 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
127
130
|
sql
|
|
128
131
|
end
|
|
129
132
|
|
|
130
|
-
def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
|
|
133
|
+
def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
|
|
131
134
|
results = arr.map do |column|
|
|
132
|
-
|
|
135
|
+
original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
|
|
136
|
+
qc = quote_column_name( original_column_name )
|
|
133
137
|
"#{qc}=EXCLUDED.#{qc}"
|
|
134
138
|
end
|
|
135
139
|
increment_locking_column!(table_name, results, locking_column)
|
|
136
140
|
results.join( ',' )
|
|
137
141
|
end
|
|
138
142
|
|
|
139
|
-
def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
|
|
143
|
+
def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
|
|
140
144
|
results = hsh.map do |column1, column2|
|
|
141
|
-
|
|
142
|
-
|
|
145
|
+
original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
|
|
146
|
+
qc1 = quote_column_name( original_column1_name )
|
|
147
|
+
|
|
148
|
+
original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
|
|
149
|
+
qc2 = quote_column_name( original_column2_name )
|
|
150
|
+
|
|
143
151
|
"#{qc1}=EXCLUDED.#{qc2}"
|
|
144
152
|
end
|
|
145
153
|
increment_locking_column!(table_name, results, locking_column)
|
|
@@ -150,7 +158,7 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
150
158
|
conflict_target = args[:conflict_target]
|
|
151
159
|
index_predicate = args[:index_predicate]
|
|
152
160
|
if conflict_target.present?
|
|
153
|
-
sql =
|
|
161
|
+
sql = "(#{Array( conflict_target ).reject( &:blank? ).join( ', ' )}) "
|
|
154
162
|
sql += "WHERE #{index_predicate} " if index_predicate
|
|
155
163
|
sql
|
|
156
164
|
end
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require "pathname"
|
|
2
4
|
require "active_record"
|
|
3
5
|
require "active_record/version"
|
|
4
6
|
|
|
5
7
|
module ActiveRecord::Import
|
|
6
|
-
ADAPTER_PATH = "activerecord-import/active_record/adapters"
|
|
8
|
+
ADAPTER_PATH = "activerecord-import/active_record/adapters"
|
|
7
9
|
|
|
8
10
|
def self.base_adapter(adapter)
|
|
9
11
|
case adapter
|
|
@@ -1,18 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require "ostruct"
|
|
2
4
|
|
|
3
5
|
module ActiveRecord::Import::ConnectionAdapters; end
|
|
4
6
|
|
|
5
|
-
module ActiveRecord::Import
|
|
7
|
+
module ActiveRecord::Import # :nodoc:
|
|
6
8
|
Result = Struct.new(:failed_instances, :num_inserts, :ids, :results)
|
|
7
9
|
|
|
8
|
-
module ImportSupport
|
|
9
|
-
def supports_import?
|
|
10
|
+
module ImportSupport # :nodoc:
|
|
11
|
+
def supports_import? # :nodoc:
|
|
10
12
|
true
|
|
11
13
|
end
|
|
12
14
|
end
|
|
13
15
|
|
|
14
|
-
module OnDuplicateKeyUpdateSupport
|
|
15
|
-
def supports_on_duplicate_key_update?
|
|
16
|
+
module OnDuplicateKeyUpdateSupport # :nodoc:
|
|
17
|
+
def supports_on_duplicate_key_update? # :nodoc:
|
|
16
18
|
true
|
|
17
19
|
end
|
|
18
20
|
end
|
|
@@ -55,7 +57,7 @@ module ActiveRecord::Import #:nodoc:
|
|
|
55
57
|
end
|
|
56
58
|
end
|
|
57
59
|
|
|
58
|
-
filter.instance_variable_set(:@attributes, attrs)
|
|
60
|
+
filter.instance_variable_set(:@attributes, attrs.flatten)
|
|
59
61
|
|
|
60
62
|
if @validate_callbacks.respond_to?(:chain, true)
|
|
61
63
|
@validate_callbacks.send(:chain).tap do |chain|
|
|
@@ -71,7 +73,7 @@ module ActiveRecord::Import #:nodoc:
|
|
|
71
73
|
end
|
|
72
74
|
|
|
73
75
|
def valid_model?(model)
|
|
74
|
-
init_validations(model.class) unless model.
|
|
76
|
+
init_validations(model.class) unless model.instance_of?(@validator_class)
|
|
75
77
|
|
|
76
78
|
validation_context = @options[:validate_with_context]
|
|
77
79
|
validation_context ||= (model.new_record? ? :create : :update)
|
|
@@ -83,7 +85,11 @@ module ActiveRecord::Import #:nodoc:
|
|
|
83
85
|
|
|
84
86
|
model.run_callbacks(:validation) do
|
|
85
87
|
if defined?(ActiveSupport::Callbacks::Filters::Environment) # ActiveRecord >= 4.1
|
|
86
|
-
runner = @validate_callbacks.compile
|
|
88
|
+
runner = if @validate_callbacks.method(:compile).arity == 0
|
|
89
|
+
@validate_callbacks.compile
|
|
90
|
+
else # ActiveRecord >= 7.1
|
|
91
|
+
@validate_callbacks.compile(nil)
|
|
92
|
+
end
|
|
87
93
|
env = ActiveSupport::Callbacks::Filters::Environment.new(model, false, nil)
|
|
88
94
|
if runner.respond_to?(:call) # ActiveRecord < 5.1
|
|
89
95
|
runner.call(env)
|
|
@@ -163,7 +169,7 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
|
163
169
|
m.public_send "#{reflection.type}=", owner.class.name if reflection.type
|
|
164
170
|
end
|
|
165
171
|
|
|
166
|
-
|
|
172
|
+
model_klass.bulk_import column_names, models, options
|
|
167
173
|
|
|
168
174
|
# supports array of hash objects
|
|
169
175
|
elsif args.last.is_a?( Array ) && args.last.first.is_a?(Hash)
|
|
@@ -202,11 +208,11 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
|
202
208
|
end
|
|
203
209
|
end
|
|
204
210
|
|
|
205
|
-
|
|
211
|
+
model_klass.bulk_import column_names, array_of_attributes, options
|
|
206
212
|
|
|
207
213
|
# supports empty array
|
|
208
214
|
elsif args.last.is_a?( Array ) && args.last.empty?
|
|
209
|
-
|
|
215
|
+
ActiveRecord::Import::Result.new([], 0, [])
|
|
210
216
|
|
|
211
217
|
# supports 2-element array and array
|
|
212
218
|
elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
|
|
@@ -237,7 +243,7 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
|
237
243
|
end
|
|
238
244
|
end
|
|
239
245
|
|
|
240
|
-
|
|
246
|
+
model_klass.bulk_import column_names, array_of_attributes, options
|
|
241
247
|
else
|
|
242
248
|
raise ArgumentError, "Invalid arguments!"
|
|
243
249
|
end
|
|
@@ -547,7 +553,7 @@ class ActiveRecord::Base
|
|
|
547
553
|
alias import! bulk_import! unless ActiveRecord::Base.respond_to? :import!
|
|
548
554
|
|
|
549
555
|
def import_helper( *args )
|
|
550
|
-
options = { validate: true, timestamps: true, track_validation_failures: false }
|
|
556
|
+
options = { model: self, validate: true, timestamps: true, track_validation_failures: false }
|
|
551
557
|
options.merge!( args.pop ) if args.last.is_a? Hash
|
|
552
558
|
# making sure that current model's primary key is used
|
|
553
559
|
options[:primary_key] = primary_key
|
|
@@ -572,7 +578,7 @@ class ActiveRecord::Base
|
|
|
572
578
|
|
|
573
579
|
if models.first.id.nil?
|
|
574
580
|
Array(primary_key).each do |c|
|
|
575
|
-
if column_names.include?(c) &&
|
|
581
|
+
if column_names.include?(c) && schema_columns_hash[c].type == :uuid
|
|
576
582
|
column_names.delete(c)
|
|
577
583
|
end
|
|
578
584
|
end
|
|
@@ -695,7 +701,11 @@ class ActiveRecord::Base
|
|
|
695
701
|
return_obj = if is_validating
|
|
696
702
|
import_with_validations( column_names, array_of_attributes, options ) do |failed_instances|
|
|
697
703
|
if models
|
|
698
|
-
models.
|
|
704
|
+
models.each_with_index do |m, i|
|
|
705
|
+
next unless m.errors.any?
|
|
706
|
+
|
|
707
|
+
failed_instances << (options[:track_validation_failures] ? [i, m] : m)
|
|
708
|
+
end
|
|
699
709
|
else
|
|
700
710
|
# create instances for each of our column/value sets
|
|
701
711
|
arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
|
|
@@ -772,7 +782,10 @@ class ActiveRecord::Base
|
|
|
772
782
|
def import_without_validations_or_callbacks( column_names, array_of_attributes, options = {} )
|
|
773
783
|
return ActiveRecord::Import::Result.new([], 0, [], []) if array_of_attributes.empty?
|
|
774
784
|
|
|
775
|
-
column_names = column_names.map
|
|
785
|
+
column_names = column_names.map do |name|
|
|
786
|
+
original_name = attribute_alias?(name) ? attribute_alias(name) : name
|
|
787
|
+
original_name.to_sym
|
|
788
|
+
end
|
|
776
789
|
scope_columns, scope_values = scope_attributes.to_a.transpose
|
|
777
790
|
|
|
778
791
|
unless scope_columns.blank?
|
|
@@ -784,15 +797,13 @@ class ActiveRecord::Base
|
|
|
784
797
|
end
|
|
785
798
|
end
|
|
786
799
|
|
|
787
|
-
if finder_needs_type_condition?
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
array_of_attributes.each { |attrs| attrs << sti_name }
|
|
791
|
-
end
|
|
800
|
+
if finder_needs_type_condition? && !column_names.include?(inheritance_column.to_sym)
|
|
801
|
+
column_names << inheritance_column.to_sym
|
|
802
|
+
array_of_attributes.each { |attrs| attrs << sti_name }
|
|
792
803
|
end
|
|
793
804
|
|
|
794
805
|
columns = column_names.each_with_index.map do |name, i|
|
|
795
|
-
column =
|
|
806
|
+
column = schema_columns_hash[name.to_s]
|
|
796
807
|
raise ActiveRecord::Import::MissingColumnError.new(name.to_s, i) if column.nil?
|
|
797
808
|
column
|
|
798
809
|
end
|
|
@@ -857,13 +868,13 @@ class ActiveRecord::Base
|
|
|
857
868
|
model.id = id
|
|
858
869
|
|
|
859
870
|
timestamps.each do |attr, value|
|
|
860
|
-
model.send(attr
|
|
871
|
+
model.send("#{attr}=", value) if model.send(attr).nil?
|
|
861
872
|
end
|
|
862
873
|
end
|
|
863
874
|
end
|
|
864
875
|
|
|
865
876
|
deserialize_value = lambda do |column, value|
|
|
866
|
-
column =
|
|
877
|
+
column = schema_columns_hash[column]
|
|
867
878
|
return value unless column
|
|
868
879
|
if respond_to?(:type_caster)
|
|
869
880
|
type = type_for_attribute(column.name)
|
|
@@ -875,19 +886,28 @@ class ActiveRecord::Base
|
|
|
875
886
|
end
|
|
876
887
|
end
|
|
877
888
|
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
889
|
+
set_value = lambda do |model, column, value|
|
|
890
|
+
val = deserialize_value.call(column, value)
|
|
891
|
+
if model.attribute_names.include?(column)
|
|
892
|
+
model.send("#{column}=", val)
|
|
893
|
+
else
|
|
894
|
+
attributes = attributes_builder.build_from_database(model.attributes.merge(column => val))
|
|
895
|
+
model.instance_variable_set(:@attributes, attributes)
|
|
896
|
+
end
|
|
897
|
+
end
|
|
898
|
+
|
|
899
|
+
columns = Array(options[:returning_columns])
|
|
900
|
+
results = Array(import_result.results)
|
|
901
|
+
if models.size == results.size
|
|
902
|
+
single_column = columns.first if columns.size == 1
|
|
903
|
+
results.each_with_index do |result, index|
|
|
882
904
|
model = models[index]
|
|
883
905
|
|
|
884
906
|
if single_column
|
|
885
|
-
|
|
886
|
-
model.send(single_column, val)
|
|
907
|
+
set_value.call(model, single_column, result)
|
|
887
908
|
else
|
|
888
909
|
columns.each_with_index do |column, col_index|
|
|
889
|
-
|
|
890
|
-
model.send("#{column}=", val)
|
|
910
|
+
set_value.call(model, column, result[col_index])
|
|
891
911
|
end
|
|
892
912
|
end
|
|
893
913
|
end
|
|
@@ -948,6 +968,14 @@ class ActiveRecord::Base
|
|
|
948
968
|
end
|
|
949
969
|
end
|
|
950
970
|
|
|
971
|
+
def schema_columns_hash
|
|
972
|
+
@schema_columns_hash ||= if respond_to?(:ignored_columns) && ignored_columns.any?
|
|
973
|
+
connection.schema_cache.columns_hash(table_name)
|
|
974
|
+
else
|
|
975
|
+
columns_hash
|
|
976
|
+
end
|
|
977
|
+
end
|
|
978
|
+
|
|
951
979
|
# We are eventually going to call Class.import <objects> so we build up a hash
|
|
952
980
|
# of class => objects to import.
|
|
953
981
|
def find_associated_objects_for_import(associated_objects_by_class, model)
|