activerecord-import 1.0.4 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/test.yaml +159 -0
- data/.gitignore +5 -0
- data/.rubocop.yml +76 -7
- data/.rubocop_todo.yml +10 -16
- data/Brewfile +3 -1
- data/CHANGELOG.md +143 -3
- data/Dockerfile +23 -0
- data/Gemfile +28 -24
- data/LICENSE +21 -56
- data/README.markdown +83 -27
- data/Rakefile +3 -0
- data/activerecord-import.gemspec +10 -5
- data/benchmarks/benchmark.rb +10 -6
- data/benchmarks/lib/base.rb +10 -5
- data/benchmarks/lib/cli_parser.rb +10 -6
- data/benchmarks/lib/float.rb +2 -0
- data/benchmarks/lib/mysql2_benchmark.rb +2 -0
- data/benchmarks/lib/output_to_csv.rb +2 -0
- data/benchmarks/lib/output_to_html.rb +4 -2
- data/benchmarks/models/test_innodb.rb +2 -0
- data/benchmarks/models/test_memory.rb +2 -0
- data/benchmarks/models/test_myisam.rb +2 -0
- data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +2 -0
- data/docker-compose.yml +34 -0
- data/gemfiles/5.2.gemfile +2 -0
- data/gemfiles/6.0.gemfile +3 -0
- data/gemfiles/6.1.gemfile +4 -1
- data/gemfiles/7.0.gemfile +4 -0
- data/gemfiles/7.1.gemfile +3 -0
- data/gemfiles/7.2.gemfile +3 -0
- data/gemfiles/8.0.gemfile +3 -0
- data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +6 -4
- data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/trilogy_adapter.rb +8 -0
- data/lib/activerecord-import/adapters/abstract_adapter.rb +9 -6
- data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql_adapter.rb +30 -21
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +68 -48
- data/lib/activerecord-import/adapters/sqlite3_adapter.rb +37 -30
- data/lib/activerecord-import/adapters/trilogy_adapter.rb +7 -0
- data/lib/activerecord-import/base.rb +3 -1
- data/lib/activerecord-import/import.rb +160 -58
- data/lib/activerecord-import/synchronize.rb +3 -1
- data/lib/activerecord-import/value_sets_parser.rb +5 -0
- data/lib/activerecord-import/version.rb +3 -1
- data/lib/activerecord-import.rb +2 -1
- data/test/adapters/jdbcmysql.rb +2 -0
- data/test/adapters/jdbcpostgresql.rb +2 -0
- data/test/adapters/jdbcsqlite3.rb +2 -0
- data/test/adapters/makara_postgis.rb +2 -0
- data/test/adapters/mysql2.rb +2 -0
- data/test/adapters/mysql2_makara.rb +2 -0
- data/test/adapters/mysql2spatial.rb +2 -0
- data/test/adapters/postgis.rb +2 -0
- data/test/adapters/postgresql.rb +2 -0
- data/test/adapters/postgresql_makara.rb +2 -0
- data/test/adapters/seamless_database_pool.rb +2 -0
- data/test/adapters/spatialite.rb +2 -0
- data/test/adapters/sqlite3.rb +2 -0
- data/test/adapters/trilogy.rb +9 -0
- data/test/database.yml.sample +7 -0
- data/test/{travis → github}/database.yml +9 -3
- data/test/import_test.rb +108 -41
- data/test/jdbcmysql/import_test.rb +5 -3
- data/test/jdbcpostgresql/import_test.rb +4 -2
- data/test/jdbcsqlite3/import_test.rb +4 -2
- data/test/makara_postgis/import_test.rb +4 -2
- data/test/models/account.rb +2 -0
- data/test/models/alarm.rb +2 -0
- data/test/models/animal.rb +8 -0
- data/test/models/author.rb +9 -0
- data/test/models/bike_maker.rb +3 -0
- data/test/models/book.rb +12 -3
- data/test/models/car.rb +2 -0
- data/test/models/card.rb +5 -0
- data/test/models/chapter.rb +2 -0
- data/test/models/composite_book.rb +19 -0
- data/test/models/composite_chapter.rb +12 -0
- data/test/models/customer.rb +18 -0
- data/test/models/deck.rb +8 -0
- data/test/models/dictionary.rb +2 -0
- data/test/models/discount.rb +2 -0
- data/test/models/end_note.rb +2 -0
- data/test/models/group.rb +2 -0
- data/test/models/order.rb +17 -0
- data/test/models/playing_card.rb +4 -0
- data/test/models/promotion.rb +2 -0
- data/test/models/question.rb +2 -0
- data/test/models/rule.rb +2 -0
- data/test/models/tag.rb +9 -1
- data/test/models/tag_alias.rb +11 -0
- data/test/models/topic.rb +8 -0
- data/test/models/user.rb +2 -0
- data/test/models/user_token.rb +2 -0
- data/test/models/vendor.rb +2 -0
- data/test/models/widget.rb +12 -3
- data/test/mysql2/import_test.rb +5 -3
- data/test/mysql2_makara/import_test.rb +5 -3
- data/test/mysqlspatial2/import_test.rb +5 -3
- data/test/postgis/import_test.rb +4 -2
- data/test/postgresql/import_test.rb +4 -2
- data/test/schema/generic_schema.rb +37 -1
- data/test/schema/jdbcpostgresql_schema.rb +3 -1
- data/test/schema/mysql2_schema.rb +2 -0
- data/test/schema/postgis_schema.rb +3 -1
- data/test/schema/postgresql_schema.rb +38 -4
- data/test/schema/sqlite3_schema.rb +2 -0
- data/test/schema/version.rb +2 -0
- data/test/sqlite3/import_test.rb +4 -2
- data/test/support/active_support/test_case_extensions.rb +3 -5
- data/test/support/assertions.rb +2 -0
- data/test/support/factories.rb +2 -0
- data/test/support/generate.rb +4 -2
- data/test/support/mysql/import_examples.rb +7 -8
- data/test/support/postgresql/import_examples.rb +121 -53
- data/test/support/shared_examples/on_duplicate_key_ignore.rb +2 -0
- data/test/support/shared_examples/on_duplicate_key_update.rb +69 -10
- data/test/support/shared_examples/recursive_import.rb +137 -1
- data/test/support/sqlite3/import_examples.rb +2 -1
- data/test/synchronize_test.rb +2 -0
- data/test/test_helper.rb +38 -24
- data/test/trilogy/import_test.rb +7 -0
- data/test/value_sets_bytes_parser_test.rb +3 -1
- data/test/value_sets_records_parser_test.rb +3 -1
- metadata +46 -22
- data/.travis.yml +0 -74
- data/gemfiles/3.2.gemfile +0 -2
- data/gemfiles/4.0.gemfile +0 -2
- data/gemfiles/4.1.gemfile +0 -2
- data/gemfiles/4.2.gemfile +0 -2
- data/gemfiles/5.0.gemfile +0 -2
- data/gemfiles/5.1.gemfile +0 -2
- data/lib/activerecord-import/mysql2.rb +0 -7
- data/lib/activerecord-import/postgresql.rb +0 -7
- data/lib/activerecord-import/sqlite3.rb +0 -7
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module ActiveRecord::Import::AbstractAdapter
|
2
4
|
module InstanceMethods
|
3
5
|
def next_value_for_sequence(sequence_name)
|
@@ -7,10 +9,11 @@ module ActiveRecord::Import::AbstractAdapter
|
|
7
9
|
def insert_many( sql, values, _options = {}, *args ) # :nodoc:
|
8
10
|
number_of_inserts = 1
|
9
11
|
|
10
|
-
base_sql, post_sql =
|
11
|
-
|
12
|
-
|
13
|
-
|
12
|
+
base_sql, post_sql = case sql
|
13
|
+
when String
|
14
|
+
[sql, '']
|
15
|
+
when Array
|
16
|
+
[sql.shift, sql.join( ' ' )]
|
14
17
|
end
|
15
18
|
|
16
19
|
sql2insert = base_sql + values.join( ',' ) + post_sql
|
@@ -45,7 +48,7 @@ module ActiveRecord::Import::AbstractAdapter
|
|
45
48
|
post_sql_statements = []
|
46
49
|
|
47
50
|
if supports_on_duplicate_key_update? && options[:on_duplicate_key_update]
|
48
|
-
post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update], options[:primary_key], options[:locking_column] )
|
51
|
+
post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update], options[:model], options[:primary_key], options[:locking_column] )
|
49
52
|
elsif logger && options[:on_duplicate_key_update]
|
50
53
|
logger.warn "Ignoring on_duplicate_key_update because it is not supported by the database."
|
51
54
|
end
|
@@ -66,7 +69,7 @@ module ActiveRecord::Import::AbstractAdapter
|
|
66
69
|
end
|
67
70
|
|
68
71
|
def supports_on_duplicate_key_update?
|
69
|
-
|
72
|
+
false
|
70
73
|
end
|
71
74
|
end
|
72
75
|
end
|
@@ -1,5 +1,8 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module ActiveRecord::Import::MysqlAdapter
|
2
4
|
include ActiveRecord::Import::ImportSupport
|
5
|
+
include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
|
3
6
|
|
4
7
|
NO_MAX_PACKET = 0
|
5
8
|
QUERY_OVERHEAD = 8 # This was shown to be true for MySQL, but it's not clear where the overhead is from.
|
@@ -10,13 +13,14 @@ module ActiveRecord::Import::MysqlAdapter
|
|
10
13
|
# the number of inserts default
|
11
14
|
number_of_inserts = 0
|
12
15
|
|
13
|
-
base_sql, post_sql =
|
14
|
-
|
15
|
-
|
16
|
-
|
16
|
+
base_sql, post_sql = case sql
|
17
|
+
when String
|
18
|
+
[sql, '']
|
19
|
+
when Array
|
20
|
+
[sql.shift, sql.join( ' ' )]
|
17
21
|
end
|
18
22
|
|
19
|
-
sql_size = QUERY_OVERHEAD + base_sql.
|
23
|
+
sql_size = QUERY_OVERHEAD + base_sql.bytesize + post_sql.bytesize
|
20
24
|
|
21
25
|
# the number of bytes the requested insert statement values will take up
|
22
26
|
values_in_bytes = values.sum(&:bytesize)
|
@@ -30,7 +34,7 @@ module ActiveRecord::Import::MysqlAdapter
|
|
30
34
|
max = max_allowed_packet
|
31
35
|
|
32
36
|
# if we can insert it all as one statement
|
33
|
-
if
|
37
|
+
if max == NO_MAX_PACKET || total_bytes <= max || options[:force_single_insert]
|
34
38
|
number_of_inserts += 1
|
35
39
|
sql2insert = base_sql + values.join( ',' ) + post_sql
|
36
40
|
insert( sql2insert, *args )
|
@@ -55,9 +59,9 @@ module ActiveRecord::Import::MysqlAdapter
|
|
55
59
|
# in a single packet
|
56
60
|
def max_allowed_packet # :nodoc:
|
57
61
|
@max_allowed_packet ||= begin
|
58
|
-
result = execute( "
|
62
|
+
result = execute( "SELECT @@max_allowed_packet" )
|
59
63
|
# original Mysql gem responds to #fetch_row while Mysql2 responds to #first
|
60
|
-
val = result.respond_to?(:fetch_row) ? result.fetch_row[
|
64
|
+
val = result.respond_to?(:fetch_row) ? result.fetch_row[0] : result.first[0]
|
61
65
|
val.to_i
|
62
66
|
end
|
63
67
|
end
|
@@ -81,14 +85,14 @@ module ActiveRecord::Import::MysqlAdapter
|
|
81
85
|
# Returns a generated ON DUPLICATE KEY UPDATE statement given the passed
|
82
86
|
# in +args+.
|
83
87
|
def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
|
84
|
-
sql = ' ON DUPLICATE KEY UPDATE '
|
85
|
-
arg = args
|
86
|
-
|
87
|
-
|
88
|
-
sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arg )
|
89
|
-
|
90
|
-
sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, arg )
|
91
|
-
|
88
|
+
sql = ' ON DUPLICATE KEY UPDATE '.dup
|
89
|
+
arg, model, _primary_key, locking_column = args
|
90
|
+
case arg
|
91
|
+
when Array
|
92
|
+
sql << sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arg )
|
93
|
+
when Hash
|
94
|
+
sql << sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, arg )
|
95
|
+
when String
|
92
96
|
sql << arg
|
93
97
|
else
|
94
98
|
raise ArgumentError, "Expected Array or Hash"
|
@@ -96,19 +100,24 @@ module ActiveRecord::Import::MysqlAdapter
|
|
96
100
|
sql
|
97
101
|
end
|
98
102
|
|
99
|
-
def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
|
103
|
+
def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
|
100
104
|
results = arr.map do |column|
|
101
|
-
|
105
|
+
original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
|
106
|
+
qc = quote_column_name( original_column_name )
|
102
107
|
"#{table_name}.#{qc}=VALUES(#{qc})"
|
103
108
|
end
|
104
109
|
increment_locking_column!(table_name, results, locking_column)
|
105
110
|
results.join( ',' )
|
106
111
|
end
|
107
112
|
|
108
|
-
def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
|
113
|
+
def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
|
109
114
|
results = hsh.map do |column1, column2|
|
110
|
-
|
111
|
-
|
115
|
+
original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
|
116
|
+
qc1 = quote_column_name( original_column1_name )
|
117
|
+
|
118
|
+
original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
|
119
|
+
qc2 = quote_column_name( original_column2_name )
|
120
|
+
|
112
121
|
"#{table_name}.#{qc1}=VALUES( #{qc2} )"
|
113
122
|
end
|
114
123
|
increment_locking_column!(table_name, results, locking_column)
|
@@ -1,63 +1,73 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module ActiveRecord::Import::PostgreSQLAdapter
|
2
4
|
include ActiveRecord::Import::ImportSupport
|
5
|
+
include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
|
3
6
|
|
4
7
|
MIN_VERSION_FOR_UPSERT = 90_500
|
5
8
|
|
6
9
|
def insert_many( sql, values, options = {}, *args ) # :nodoc:
|
7
10
|
number_of_inserts = 1
|
8
|
-
returned_values =
|
11
|
+
returned_values = {}
|
9
12
|
ids = []
|
10
13
|
results = []
|
11
14
|
|
12
|
-
base_sql, post_sql =
|
13
|
-
|
14
|
-
|
15
|
-
|
15
|
+
base_sql, post_sql = case sql
|
16
|
+
when String
|
17
|
+
[sql, '']
|
18
|
+
when Array
|
19
|
+
[sql.shift, sql.join( ' ' )]
|
16
20
|
end
|
17
21
|
|
18
22
|
sql2insert = base_sql + values.join( ',' ) + post_sql
|
19
23
|
|
20
|
-
|
21
|
-
if
|
24
|
+
selections = returning_selections(options)
|
25
|
+
if selections.blank? || (options[:no_returning] && !options[:recursive])
|
22
26
|
insert( sql2insert, *args )
|
23
27
|
else
|
24
|
-
returned_values = if
|
28
|
+
returned_values = if selections.size > 1
|
25
29
|
# Select composite columns
|
26
|
-
|
30
|
+
db_result = select_all( sql2insert, *args )
|
31
|
+
{ values: db_result.rows, columns: db_result.columns }
|
27
32
|
else
|
28
|
-
select_values( sql2insert, *args )
|
33
|
+
{ values: select_values( sql2insert, *args ) }
|
29
34
|
end
|
30
|
-
|
35
|
+
clear_query_cache if query_cache_enabled
|
31
36
|
end
|
32
37
|
|
33
38
|
if options[:returning].blank?
|
34
|
-
ids = returned_values
|
39
|
+
ids = Array(returned_values[:values])
|
35
40
|
elsif options[:primary_key].blank?
|
36
|
-
|
41
|
+
options[:returning_columns] ||= returned_values[:columns]
|
42
|
+
results = Array(returned_values[:values])
|
37
43
|
else
|
38
44
|
# split primary key and returning columns
|
39
|
-
ids, results = split_ids_and_results(returned_values,
|
45
|
+
ids, results, options[:returning_columns] = split_ids_and_results(returned_values, options)
|
40
46
|
end
|
41
47
|
|
42
48
|
ActiveRecord::Import::Result.new([], number_of_inserts, ids, results)
|
43
49
|
end
|
44
50
|
|
45
|
-
def split_ids_and_results(
|
51
|
+
def split_ids_and_results( selections, options )
|
46
52
|
ids = []
|
47
|
-
|
53
|
+
returning_values = []
|
54
|
+
|
55
|
+
columns = Array(selections[:columns])
|
56
|
+
values = Array(selections[:values])
|
48
57
|
id_indexes = Array(options[:primary_key]).map { |key| columns.index(key) }
|
49
|
-
|
58
|
+
returning_columns = columns.reject.with_index { |_, index| id_indexes.include?(index) }
|
59
|
+
returning_indexes = returning_columns.map { |column| columns.index(column) }
|
50
60
|
|
51
61
|
values.each do |value|
|
52
62
|
value_array = Array(value)
|
53
|
-
ids << id_indexes.map { |
|
54
|
-
|
63
|
+
ids << id_indexes.map { |index| value_array[index] }
|
64
|
+
returning_values << returning_indexes.map { |index| value_array[index] }
|
55
65
|
end
|
56
66
|
|
57
67
|
ids.map!(&:first) if id_indexes.size == 1
|
58
|
-
|
68
|
+
returning_values.map!(&:first) if returning_columns.size == 1
|
59
69
|
|
60
|
-
[ids,
|
70
|
+
[ids, returning_values, returning_columns]
|
61
71
|
end
|
62
72
|
|
63
73
|
def next_value_for_sequence(sequence_name)
|
@@ -78,31 +88,37 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
78
88
|
|
79
89
|
sql += super(table_name, options)
|
80
90
|
|
81
|
-
|
82
|
-
unless
|
83
|
-
sql << " RETURNING
|
91
|
+
selections = returning_selections(options)
|
92
|
+
unless selections.blank? || (options[:no_returning] && !options[:recursive])
|
93
|
+
sql << " RETURNING #{selections.join(', ')}"
|
84
94
|
end
|
85
95
|
|
86
96
|
sql
|
87
97
|
end
|
88
98
|
|
89
|
-
def
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
99
|
+
def returning_selections(options)
|
100
|
+
selections = []
|
101
|
+
column_names = Array(options[:model].column_names)
|
102
|
+
|
103
|
+
selections += Array(options[:primary_key]) if options[:primary_key].present?
|
104
|
+
selections += Array(options[:returning]) if options[:returning].present?
|
105
|
+
|
106
|
+
selections.map do |selection|
|
107
|
+
column_names.include?(selection.to_s) ? "\"#{selection}\"" : selection
|
108
|
+
end
|
94
109
|
end
|
95
110
|
|
96
111
|
# Add a column to be updated on duplicate key update
|
97
112
|
def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
|
98
113
|
arg = options[:on_duplicate_key_update]
|
99
|
-
|
114
|
+
case arg
|
115
|
+
when Hash
|
100
116
|
columns = arg.fetch( :columns ) { arg[:columns] = [] }
|
101
117
|
case columns
|
102
118
|
when Array then columns << column.to_sym unless columns.include?( column.to_sym )
|
103
119
|
when Hash then columns[column.to_sym] = column.to_sym
|
104
120
|
end
|
105
|
-
|
121
|
+
when Array
|
106
122
|
arg << column.to_sym unless arg.include?( column.to_sym )
|
107
123
|
end
|
108
124
|
end
|
@@ -118,11 +134,11 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
118
134
|
# Returns a generated ON CONFLICT DO UPDATE statement given the passed
|
119
135
|
# in +args+.
|
120
136
|
def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
|
121
|
-
arg, primary_key, locking_column = args
|
137
|
+
arg, model, primary_key, locking_column = args
|
122
138
|
arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
|
123
139
|
return unless arg.is_a?( Hash )
|
124
140
|
|
125
|
-
sql = ' ON CONFLICT '
|
141
|
+
sql = ' ON CONFLICT '.dup
|
126
142
|
conflict_target = sql_for_conflict_target( arg )
|
127
143
|
|
128
144
|
columns = arg.fetch( :columns, [] )
|
@@ -137,11 +153,12 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
137
153
|
end
|
138
154
|
|
139
155
|
sql << "#{conflict_target}DO UPDATE SET "
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
156
|
+
case columns
|
157
|
+
when Array
|
158
|
+
sql << sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, columns )
|
159
|
+
when Hash
|
160
|
+
sql << sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, columns )
|
161
|
+
when String
|
145
162
|
sql << columns
|
146
163
|
else
|
147
164
|
raise ArgumentError, 'Expected :columns to be an Array or Hash'
|
@@ -152,19 +169,24 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
152
169
|
sql
|
153
170
|
end
|
154
171
|
|
155
|
-
def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
|
172
|
+
def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
|
156
173
|
results = arr.map do |column|
|
157
|
-
|
174
|
+
original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
|
175
|
+
qc = quote_column_name( original_column_name )
|
158
176
|
"#{qc}=EXCLUDED.#{qc}"
|
159
177
|
end
|
160
178
|
increment_locking_column!(table_name, results, locking_column)
|
161
179
|
results.join( ',' )
|
162
180
|
end
|
163
181
|
|
164
|
-
def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
|
182
|
+
def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
|
165
183
|
results = hsh.map do |column1, column2|
|
166
|
-
|
167
|
-
|
184
|
+
original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
|
185
|
+
qc1 = quote_column_name( original_column1_name )
|
186
|
+
|
187
|
+
original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
|
188
|
+
qc2 = quote_column_name( original_column2_name )
|
189
|
+
|
168
190
|
"#{qc1}=EXCLUDED.#{qc2}"
|
169
191
|
end
|
170
192
|
increment_locking_column!(table_name, results, locking_column)
|
@@ -178,9 +200,9 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
178
200
|
if constraint_name.present?
|
179
201
|
"ON CONSTRAINT #{constraint_name} "
|
180
202
|
elsif conflict_target.present?
|
181
|
-
|
182
|
-
|
183
|
-
|
203
|
+
sql = "(#{Array( conflict_target ).reject( &:blank? ).join( ', ' )}) "
|
204
|
+
sql += "WHERE #{index_predicate} " if index_predicate
|
205
|
+
sql
|
184
206
|
end
|
185
207
|
end
|
186
208
|
|
@@ -202,8 +224,6 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
202
224
|
true
|
203
225
|
end
|
204
226
|
|
205
|
-
private
|
206
|
-
|
207
227
|
def database_version
|
208
228
|
defined?(postgresql_version) ? postgresql_version : super
|
209
229
|
end
|
@@ -1,8 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module ActiveRecord::Import::SQLite3Adapter
|
2
4
|
include ActiveRecord::Import::ImportSupport
|
5
|
+
include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
|
3
6
|
|
4
|
-
MIN_VERSION_FOR_IMPORT = "3.7.11"
|
5
|
-
MIN_VERSION_FOR_UPSERT = "3.24.0"
|
7
|
+
MIN_VERSION_FOR_IMPORT = "3.7.11"
|
8
|
+
MIN_VERSION_FOR_UPSERT = "3.24.0"
|
6
9
|
SQLITE_LIMIT_COMPOUND_SELECT = 500
|
7
10
|
|
8
11
|
# Override our conformance to ActiveRecord::Import::ImportSupport interface
|
@@ -21,10 +24,11 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
21
24
|
def insert_many( sql, values, _options = {}, *args ) # :nodoc:
|
22
25
|
number_of_inserts = 0
|
23
26
|
|
24
|
-
base_sql, post_sql =
|
25
|
-
|
26
|
-
|
27
|
-
|
27
|
+
base_sql, post_sql = case sql
|
28
|
+
when String
|
29
|
+
[sql, '']
|
30
|
+
when Array
|
31
|
+
[sql.shift, sql.join( ' ' )]
|
28
32
|
end
|
29
33
|
|
30
34
|
value_sets = ::ActiveRecord::Import::ValueSetsRecordsParser.parse(values,
|
@@ -53,11 +57,9 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
53
57
|
def post_sql_statements( table_name, options ) # :nodoc:
|
54
58
|
sql = []
|
55
59
|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
sql << sql_for_on_duplicate_key_ignore( options[:on_duplicate_key_ignore] )
|
60
|
-
end
|
60
|
+
# Options :recursive and :on_duplicate_key_ignore are mutually exclusive
|
61
|
+
if supports_on_duplicate_key_update? && ((options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:on_duplicate_key_update])
|
62
|
+
sql << sql_for_on_duplicate_key_ignore( options[:on_duplicate_key_ignore] )
|
61
63
|
end
|
62
64
|
|
63
65
|
sql + super
|
@@ -70,13 +72,14 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
70
72
|
# Add a column to be updated on duplicate key update
|
71
73
|
def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
|
72
74
|
arg = options[:on_duplicate_key_update]
|
73
|
-
|
75
|
+
case arg
|
76
|
+
when Hash
|
74
77
|
columns = arg.fetch( :columns ) { arg[:columns] = [] }
|
75
78
|
case columns
|
76
79
|
when Array then columns << column.to_sym unless columns.include?( column.to_sym )
|
77
80
|
when Hash then columns[column.to_sym] = column.to_sym
|
78
81
|
end
|
79
|
-
|
82
|
+
when Array
|
80
83
|
arg << column.to_sym unless arg.include?( column.to_sym )
|
81
84
|
end
|
82
85
|
end
|
@@ -92,11 +95,11 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
92
95
|
# Returns a generated ON CONFLICT DO UPDATE statement given the passed
|
93
96
|
# in +args+.
|
94
97
|
def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
|
95
|
-
arg, primary_key, locking_column = args
|
98
|
+
arg, model, primary_key, locking_column = args
|
96
99
|
arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
|
97
100
|
return unless arg.is_a?( Hash )
|
98
101
|
|
99
|
-
sql = ' ON CONFLICT '
|
102
|
+
sql = ' ON CONFLICT '.dup
|
100
103
|
conflict_target = sql_for_conflict_target( arg )
|
101
104
|
|
102
105
|
columns = arg.fetch( :columns, [] )
|
@@ -111,11 +114,12 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
111
114
|
end
|
112
115
|
|
113
116
|
sql << "#{conflict_target}DO UPDATE SET "
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
117
|
+
case columns
|
118
|
+
when Array
|
119
|
+
sql << sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, columns )
|
120
|
+
when Hash
|
121
|
+
sql << sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, columns )
|
122
|
+
when String
|
119
123
|
sql << columns
|
120
124
|
else
|
121
125
|
raise ArgumentError, 'Expected :columns to be an Array or Hash'
|
@@ -126,19 +130,24 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
126
130
|
sql
|
127
131
|
end
|
128
132
|
|
129
|
-
def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
|
133
|
+
def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
|
130
134
|
results = arr.map do |column|
|
131
|
-
|
135
|
+
original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
|
136
|
+
qc = quote_column_name( original_column_name )
|
132
137
|
"#{qc}=EXCLUDED.#{qc}"
|
133
138
|
end
|
134
139
|
increment_locking_column!(table_name, results, locking_column)
|
135
140
|
results.join( ',' )
|
136
141
|
end
|
137
142
|
|
138
|
-
def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
|
143
|
+
def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
|
139
144
|
results = hsh.map do |column1, column2|
|
140
|
-
|
141
|
-
|
145
|
+
original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
|
146
|
+
qc1 = quote_column_name( original_column1_name )
|
147
|
+
|
148
|
+
original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
|
149
|
+
qc2 = quote_column_name( original_column2_name )
|
150
|
+
|
142
151
|
"#{qc1}=EXCLUDED.#{qc2}"
|
143
152
|
end
|
144
153
|
increment_locking_column!(table_name, results, locking_column)
|
@@ -149,9 +158,9 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
149
158
|
conflict_target = args[:conflict_target]
|
150
159
|
index_predicate = args[:index_predicate]
|
151
160
|
if conflict_target.present?
|
152
|
-
|
153
|
-
|
154
|
-
|
161
|
+
sql = "(#{Array( conflict_target ).reject( &:blank? ).join( ', ' )}) "
|
162
|
+
sql += "WHERE #{index_predicate} " if index_predicate
|
163
|
+
sql
|
155
164
|
end
|
156
165
|
end
|
157
166
|
|
@@ -165,8 +174,6 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
165
174
|
exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key')
|
166
175
|
end
|
167
176
|
|
168
|
-
private
|
169
|
-
|
170
177
|
def database_version
|
171
178
|
defined?(sqlite_version) ? sqlite_version : super
|
172
179
|
end
|
@@ -1,9 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "pathname"
|
2
4
|
require "active_record"
|
3
5
|
require "active_record/version"
|
4
6
|
|
5
7
|
module ActiveRecord::Import
|
6
|
-
ADAPTER_PATH = "activerecord-import/active_record/adapters"
|
8
|
+
ADAPTER_PATH = "activerecord-import/active_record/adapters"
|
7
9
|
|
8
10
|
def self.base_adapter(adapter)
|
9
11
|
case adapter
|