activerecord-import 1.0.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yaml +113 -0
- data/.gitignore +1 -0
- data/.rubocop.yml +74 -8
- data/.rubocop_todo.yml +6 -16
- data/Brewfile +3 -1
- data/CHANGELOG.md +115 -3
- data/Gemfile +12 -10
- data/LICENSE +21 -56
- data/README.markdown +71 -60
- data/Rakefile +2 -0
- data/activerecord-import.gemspec +6 -5
- data/benchmarks/benchmark.rb +10 -4
- data/benchmarks/lib/base.rb +4 -2
- data/benchmarks/lib/cli_parser.rb +4 -2
- data/benchmarks/lib/float.rb +2 -0
- data/benchmarks/lib/mysql2_benchmark.rb +2 -0
- data/benchmarks/lib/output_to_csv.rb +2 -0
- data/benchmarks/lib/output_to_html.rb +4 -2
- data/benchmarks/models/test_innodb.rb +2 -0
- data/benchmarks/models/test_memory.rb +2 -0
- data/benchmarks/models/test_myisam.rb +2 -0
- data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +2 -0
- data/gemfiles/4.2.gemfile +2 -0
- data/gemfiles/5.0.gemfile +2 -0
- data/gemfiles/5.1.gemfile +2 -0
- data/gemfiles/5.2.gemfile +2 -0
- data/gemfiles/6.0.gemfile +4 -1
- data/gemfiles/6.1.gemfile +4 -1
- data/gemfiles/7.0.gemfile +4 -0
- data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +6 -4
- data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/abstract_adapter.rb +14 -5
- data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql_adapter.rb +33 -25
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +69 -56
- data/lib/activerecord-import/adapters/sqlite3_adapter.rb +39 -39
- data/lib/activerecord-import/base.rb +10 -2
- data/lib/activerecord-import/import.rb +143 -62
- data/lib/activerecord-import/mysql2.rb +2 -0
- data/lib/activerecord-import/postgresql.rb +2 -0
- data/lib/activerecord-import/sqlite3.rb +2 -0
- data/lib/activerecord-import/synchronize.rb +3 -1
- data/lib/activerecord-import/value_sets_parser.rb +5 -0
- data/lib/activerecord-import/version.rb +3 -1
- data/lib/activerecord-import.rb +3 -1
- data/test/adapters/jdbcmysql.rb +2 -0
- data/test/adapters/jdbcpostgresql.rb +2 -0
- data/test/adapters/jdbcsqlite3.rb +2 -0
- data/test/adapters/makara_postgis.rb +2 -0
- data/test/adapters/mysql2.rb +2 -0
- data/test/adapters/mysql2_makara.rb +2 -0
- data/test/adapters/mysql2spatial.rb +2 -0
- data/test/adapters/postgis.rb +2 -0
- data/test/adapters/postgresql.rb +2 -0
- data/test/adapters/postgresql_makara.rb +2 -0
- data/test/adapters/seamless_database_pool.rb +2 -0
- data/test/adapters/spatialite.rb +2 -0
- data/test/adapters/sqlite3.rb +2 -0
- data/test/{travis → github}/database.yml +3 -1
- data/test/import_test.rb +93 -2
- data/test/jdbcmysql/import_test.rb +5 -3
- data/test/jdbcpostgresql/import_test.rb +4 -2
- data/test/jdbcsqlite3/import_test.rb +4 -2
- data/test/makara_postgis/import_test.rb +4 -2
- data/test/models/account.rb +2 -0
- data/test/models/alarm.rb +2 -0
- data/test/models/animal.rb +8 -0
- data/test/models/bike_maker.rb +3 -0
- data/test/models/book.rb +2 -0
- data/test/models/car.rb +2 -0
- data/test/models/card.rb +5 -0
- data/test/models/chapter.rb +2 -0
- data/test/models/customer.rb +8 -0
- data/test/models/deck.rb +8 -0
- data/test/models/dictionary.rb +2 -0
- data/test/models/discount.rb +2 -0
- data/test/models/end_note.rb +2 -0
- data/test/models/group.rb +2 -0
- data/test/models/order.rb +8 -0
- data/test/models/playing_card.rb +4 -0
- data/test/models/promotion.rb +2 -0
- data/test/models/question.rb +2 -0
- data/test/models/rule.rb +2 -0
- data/test/models/tag.rb +3 -0
- data/test/models/tag_alias.rb +5 -0
- data/test/models/topic.rb +7 -0
- data/test/models/user.rb +2 -0
- data/test/models/user_token.rb +2 -0
- data/test/models/vendor.rb +2 -0
- data/test/models/widget.rb +2 -0
- data/test/mysql2/import_test.rb +5 -3
- data/test/mysql2_makara/import_test.rb +5 -3
- data/test/mysqlspatial2/import_test.rb +5 -3
- data/test/postgis/import_test.rb +4 -2
- data/test/postgresql/import_test.rb +4 -2
- data/test/schema/generic_schema.rb +34 -0
- data/test/schema/jdbcpostgresql_schema.rb +3 -1
- data/test/schema/mysql2_schema.rb +2 -0
- data/test/schema/postgis_schema.rb +3 -1
- data/test/schema/postgresql_schema.rb +16 -0
- data/test/schema/sqlite3_schema.rb +2 -0
- data/test/schema/version.rb +2 -0
- data/test/sqlite3/import_test.rb +4 -2
- data/test/support/active_support/test_case_extensions.rb +2 -0
- data/test/support/assertions.rb +2 -0
- data/test/support/factories.rb +2 -0
- data/test/support/generate.rb +4 -2
- data/test/support/mysql/import_examples.rb +2 -1
- data/test/support/postgresql/import_examples.rb +96 -2
- data/test/support/shared_examples/on_duplicate_key_ignore.rb +2 -0
- data/test/support/shared_examples/on_duplicate_key_update.rb +50 -9
- data/test/support/shared_examples/recursive_import.rb +32 -1
- data/test/support/sqlite3/import_examples.rb +2 -1
- data/test/synchronize_test.rb +2 -0
- data/test/test_helper.rb +30 -5
- data/test/value_sets_bytes_parser_test.rb +3 -1
- data/test/value_sets_records_parser_test.rb +3 -1
- metadata +27 -16
- data/.travis.yml +0 -70
- data/gemfiles/3.2.gemfile +0 -2
- data/gemfiles/4.0.gemfile +0 -2
- data/gemfiles/4.1.gemfile +0 -2
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module ActiveRecord::Import::AbstractAdapter
|
|
2
4
|
module InstanceMethods
|
|
3
5
|
def next_value_for_sequence(sequence_name)
|
|
@@ -7,10 +9,11 @@ module ActiveRecord::Import::AbstractAdapter
|
|
|
7
9
|
def insert_many( sql, values, _options = {}, *args ) # :nodoc:
|
|
8
10
|
number_of_inserts = 1
|
|
9
11
|
|
|
10
|
-
base_sql, post_sql =
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
12
|
+
base_sql, post_sql = case sql
|
|
13
|
+
when String
|
|
14
|
+
[sql, '']
|
|
15
|
+
when Array
|
|
16
|
+
[sql.shift, sql.join( ' ' )]
|
|
14
17
|
end
|
|
15
18
|
|
|
16
19
|
sql2insert = base_sql + values.join( ',' ) + post_sql
|
|
@@ -45,7 +48,7 @@ module ActiveRecord::Import::AbstractAdapter
|
|
|
45
48
|
post_sql_statements = []
|
|
46
49
|
|
|
47
50
|
if supports_on_duplicate_key_update? && options[:on_duplicate_key_update]
|
|
48
|
-
post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update], options[:primary_key], options[:locking_column] )
|
|
51
|
+
post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update], options[:model], options[:primary_key], options[:locking_column] )
|
|
49
52
|
elsif logger && options[:on_duplicate_key_update]
|
|
50
53
|
logger.warn "Ignoring on_duplicate_key_update because it is not supported by the database."
|
|
51
54
|
end
|
|
@@ -59,6 +62,12 @@ module ActiveRecord::Import::AbstractAdapter
|
|
|
59
62
|
post_sql_statements
|
|
60
63
|
end
|
|
61
64
|
|
|
65
|
+
def increment_locking_column!(table_name, results, locking_column)
|
|
66
|
+
if locking_column.present?
|
|
67
|
+
results << "\"#{locking_column}\"=#{table_name}.\"#{locking_column}\"+1"
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
62
71
|
def supports_on_duplicate_key_update?
|
|
63
72
|
false
|
|
64
73
|
end
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module ActiveRecord::Import::MysqlAdapter
|
|
2
4
|
include ActiveRecord::Import::ImportSupport
|
|
3
5
|
include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
|
|
@@ -11,13 +13,14 @@ module ActiveRecord::Import::MysqlAdapter
|
|
|
11
13
|
# the number of inserts default
|
|
12
14
|
number_of_inserts = 0
|
|
13
15
|
|
|
14
|
-
base_sql, post_sql =
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
16
|
+
base_sql, post_sql = case sql
|
|
17
|
+
when String
|
|
18
|
+
[sql, '']
|
|
19
|
+
when Array
|
|
20
|
+
[sql.shift, sql.join( ' ' )]
|
|
18
21
|
end
|
|
19
22
|
|
|
20
|
-
sql_size = QUERY_OVERHEAD + base_sql.
|
|
23
|
+
sql_size = QUERY_OVERHEAD + base_sql.bytesize + post_sql.bytesize
|
|
21
24
|
|
|
22
25
|
# the number of bytes the requested insert statement values will take up
|
|
23
26
|
values_in_bytes = values.sum(&:bytesize)
|
|
@@ -31,7 +34,7 @@ module ActiveRecord::Import::MysqlAdapter
|
|
|
31
34
|
max = max_allowed_packet
|
|
32
35
|
|
|
33
36
|
# if we can insert it all as one statement
|
|
34
|
-
if
|
|
37
|
+
if max == NO_MAX_PACKET || total_bytes <= max || options[:force_single_insert]
|
|
35
38
|
number_of_inserts += 1
|
|
36
39
|
sql2insert = base_sql + values.join( ',' ) + post_sql
|
|
37
40
|
insert( sql2insert, *args )
|
|
@@ -56,9 +59,9 @@ module ActiveRecord::Import::MysqlAdapter
|
|
|
56
59
|
# in a single packet
|
|
57
60
|
def max_allowed_packet # :nodoc:
|
|
58
61
|
@max_allowed_packet ||= begin
|
|
59
|
-
result = execute( "
|
|
62
|
+
result = execute( "SELECT @@max_allowed_packet" )
|
|
60
63
|
# original Mysql gem responds to #fetch_row while Mysql2 responds to #first
|
|
61
|
-
val = result.respond_to?(:fetch_row) ? result.fetch_row[
|
|
64
|
+
val = result.respond_to?(:fetch_row) ? result.fetch_row[0] : result.first[0]
|
|
62
65
|
val.to_i
|
|
63
66
|
end
|
|
64
67
|
end
|
|
@@ -82,14 +85,14 @@ module ActiveRecord::Import::MysqlAdapter
|
|
|
82
85
|
# Returns a generated ON DUPLICATE KEY UPDATE statement given the passed
|
|
83
86
|
# in +args+.
|
|
84
87
|
def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
|
|
85
|
-
sql = ' ON DUPLICATE KEY UPDATE '
|
|
86
|
-
arg = args
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arg )
|
|
90
|
-
|
|
91
|
-
sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, arg )
|
|
92
|
-
|
|
88
|
+
sql = ' ON DUPLICATE KEY UPDATE '.dup
|
|
89
|
+
arg, model, _primary_key, locking_column = args
|
|
90
|
+
case arg
|
|
91
|
+
when Array
|
|
92
|
+
sql << sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arg )
|
|
93
|
+
when Hash
|
|
94
|
+
sql << sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, arg )
|
|
95
|
+
when String
|
|
93
96
|
sql << arg
|
|
94
97
|
else
|
|
95
98
|
raise ArgumentError, "Expected Array or Hash"
|
|
@@ -97,22 +100,27 @@ module ActiveRecord::Import::MysqlAdapter
|
|
|
97
100
|
sql
|
|
98
101
|
end
|
|
99
102
|
|
|
100
|
-
def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
|
|
103
|
+
def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
|
|
101
104
|
results = arr.map do |column|
|
|
102
|
-
|
|
105
|
+
original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
|
|
106
|
+
qc = quote_column_name( original_column_name )
|
|
103
107
|
"#{table_name}.#{qc}=VALUES(#{qc})"
|
|
104
108
|
end
|
|
105
|
-
increment_locking_column!(
|
|
109
|
+
increment_locking_column!(table_name, results, locking_column)
|
|
106
110
|
results.join( ',' )
|
|
107
111
|
end
|
|
108
112
|
|
|
109
|
-
def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
|
|
113
|
+
def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
|
|
110
114
|
results = hsh.map do |column1, column2|
|
|
111
|
-
|
|
112
|
-
|
|
115
|
+
original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
|
|
116
|
+
qc1 = quote_column_name( original_column1_name )
|
|
117
|
+
|
|
118
|
+
original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
|
|
119
|
+
qc2 = quote_column_name( original_column2_name )
|
|
120
|
+
|
|
113
121
|
"#{table_name}.#{qc1}=VALUES( #{qc2} )"
|
|
114
122
|
end
|
|
115
|
-
increment_locking_column!(
|
|
123
|
+
increment_locking_column!(table_name, results, locking_column)
|
|
116
124
|
results.join( ',')
|
|
117
125
|
end
|
|
118
126
|
|
|
@@ -121,9 +129,9 @@ module ActiveRecord::Import::MysqlAdapter
|
|
|
121
129
|
exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('Duplicate entry')
|
|
122
130
|
end
|
|
123
131
|
|
|
124
|
-
def increment_locking_column!(
|
|
132
|
+
def increment_locking_column!(table_name, results, locking_column)
|
|
125
133
|
if locking_column.present?
|
|
126
|
-
results << "
|
|
134
|
+
results << "`#{locking_column}`=#{table_name}.`#{locking_column}`+1"
|
|
127
135
|
end
|
|
128
136
|
end
|
|
129
137
|
end
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module ActiveRecord::Import::PostgreSQLAdapter
|
|
2
4
|
include ActiveRecord::Import::ImportSupport
|
|
3
5
|
include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
|
|
@@ -6,59 +8,66 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
|
6
8
|
|
|
7
9
|
def insert_many( sql, values, options = {}, *args ) # :nodoc:
|
|
8
10
|
number_of_inserts = 1
|
|
9
|
-
returned_values =
|
|
11
|
+
returned_values = {}
|
|
10
12
|
ids = []
|
|
11
13
|
results = []
|
|
12
14
|
|
|
13
|
-
base_sql, post_sql =
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
base_sql, post_sql = case sql
|
|
16
|
+
when String
|
|
17
|
+
[sql, '']
|
|
18
|
+
when Array
|
|
19
|
+
[sql.shift, sql.join( ' ' )]
|
|
17
20
|
end
|
|
18
21
|
|
|
19
22
|
sql2insert = base_sql + values.join( ',' ) + post_sql
|
|
20
23
|
|
|
21
|
-
|
|
22
|
-
if
|
|
24
|
+
selections = returning_selections(options)
|
|
25
|
+
if selections.blank? || (options[:no_returning] && !options[:recursive])
|
|
23
26
|
insert( sql2insert, *args )
|
|
24
27
|
else
|
|
25
|
-
returned_values = if
|
|
28
|
+
returned_values = if selections.size > 1
|
|
26
29
|
# Select composite columns
|
|
27
|
-
|
|
30
|
+
db_result = select_all( sql2insert, *args )
|
|
31
|
+
{ values: db_result.rows, columns: db_result.columns }
|
|
28
32
|
else
|
|
29
|
-
select_values( sql2insert, *args )
|
|
33
|
+
{ values: select_values( sql2insert, *args ) }
|
|
30
34
|
end
|
|
31
|
-
|
|
35
|
+
clear_query_cache if query_cache_enabled
|
|
32
36
|
end
|
|
33
37
|
|
|
34
38
|
if options[:returning].blank?
|
|
35
|
-
ids = returned_values
|
|
39
|
+
ids = Array(returned_values[:values])
|
|
36
40
|
elsif options[:primary_key].blank?
|
|
37
|
-
|
|
41
|
+
options[:returning_columns] ||= returned_values[:columns]
|
|
42
|
+
results = Array(returned_values[:values])
|
|
38
43
|
else
|
|
39
44
|
# split primary key and returning columns
|
|
40
|
-
ids, results = split_ids_and_results(returned_values,
|
|
45
|
+
ids, results, options[:returning_columns] = split_ids_and_results(returned_values, options)
|
|
41
46
|
end
|
|
42
47
|
|
|
43
48
|
ActiveRecord::Import::Result.new([], number_of_inserts, ids, results)
|
|
44
49
|
end
|
|
45
50
|
|
|
46
|
-
def split_ids_and_results(
|
|
51
|
+
def split_ids_and_results( selections, options )
|
|
47
52
|
ids = []
|
|
48
|
-
|
|
53
|
+
returning_values = []
|
|
54
|
+
|
|
55
|
+
columns = Array(selections[:columns])
|
|
56
|
+
values = Array(selections[:values])
|
|
49
57
|
id_indexes = Array(options[:primary_key]).map { |key| columns.index(key) }
|
|
50
|
-
|
|
58
|
+
returning_columns = columns.reject.with_index { |_, index| id_indexes.include?(index) }
|
|
59
|
+
returning_indexes = returning_columns.map { |column| columns.index(column) }
|
|
51
60
|
|
|
52
61
|
values.each do |value|
|
|
53
62
|
value_array = Array(value)
|
|
54
|
-
ids << id_indexes.map { |
|
|
55
|
-
|
|
63
|
+
ids << id_indexes.map { |index| value_array[index] }
|
|
64
|
+
returning_values << returning_indexes.map { |index| value_array[index] }
|
|
56
65
|
end
|
|
57
66
|
|
|
58
67
|
ids.map!(&:first) if id_indexes.size == 1
|
|
59
|
-
|
|
68
|
+
returning_values.map!(&:first) if returning_columns.size == 1
|
|
60
69
|
|
|
61
|
-
[ids,
|
|
70
|
+
[ids, returning_values, returning_columns]
|
|
62
71
|
end
|
|
63
72
|
|
|
64
73
|
def next_value_for_sequence(sequence_name)
|
|
@@ -79,31 +88,37 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
|
79
88
|
|
|
80
89
|
sql += super(table_name, options)
|
|
81
90
|
|
|
82
|
-
|
|
83
|
-
unless
|
|
84
|
-
sql << " RETURNING
|
|
91
|
+
selections = returning_selections(options)
|
|
92
|
+
unless selections.blank? || (options[:no_returning] && !options[:recursive])
|
|
93
|
+
sql << " RETURNING #{selections.join(', ')}"
|
|
85
94
|
end
|
|
86
95
|
|
|
87
96
|
sql
|
|
88
97
|
end
|
|
89
98
|
|
|
90
|
-
def
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
99
|
+
def returning_selections(options)
|
|
100
|
+
selections = []
|
|
101
|
+
column_names = Array(options[:model].column_names)
|
|
102
|
+
|
|
103
|
+
selections += Array(options[:primary_key]) if options[:primary_key].present?
|
|
104
|
+
selections += Array(options[:returning]) if options[:returning].present?
|
|
105
|
+
|
|
106
|
+
selections.map do |selection|
|
|
107
|
+
column_names.include?(selection.to_s) ? "\"#{selection}\"" : selection
|
|
108
|
+
end
|
|
95
109
|
end
|
|
96
110
|
|
|
97
111
|
# Add a column to be updated on duplicate key update
|
|
98
112
|
def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
|
|
99
113
|
arg = options[:on_duplicate_key_update]
|
|
100
|
-
|
|
114
|
+
case arg
|
|
115
|
+
when Hash
|
|
101
116
|
columns = arg.fetch( :columns ) { arg[:columns] = [] }
|
|
102
117
|
case columns
|
|
103
118
|
when Array then columns << column.to_sym unless columns.include?( column.to_sym )
|
|
104
119
|
when Hash then columns[column.to_sym] = column.to_sym
|
|
105
120
|
end
|
|
106
|
-
|
|
121
|
+
when Array
|
|
107
122
|
arg << column.to_sym unless arg.include?( column.to_sym )
|
|
108
123
|
end
|
|
109
124
|
end
|
|
@@ -119,11 +134,11 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
|
119
134
|
# Returns a generated ON CONFLICT DO UPDATE statement given the passed
|
|
120
135
|
# in +args+.
|
|
121
136
|
def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
|
|
122
|
-
arg, primary_key, locking_column = args
|
|
137
|
+
arg, model, primary_key, locking_column = args
|
|
123
138
|
arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
|
|
124
139
|
return unless arg.is_a?( Hash )
|
|
125
140
|
|
|
126
|
-
sql = ' ON CONFLICT '
|
|
141
|
+
sql = ' ON CONFLICT '.dup
|
|
127
142
|
conflict_target = sql_for_conflict_target( arg )
|
|
128
143
|
|
|
129
144
|
columns = arg.fetch( :columns, [] )
|
|
@@ -138,11 +153,12 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
|
138
153
|
end
|
|
139
154
|
|
|
140
155
|
sql << "#{conflict_target}DO UPDATE SET "
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
156
|
+
case columns
|
|
157
|
+
when Array
|
|
158
|
+
sql << sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, columns )
|
|
159
|
+
when Hash
|
|
160
|
+
sql << sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, columns )
|
|
161
|
+
when String
|
|
146
162
|
sql << columns
|
|
147
163
|
else
|
|
148
164
|
raise ArgumentError, 'Expected :columns to be an Array or Hash'
|
|
@@ -153,22 +169,27 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
|
153
169
|
sql
|
|
154
170
|
end
|
|
155
171
|
|
|
156
|
-
def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
|
|
172
|
+
def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
|
|
157
173
|
results = arr.map do |column|
|
|
158
|
-
|
|
174
|
+
original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
|
|
175
|
+
qc = quote_column_name( original_column_name )
|
|
159
176
|
"#{qc}=EXCLUDED.#{qc}"
|
|
160
177
|
end
|
|
161
|
-
increment_locking_column!(results, locking_column)
|
|
178
|
+
increment_locking_column!(table_name, results, locking_column)
|
|
162
179
|
results.join( ',' )
|
|
163
180
|
end
|
|
164
181
|
|
|
165
|
-
def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
|
|
182
|
+
def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
|
|
166
183
|
results = hsh.map do |column1, column2|
|
|
167
|
-
|
|
168
|
-
|
|
184
|
+
original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
|
|
185
|
+
qc1 = quote_column_name( original_column1_name )
|
|
186
|
+
|
|
187
|
+
original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
|
|
188
|
+
qc2 = quote_column_name( original_column2_name )
|
|
189
|
+
|
|
169
190
|
"#{qc1}=EXCLUDED.#{qc2}"
|
|
170
191
|
end
|
|
171
|
-
increment_locking_column!(results, locking_column)
|
|
192
|
+
increment_locking_column!(table_name, results, locking_column)
|
|
172
193
|
results.join( ',' )
|
|
173
194
|
end
|
|
174
195
|
|
|
@@ -179,9 +200,9 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
|
179
200
|
if constraint_name.present?
|
|
180
201
|
"ON CONSTRAINT #{constraint_name} "
|
|
181
202
|
elsif conflict_target.present?
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
203
|
+
sql = "(#{Array( conflict_target ).reject( &:blank? ).join( ', ' )}) "
|
|
204
|
+
sql += "WHERE #{index_predicate} " if index_predicate
|
|
205
|
+
sql
|
|
185
206
|
end
|
|
186
207
|
end
|
|
187
208
|
|
|
@@ -203,14 +224,6 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
|
203
224
|
true
|
|
204
225
|
end
|
|
205
226
|
|
|
206
|
-
def increment_locking_column!(results, locking_column)
|
|
207
|
-
if locking_column.present?
|
|
208
|
-
results << "\"#{locking_column}\"=EXCLUDED.\"#{locking_column}\"+1"
|
|
209
|
-
end
|
|
210
|
-
end
|
|
211
|
-
|
|
212
|
-
private
|
|
213
|
-
|
|
214
227
|
def database_version
|
|
215
228
|
defined?(postgresql_version) ? postgresql_version : super
|
|
216
229
|
end
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module ActiveRecord::Import::SQLite3Adapter
|
|
2
4
|
include ActiveRecord::Import::ImportSupport
|
|
3
5
|
include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
|
|
4
6
|
|
|
5
|
-
MIN_VERSION_FOR_IMPORT = "3.7.11"
|
|
6
|
-
MIN_VERSION_FOR_UPSERT = "3.24.0"
|
|
7
|
+
MIN_VERSION_FOR_IMPORT = "3.7.11"
|
|
8
|
+
MIN_VERSION_FOR_UPSERT = "3.24.0"
|
|
7
9
|
SQLITE_LIMIT_COMPOUND_SELECT = 500
|
|
8
10
|
|
|
9
11
|
# Override our conformance to ActiveRecord::Import::ImportSupport interface
|
|
@@ -22,10 +24,11 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
22
24
|
def insert_many( sql, values, _options = {}, *args ) # :nodoc:
|
|
23
25
|
number_of_inserts = 0
|
|
24
26
|
|
|
25
|
-
base_sql, post_sql =
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
27
|
+
base_sql, post_sql = case sql
|
|
28
|
+
when String
|
|
29
|
+
[sql, '']
|
|
30
|
+
when Array
|
|
31
|
+
[sql.shift, sql.join( ' ' )]
|
|
29
32
|
end
|
|
30
33
|
|
|
31
34
|
value_sets = ::ActiveRecord::Import::ValueSetsRecordsParser.parse(values,
|
|
@@ -54,11 +57,9 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
54
57
|
def post_sql_statements( table_name, options ) # :nodoc:
|
|
55
58
|
sql = []
|
|
56
59
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
sql << sql_for_on_duplicate_key_ignore( options[:on_duplicate_key_ignore] )
|
|
61
|
-
end
|
|
60
|
+
# Options :recursive and :on_duplicate_key_ignore are mutually exclusive
|
|
61
|
+
if supports_on_duplicate_key_update? && ((options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:on_duplicate_key_update])
|
|
62
|
+
sql << sql_for_on_duplicate_key_ignore( options[:on_duplicate_key_ignore] )
|
|
62
63
|
end
|
|
63
64
|
|
|
64
65
|
sql + super
|
|
@@ -71,13 +72,14 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
71
72
|
# Add a column to be updated on duplicate key update
|
|
72
73
|
def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
|
|
73
74
|
arg = options[:on_duplicate_key_update]
|
|
74
|
-
|
|
75
|
+
case arg
|
|
76
|
+
when Hash
|
|
75
77
|
columns = arg.fetch( :columns ) { arg[:columns] = [] }
|
|
76
78
|
case columns
|
|
77
79
|
when Array then columns << column.to_sym unless columns.include?( column.to_sym )
|
|
78
80
|
when Hash then columns[column.to_sym] = column.to_sym
|
|
79
81
|
end
|
|
80
|
-
|
|
82
|
+
when Array
|
|
81
83
|
arg << column.to_sym unless arg.include?( column.to_sym )
|
|
82
84
|
end
|
|
83
85
|
end
|
|
@@ -92,12 +94,12 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
92
94
|
|
|
93
95
|
# Returns a generated ON CONFLICT DO UPDATE statement given the passed
|
|
94
96
|
# in +args+.
|
|
95
|
-
def sql_for_on_duplicate_key_update(
|
|
96
|
-
arg, primary_key, locking_column = args
|
|
97
|
+
def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
|
|
98
|
+
arg, model, primary_key, locking_column = args
|
|
97
99
|
arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
|
|
98
100
|
return unless arg.is_a?( Hash )
|
|
99
101
|
|
|
100
|
-
sql = ' ON CONFLICT '
|
|
102
|
+
sql = ' ON CONFLICT '.dup
|
|
101
103
|
conflict_target = sql_for_conflict_target( arg )
|
|
102
104
|
|
|
103
105
|
columns = arg.fetch( :columns, [] )
|
|
@@ -112,11 +114,12 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
112
114
|
end
|
|
113
115
|
|
|
114
116
|
sql << "#{conflict_target}DO UPDATE SET "
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
117
|
+
case columns
|
|
118
|
+
when Array
|
|
119
|
+
sql << sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, columns )
|
|
120
|
+
when Hash
|
|
121
|
+
sql << sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, columns )
|
|
122
|
+
when String
|
|
120
123
|
sql << columns
|
|
121
124
|
else
|
|
122
125
|
raise ArgumentError, 'Expected :columns to be an Array or Hash'
|
|
@@ -127,22 +130,27 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
127
130
|
sql
|
|
128
131
|
end
|
|
129
132
|
|
|
130
|
-
def sql_for_on_duplicate_key_update_as_array( locking_column, arr ) # :nodoc:
|
|
133
|
+
def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
|
|
131
134
|
results = arr.map do |column|
|
|
132
|
-
|
|
135
|
+
original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
|
|
136
|
+
qc = quote_column_name( original_column_name )
|
|
133
137
|
"#{qc}=EXCLUDED.#{qc}"
|
|
134
138
|
end
|
|
135
|
-
increment_locking_column!(results, locking_column)
|
|
139
|
+
increment_locking_column!(table_name, results, locking_column)
|
|
136
140
|
results.join( ',' )
|
|
137
141
|
end
|
|
138
142
|
|
|
139
|
-
def sql_for_on_duplicate_key_update_as_hash( locking_column, hsh ) # :nodoc:
|
|
143
|
+
def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
|
|
140
144
|
results = hsh.map do |column1, column2|
|
|
141
|
-
|
|
142
|
-
|
|
145
|
+
original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
|
|
146
|
+
qc1 = quote_column_name( original_column1_name )
|
|
147
|
+
|
|
148
|
+
original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
|
|
149
|
+
qc2 = quote_column_name( original_column2_name )
|
|
150
|
+
|
|
143
151
|
"#{qc1}=EXCLUDED.#{qc2}"
|
|
144
152
|
end
|
|
145
|
-
increment_locking_column!(results, locking_column)
|
|
153
|
+
increment_locking_column!(table_name, results, locking_column)
|
|
146
154
|
results.join( ',' )
|
|
147
155
|
end
|
|
148
156
|
|
|
@@ -150,9 +158,9 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
150
158
|
conflict_target = args[:conflict_target]
|
|
151
159
|
index_predicate = args[:index_predicate]
|
|
152
160
|
if conflict_target.present?
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
161
|
+
sql = "(#{Array( conflict_target ).reject( &:blank? ).join( ', ' )}) "
|
|
162
|
+
sql += "WHERE #{index_predicate} " if index_predicate
|
|
163
|
+
sql
|
|
156
164
|
end
|
|
157
165
|
end
|
|
158
166
|
|
|
@@ -166,14 +174,6 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
166
174
|
exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key')
|
|
167
175
|
end
|
|
168
176
|
|
|
169
|
-
def increment_locking_column!(results, locking_column)
|
|
170
|
-
if locking_column.present?
|
|
171
|
-
results << "\"#{locking_column}\"=EXCLUDED.\"#{locking_column}\"+1"
|
|
172
|
-
end
|
|
173
|
-
end
|
|
174
|
-
|
|
175
|
-
private
|
|
176
|
-
|
|
177
177
|
def database_version
|
|
178
178
|
defined?(sqlite_version) ? sqlite_version : super
|
|
179
179
|
end
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require "pathname"
|
|
2
4
|
require "active_record"
|
|
3
5
|
require "active_record/version"
|
|
4
6
|
|
|
5
7
|
module ActiveRecord::Import
|
|
6
|
-
ADAPTER_PATH = "activerecord-import/active_record/adapters"
|
|
8
|
+
ADAPTER_PATH = "activerecord-import/active_record/adapters"
|
|
7
9
|
|
|
8
10
|
def self.base_adapter(adapter)
|
|
9
11
|
case adapter
|
|
@@ -27,7 +29,13 @@ module ActiveRecord::Import
|
|
|
27
29
|
|
|
28
30
|
# Loads the import functionality for the passed in ActiveRecord connection
|
|
29
31
|
def self.load_from_connection_pool(connection_pool)
|
|
30
|
-
|
|
32
|
+
adapter =
|
|
33
|
+
if connection_pool.respond_to?(:db_config) # ActiveRecord >= 6.1
|
|
34
|
+
connection_pool.db_config.adapter
|
|
35
|
+
else
|
|
36
|
+
connection_pool.spec.config[:adapter]
|
|
37
|
+
end
|
|
38
|
+
require_adapter adapter
|
|
31
39
|
end
|
|
32
40
|
end
|
|
33
41
|
|