activerecord-import 1.0.4 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yaml +159 -0
- data/.gitignore +5 -0
- data/.rubocop.yml +76 -7
- data/.rubocop_todo.yml +10 -16
- data/Brewfile +3 -1
- data/CHANGELOG.md +143 -3
- data/Dockerfile +23 -0
- data/Gemfile +28 -24
- data/LICENSE +21 -56
- data/README.markdown +83 -27
- data/Rakefile +3 -0
- data/activerecord-import.gemspec +10 -5
- data/benchmarks/benchmark.rb +10 -6
- data/benchmarks/lib/base.rb +10 -5
- data/benchmarks/lib/cli_parser.rb +10 -6
- data/benchmarks/lib/float.rb +2 -0
- data/benchmarks/lib/mysql2_benchmark.rb +2 -0
- data/benchmarks/lib/output_to_csv.rb +2 -0
- data/benchmarks/lib/output_to_html.rb +4 -2
- data/benchmarks/models/test_innodb.rb +2 -0
- data/benchmarks/models/test_memory.rb +2 -0
- data/benchmarks/models/test_myisam.rb +2 -0
- data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +2 -0
- data/docker-compose.yml +34 -0
- data/gemfiles/5.2.gemfile +2 -0
- data/gemfiles/6.0.gemfile +3 -0
- data/gemfiles/6.1.gemfile +4 -1
- data/gemfiles/7.0.gemfile +4 -0
- data/gemfiles/7.1.gemfile +3 -0
- data/gemfiles/7.2.gemfile +3 -0
- data/gemfiles/8.0.gemfile +3 -0
- data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +6 -4
- data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/trilogy_adapter.rb +8 -0
- data/lib/activerecord-import/adapters/abstract_adapter.rb +9 -6
- data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql_adapter.rb +30 -21
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +68 -48
- data/lib/activerecord-import/adapters/sqlite3_adapter.rb +37 -30
- data/lib/activerecord-import/adapters/trilogy_adapter.rb +7 -0
- data/lib/activerecord-import/base.rb +3 -1
- data/lib/activerecord-import/import.rb +160 -58
- data/lib/activerecord-import/synchronize.rb +3 -1
- data/lib/activerecord-import/value_sets_parser.rb +5 -0
- data/lib/activerecord-import/version.rb +3 -1
- data/lib/activerecord-import.rb +2 -1
- data/test/adapters/jdbcmysql.rb +2 -0
- data/test/adapters/jdbcpostgresql.rb +2 -0
- data/test/adapters/jdbcsqlite3.rb +2 -0
- data/test/adapters/makara_postgis.rb +2 -0
- data/test/adapters/mysql2.rb +2 -0
- data/test/adapters/mysql2_makara.rb +2 -0
- data/test/adapters/mysql2spatial.rb +2 -0
- data/test/adapters/postgis.rb +2 -0
- data/test/adapters/postgresql.rb +2 -0
- data/test/adapters/postgresql_makara.rb +2 -0
- data/test/adapters/seamless_database_pool.rb +2 -0
- data/test/adapters/spatialite.rb +2 -0
- data/test/adapters/sqlite3.rb +2 -0
- data/test/adapters/trilogy.rb +9 -0
- data/test/database.yml.sample +7 -0
- data/test/{travis → github}/database.yml +9 -3
- data/test/import_test.rb +108 -41
- data/test/jdbcmysql/import_test.rb +5 -3
- data/test/jdbcpostgresql/import_test.rb +4 -2
- data/test/jdbcsqlite3/import_test.rb +4 -2
- data/test/makara_postgis/import_test.rb +4 -2
- data/test/models/account.rb +2 -0
- data/test/models/alarm.rb +2 -0
- data/test/models/animal.rb +8 -0
- data/test/models/author.rb +9 -0
- data/test/models/bike_maker.rb +3 -0
- data/test/models/book.rb +12 -3
- data/test/models/car.rb +2 -0
- data/test/models/card.rb +5 -0
- data/test/models/chapter.rb +2 -0
- data/test/models/composite_book.rb +19 -0
- data/test/models/composite_chapter.rb +12 -0
- data/test/models/customer.rb +18 -0
- data/test/models/deck.rb +8 -0
- data/test/models/dictionary.rb +2 -0
- data/test/models/discount.rb +2 -0
- data/test/models/end_note.rb +2 -0
- data/test/models/group.rb +2 -0
- data/test/models/order.rb +17 -0
- data/test/models/playing_card.rb +4 -0
- data/test/models/promotion.rb +2 -0
- data/test/models/question.rb +2 -0
- data/test/models/rule.rb +2 -0
- data/test/models/tag.rb +9 -1
- data/test/models/tag_alias.rb +11 -0
- data/test/models/topic.rb +8 -0
- data/test/models/user.rb +2 -0
- data/test/models/user_token.rb +2 -0
- data/test/models/vendor.rb +2 -0
- data/test/models/widget.rb +12 -3
- data/test/mysql2/import_test.rb +5 -3
- data/test/mysql2_makara/import_test.rb +5 -3
- data/test/mysqlspatial2/import_test.rb +5 -3
- data/test/postgis/import_test.rb +4 -2
- data/test/postgresql/import_test.rb +4 -2
- data/test/schema/generic_schema.rb +37 -1
- data/test/schema/jdbcpostgresql_schema.rb +3 -1
- data/test/schema/mysql2_schema.rb +2 -0
- data/test/schema/postgis_schema.rb +3 -1
- data/test/schema/postgresql_schema.rb +38 -4
- data/test/schema/sqlite3_schema.rb +2 -0
- data/test/schema/version.rb +2 -0
- data/test/sqlite3/import_test.rb +4 -2
- data/test/support/active_support/test_case_extensions.rb +3 -5
- data/test/support/assertions.rb +2 -0
- data/test/support/factories.rb +2 -0
- data/test/support/generate.rb +4 -2
- data/test/support/mysql/import_examples.rb +7 -8
- data/test/support/postgresql/import_examples.rb +121 -53
- data/test/support/shared_examples/on_duplicate_key_ignore.rb +2 -0
- data/test/support/shared_examples/on_duplicate_key_update.rb +69 -10
- data/test/support/shared_examples/recursive_import.rb +137 -1
- data/test/support/sqlite3/import_examples.rb +2 -1
- data/test/synchronize_test.rb +2 -0
- data/test/test_helper.rb +38 -24
- data/test/trilogy/import_test.rb +7 -0
- data/test/value_sets_bytes_parser_test.rb +3 -1
- data/test/value_sets_records_parser_test.rb +3 -1
- metadata +46 -22
- data/.travis.yml +0 -74
- data/gemfiles/3.2.gemfile +0 -2
- data/gemfiles/4.0.gemfile +0 -2
- data/gemfiles/4.1.gemfile +0 -2
- data/gemfiles/4.2.gemfile +0 -2
- data/gemfiles/5.0.gemfile +0 -2
- data/gemfiles/5.1.gemfile +0 -2
- data/lib/activerecord-import/mysql2.rb +0 -7
- data/lib/activerecord-import/postgresql.rb +0 -7
- data/lib/activerecord-import/sqlite3.rb +0 -7
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module ActiveRecord::Import::AbstractAdapter
|
|
2
4
|
module InstanceMethods
|
|
3
5
|
def next_value_for_sequence(sequence_name)
|
|
@@ -7,10 +9,11 @@ module ActiveRecord::Import::AbstractAdapter
|
|
|
7
9
|
def insert_many( sql, values, _options = {}, *args ) # :nodoc:
|
|
8
10
|
number_of_inserts = 1
|
|
9
11
|
|
|
10
|
-
base_sql, post_sql =
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
12
|
+
base_sql, post_sql = case sql
|
|
13
|
+
when String
|
|
14
|
+
[sql, '']
|
|
15
|
+
when Array
|
|
16
|
+
[sql.shift, sql.join( ' ' )]
|
|
14
17
|
end
|
|
15
18
|
|
|
16
19
|
sql2insert = base_sql + values.join( ',' ) + post_sql
|
|
@@ -45,7 +48,7 @@ module ActiveRecord::Import::AbstractAdapter
|
|
|
45
48
|
post_sql_statements = []
|
|
46
49
|
|
|
47
50
|
if supports_on_duplicate_key_update? && options[:on_duplicate_key_update]
|
|
48
|
-
post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update], options[:primary_key], options[:locking_column] )
|
|
51
|
+
post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update], options[:model], options[:primary_key], options[:locking_column] )
|
|
49
52
|
elsif logger && options[:on_duplicate_key_update]
|
|
50
53
|
logger.warn "Ignoring on_duplicate_key_update because it is not supported by the database."
|
|
51
54
|
end
|
|
@@ -66,7 +69,7 @@ module ActiveRecord::Import::AbstractAdapter
|
|
|
66
69
|
end
|
|
67
70
|
|
|
68
71
|
def supports_on_duplicate_key_update?
|
|
69
|
-
|
|
72
|
+
false
|
|
70
73
|
end
|
|
71
74
|
end
|
|
72
75
|
end
|
|
@@ -1,5 +1,8 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module ActiveRecord::Import::MysqlAdapter
|
|
2
4
|
include ActiveRecord::Import::ImportSupport
|
|
5
|
+
include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
|
|
3
6
|
|
|
4
7
|
NO_MAX_PACKET = 0
|
|
5
8
|
QUERY_OVERHEAD = 8 # This was shown to be true for MySQL, but it's not clear where the overhead is from.
|
|
@@ -10,13 +13,14 @@ module ActiveRecord::Import::MysqlAdapter
|
|
|
10
13
|
# the number of inserts default
|
|
11
14
|
number_of_inserts = 0
|
|
12
15
|
|
|
13
|
-
base_sql, post_sql =
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
16
|
+
base_sql, post_sql = case sql
|
|
17
|
+
when String
|
|
18
|
+
[sql, '']
|
|
19
|
+
when Array
|
|
20
|
+
[sql.shift, sql.join( ' ' )]
|
|
17
21
|
end
|
|
18
22
|
|
|
19
|
-
sql_size = QUERY_OVERHEAD + base_sql.
|
|
23
|
+
sql_size = QUERY_OVERHEAD + base_sql.bytesize + post_sql.bytesize
|
|
20
24
|
|
|
21
25
|
# the number of bytes the requested insert statement values will take up
|
|
22
26
|
values_in_bytes = values.sum(&:bytesize)
|
|
@@ -30,7 +34,7 @@ module ActiveRecord::Import::MysqlAdapter
|
|
|
30
34
|
max = max_allowed_packet
|
|
31
35
|
|
|
32
36
|
# if we can insert it all as one statement
|
|
33
|
-
if
|
|
37
|
+
if max == NO_MAX_PACKET || total_bytes <= max || options[:force_single_insert]
|
|
34
38
|
number_of_inserts += 1
|
|
35
39
|
sql2insert = base_sql + values.join( ',' ) + post_sql
|
|
36
40
|
insert( sql2insert, *args )
|
|
@@ -55,9 +59,9 @@ module ActiveRecord::Import::MysqlAdapter
|
|
|
55
59
|
# in a single packet
|
|
56
60
|
def max_allowed_packet # :nodoc:
|
|
57
61
|
@max_allowed_packet ||= begin
|
|
58
|
-
result = execute( "
|
|
62
|
+
result = execute( "SELECT @@max_allowed_packet" )
|
|
59
63
|
# original Mysql gem responds to #fetch_row while Mysql2 responds to #first
|
|
60
|
-
val = result.respond_to?(:fetch_row) ? result.fetch_row[
|
|
64
|
+
val = result.respond_to?(:fetch_row) ? result.fetch_row[0] : result.first[0]
|
|
61
65
|
val.to_i
|
|
62
66
|
end
|
|
63
67
|
end
|
|
@@ -81,14 +85,14 @@ module ActiveRecord::Import::MysqlAdapter
|
|
|
81
85
|
# Returns a generated ON DUPLICATE KEY UPDATE statement given the passed
|
|
82
86
|
# in +args+.
|
|
83
87
|
def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
|
|
84
|
-
sql = ' ON DUPLICATE KEY UPDATE '
|
|
85
|
-
arg = args
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arg )
|
|
89
|
-
|
|
90
|
-
sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, arg )
|
|
91
|
-
|
|
88
|
+
sql = ' ON DUPLICATE KEY UPDATE '.dup
|
|
89
|
+
arg, model, _primary_key, locking_column = args
|
|
90
|
+
case arg
|
|
91
|
+
when Array
|
|
92
|
+
sql << sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arg )
|
|
93
|
+
when Hash
|
|
94
|
+
sql << sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, arg )
|
|
95
|
+
when String
|
|
92
96
|
sql << arg
|
|
93
97
|
else
|
|
94
98
|
raise ArgumentError, "Expected Array or Hash"
|
|
@@ -96,19 +100,24 @@ module ActiveRecord::Import::MysqlAdapter
|
|
|
96
100
|
sql
|
|
97
101
|
end
|
|
98
102
|
|
|
99
|
-
def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
|
|
103
|
+
def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
|
|
100
104
|
results = arr.map do |column|
|
|
101
|
-
|
|
105
|
+
original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
|
|
106
|
+
qc = quote_column_name( original_column_name )
|
|
102
107
|
"#{table_name}.#{qc}=VALUES(#{qc})"
|
|
103
108
|
end
|
|
104
109
|
increment_locking_column!(table_name, results, locking_column)
|
|
105
110
|
results.join( ',' )
|
|
106
111
|
end
|
|
107
112
|
|
|
108
|
-
def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
|
|
113
|
+
def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
|
|
109
114
|
results = hsh.map do |column1, column2|
|
|
110
|
-
|
|
111
|
-
|
|
115
|
+
original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
|
|
116
|
+
qc1 = quote_column_name( original_column1_name )
|
|
117
|
+
|
|
118
|
+
original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
|
|
119
|
+
qc2 = quote_column_name( original_column2_name )
|
|
120
|
+
|
|
112
121
|
"#{table_name}.#{qc1}=VALUES( #{qc2} )"
|
|
113
122
|
end
|
|
114
123
|
increment_locking_column!(table_name, results, locking_column)
|
|
@@ -1,63 +1,73 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module ActiveRecord::Import::PostgreSQLAdapter
|
|
2
4
|
include ActiveRecord::Import::ImportSupport
|
|
5
|
+
include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
|
|
3
6
|
|
|
4
7
|
MIN_VERSION_FOR_UPSERT = 90_500
|
|
5
8
|
|
|
6
9
|
def insert_many( sql, values, options = {}, *args ) # :nodoc:
|
|
7
10
|
number_of_inserts = 1
|
|
8
|
-
returned_values =
|
|
11
|
+
returned_values = {}
|
|
9
12
|
ids = []
|
|
10
13
|
results = []
|
|
11
14
|
|
|
12
|
-
base_sql, post_sql =
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
15
|
+
base_sql, post_sql = case sql
|
|
16
|
+
when String
|
|
17
|
+
[sql, '']
|
|
18
|
+
when Array
|
|
19
|
+
[sql.shift, sql.join( ' ' )]
|
|
16
20
|
end
|
|
17
21
|
|
|
18
22
|
sql2insert = base_sql + values.join( ',' ) + post_sql
|
|
19
23
|
|
|
20
|
-
|
|
21
|
-
if
|
|
24
|
+
selections = returning_selections(options)
|
|
25
|
+
if selections.blank? || (options[:no_returning] && !options[:recursive])
|
|
22
26
|
insert( sql2insert, *args )
|
|
23
27
|
else
|
|
24
|
-
returned_values = if
|
|
28
|
+
returned_values = if selections.size > 1
|
|
25
29
|
# Select composite columns
|
|
26
|
-
|
|
30
|
+
db_result = select_all( sql2insert, *args )
|
|
31
|
+
{ values: db_result.rows, columns: db_result.columns }
|
|
27
32
|
else
|
|
28
|
-
select_values( sql2insert, *args )
|
|
33
|
+
{ values: select_values( sql2insert, *args ) }
|
|
29
34
|
end
|
|
30
|
-
|
|
35
|
+
clear_query_cache if query_cache_enabled
|
|
31
36
|
end
|
|
32
37
|
|
|
33
38
|
if options[:returning].blank?
|
|
34
|
-
ids = returned_values
|
|
39
|
+
ids = Array(returned_values[:values])
|
|
35
40
|
elsif options[:primary_key].blank?
|
|
36
|
-
|
|
41
|
+
options[:returning_columns] ||= returned_values[:columns]
|
|
42
|
+
results = Array(returned_values[:values])
|
|
37
43
|
else
|
|
38
44
|
# split primary key and returning columns
|
|
39
|
-
ids, results = split_ids_and_results(returned_values,
|
|
45
|
+
ids, results, options[:returning_columns] = split_ids_and_results(returned_values, options)
|
|
40
46
|
end
|
|
41
47
|
|
|
42
48
|
ActiveRecord::Import::Result.new([], number_of_inserts, ids, results)
|
|
43
49
|
end
|
|
44
50
|
|
|
45
|
-
def split_ids_and_results(
|
|
51
|
+
def split_ids_and_results( selections, options )
|
|
46
52
|
ids = []
|
|
47
|
-
|
|
53
|
+
returning_values = []
|
|
54
|
+
|
|
55
|
+
columns = Array(selections[:columns])
|
|
56
|
+
values = Array(selections[:values])
|
|
48
57
|
id_indexes = Array(options[:primary_key]).map { |key| columns.index(key) }
|
|
49
|
-
|
|
58
|
+
returning_columns = columns.reject.with_index { |_, index| id_indexes.include?(index) }
|
|
59
|
+
returning_indexes = returning_columns.map { |column| columns.index(column) }
|
|
50
60
|
|
|
51
61
|
values.each do |value|
|
|
52
62
|
value_array = Array(value)
|
|
53
|
-
ids << id_indexes.map { |
|
|
54
|
-
|
|
63
|
+
ids << id_indexes.map { |index| value_array[index] }
|
|
64
|
+
returning_values << returning_indexes.map { |index| value_array[index] }
|
|
55
65
|
end
|
|
56
66
|
|
|
57
67
|
ids.map!(&:first) if id_indexes.size == 1
|
|
58
|
-
|
|
68
|
+
returning_values.map!(&:first) if returning_columns.size == 1
|
|
59
69
|
|
|
60
|
-
[ids,
|
|
70
|
+
[ids, returning_values, returning_columns]
|
|
61
71
|
end
|
|
62
72
|
|
|
63
73
|
def next_value_for_sequence(sequence_name)
|
|
@@ -78,31 +88,37 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
|
78
88
|
|
|
79
89
|
sql += super(table_name, options)
|
|
80
90
|
|
|
81
|
-
|
|
82
|
-
unless
|
|
83
|
-
sql << " RETURNING
|
|
91
|
+
selections = returning_selections(options)
|
|
92
|
+
unless selections.blank? || (options[:no_returning] && !options[:recursive])
|
|
93
|
+
sql << " RETURNING #{selections.join(', ')}"
|
|
84
94
|
end
|
|
85
95
|
|
|
86
96
|
sql
|
|
87
97
|
end
|
|
88
98
|
|
|
89
|
-
def
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
99
|
+
def returning_selections(options)
|
|
100
|
+
selections = []
|
|
101
|
+
column_names = Array(options[:model].column_names)
|
|
102
|
+
|
|
103
|
+
selections += Array(options[:primary_key]) if options[:primary_key].present?
|
|
104
|
+
selections += Array(options[:returning]) if options[:returning].present?
|
|
105
|
+
|
|
106
|
+
selections.map do |selection|
|
|
107
|
+
column_names.include?(selection.to_s) ? "\"#{selection}\"" : selection
|
|
108
|
+
end
|
|
94
109
|
end
|
|
95
110
|
|
|
96
111
|
# Add a column to be updated on duplicate key update
|
|
97
112
|
def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
|
|
98
113
|
arg = options[:on_duplicate_key_update]
|
|
99
|
-
|
|
114
|
+
case arg
|
|
115
|
+
when Hash
|
|
100
116
|
columns = arg.fetch( :columns ) { arg[:columns] = [] }
|
|
101
117
|
case columns
|
|
102
118
|
when Array then columns << column.to_sym unless columns.include?( column.to_sym )
|
|
103
119
|
when Hash then columns[column.to_sym] = column.to_sym
|
|
104
120
|
end
|
|
105
|
-
|
|
121
|
+
when Array
|
|
106
122
|
arg << column.to_sym unless arg.include?( column.to_sym )
|
|
107
123
|
end
|
|
108
124
|
end
|
|
@@ -118,11 +134,11 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
|
118
134
|
# Returns a generated ON CONFLICT DO UPDATE statement given the passed
|
|
119
135
|
# in +args+.
|
|
120
136
|
def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
|
|
121
|
-
arg, primary_key, locking_column = args
|
|
137
|
+
arg, model, primary_key, locking_column = args
|
|
122
138
|
arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
|
|
123
139
|
return unless arg.is_a?( Hash )
|
|
124
140
|
|
|
125
|
-
sql = ' ON CONFLICT '
|
|
141
|
+
sql = ' ON CONFLICT '.dup
|
|
126
142
|
conflict_target = sql_for_conflict_target( arg )
|
|
127
143
|
|
|
128
144
|
columns = arg.fetch( :columns, [] )
|
|
@@ -137,11 +153,12 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
|
137
153
|
end
|
|
138
154
|
|
|
139
155
|
sql << "#{conflict_target}DO UPDATE SET "
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
156
|
+
case columns
|
|
157
|
+
when Array
|
|
158
|
+
sql << sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, columns )
|
|
159
|
+
when Hash
|
|
160
|
+
sql << sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, columns )
|
|
161
|
+
when String
|
|
145
162
|
sql << columns
|
|
146
163
|
else
|
|
147
164
|
raise ArgumentError, 'Expected :columns to be an Array or Hash'
|
|
@@ -152,19 +169,24 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
|
152
169
|
sql
|
|
153
170
|
end
|
|
154
171
|
|
|
155
|
-
def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
|
|
172
|
+
def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
|
|
156
173
|
results = arr.map do |column|
|
|
157
|
-
|
|
174
|
+
original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
|
|
175
|
+
qc = quote_column_name( original_column_name )
|
|
158
176
|
"#{qc}=EXCLUDED.#{qc}"
|
|
159
177
|
end
|
|
160
178
|
increment_locking_column!(table_name, results, locking_column)
|
|
161
179
|
results.join( ',' )
|
|
162
180
|
end
|
|
163
181
|
|
|
164
|
-
def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
|
|
182
|
+
def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
|
|
165
183
|
results = hsh.map do |column1, column2|
|
|
166
|
-
|
|
167
|
-
|
|
184
|
+
original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
|
|
185
|
+
qc1 = quote_column_name( original_column1_name )
|
|
186
|
+
|
|
187
|
+
original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
|
|
188
|
+
qc2 = quote_column_name( original_column2_name )
|
|
189
|
+
|
|
168
190
|
"#{qc1}=EXCLUDED.#{qc2}"
|
|
169
191
|
end
|
|
170
192
|
increment_locking_column!(table_name, results, locking_column)
|
|
@@ -178,9 +200,9 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
|
178
200
|
if constraint_name.present?
|
|
179
201
|
"ON CONSTRAINT #{constraint_name} "
|
|
180
202
|
elsif conflict_target.present?
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
203
|
+
sql = "(#{Array( conflict_target ).reject( &:blank? ).join( ', ' )}) "
|
|
204
|
+
sql += "WHERE #{index_predicate} " if index_predicate
|
|
205
|
+
sql
|
|
184
206
|
end
|
|
185
207
|
end
|
|
186
208
|
|
|
@@ -202,8 +224,6 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
|
202
224
|
true
|
|
203
225
|
end
|
|
204
226
|
|
|
205
|
-
private
|
|
206
|
-
|
|
207
227
|
def database_version
|
|
208
228
|
defined?(postgresql_version) ? postgresql_version : super
|
|
209
229
|
end
|
|
@@ -1,8 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module ActiveRecord::Import::SQLite3Adapter
|
|
2
4
|
include ActiveRecord::Import::ImportSupport
|
|
5
|
+
include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
|
|
3
6
|
|
|
4
|
-
MIN_VERSION_FOR_IMPORT = "3.7.11"
|
|
5
|
-
MIN_VERSION_FOR_UPSERT = "3.24.0"
|
|
7
|
+
MIN_VERSION_FOR_IMPORT = "3.7.11"
|
|
8
|
+
MIN_VERSION_FOR_UPSERT = "3.24.0"
|
|
6
9
|
SQLITE_LIMIT_COMPOUND_SELECT = 500
|
|
7
10
|
|
|
8
11
|
# Override our conformance to ActiveRecord::Import::ImportSupport interface
|
|
@@ -21,10 +24,11 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
21
24
|
def insert_many( sql, values, _options = {}, *args ) # :nodoc:
|
|
22
25
|
number_of_inserts = 0
|
|
23
26
|
|
|
24
|
-
base_sql, post_sql =
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
27
|
+
base_sql, post_sql = case sql
|
|
28
|
+
when String
|
|
29
|
+
[sql, '']
|
|
30
|
+
when Array
|
|
31
|
+
[sql.shift, sql.join( ' ' )]
|
|
28
32
|
end
|
|
29
33
|
|
|
30
34
|
value_sets = ::ActiveRecord::Import::ValueSetsRecordsParser.parse(values,
|
|
@@ -53,11 +57,9 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
53
57
|
def post_sql_statements( table_name, options ) # :nodoc:
|
|
54
58
|
sql = []
|
|
55
59
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
sql << sql_for_on_duplicate_key_ignore( options[:on_duplicate_key_ignore] )
|
|
60
|
-
end
|
|
60
|
+
# Options :recursive and :on_duplicate_key_ignore are mutually exclusive
|
|
61
|
+
if supports_on_duplicate_key_update? && ((options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:on_duplicate_key_update])
|
|
62
|
+
sql << sql_for_on_duplicate_key_ignore( options[:on_duplicate_key_ignore] )
|
|
61
63
|
end
|
|
62
64
|
|
|
63
65
|
sql + super
|
|
@@ -70,13 +72,14 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
70
72
|
# Add a column to be updated on duplicate key update
|
|
71
73
|
def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
|
|
72
74
|
arg = options[:on_duplicate_key_update]
|
|
73
|
-
|
|
75
|
+
case arg
|
|
76
|
+
when Hash
|
|
74
77
|
columns = arg.fetch( :columns ) { arg[:columns] = [] }
|
|
75
78
|
case columns
|
|
76
79
|
when Array then columns << column.to_sym unless columns.include?( column.to_sym )
|
|
77
80
|
when Hash then columns[column.to_sym] = column.to_sym
|
|
78
81
|
end
|
|
79
|
-
|
|
82
|
+
when Array
|
|
80
83
|
arg << column.to_sym unless arg.include?( column.to_sym )
|
|
81
84
|
end
|
|
82
85
|
end
|
|
@@ -92,11 +95,11 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
92
95
|
# Returns a generated ON CONFLICT DO UPDATE statement given the passed
|
|
93
96
|
# in +args+.
|
|
94
97
|
def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
|
|
95
|
-
arg, primary_key, locking_column = args
|
|
98
|
+
arg, model, primary_key, locking_column = args
|
|
96
99
|
arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
|
|
97
100
|
return unless arg.is_a?( Hash )
|
|
98
101
|
|
|
99
|
-
sql = ' ON CONFLICT '
|
|
102
|
+
sql = ' ON CONFLICT '.dup
|
|
100
103
|
conflict_target = sql_for_conflict_target( arg )
|
|
101
104
|
|
|
102
105
|
columns = arg.fetch( :columns, [] )
|
|
@@ -111,11 +114,12 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
111
114
|
end
|
|
112
115
|
|
|
113
116
|
sql << "#{conflict_target}DO UPDATE SET "
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
117
|
+
case columns
|
|
118
|
+
when Array
|
|
119
|
+
sql << sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, columns )
|
|
120
|
+
when Hash
|
|
121
|
+
sql << sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, columns )
|
|
122
|
+
when String
|
|
119
123
|
sql << columns
|
|
120
124
|
else
|
|
121
125
|
raise ArgumentError, 'Expected :columns to be an Array or Hash'
|
|
@@ -126,19 +130,24 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
126
130
|
sql
|
|
127
131
|
end
|
|
128
132
|
|
|
129
|
-
def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
|
|
133
|
+
def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
|
|
130
134
|
results = arr.map do |column|
|
|
131
|
-
|
|
135
|
+
original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
|
|
136
|
+
qc = quote_column_name( original_column_name )
|
|
132
137
|
"#{qc}=EXCLUDED.#{qc}"
|
|
133
138
|
end
|
|
134
139
|
increment_locking_column!(table_name, results, locking_column)
|
|
135
140
|
results.join( ',' )
|
|
136
141
|
end
|
|
137
142
|
|
|
138
|
-
def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
|
|
143
|
+
def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
|
|
139
144
|
results = hsh.map do |column1, column2|
|
|
140
|
-
|
|
141
|
-
|
|
145
|
+
original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
|
|
146
|
+
qc1 = quote_column_name( original_column1_name )
|
|
147
|
+
|
|
148
|
+
original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
|
|
149
|
+
qc2 = quote_column_name( original_column2_name )
|
|
150
|
+
|
|
142
151
|
"#{qc1}=EXCLUDED.#{qc2}"
|
|
143
152
|
end
|
|
144
153
|
increment_locking_column!(table_name, results, locking_column)
|
|
@@ -149,9 +158,9 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
149
158
|
conflict_target = args[:conflict_target]
|
|
150
159
|
index_predicate = args[:index_predicate]
|
|
151
160
|
if conflict_target.present?
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
161
|
+
sql = "(#{Array( conflict_target ).reject( &:blank? ).join( ', ' )}) "
|
|
162
|
+
sql += "WHERE #{index_predicate} " if index_predicate
|
|
163
|
+
sql
|
|
155
164
|
end
|
|
156
165
|
end
|
|
157
166
|
|
|
@@ -165,8 +174,6 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
165
174
|
exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key')
|
|
166
175
|
end
|
|
167
176
|
|
|
168
|
-
private
|
|
169
|
-
|
|
170
177
|
def database_version
|
|
171
178
|
defined?(sqlite_version) ? sqlite_version : super
|
|
172
179
|
end
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require "pathname"
|
|
2
4
|
require "active_record"
|
|
3
5
|
require "active_record/version"
|
|
4
6
|
|
|
5
7
|
module ActiveRecord::Import
|
|
6
|
-
ADAPTER_PATH = "activerecord-import/active_record/adapters"
|
|
8
|
+
ADAPTER_PATH = "activerecord-import/active_record/adapters"
|
|
7
9
|
|
|
8
10
|
def self.base_adapter(adapter)
|
|
9
11
|
case adapter
|