activerecord-import 0.10.0 → 1.0.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gitignore +1 -0
- data/.rubocop.yml +49 -0
- data/.rubocop_todo.yml +36 -0
- data/.travis.yml +64 -8
- data/CHANGELOG.md +475 -0
- data/Gemfile +32 -15
- data/LICENSE +21 -56
- data/README.markdown +564 -35
- data/Rakefile +20 -3
- data/activerecord-import.gemspec +7 -7
- data/benchmarks/README +2 -2
- data/benchmarks/benchmark.rb +68 -64
- data/benchmarks/lib/base.rb +138 -137
- data/benchmarks/lib/cli_parser.rb +107 -103
- data/benchmarks/lib/{mysql_benchmark.rb → mysql2_benchmark.rb} +19 -22
- data/benchmarks/lib/output_to_csv.rb +5 -4
- data/benchmarks/lib/output_to_html.rb +8 -13
- data/benchmarks/models/test_innodb.rb +1 -1
- data/benchmarks/models/test_memory.rb +1 -1
- data/benchmarks/models/test_myisam.rb +1 -1
- data/benchmarks/schema/mysql2_schema.rb +16 -0
- data/gemfiles/3.2.gemfile +2 -4
- data/gemfiles/4.0.gemfile +2 -4
- data/gemfiles/4.1.gemfile +2 -4
- data/gemfiles/4.2.gemfile +2 -4
- data/gemfiles/5.0.gemfile +2 -0
- data/gemfiles/5.1.gemfile +2 -0
- data/gemfiles/5.2.gemfile +2 -0
- data/gemfiles/6.0.gemfile +2 -0
- data/gemfiles/6.1.gemfile +1 -0
- data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +6 -0
- data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +0 -1
- data/lib/activerecord-import/adapters/abstract_adapter.rb +23 -17
- data/lib/activerecord-import/adapters/mysql_adapter.rb +52 -25
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +187 -10
- data/lib/activerecord-import/adapters/sqlite3_adapter.rb +148 -17
- data/lib/activerecord-import/base.rb +15 -9
- data/lib/activerecord-import/import.rb +740 -191
- data/lib/activerecord-import/synchronize.rb +21 -21
- data/lib/activerecord-import/value_sets_parser.rb +33 -8
- data/lib/activerecord-import/version.rb +1 -1
- data/lib/activerecord-import.rb +4 -15
- data/test/adapters/jdbcsqlite3.rb +1 -0
- data/test/adapters/makara_postgis.rb +1 -0
- data/test/adapters/mysql2_makara.rb +1 -0
- data/test/adapters/mysql2spatial.rb +1 -1
- data/test/adapters/postgis.rb +1 -1
- data/test/adapters/postgresql.rb +1 -1
- data/test/adapters/postgresql_makara.rb +1 -0
- data/test/adapters/spatialite.rb +1 -1
- data/test/adapters/sqlite3.rb +1 -1
- data/test/database.yml.sample +13 -18
- data/test/import_test.rb +608 -89
- data/test/jdbcmysql/import_test.rb +2 -3
- data/test/jdbcpostgresql/import_test.rb +0 -2
- data/test/jdbcsqlite3/import_test.rb +4 -0
- data/test/makara_postgis/import_test.rb +8 -0
- data/test/models/account.rb +3 -0
- data/test/models/alarm.rb +2 -0
- data/test/models/animal.rb +6 -0
- data/test/models/bike_maker.rb +7 -0
- data/test/models/book.rb +7 -6
- data/test/models/car.rb +3 -0
- data/test/models/chapter.rb +2 -2
- data/test/models/dictionary.rb +4 -0
- data/test/models/discount.rb +3 -0
- data/test/models/end_note.rb +2 -2
- data/test/models/promotion.rb +3 -0
- data/test/models/question.rb +3 -0
- data/test/models/rule.rb +3 -0
- data/test/models/tag.rb +4 -0
- data/test/models/topic.rb +17 -3
- data/test/models/user.rb +3 -0
- data/test/models/user_token.rb +4 -0
- data/test/models/vendor.rb +7 -0
- data/test/models/widget.rb +19 -2
- data/test/mysql2/import_test.rb +2 -3
- data/test/{em_mysql2 → mysql2_makara}/import_test.rb +1 -1
- data/test/mysqlspatial2/import_test.rb +2 -2
- data/test/postgis/import_test.rb +5 -1
- data/test/schema/generic_schema.rb +159 -85
- data/test/schema/jdbcpostgresql_schema.rb +1 -0
- data/test/schema/mysql2_schema.rb +19 -0
- data/test/schema/postgis_schema.rb +1 -0
- data/test/schema/postgresql_schema.rb +61 -0
- data/test/schema/sqlite3_schema.rb +13 -0
- data/test/sqlite3/import_test.rb +2 -50
- data/test/support/active_support/test_case_extensions.rb +21 -13
- data/test/support/{mysql/assertions.rb → assertions.rb} +20 -2
- data/test/support/factories.rb +39 -14
- data/test/support/generate.rb +10 -10
- data/test/support/mysql/import_examples.rb +49 -98
- data/test/support/postgresql/import_examples.rb +535 -57
- data/test/support/shared_examples/on_duplicate_key_ignore.rb +43 -0
- data/test/support/shared_examples/on_duplicate_key_update.rb +378 -0
- data/test/support/shared_examples/recursive_import.rb +225 -0
- data/test/support/sqlite3/import_examples.rb +231 -0
- data/test/synchronize_test.rb +10 -2
- data/test/test_helper.rb +36 -8
- data/test/travis/database.yml +26 -17
- data/test/value_sets_bytes_parser_test.rb +25 -17
- data/test/value_sets_records_parser_test.rb +6 -6
- metadata +86 -42
- data/benchmarks/boot.rb +0 -18
- data/benchmarks/schema/mysql_schema.rb +0 -16
- data/gemfiles/3.1.gemfile +0 -4
- data/lib/activerecord-import/active_record/adapters/em_mysql2_adapter.rb +0 -8
- data/lib/activerecord-import/active_record/adapters/mysql_adapter.rb +0 -6
- data/lib/activerecord-import/em_mysql2.rb +0 -7
- data/lib/activerecord-import/mysql.rb +0 -7
- data/test/adapters/em_mysql2.rb +0 -1
- data/test/adapters/mysql.rb +0 -1
- data/test/adapters/mysqlspatial.rb +0 -1
- data/test/mysql/import_test.rb +0 -6
- data/test/mysqlspatial/import_test.rb +0 -6
- data/test/schema/mysql_schema.rb +0 -18
- data/test/travis/build.sh +0 -30
@@ -3,27 +3,27 @@ module ActiveRecord::Import::MysqlAdapter
|
|
3
3
|
include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
|
4
4
|
|
5
5
|
NO_MAX_PACKET = 0
|
6
|
-
QUERY_OVERHEAD = 8 #This was shown to be true for MySQL, but it's not clear where the overhead is from.
|
6
|
+
QUERY_OVERHEAD = 8 # This was shown to be true for MySQL, but it's not clear where the overhead is from.
|
7
7
|
|
8
8
|
# +sql+ can be a single string or an array. If it is an array all
|
9
9
|
# elements that are in position >= 1 will be appended to the final SQL.
|
10
|
-
def insert_many( sql, values, *args ) # :nodoc:
|
10
|
+
def insert_many( sql, values, options = {}, *args ) # :nodoc:
|
11
11
|
# the number of inserts default
|
12
12
|
number_of_inserts = 0
|
13
13
|
|
14
|
-
base_sql,post_sql = if sql.is_a?( String )
|
15
|
-
[
|
14
|
+
base_sql, post_sql = if sql.is_a?( String )
|
15
|
+
[sql, '']
|
16
16
|
elsif sql.is_a?( Array )
|
17
|
-
[
|
17
|
+
[sql.shift, sql.join( ' ' )]
|
18
18
|
end
|
19
19
|
|
20
20
|
sql_size = QUERY_OVERHEAD + base_sql.size + post_sql.size
|
21
21
|
|
22
22
|
# the number of bytes the requested insert statement values will take up
|
23
|
-
values_in_bytes = values.sum
|
23
|
+
values_in_bytes = values.sum(&:bytesize)
|
24
24
|
|
25
25
|
# the number of bytes (commas) it will take to comma separate our values
|
26
|
-
comma_separated_bytes = values.size-1
|
26
|
+
comma_separated_bytes = values.size - 1
|
27
27
|
|
28
28
|
# the total number of bytes required if this statement is one statement
|
29
29
|
total_bytes = sql_size + values_in_bytes + comma_separated_bytes
|
@@ -31,72 +31,99 @@ module ActiveRecord::Import::MysqlAdapter
|
|
31
31
|
max = max_allowed_packet
|
32
32
|
|
33
33
|
# if we can insert it all as one statement
|
34
|
-
if NO_MAX_PACKET == max
|
34
|
+
if NO_MAX_PACKET == max || total_bytes <= max || options[:force_single_insert]
|
35
35
|
number_of_inserts += 1
|
36
36
|
sql2insert = base_sql + values.join( ',' ) + post_sql
|
37
37
|
insert( sql2insert, *args )
|
38
38
|
else
|
39
39
|
value_sets = ::ActiveRecord::Import::ValueSetsBytesParser.parse(values,
|
40
|
-
:
|
41
|
-
:
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
40
|
+
reserved_bytes: sql_size,
|
41
|
+
max_bytes: max)
|
42
|
+
|
43
|
+
transaction(requires_new: true) do
|
44
|
+
value_sets.each do |value_set|
|
45
|
+
number_of_inserts += 1
|
46
|
+
sql2insert = base_sql + value_set.join( ',' ) + post_sql
|
47
|
+
insert( sql2insert, *args )
|
48
|
+
end
|
46
49
|
end
|
47
50
|
end
|
48
51
|
|
49
|
-
[number_of_inserts,[]]
|
52
|
+
ActiveRecord::Import::Result.new([], number_of_inserts, [], [])
|
50
53
|
end
|
51
54
|
|
52
55
|
# Returns the maximum number of bytes that the server will allow
|
53
56
|
# in a single packet
|
54
57
|
def max_allowed_packet # :nodoc:
|
55
58
|
@max_allowed_packet ||= begin
|
56
|
-
result = execute( "
|
59
|
+
result = execute( "SELECT @@max_allowed_packet" )
|
57
60
|
# original Mysql gem responds to #fetch_row while Mysql2 responds to #first
|
58
|
-
val = result.respond_to?(:fetch_row) ? result.fetch_row[
|
61
|
+
val = result.respond_to?(:fetch_row) ? result.fetch_row[0] : result.first[0]
|
59
62
|
val.to_i
|
60
63
|
end
|
61
64
|
end
|
62
65
|
|
66
|
+
def pre_sql_statements( options)
|
67
|
+
sql = []
|
68
|
+
sql << "IGNORE" if options[:ignore] || options[:on_duplicate_key_ignore]
|
69
|
+
sql + super
|
70
|
+
end
|
71
|
+
|
72
|
+
# Add a column to be updated on duplicate key update
|
73
|
+
def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
|
74
|
+
if (columns = options[:on_duplicate_key_update])
|
75
|
+
case columns
|
76
|
+
when Array then columns << column.to_sym unless columns.include?(column.to_sym)
|
77
|
+
when Hash then columns[column.to_sym] = column.to_sym
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
63
82
|
# Returns a generated ON DUPLICATE KEY UPDATE statement given the passed
|
64
83
|
# in +args+.
|
65
84
|
def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
|
66
85
|
sql = ' ON DUPLICATE KEY UPDATE '
|
67
86
|
arg = args.first
|
87
|
+
locking_column = args.last
|
68
88
|
if arg.is_a?( Array )
|
69
|
-
sql << sql_for_on_duplicate_key_update_as_array( table_name, arg )
|
89
|
+
sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arg )
|
70
90
|
elsif arg.is_a?( Hash )
|
71
|
-
sql << sql_for_on_duplicate_key_update_as_hash( table_name, arg )
|
91
|
+
sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, arg )
|
72
92
|
elsif arg.is_a?( String )
|
73
93
|
sql << arg
|
74
94
|
else
|
75
|
-
raise ArgumentError
|
95
|
+
raise ArgumentError, "Expected Array or Hash"
|
76
96
|
end
|
77
97
|
sql
|
78
98
|
end
|
79
99
|
|
80
|
-
def sql_for_on_duplicate_key_update_as_array( table_name, arr )
|
100
|
+
def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
|
81
101
|
results = arr.map do |column|
|
82
102
|
qc = quote_column_name( column )
|
83
103
|
"#{table_name}.#{qc}=VALUES(#{qc})"
|
84
104
|
end
|
105
|
+
increment_locking_column!(table_name, results, locking_column)
|
85
106
|
results.join( ',' )
|
86
107
|
end
|
87
108
|
|
88
|
-
def sql_for_on_duplicate_key_update_as_hash( table_name, hsh ) # :nodoc:
|
89
|
-
sql = ' ON DUPLICATE KEY UPDATE '
|
109
|
+
def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
|
90
110
|
results = hsh.map do |column1, column2|
|
91
111
|
qc1 = quote_column_name( column1 )
|
92
112
|
qc2 = quote_column_name( column2 )
|
93
113
|
"#{table_name}.#{qc1}=VALUES( #{qc2} )"
|
94
114
|
end
|
115
|
+
increment_locking_column!(table_name, results, locking_column)
|
95
116
|
results.join( ',')
|
96
117
|
end
|
97
118
|
|
98
|
-
#
|
99
|
-
def duplicate_key_update_error?(exception)# :nodoc:
|
119
|
+
# Return true if the statement is a duplicate key record error
|
120
|
+
def duplicate_key_update_error?(exception) # :nodoc:
|
100
121
|
exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('Duplicate entry')
|
101
122
|
end
|
123
|
+
|
124
|
+
def increment_locking_column!(table_name, results, locking_column)
|
125
|
+
if locking_column.present?
|
126
|
+
results << "`#{locking_column}`=#{table_name}.`#{locking_column}`+1"
|
127
|
+
end
|
128
|
+
end
|
102
129
|
end
|
@@ -1,19 +1,64 @@
|
|
1
1
|
module ActiveRecord::Import::PostgreSQLAdapter
|
2
2
|
include ActiveRecord::Import::ImportSupport
|
3
|
+
include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
|
3
4
|
|
4
|
-
|
5
|
+
MIN_VERSION_FOR_UPSERT = 90_500
|
6
|
+
|
7
|
+
def insert_many( sql, values, options = {}, *args ) # :nodoc:
|
5
8
|
number_of_inserts = 1
|
9
|
+
returned_values = []
|
10
|
+
ids = []
|
11
|
+
results = []
|
6
12
|
|
7
|
-
base_sql,post_sql = if sql.is_a?( String )
|
8
|
-
[
|
13
|
+
base_sql, post_sql = if sql.is_a?( String )
|
14
|
+
[sql, '']
|
9
15
|
elsif sql.is_a?( Array )
|
10
|
-
[
|
16
|
+
[sql.shift, sql.join( ' ' )]
|
11
17
|
end
|
12
18
|
|
13
19
|
sql2insert = base_sql + values.join( ',' ) + post_sql
|
14
|
-
ids = select_values( sql2insert, *args )
|
15
20
|
|
16
|
-
|
21
|
+
columns = returning_columns(options)
|
22
|
+
if columns.blank? || (options[:no_returning] && !options[:recursive])
|
23
|
+
insert( sql2insert, *args )
|
24
|
+
else
|
25
|
+
returned_values = if columns.size > 1
|
26
|
+
# Select composite columns
|
27
|
+
select_rows( sql2insert, *args )
|
28
|
+
else
|
29
|
+
select_values( sql2insert, *args )
|
30
|
+
end
|
31
|
+
clear_query_cache if query_cache_enabled
|
32
|
+
end
|
33
|
+
|
34
|
+
if options[:returning].blank?
|
35
|
+
ids = returned_values
|
36
|
+
elsif options[:primary_key].blank?
|
37
|
+
results = returned_values
|
38
|
+
else
|
39
|
+
# split primary key and returning columns
|
40
|
+
ids, results = split_ids_and_results(returned_values, columns, options)
|
41
|
+
end
|
42
|
+
|
43
|
+
ActiveRecord::Import::Result.new([], number_of_inserts, ids, results)
|
44
|
+
end
|
45
|
+
|
46
|
+
def split_ids_and_results(values, columns, options)
|
47
|
+
ids = []
|
48
|
+
results = []
|
49
|
+
id_indexes = Array(options[:primary_key]).map { |key| columns.index(key) }
|
50
|
+
returning_indexes = Array(options[:returning]).map { |key| columns.index(key) }
|
51
|
+
|
52
|
+
values.each do |value|
|
53
|
+
value_array = Array(value)
|
54
|
+
ids << id_indexes.map { |i| value_array[i] }
|
55
|
+
results << returning_indexes.map { |i| value_array[i] }
|
56
|
+
end
|
57
|
+
|
58
|
+
ids.map!(&:first) if id_indexes.size == 1
|
59
|
+
results.map!(&:first) if returning_indexes.size == 1
|
60
|
+
|
61
|
+
[ids, results]
|
17
62
|
end
|
18
63
|
|
19
64
|
def next_value_for_sequence(sequence_name)
|
@@ -21,14 +66,146 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
21
66
|
end
|
22
67
|
|
23
68
|
def post_sql_statements( table_name, options ) # :nodoc:
|
24
|
-
|
25
|
-
|
69
|
+
sql = []
|
70
|
+
|
71
|
+
if supports_on_duplicate_key_update?
|
72
|
+
# Options :recursive and :on_duplicate_key_ignore are mutually exclusive
|
73
|
+
if (options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:on_duplicate_key_update] && !options[:recursive]
|
74
|
+
sql << sql_for_on_duplicate_key_ignore( table_name, options[:on_duplicate_key_ignore] )
|
75
|
+
end
|
76
|
+
elsif logger && options[:on_duplicate_key_ignore] && !options[:on_duplicate_key_update]
|
77
|
+
logger.warn "Ignoring on_duplicate_key_ignore because it is not supported by the database."
|
78
|
+
end
|
79
|
+
|
80
|
+
sql += super(table_name, options)
|
81
|
+
|
82
|
+
columns = returning_columns(options)
|
83
|
+
unless columns.blank? || (options[:no_returning] && !options[:recursive])
|
84
|
+
sql << " RETURNING \"#{columns.join('", "')}\""
|
85
|
+
end
|
86
|
+
|
87
|
+
sql
|
88
|
+
end
|
89
|
+
|
90
|
+
def returning_columns(options)
|
91
|
+
columns = []
|
92
|
+
columns += Array(options[:primary_key]) if options[:primary_key].present?
|
93
|
+
columns |= Array(options[:returning]) if options[:returning].present?
|
94
|
+
columns
|
95
|
+
end
|
96
|
+
|
97
|
+
# Add a column to be updated on duplicate key update
|
98
|
+
def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
|
99
|
+
arg = options[:on_duplicate_key_update]
|
100
|
+
if arg.is_a?( Hash )
|
101
|
+
columns = arg.fetch( :columns ) { arg[:columns] = [] }
|
102
|
+
case columns
|
103
|
+
when Array then columns << column.to_sym unless columns.include?( column.to_sym )
|
104
|
+
when Hash then columns[column.to_sym] = column.to_sym
|
105
|
+
end
|
106
|
+
elsif arg.is_a?( Array )
|
107
|
+
arg << column.to_sym unless arg.include?( column.to_sym )
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
# Returns a generated ON CONFLICT DO NOTHING statement given the passed
|
112
|
+
# in +args+.
|
113
|
+
def sql_for_on_duplicate_key_ignore( table_name, *args ) # :nodoc:
|
114
|
+
arg = args.first
|
115
|
+
conflict_target = sql_for_conflict_target( arg ) if arg.is_a?( Hash )
|
116
|
+
" ON CONFLICT #{conflict_target}DO NOTHING"
|
117
|
+
end
|
118
|
+
|
119
|
+
# Returns a generated ON CONFLICT DO UPDATE statement given the passed
|
120
|
+
# in +args+.
|
121
|
+
def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
|
122
|
+
arg, primary_key, locking_column = args
|
123
|
+
arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
|
124
|
+
return unless arg.is_a?( Hash )
|
125
|
+
|
126
|
+
sql = ' ON CONFLICT '
|
127
|
+
conflict_target = sql_for_conflict_target( arg )
|
128
|
+
|
129
|
+
columns = arg.fetch( :columns, [] )
|
130
|
+
condition = arg[:condition]
|
131
|
+
if columns.respond_to?( :empty? ) && columns.empty?
|
132
|
+
return sql << "#{conflict_target}DO NOTHING"
|
133
|
+
end
|
134
|
+
|
135
|
+
conflict_target ||= sql_for_default_conflict_target( table_name, primary_key )
|
136
|
+
unless conflict_target
|
137
|
+
raise ArgumentError, 'Expected :conflict_target or :constraint_name to be specified'
|
138
|
+
end
|
139
|
+
|
140
|
+
sql << "#{conflict_target}DO UPDATE SET "
|
141
|
+
if columns.is_a?( Array )
|
142
|
+
sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, columns )
|
143
|
+
elsif columns.is_a?( Hash )
|
144
|
+
sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, columns )
|
145
|
+
elsif columns.is_a?( String )
|
146
|
+
sql << columns
|
26
147
|
else
|
27
|
-
|
148
|
+
raise ArgumentError, 'Expected :columns to be an Array or Hash'
|
28
149
|
end
|
150
|
+
|
151
|
+
sql << " WHERE #{condition}" if condition.present?
|
152
|
+
|
153
|
+
sql
|
29
154
|
end
|
30
155
|
|
31
|
-
def
|
156
|
+
def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
|
157
|
+
results = arr.map do |column|
|
158
|
+
qc = quote_column_name( column )
|
159
|
+
"#{qc}=EXCLUDED.#{qc}"
|
160
|
+
end
|
161
|
+
increment_locking_column!(table_name, results, locking_column)
|
162
|
+
results.join( ',' )
|
163
|
+
end
|
164
|
+
|
165
|
+
def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
|
166
|
+
results = hsh.map do |column1, column2|
|
167
|
+
qc1 = quote_column_name( column1 )
|
168
|
+
qc2 = quote_column_name( column2 )
|
169
|
+
"#{qc1}=EXCLUDED.#{qc2}"
|
170
|
+
end
|
171
|
+
increment_locking_column!(table_name, results, locking_column)
|
172
|
+
results.join( ',' )
|
173
|
+
end
|
174
|
+
|
175
|
+
def sql_for_conflict_target( args = {} )
|
176
|
+
constraint_name = args[:constraint_name]
|
177
|
+
conflict_target = args[:conflict_target]
|
178
|
+
index_predicate = args[:index_predicate]
|
179
|
+
if constraint_name.present?
|
180
|
+
"ON CONSTRAINT #{constraint_name} "
|
181
|
+
elsif conflict_target.present?
|
182
|
+
'(' << Array( conflict_target ).reject( &:blank? ).join( ', ' ) << ') '.tap do |sql|
|
183
|
+
sql << "WHERE #{index_predicate} " if index_predicate
|
184
|
+
end
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
def sql_for_default_conflict_target( table_name, primary_key )
|
189
|
+
conflict_target = Array(primary_key).join(', ')
|
190
|
+
"(#{conflict_target}) " if conflict_target.present?
|
191
|
+
end
|
192
|
+
|
193
|
+
# Return true if the statement is a duplicate key record error
|
194
|
+
def duplicate_key_update_error?(exception) # :nodoc:
|
195
|
+
exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key')
|
196
|
+
end
|
197
|
+
|
198
|
+
def supports_on_duplicate_key_update?
|
199
|
+
database_version >= MIN_VERSION_FOR_UPSERT
|
200
|
+
end
|
201
|
+
|
202
|
+
def supports_setting_primary_key_of_imported_objects?
|
32
203
|
true
|
33
204
|
end
|
205
|
+
|
206
|
+
private
|
207
|
+
|
208
|
+
def database_version
|
209
|
+
defined?(postgresql_version) ? postgresql_version : super
|
210
|
+
end
|
34
211
|
end
|
@@ -1,43 +1,174 @@
|
|
1
1
|
module ActiveRecord::Import::SQLite3Adapter
|
2
2
|
include ActiveRecord::Import::ImportSupport
|
3
|
+
include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
|
3
4
|
|
4
|
-
MIN_VERSION_FOR_IMPORT = "3.7.11"
|
5
|
+
MIN_VERSION_FOR_IMPORT = "3.7.11".freeze
|
6
|
+
MIN_VERSION_FOR_UPSERT = "3.24.0".freeze
|
5
7
|
SQLITE_LIMIT_COMPOUND_SELECT = 500
|
6
8
|
|
7
9
|
# Override our conformance to ActiveRecord::Import::ImportSupport interface
|
8
10
|
# to ensure that we only support import in supported version of SQLite.
|
9
11
|
# Which INSERT statements with multiple value sets was introduced in 3.7.11.
|
10
|
-
def supports_import?
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
12
|
+
def supports_import?
|
13
|
+
database_version >= MIN_VERSION_FOR_IMPORT
|
14
|
+
end
|
15
|
+
|
16
|
+
def supports_on_duplicate_key_update?
|
17
|
+
database_version >= MIN_VERSION_FOR_UPSERT
|
16
18
|
end
|
17
19
|
|
18
20
|
# +sql+ can be a single string or an array. If it is an array all
|
19
21
|
# elements that are in position >= 1 will be appended to the final SQL.
|
20
|
-
def insert_many(sql, values, *args) # :nodoc:
|
22
|
+
def insert_many( sql, values, _options = {}, *args ) # :nodoc:
|
21
23
|
number_of_inserts = 0
|
22
|
-
|
23
|
-
|
24
|
+
|
25
|
+
base_sql, post_sql = if sql.is_a?( String )
|
26
|
+
[sql, '']
|
24
27
|
elsif sql.is_a?( Array )
|
25
|
-
[
|
28
|
+
[sql.shift, sql.join( ' ' )]
|
26
29
|
end
|
27
30
|
|
28
31
|
value_sets = ::ActiveRecord::Import::ValueSetsRecordsParser.parse(values,
|
29
|
-
:
|
32
|
+
max_records: SQLITE_LIMIT_COMPOUND_SELECT)
|
30
33
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
34
|
+
transaction(requires_new: true) do
|
35
|
+
value_sets.each do |value_set|
|
36
|
+
number_of_inserts += 1
|
37
|
+
sql2insert = base_sql + value_set.join( ',' ) + post_sql
|
38
|
+
insert( sql2insert, *args )
|
39
|
+
end
|
35
40
|
end
|
36
41
|
|
37
|
-
[number_of_inserts,[]]
|
42
|
+
ActiveRecord::Import::Result.new([], number_of_inserts, [], [])
|
43
|
+
end
|
44
|
+
|
45
|
+
def pre_sql_statements( options )
|
46
|
+
sql = []
|
47
|
+
# Options :recursive and :on_duplicate_key_ignore are mutually exclusive
|
48
|
+
if !supports_on_duplicate_key_update? && (options[:ignore] || options[:on_duplicate_key_ignore])
|
49
|
+
sql << "OR IGNORE"
|
50
|
+
end
|
51
|
+
sql + super
|
52
|
+
end
|
53
|
+
|
54
|
+
def post_sql_statements( table_name, options ) # :nodoc:
|
55
|
+
sql = []
|
56
|
+
|
57
|
+
if supports_on_duplicate_key_update?
|
58
|
+
# Options :recursive and :on_duplicate_key_ignore are mutually exclusive
|
59
|
+
if (options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:on_duplicate_key_update]
|
60
|
+
sql << sql_for_on_duplicate_key_ignore( options[:on_duplicate_key_ignore] )
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
sql + super
|
38
65
|
end
|
39
66
|
|
40
67
|
def next_value_for_sequence(sequence_name)
|
41
68
|
%{nextval('#{sequence_name}')}
|
42
69
|
end
|
70
|
+
|
71
|
+
# Add a column to be updated on duplicate key update
|
72
|
+
def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
|
73
|
+
arg = options[:on_duplicate_key_update]
|
74
|
+
if arg.is_a?( Hash )
|
75
|
+
columns = arg.fetch( :columns ) { arg[:columns] = [] }
|
76
|
+
case columns
|
77
|
+
when Array then columns << column.to_sym unless columns.include?( column.to_sym )
|
78
|
+
when Hash then columns[column.to_sym] = column.to_sym
|
79
|
+
end
|
80
|
+
elsif arg.is_a?( Array )
|
81
|
+
arg << column.to_sym unless arg.include?( column.to_sym )
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
# Returns a generated ON CONFLICT DO NOTHING statement given the passed
|
86
|
+
# in +args+.
|
87
|
+
def sql_for_on_duplicate_key_ignore( *args ) # :nodoc:
|
88
|
+
arg = args.first
|
89
|
+
conflict_target = sql_for_conflict_target( arg ) if arg.is_a?( Hash )
|
90
|
+
" ON CONFLICT #{conflict_target}DO NOTHING"
|
91
|
+
end
|
92
|
+
|
93
|
+
# Returns a generated ON CONFLICT DO UPDATE statement given the passed
|
94
|
+
# in +args+.
|
95
|
+
def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
|
96
|
+
arg, primary_key, locking_column = args
|
97
|
+
arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
|
98
|
+
return unless arg.is_a?( Hash )
|
99
|
+
|
100
|
+
sql = ' ON CONFLICT '
|
101
|
+
conflict_target = sql_for_conflict_target( arg )
|
102
|
+
|
103
|
+
columns = arg.fetch( :columns, [] )
|
104
|
+
condition = arg[:condition]
|
105
|
+
if columns.respond_to?( :empty? ) && columns.empty?
|
106
|
+
return sql << "#{conflict_target}DO NOTHING"
|
107
|
+
end
|
108
|
+
|
109
|
+
conflict_target ||= sql_for_default_conflict_target( primary_key )
|
110
|
+
unless conflict_target
|
111
|
+
raise ArgumentError, 'Expected :conflict_target to be specified'
|
112
|
+
end
|
113
|
+
|
114
|
+
sql << "#{conflict_target}DO UPDATE SET "
|
115
|
+
if columns.is_a?( Array )
|
116
|
+
sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, columns )
|
117
|
+
elsif columns.is_a?( Hash )
|
118
|
+
sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, columns )
|
119
|
+
elsif columns.is_a?( String )
|
120
|
+
sql << columns
|
121
|
+
else
|
122
|
+
raise ArgumentError, 'Expected :columns to be an Array or Hash'
|
123
|
+
end
|
124
|
+
|
125
|
+
sql << " WHERE #{condition}" if condition.present?
|
126
|
+
|
127
|
+
sql
|
128
|
+
end
|
129
|
+
|
130
|
+
def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
|
131
|
+
results = arr.map do |column|
|
132
|
+
qc = quote_column_name( column )
|
133
|
+
"#{qc}=EXCLUDED.#{qc}"
|
134
|
+
end
|
135
|
+
increment_locking_column!(table_name, results, locking_column)
|
136
|
+
results.join( ',' )
|
137
|
+
end
|
138
|
+
|
139
|
+
def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
|
140
|
+
results = hsh.map do |column1, column2|
|
141
|
+
qc1 = quote_column_name( column1 )
|
142
|
+
qc2 = quote_column_name( column2 )
|
143
|
+
"#{qc1}=EXCLUDED.#{qc2}"
|
144
|
+
end
|
145
|
+
increment_locking_column!(table_name, results, locking_column)
|
146
|
+
results.join( ',' )
|
147
|
+
end
|
148
|
+
|
149
|
+
def sql_for_conflict_target( args = {} )
|
150
|
+
conflict_target = args[:conflict_target]
|
151
|
+
index_predicate = args[:index_predicate]
|
152
|
+
if conflict_target.present?
|
153
|
+
'(' << Array( conflict_target ).reject( &:blank? ).join( ', ' ) << ') '.tap do |sql|
|
154
|
+
sql << "WHERE #{index_predicate} " if index_predicate
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
def sql_for_default_conflict_target( primary_key )
|
160
|
+
conflict_target = Array(primary_key).join(', ')
|
161
|
+
"(#{conflict_target}) " if conflict_target.present?
|
162
|
+
end
|
163
|
+
|
164
|
+
# Return true if the statement is a duplicate key record error
|
165
|
+
def duplicate_key_update_error?(exception) # :nodoc:
|
166
|
+
exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key')
|
167
|
+
end
|
168
|
+
|
169
|
+
private
|
170
|
+
|
171
|
+
def database_version
|
172
|
+
defined?(sqlite_version) ? sqlite_version : super
|
173
|
+
end
|
43
174
|
end
|
@@ -3,31 +3,37 @@ require "active_record"
|
|
3
3
|
require "active_record/version"
|
4
4
|
|
5
5
|
module ActiveRecord::Import
|
6
|
-
|
6
|
+
ADAPTER_PATH = "activerecord-import/active_record/adapters".freeze
|
7
7
|
|
8
8
|
def self.base_adapter(adapter)
|
9
9
|
case adapter
|
10
|
-
when '
|
10
|
+
when 'mysql2_makara' then 'mysql2'
|
11
11
|
when 'mysql2spatial' then 'mysql2'
|
12
12
|
when 'spatialite' then 'sqlite3'
|
13
|
+
when 'postgresql_makara' then 'postgresql'
|
14
|
+
when 'makara_postgis' then 'postgresql'
|
13
15
|
when 'postgis' then 'postgresql'
|
16
|
+
when 'cockroachdb' then 'postgresql'
|
14
17
|
else adapter
|
15
18
|
end
|
16
19
|
end
|
17
20
|
|
18
21
|
# Loads the import functionality for a specific database adapter
|
19
22
|
def self.require_adapter(adapter)
|
20
|
-
require File.join(
|
21
|
-
|
22
|
-
|
23
|
-
rescue LoadError
|
24
|
-
# fallback
|
25
|
-
end
|
23
|
+
require File.join(ADAPTER_PATH, "/#{base_adapter(adapter)}_adapter")
|
24
|
+
rescue LoadError
|
25
|
+
# fallback
|
26
26
|
end
|
27
27
|
|
28
28
|
# Loads the import functionality for the passed in ActiveRecord connection
|
29
29
|
def self.load_from_connection_pool(connection_pool)
|
30
|
-
|
30
|
+
adapter =
|
31
|
+
if connection_pool.respond_to?(:db_config) # ActiveRecord >= 6.1
|
32
|
+
connection_pool.db_config.adapter
|
33
|
+
else
|
34
|
+
connection_pool.spec.config[:adapter]
|
35
|
+
end
|
36
|
+
require_adapter adapter
|
31
37
|
end
|
32
38
|
end
|
33
39
|
|