activerecord-import 1.0.2 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yaml +151 -0
- data/.gitignore +5 -0
- data/.rubocop.yml +74 -8
- data/.rubocop_todo.yml +10 -16
- data/Brewfile +3 -1
- data/CHANGELOG.md +138 -3
- data/Dockerfile +23 -0
- data/Gemfile +24 -14
- data/LICENSE +21 -56
- data/README.markdown +108 -60
- data/Rakefile +3 -0
- data/activerecord-import.gemspec +6 -5
- data/benchmarks/benchmark.rb +10 -4
- data/benchmarks/lib/base.rb +4 -2
- data/benchmarks/lib/cli_parser.rb +4 -2
- data/benchmarks/lib/float.rb +2 -0
- data/benchmarks/lib/mysql2_benchmark.rb +2 -0
- data/benchmarks/lib/output_to_csv.rb +2 -0
- data/benchmarks/lib/output_to_html.rb +4 -2
- data/benchmarks/models/test_innodb.rb +2 -0
- data/benchmarks/models/test_memory.rb +2 -0
- data/benchmarks/models/test_myisam.rb +2 -0
- data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +2 -0
- data/docker-compose.yml +34 -0
- data/gemfiles/4.2.gemfile +2 -0
- data/gemfiles/5.0.gemfile +2 -0
- data/gemfiles/5.1.gemfile +2 -0
- data/gemfiles/5.2.gemfile +2 -0
- data/gemfiles/6.0.gemfile +4 -1
- data/gemfiles/6.1.gemfile +4 -1
- data/gemfiles/7.0.gemfile +4 -0
- data/gemfiles/7.1.gemfile +3 -0
- data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +6 -4
- data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/trilogy_adapter.rb +8 -0
- data/lib/activerecord-import/adapters/abstract_adapter.rb +14 -5
- data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql_adapter.rb +33 -25
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +69 -56
- data/lib/activerecord-import/adapters/sqlite3_adapter.rb +39 -39
- data/lib/activerecord-import/adapters/trilogy_adapter.rb +7 -0
- data/lib/activerecord-import/base.rb +10 -2
- data/lib/activerecord-import/import.rb +162 -65
- data/lib/activerecord-import/mysql2.rb +2 -0
- data/lib/activerecord-import/postgresql.rb +2 -0
- data/lib/activerecord-import/sqlite3.rb +2 -0
- data/lib/activerecord-import/synchronize.rb +3 -1
- data/lib/activerecord-import/value_sets_parser.rb +5 -0
- data/lib/activerecord-import/version.rb +3 -1
- data/lib/activerecord-import.rb +2 -1
- data/test/adapters/jdbcmysql.rb +2 -0
- data/test/adapters/jdbcpostgresql.rb +2 -0
- data/test/adapters/jdbcsqlite3.rb +2 -0
- data/test/adapters/makara_postgis.rb +2 -0
- data/test/adapters/mysql2.rb +2 -0
- data/test/adapters/mysql2_makara.rb +2 -0
- data/test/adapters/mysql2spatial.rb +2 -0
- data/test/adapters/postgis.rb +2 -0
- data/test/adapters/postgresql.rb +2 -0
- data/test/adapters/postgresql_makara.rb +2 -0
- data/test/adapters/seamless_database_pool.rb +2 -0
- data/test/adapters/spatialite.rb +2 -0
- data/test/adapters/sqlite3.rb +2 -0
- data/test/adapters/trilogy.rb +9 -0
- data/test/database.yml.sample +7 -0
- data/test/{travis → github}/database.yml +7 -1
- data/test/import_test.rb +93 -2
- data/test/jdbcmysql/import_test.rb +5 -3
- data/test/jdbcpostgresql/import_test.rb +4 -2
- data/test/jdbcsqlite3/import_test.rb +4 -2
- data/test/makara_postgis/import_test.rb +4 -2
- data/test/models/account.rb +2 -0
- data/test/models/alarm.rb +2 -0
- data/test/models/animal.rb +8 -0
- data/test/models/author.rb +7 -0
- data/test/models/bike_maker.rb +3 -0
- data/test/models/book.rb +7 -2
- data/test/models/car.rb +2 -0
- data/test/models/card.rb +5 -0
- data/test/models/chapter.rb +2 -0
- data/test/models/composite_book.rb +19 -0
- data/test/models/composite_chapter.rb +9 -0
- data/test/models/customer.rb +18 -0
- data/test/models/deck.rb +8 -0
- data/test/models/dictionary.rb +2 -0
- data/test/models/discount.rb +2 -0
- data/test/models/end_note.rb +2 -0
- data/test/models/group.rb +2 -0
- data/test/models/order.rb +17 -0
- data/test/models/playing_card.rb +4 -0
- data/test/models/promotion.rb +2 -0
- data/test/models/question.rb +2 -0
- data/test/models/rule.rb +2 -0
- data/test/models/tag.rb +9 -1
- data/test/models/tag_alias.rb +11 -0
- data/test/models/topic.rb +7 -0
- data/test/models/user.rb +2 -0
- data/test/models/user_token.rb +2 -0
- data/test/models/vendor.rb +2 -0
- data/test/models/widget.rb +2 -0
- data/test/mysql2/import_test.rb +5 -3
- data/test/mysql2_makara/import_test.rb +5 -3
- data/test/mysqlspatial2/import_test.rb +5 -3
- data/test/postgis/import_test.rb +4 -2
- data/test/postgresql/import_test.rb +4 -2
- data/test/schema/generic_schema.rb +37 -1
- data/test/schema/jdbcpostgresql_schema.rb +3 -1
- data/test/schema/mysql2_schema.rb +2 -0
- data/test/schema/postgis_schema.rb +3 -1
- data/test/schema/postgresql_schema.rb +47 -0
- data/test/schema/sqlite3_schema.rb +2 -0
- data/test/schema/version.rb +2 -0
- data/test/sqlite3/import_test.rb +4 -2
- data/test/support/active_support/test_case_extensions.rb +2 -0
- data/test/support/assertions.rb +2 -0
- data/test/support/factories.rb +2 -0
- data/test/support/generate.rb +4 -2
- data/test/support/mysql/import_examples.rb +2 -1
- data/test/support/postgresql/import_examples.rb +108 -2
- data/test/support/shared_examples/on_duplicate_key_ignore.rb +2 -0
- data/test/support/shared_examples/on_duplicate_key_update.rb +78 -9
- data/test/support/shared_examples/recursive_import.rb +98 -1
- data/test/support/sqlite3/import_examples.rb +2 -1
- data/test/synchronize_test.rb +2 -0
- data/test/test_helper.rb +33 -6
- data/test/trilogy/import_test.rb +7 -0
- data/test/value_sets_bytes_parser_test.rb +3 -1
- data/test/value_sets_records_parser_test.rb +3 -1
- metadata +42 -16
- data/.travis.yml +0 -70
- data/gemfiles/3.2.gemfile +0 -2
- data/gemfiles/4.0.gemfile +0 -2
- data/gemfiles/4.1.gemfile +0 -2
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module ActiveRecord::Import::SQLite3Adapter
|
|
2
4
|
include ActiveRecord::Import::ImportSupport
|
|
3
5
|
include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
|
|
4
6
|
|
|
5
|
-
MIN_VERSION_FOR_IMPORT = "3.7.11"
|
|
6
|
-
MIN_VERSION_FOR_UPSERT = "3.24.0"
|
|
7
|
+
MIN_VERSION_FOR_IMPORT = "3.7.11"
|
|
8
|
+
MIN_VERSION_FOR_UPSERT = "3.24.0"
|
|
7
9
|
SQLITE_LIMIT_COMPOUND_SELECT = 500
|
|
8
10
|
|
|
9
11
|
# Override our conformance to ActiveRecord::Import::ImportSupport interface
|
|
@@ -22,10 +24,11 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
22
24
|
def insert_many( sql, values, _options = {}, *args ) # :nodoc:
|
|
23
25
|
number_of_inserts = 0
|
|
24
26
|
|
|
25
|
-
base_sql, post_sql =
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
27
|
+
base_sql, post_sql = case sql
|
|
28
|
+
when String
|
|
29
|
+
[sql, '']
|
|
30
|
+
when Array
|
|
31
|
+
[sql.shift, sql.join( ' ' )]
|
|
29
32
|
end
|
|
30
33
|
|
|
31
34
|
value_sets = ::ActiveRecord::Import::ValueSetsRecordsParser.parse(values,
|
|
@@ -54,11 +57,9 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
54
57
|
def post_sql_statements( table_name, options ) # :nodoc:
|
|
55
58
|
sql = []
|
|
56
59
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
sql << sql_for_on_duplicate_key_ignore( options[:on_duplicate_key_ignore] )
|
|
61
|
-
end
|
|
60
|
+
# Options :recursive and :on_duplicate_key_ignore are mutually exclusive
|
|
61
|
+
if supports_on_duplicate_key_update? && ((options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:on_duplicate_key_update])
|
|
62
|
+
sql << sql_for_on_duplicate_key_ignore( options[:on_duplicate_key_ignore] )
|
|
62
63
|
end
|
|
63
64
|
|
|
64
65
|
sql + super
|
|
@@ -71,13 +72,14 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
71
72
|
# Add a column to be updated on duplicate key update
|
|
72
73
|
def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
|
|
73
74
|
arg = options[:on_duplicate_key_update]
|
|
74
|
-
|
|
75
|
+
case arg
|
|
76
|
+
when Hash
|
|
75
77
|
columns = arg.fetch( :columns ) { arg[:columns] = [] }
|
|
76
78
|
case columns
|
|
77
79
|
when Array then columns << column.to_sym unless columns.include?( column.to_sym )
|
|
78
80
|
when Hash then columns[column.to_sym] = column.to_sym
|
|
79
81
|
end
|
|
80
|
-
|
|
82
|
+
when Array
|
|
81
83
|
arg << column.to_sym unless arg.include?( column.to_sym )
|
|
82
84
|
end
|
|
83
85
|
end
|
|
@@ -92,12 +94,12 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
92
94
|
|
|
93
95
|
# Returns a generated ON CONFLICT DO UPDATE statement given the passed
|
|
94
96
|
# in +args+.
|
|
95
|
-
def sql_for_on_duplicate_key_update(
|
|
96
|
-
arg, primary_key, locking_column = args
|
|
97
|
+
def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
|
|
98
|
+
arg, model, primary_key, locking_column = args
|
|
97
99
|
arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
|
|
98
100
|
return unless arg.is_a?( Hash )
|
|
99
101
|
|
|
100
|
-
sql = ' ON CONFLICT '
|
|
102
|
+
sql = ' ON CONFLICT '.dup
|
|
101
103
|
conflict_target = sql_for_conflict_target( arg )
|
|
102
104
|
|
|
103
105
|
columns = arg.fetch( :columns, [] )
|
|
@@ -112,11 +114,12 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
112
114
|
end
|
|
113
115
|
|
|
114
116
|
sql << "#{conflict_target}DO UPDATE SET "
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
117
|
+
case columns
|
|
118
|
+
when Array
|
|
119
|
+
sql << sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, columns )
|
|
120
|
+
when Hash
|
|
121
|
+
sql << sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, columns )
|
|
122
|
+
when String
|
|
120
123
|
sql << columns
|
|
121
124
|
else
|
|
122
125
|
raise ArgumentError, 'Expected :columns to be an Array or Hash'
|
|
@@ -127,22 +130,27 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
127
130
|
sql
|
|
128
131
|
end
|
|
129
132
|
|
|
130
|
-
def sql_for_on_duplicate_key_update_as_array( locking_column, arr ) # :nodoc:
|
|
133
|
+
def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
|
|
131
134
|
results = arr.map do |column|
|
|
132
|
-
|
|
135
|
+
original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
|
|
136
|
+
qc = quote_column_name( original_column_name )
|
|
133
137
|
"#{qc}=EXCLUDED.#{qc}"
|
|
134
138
|
end
|
|
135
|
-
increment_locking_column!(results, locking_column)
|
|
139
|
+
increment_locking_column!(table_name, results, locking_column)
|
|
136
140
|
results.join( ',' )
|
|
137
141
|
end
|
|
138
142
|
|
|
139
|
-
def sql_for_on_duplicate_key_update_as_hash( locking_column, hsh ) # :nodoc:
|
|
143
|
+
def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
|
|
140
144
|
results = hsh.map do |column1, column2|
|
|
141
|
-
|
|
142
|
-
|
|
145
|
+
original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
|
|
146
|
+
qc1 = quote_column_name( original_column1_name )
|
|
147
|
+
|
|
148
|
+
original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
|
|
149
|
+
qc2 = quote_column_name( original_column2_name )
|
|
150
|
+
|
|
143
151
|
"#{qc1}=EXCLUDED.#{qc2}"
|
|
144
152
|
end
|
|
145
|
-
increment_locking_column!(results, locking_column)
|
|
153
|
+
increment_locking_column!(table_name, results, locking_column)
|
|
146
154
|
results.join( ',' )
|
|
147
155
|
end
|
|
148
156
|
|
|
@@ -150,9 +158,9 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
150
158
|
conflict_target = args[:conflict_target]
|
|
151
159
|
index_predicate = args[:index_predicate]
|
|
152
160
|
if conflict_target.present?
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
161
|
+
sql = "(#{Array( conflict_target ).reject( &:blank? ).join( ', ' )}) "
|
|
162
|
+
sql += "WHERE #{index_predicate} " if index_predicate
|
|
163
|
+
sql
|
|
156
164
|
end
|
|
157
165
|
end
|
|
158
166
|
|
|
@@ -166,14 +174,6 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
|
166
174
|
exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key')
|
|
167
175
|
end
|
|
168
176
|
|
|
169
|
-
def increment_locking_column!(results, locking_column)
|
|
170
|
-
if locking_column.present?
|
|
171
|
-
results << "\"#{locking_column}\"=EXCLUDED.\"#{locking_column}\"+1"
|
|
172
|
-
end
|
|
173
|
-
end
|
|
174
|
-
|
|
175
|
-
private
|
|
176
|
-
|
|
177
177
|
def database_version
|
|
178
178
|
defined?(sqlite_version) ? sqlite_version : super
|
|
179
179
|
end
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require "pathname"
|
|
2
4
|
require "active_record"
|
|
3
5
|
require "active_record/version"
|
|
4
6
|
|
|
5
7
|
module ActiveRecord::Import
|
|
6
|
-
ADAPTER_PATH = "activerecord-import/active_record/adapters"
|
|
8
|
+
ADAPTER_PATH = "activerecord-import/active_record/adapters"
|
|
7
9
|
|
|
8
10
|
def self.base_adapter(adapter)
|
|
9
11
|
case adapter
|
|
@@ -27,7 +29,13 @@ module ActiveRecord::Import
|
|
|
27
29
|
|
|
28
30
|
# Loads the import functionality for the passed in ActiveRecord connection
|
|
29
31
|
def self.load_from_connection_pool(connection_pool)
|
|
30
|
-
|
|
32
|
+
adapter =
|
|
33
|
+
if connection_pool.respond_to?(:db_config) # ActiveRecord >= 6.1
|
|
34
|
+
connection_pool.db_config.adapter
|
|
35
|
+
else
|
|
36
|
+
connection_pool.spec.config[:adapter]
|
|
37
|
+
end
|
|
38
|
+
require_adapter adapter
|
|
31
39
|
end
|
|
32
40
|
end
|
|
33
41
|
|
|
@@ -1,18 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require "ostruct"
|
|
2
4
|
|
|
3
5
|
module ActiveRecord::Import::ConnectionAdapters; end
|
|
4
6
|
|
|
5
|
-
module ActiveRecord::Import
|
|
7
|
+
module ActiveRecord::Import # :nodoc:
|
|
6
8
|
Result = Struct.new(:failed_instances, :num_inserts, :ids, :results)
|
|
7
9
|
|
|
8
|
-
module ImportSupport
|
|
9
|
-
def supports_import?
|
|
10
|
+
module ImportSupport # :nodoc:
|
|
11
|
+
def supports_import? # :nodoc:
|
|
10
12
|
true
|
|
11
13
|
end
|
|
12
14
|
end
|
|
13
15
|
|
|
14
|
-
module OnDuplicateKeyUpdateSupport
|
|
15
|
-
def supports_on_duplicate_key_update?
|
|
16
|
+
module OnDuplicateKeyUpdateSupport # :nodoc:
|
|
17
|
+
def supports_on_duplicate_key_update? # :nodoc:
|
|
16
18
|
true
|
|
17
19
|
end
|
|
18
20
|
end
|
|
@@ -34,7 +36,7 @@ module ActiveRecord::Import #:nodoc:
|
|
|
34
36
|
@validate_callbacks = klass._validate_callbacks.dup
|
|
35
37
|
|
|
36
38
|
@validate_callbacks.each_with_index do |callback, i|
|
|
37
|
-
filter = callback.raw_filter
|
|
39
|
+
filter = callback.respond_to?(:raw_filter) ? callback.raw_filter : callback.filter
|
|
38
40
|
next unless filter.class.name =~ /Validations::PresenceValidator/ ||
|
|
39
41
|
(!@options[:validate_uniqueness] &&
|
|
40
42
|
filter.is_a?(ActiveRecord::Validations::UniquenessValidator))
|
|
@@ -49,13 +51,13 @@ module ActiveRecord::Import #:nodoc:
|
|
|
49
51
|
associations = klass.reflect_on_all_associations(:belongs_to)
|
|
50
52
|
associations.each do |assoc|
|
|
51
53
|
if (index = attrs.index(assoc.name))
|
|
52
|
-
key = assoc.foreign_key.to_sym
|
|
54
|
+
key = assoc.foreign_key.is_a?(Array) ? assoc.foreign_key.map(&:to_sym) : assoc.foreign_key.to_sym
|
|
53
55
|
attrs[index] = key unless attrs.include?(key)
|
|
54
56
|
end
|
|
55
57
|
end
|
|
56
58
|
end
|
|
57
59
|
|
|
58
|
-
filter.instance_variable_set(:@attributes, attrs)
|
|
60
|
+
filter.instance_variable_set(:@attributes, attrs.flatten)
|
|
59
61
|
|
|
60
62
|
if @validate_callbacks.respond_to?(:chain, true)
|
|
61
63
|
@validate_callbacks.send(:chain).tap do |chain|
|
|
@@ -71,7 +73,7 @@ module ActiveRecord::Import #:nodoc:
|
|
|
71
73
|
end
|
|
72
74
|
|
|
73
75
|
def valid_model?(model)
|
|
74
|
-
init_validations(model.class) unless model.
|
|
76
|
+
init_validations(model.class) unless model.instance_of?(@validator_class)
|
|
75
77
|
|
|
76
78
|
validation_context = @options[:validate_with_context]
|
|
77
79
|
validation_context ||= (model.new_record? ? :create : :update)
|
|
@@ -83,7 +85,11 @@ module ActiveRecord::Import #:nodoc:
|
|
|
83
85
|
|
|
84
86
|
model.run_callbacks(:validation) do
|
|
85
87
|
if defined?(ActiveSupport::Callbacks::Filters::Environment) # ActiveRecord >= 4.1
|
|
86
|
-
runner = @validate_callbacks.compile
|
|
88
|
+
runner = if @validate_callbacks.method(:compile).arity == 0
|
|
89
|
+
@validate_callbacks.compile
|
|
90
|
+
else # ActiveRecord >= 7.1
|
|
91
|
+
@validate_callbacks.compile(nil)
|
|
92
|
+
end
|
|
87
93
|
env = ActiveSupport::Callbacks::Filters::Environment.new(model, false, nil)
|
|
88
94
|
if runner.respond_to?(:call) # ActiveRecord < 5.1
|
|
89
95
|
runner.call(env)
|
|
@@ -163,7 +169,7 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
|
163
169
|
m.public_send "#{reflection.type}=", owner.class.name if reflection.type
|
|
164
170
|
end
|
|
165
171
|
|
|
166
|
-
|
|
172
|
+
model_klass.bulk_import column_names, models, options
|
|
167
173
|
|
|
168
174
|
# supports array of hash objects
|
|
169
175
|
elsif args.last.is_a?( Array ) && args.last.first.is_a?(Hash)
|
|
@@ -202,11 +208,11 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
|
202
208
|
end
|
|
203
209
|
end
|
|
204
210
|
|
|
205
|
-
|
|
211
|
+
model_klass.bulk_import column_names, array_of_attributes, options
|
|
206
212
|
|
|
207
213
|
# supports empty array
|
|
208
214
|
elsif args.last.is_a?( Array ) && args.last.empty?
|
|
209
|
-
|
|
215
|
+
ActiveRecord::Import::Result.new([], 0, [])
|
|
210
216
|
|
|
211
217
|
# supports 2-element array and array
|
|
212
218
|
elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
|
|
@@ -237,7 +243,7 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
|
237
243
|
end
|
|
238
244
|
end
|
|
239
245
|
|
|
240
|
-
|
|
246
|
+
model_klass.bulk_import column_names, array_of_attributes, options
|
|
241
247
|
else
|
|
242
248
|
raise ArgumentError, "Invalid arguments!"
|
|
243
249
|
end
|
|
@@ -245,16 +251,17 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
|
245
251
|
alias import bulk_import unless respond_to? :import
|
|
246
252
|
end
|
|
247
253
|
|
|
254
|
+
module ActiveRecord::Import::Connection
|
|
255
|
+
def establish_connection(args = nil)
|
|
256
|
+
conn = super(args)
|
|
257
|
+
ActiveRecord::Import.load_from_connection_pool connection_pool
|
|
258
|
+
conn
|
|
259
|
+
end
|
|
260
|
+
end
|
|
261
|
+
|
|
248
262
|
class ActiveRecord::Base
|
|
249
263
|
class << self
|
|
250
|
-
|
|
251
|
-
conn = establish_connection_without_activerecord_import(*args)
|
|
252
|
-
ActiveRecord::Import.load_from_connection_pool connection_pool
|
|
253
|
-
conn
|
|
254
|
-
end
|
|
255
|
-
|
|
256
|
-
alias establish_connection_without_activerecord_import establish_connection
|
|
257
|
-
alias establish_connection establish_connection_with_activerecord_import
|
|
264
|
+
prepend ActiveRecord::Import::Connection
|
|
258
265
|
|
|
259
266
|
# Returns true if the current database connection adapter
|
|
260
267
|
# supports import functionality, otherwise returns false.
|
|
@@ -546,11 +553,11 @@ class ActiveRecord::Base
|
|
|
546
553
|
alias import! bulk_import! unless ActiveRecord::Base.respond_to? :import!
|
|
547
554
|
|
|
548
555
|
def import_helper( *args )
|
|
549
|
-
options = { validate: true, timestamps: true }
|
|
556
|
+
options = { model: self, validate: true, timestamps: true, track_validation_failures: false }
|
|
550
557
|
options.merge!( args.pop ) if args.last.is_a? Hash
|
|
551
558
|
# making sure that current model's primary key is used
|
|
552
559
|
options[:primary_key] = primary_key
|
|
553
|
-
options[:locking_column] = locking_column if
|
|
560
|
+
options[:locking_column] = locking_column if locking_enabled?
|
|
554
561
|
|
|
555
562
|
is_validating = options[:validate_with_context].present? ? true : options[:validate]
|
|
556
563
|
validator = ActiveRecord::Import::Validator.new(self, options)
|
|
@@ -571,7 +578,7 @@ class ActiveRecord::Base
|
|
|
571
578
|
|
|
572
579
|
if models.first.id.nil?
|
|
573
580
|
Array(primary_key).each do |c|
|
|
574
|
-
if column_names.include?(c) &&
|
|
581
|
+
if column_names.include?(c) && schema_columns_hash[c].type == :uuid
|
|
575
582
|
column_names.delete(c)
|
|
576
583
|
end
|
|
577
584
|
end
|
|
@@ -581,7 +588,7 @@ class ActiveRecord::Base
|
|
|
581
588
|
if respond_to?(:timestamp_attributes_for_update, true)
|
|
582
589
|
send(:timestamp_attributes_for_update).map(&:to_sym)
|
|
583
590
|
else
|
|
584
|
-
|
|
591
|
+
allocate.send(:timestamp_attributes_for_update_in_model)
|
|
585
592
|
end
|
|
586
593
|
end
|
|
587
594
|
|
|
@@ -630,7 +637,7 @@ class ActiveRecord::Base
|
|
|
630
637
|
end
|
|
631
638
|
# supports empty array
|
|
632
639
|
elsif args.last.is_a?( Array ) && args.last.empty?
|
|
633
|
-
return ActiveRecord::Import::Result.new([], 0, [])
|
|
640
|
+
return ActiveRecord::Import::Result.new([], 0, [], [])
|
|
634
641
|
# supports 2-element array and array
|
|
635
642
|
elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
|
|
636
643
|
|
|
@@ -694,7 +701,11 @@ class ActiveRecord::Base
|
|
|
694
701
|
return_obj = if is_validating
|
|
695
702
|
import_with_validations( column_names, array_of_attributes, options ) do |failed_instances|
|
|
696
703
|
if models
|
|
697
|
-
models.
|
|
704
|
+
models.each_with_index do |m, i|
|
|
705
|
+
next unless m.errors.any?
|
|
706
|
+
|
|
707
|
+
failed_instances << (options[:track_validation_failures] ? [i, m] : m)
|
|
708
|
+
end
|
|
698
709
|
else
|
|
699
710
|
# create instances for each of our column/value sets
|
|
700
711
|
arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
|
|
@@ -702,14 +713,18 @@ class ActiveRecord::Base
|
|
|
702
713
|
# keep track of the instance and the position it is currently at. if this fails
|
|
703
714
|
# validation we'll use the index to remove it from the array_of_attributes
|
|
704
715
|
arr.each_with_index do |hsh, i|
|
|
705
|
-
|
|
706
|
-
|
|
716
|
+
# utilize block initializer syntax to prevent failure when 'mass_assignment_sanitizer = :strict'
|
|
717
|
+
model = new do |m|
|
|
718
|
+
hsh.each_pair { |k, v| m[k] = v }
|
|
719
|
+
end
|
|
720
|
+
|
|
707
721
|
next if validator.valid_model?(model)
|
|
708
722
|
raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
|
|
723
|
+
|
|
709
724
|
array_of_attributes[i] = nil
|
|
710
725
|
failure = model.dup
|
|
711
726
|
failure.errors.send(:initialize_dup, model.errors)
|
|
712
|
-
failed_instances << failure
|
|
727
|
+
failed_instances << (options[:track_validation_failures] ? [i, failure] : failure )
|
|
713
728
|
end
|
|
714
729
|
array_of_attributes.compact!
|
|
715
730
|
end
|
|
@@ -729,7 +744,10 @@ class ActiveRecord::Base
|
|
|
729
744
|
set_attributes_and_mark_clean(models, return_obj, timestamps, options)
|
|
730
745
|
|
|
731
746
|
# if there are auto-save associations on the models we imported that are new, import them as well
|
|
732
|
-
|
|
747
|
+
if options[:recursive]
|
|
748
|
+
options[:on_duplicate_key_update] = on_duplicate_key_update unless on_duplicate_key_update.nil?
|
|
749
|
+
import_associations(models, options.dup.merge(validate: false))
|
|
750
|
+
end
|
|
733
751
|
end
|
|
734
752
|
|
|
735
753
|
return_obj
|
|
@@ -764,27 +782,29 @@ class ActiveRecord::Base
|
|
|
764
782
|
def import_without_validations_or_callbacks( column_names, array_of_attributes, options = {} )
|
|
765
783
|
return ActiveRecord::Import::Result.new([], 0, [], []) if array_of_attributes.empty?
|
|
766
784
|
|
|
767
|
-
column_names = column_names.map
|
|
785
|
+
column_names = column_names.map do |name|
|
|
786
|
+
original_name = attribute_alias?(name) ? attribute_alias(name) : name
|
|
787
|
+
original_name.to_sym
|
|
788
|
+
end
|
|
768
789
|
scope_columns, scope_values = scope_attributes.to_a.transpose
|
|
769
790
|
|
|
770
791
|
unless scope_columns.blank?
|
|
771
792
|
scope_columns.zip(scope_values).each do |name, value|
|
|
772
793
|
name_as_sym = name.to_sym
|
|
773
|
-
next if column_names.include?(name_as_sym)
|
|
774
|
-
|
|
775
|
-
is_sti = (name_as_sym == inheritance_column.to_sym && self < base_class)
|
|
776
|
-
value = Array(value).first if is_sti
|
|
777
|
-
|
|
794
|
+
next if column_names.include?(name_as_sym) || name_as_sym == inheritance_column.to_sym
|
|
778
795
|
column_names << name_as_sym
|
|
779
796
|
array_of_attributes.each { |attrs| attrs << value }
|
|
780
797
|
end
|
|
781
798
|
end
|
|
782
799
|
|
|
783
|
-
|
|
784
|
-
|
|
800
|
+
if finder_needs_type_condition? && !column_names.include?(inheritance_column.to_sym)
|
|
801
|
+
column_names << inheritance_column.to_sym
|
|
802
|
+
array_of_attributes.each { |attrs| attrs << sti_name }
|
|
803
|
+
end
|
|
785
804
|
|
|
805
|
+
columns = column_names.each_with_index.map do |name, i|
|
|
806
|
+
column = schema_columns_hash[name.to_s]
|
|
786
807
|
raise ActiveRecord::Import::MissingColumnError.new(name.to_s, i) if column.nil?
|
|
787
|
-
|
|
788
808
|
column
|
|
789
809
|
end
|
|
790
810
|
|
|
@@ -800,17 +820,29 @@ class ActiveRecord::Base
|
|
|
800
820
|
if supports_import?
|
|
801
821
|
# generate the sql
|
|
802
822
|
post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
|
|
823
|
+
import_size = values_sql.size
|
|
824
|
+
|
|
825
|
+
batch_size = options[:batch_size] || import_size
|
|
826
|
+
run_proc = options[:batch_size].to_i.positive? && options[:batch_progress].respond_to?( :call )
|
|
827
|
+
progress_proc = options[:batch_progress]
|
|
828
|
+
current_batch = 0
|
|
829
|
+
batches = (import_size / batch_size.to_f).ceil
|
|
803
830
|
|
|
804
|
-
batch_size = options[:batch_size] || values_sql.size
|
|
805
831
|
values_sql.each_slice(batch_size) do |batch_values|
|
|
832
|
+
batch_started_at = Time.now.to_i
|
|
833
|
+
|
|
806
834
|
# perform the inserts
|
|
807
835
|
result = connection.insert_many( [insert_sql, post_sql_statements].flatten,
|
|
808
836
|
batch_values,
|
|
809
837
|
options,
|
|
810
|
-
"#{model_name} Create Many
|
|
838
|
+
"#{model_name} Create Many" )
|
|
839
|
+
|
|
811
840
|
number_inserted += result.num_inserts
|
|
812
841
|
ids += result.ids
|
|
813
842
|
results += result.results
|
|
843
|
+
current_batch += 1
|
|
844
|
+
|
|
845
|
+
progress_proc.call(import_size, batches, current_batch, Time.now.to_i - batch_started_at) if run_proc
|
|
814
846
|
end
|
|
815
847
|
else
|
|
816
848
|
transaction(requires_new: true) do
|
|
@@ -825,6 +857,15 @@ class ActiveRecord::Base
|
|
|
825
857
|
|
|
826
858
|
private
|
|
827
859
|
|
|
860
|
+
def associated_options(options, associated_class)
|
|
861
|
+
return options unless options.key?(:recursive_on_duplicate_key_update)
|
|
862
|
+
|
|
863
|
+
table_name = associated_class.arel_table.name.to_sym
|
|
864
|
+
options.merge(
|
|
865
|
+
on_duplicate_key_update: options[:recursive_on_duplicate_key_update][table_name]
|
|
866
|
+
)
|
|
867
|
+
end
|
|
868
|
+
|
|
828
869
|
def set_attributes_and_mark_clean(models, import_result, timestamps, options)
|
|
829
870
|
return if models.nil?
|
|
830
871
|
models -= import_result.failed_instances
|
|
@@ -836,22 +877,46 @@ class ActiveRecord::Base
|
|
|
836
877
|
model.id = id
|
|
837
878
|
|
|
838
879
|
timestamps.each do |attr, value|
|
|
839
|
-
model.send(attr
|
|
880
|
+
model.send("#{attr}=", value) if model.send(attr).nil?
|
|
840
881
|
end
|
|
841
882
|
end
|
|
842
883
|
end
|
|
843
884
|
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
885
|
+
deserialize_value = lambda do |column, value|
|
|
886
|
+
column = schema_columns_hash[column]
|
|
887
|
+
return value unless column
|
|
888
|
+
if respond_to?(:type_caster)
|
|
889
|
+
type = type_for_attribute(column.name)
|
|
890
|
+
type.deserialize(value)
|
|
891
|
+
elsif column.respond_to?(:type_cast_from_database)
|
|
892
|
+
column.type_cast_from_database(value)
|
|
893
|
+
else
|
|
894
|
+
value
|
|
895
|
+
end
|
|
896
|
+
end
|
|
897
|
+
|
|
898
|
+
set_value = lambda do |model, column, value|
|
|
899
|
+
val = deserialize_value.call(column, value)
|
|
900
|
+
if model.attribute_names.include?(column)
|
|
901
|
+
model.send("#{column}=", val)
|
|
902
|
+
else
|
|
903
|
+
attributes = attributes_builder.build_from_database(model.attributes.merge(column => val))
|
|
904
|
+
model.instance_variable_set(:@attributes, attributes)
|
|
905
|
+
end
|
|
906
|
+
end
|
|
907
|
+
|
|
908
|
+
columns = Array(options[:returning_columns])
|
|
909
|
+
results = Array(import_result.results)
|
|
910
|
+
if models.size == results.size
|
|
911
|
+
single_column = columns.first if columns.size == 1
|
|
912
|
+
results.each_with_index do |result, index|
|
|
848
913
|
model = models[index]
|
|
849
914
|
|
|
850
915
|
if single_column
|
|
851
|
-
|
|
916
|
+
set_value.call(model, single_column, result)
|
|
852
917
|
else
|
|
853
918
|
columns.each_with_index do |column, col_index|
|
|
854
|
-
|
|
919
|
+
set_value.call(model, column, result[col_index])
|
|
855
920
|
end
|
|
856
921
|
end
|
|
857
922
|
end
|
|
@@ -872,16 +937,22 @@ class ActiveRecord::Base
|
|
|
872
937
|
|
|
873
938
|
# Sync belongs_to association ids with foreign key field
|
|
874
939
|
def load_association_ids(model)
|
|
940
|
+
changed_columns = model.changed
|
|
875
941
|
association_reflections = model.class.reflect_on_all_associations(:belongs_to)
|
|
876
942
|
association_reflections.each do |association_reflection|
|
|
877
|
-
column_name = association_reflection.foreign_key
|
|
878
943
|
next if association_reflection.options[:polymorphic]
|
|
879
|
-
association = model.association(association_reflection.name)
|
|
880
|
-
association = association.target
|
|
881
|
-
next if association.blank? || model.public_send(column_name).present?
|
|
882
944
|
|
|
883
|
-
|
|
884
|
-
|
|
945
|
+
column_names = Array(association_reflection.foreign_key).map(&:to_s)
|
|
946
|
+
column_names.each_with_index do |column_name, column_index|
|
|
947
|
+
next if changed_columns.include?(column_name)
|
|
948
|
+
|
|
949
|
+
association = model.association(association_reflection.name)
|
|
950
|
+
association = association.target
|
|
951
|
+
next if association.blank? || model.public_send(column_name).present?
|
|
952
|
+
|
|
953
|
+
association_primary_key = Array(association_reflection.association_primary_key.tr("[]:", "").split(", "))[column_index]
|
|
954
|
+
model.public_send("#{column_name}=", association.send(association_primary_key))
|
|
955
|
+
end
|
|
885
956
|
end
|
|
886
957
|
end
|
|
887
958
|
|
|
@@ -894,17 +965,30 @@ class ActiveRecord::Base
|
|
|
894
965
|
associated_objects_by_class = {}
|
|
895
966
|
models.each { |model| find_associated_objects_for_import(associated_objects_by_class, model) }
|
|
896
967
|
|
|
897
|
-
# :on_duplicate_key_update
|
|
898
|
-
options.delete(:on_duplicate_key_update)
|
|
968
|
+
# :on_duplicate_key_update only supported for all fields
|
|
969
|
+
options.delete(:on_duplicate_key_update) unless options[:on_duplicate_key_update] == :all
|
|
970
|
+
# :returning not supported for associations
|
|
899
971
|
options.delete(:returning)
|
|
900
972
|
|
|
901
973
|
associated_objects_by_class.each_value do |associations|
|
|
902
974
|
associations.each_value do |associated_records|
|
|
903
|
-
|
|
975
|
+
next if associated_records.empty?
|
|
976
|
+
|
|
977
|
+
associated_class = associated_records.first.class
|
|
978
|
+
associated_class.bulk_import(associated_records,
|
|
979
|
+
associated_options(options, associated_class))
|
|
904
980
|
end
|
|
905
981
|
end
|
|
906
982
|
end
|
|
907
983
|
|
|
984
|
+
def schema_columns_hash
|
|
985
|
+
if respond_to?(:ignored_columns) && ignored_columns.any?
|
|
986
|
+
connection.schema_cache.columns_hash(table_name)
|
|
987
|
+
else
|
|
988
|
+
columns_hash
|
|
989
|
+
end
|
|
990
|
+
end
|
|
991
|
+
|
|
908
992
|
# We are eventually going to call Class.import <objects> so we build up a hash
|
|
909
993
|
# of class => objects to import.
|
|
910
994
|
def find_associated_objects_for_import(associated_objects_by_class, model)
|
|
@@ -925,10 +1009,18 @@ class ActiveRecord::Base
|
|
|
925
1009
|
|
|
926
1010
|
changed_objects = association.select { |a| a.new_record? || a.changed? }
|
|
927
1011
|
changed_objects.each do |child|
|
|
928
|
-
|
|
1012
|
+
Array(association_reflection.inverse_of&.foreign_key || association_reflection.foreign_key).each_with_index do |column, index|
|
|
1013
|
+
child.public_send("#{column}=", Array(model.id)[index])
|
|
1014
|
+
end
|
|
1015
|
+
|
|
929
1016
|
# For polymorphic associations
|
|
1017
|
+
association_name = if model.class.respond_to?(:polymorphic_name)
|
|
1018
|
+
model.class.polymorphic_name
|
|
1019
|
+
else
|
|
1020
|
+
model.class.base_class
|
|
1021
|
+
end
|
|
930
1022
|
association_reflection.type.try do |type|
|
|
931
|
-
child.public_send("#{type}=",
|
|
1023
|
+
child.public_send("#{type}=", association_name)
|
|
932
1024
|
end
|
|
933
1025
|
end
|
|
934
1026
|
associated_objects_by_class[model.class.name][association_reflection.name].concat changed_objects
|
|
@@ -955,7 +1047,7 @@ class ActiveRecord::Base
|
|
|
955
1047
|
elsif column
|
|
956
1048
|
if respond_to?(:type_caster) # Rails 5.0 and higher
|
|
957
1049
|
type = type_for_attribute(column.name)
|
|
958
|
-
val = type.type == :boolean ? type.cast(val) : type.serialize(val)
|
|
1050
|
+
val = !type.respond_to?(:subtype) && type.type == :boolean ? type.cast(val) : type.serialize(val)
|
|
959
1051
|
connection_memo.quote(val)
|
|
960
1052
|
elsif column.respond_to?(:type_cast_from_user) # Rails 4.2
|
|
961
1053
|
connection_memo.quote(column.type_cast_from_user(val), column)
|
|
@@ -964,7 +1056,7 @@ class ActiveRecord::Base
|
|
|
964
1056
|
val = serialized_attributes[column.name].dump(val)
|
|
965
1057
|
end
|
|
966
1058
|
# Fixes #443 to support binary (i.e. bytea) columns on PG
|
|
967
|
-
val = column.type_cast(val) unless column.type.to_sym == :binary
|
|
1059
|
+
val = column.type_cast(val) unless column.type && column.type.to_sym == :binary
|
|
968
1060
|
connection_memo.quote(val, column)
|
|
969
1061
|
end
|
|
970
1062
|
else
|
|
@@ -983,13 +1075,18 @@ class ActiveRecord::Base
|
|
|
983
1075
|
timestamp_columns[:create] = timestamp_attributes_for_create_in_model
|
|
984
1076
|
timestamp_columns[:update] = timestamp_attributes_for_update_in_model
|
|
985
1077
|
else
|
|
986
|
-
instance =
|
|
1078
|
+
instance = allocate
|
|
987
1079
|
timestamp_columns[:create] = instance.send(:timestamp_attributes_for_create_in_model)
|
|
988
1080
|
timestamp_columns[:update] = instance.send(:timestamp_attributes_for_update_in_model)
|
|
989
1081
|
end
|
|
990
1082
|
|
|
991
1083
|
# use tz as set in ActiveRecord::Base
|
|
992
|
-
|
|
1084
|
+
default_timezone = if ActiveRecord.respond_to?(:default_timezone)
|
|
1085
|
+
ActiveRecord.default_timezone
|
|
1086
|
+
else
|
|
1087
|
+
ActiveRecord::Base.default_timezone
|
|
1088
|
+
end
|
|
1089
|
+
timestamp = default_timezone == :utc ? Time.now.utc : Time.now
|
|
993
1090
|
|
|
994
1091
|
[:create, :update].each do |action|
|
|
995
1092
|
timestamp_columns[action].each do |column|
|