activerecord-import 1.0.2 → 1.7.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (141) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yaml +151 -0
  3. data/.gitignore +5 -0
  4. data/.rubocop.yml +74 -8
  5. data/.rubocop_todo.yml +10 -16
  6. data/Brewfile +3 -1
  7. data/CHANGELOG.md +138 -3
  8. data/Dockerfile +23 -0
  9. data/Gemfile +24 -14
  10. data/LICENSE +21 -56
  11. data/README.markdown +108 -60
  12. data/Rakefile +3 -0
  13. data/activerecord-import.gemspec +6 -5
  14. data/benchmarks/benchmark.rb +10 -4
  15. data/benchmarks/lib/base.rb +4 -2
  16. data/benchmarks/lib/cli_parser.rb +4 -2
  17. data/benchmarks/lib/float.rb +2 -0
  18. data/benchmarks/lib/mysql2_benchmark.rb +2 -0
  19. data/benchmarks/lib/output_to_csv.rb +2 -0
  20. data/benchmarks/lib/output_to_html.rb +4 -2
  21. data/benchmarks/models/test_innodb.rb +2 -0
  22. data/benchmarks/models/test_memory.rb +2 -0
  23. data/benchmarks/models/test_myisam.rb +2 -0
  24. data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +2 -0
  25. data/docker-compose.yml +34 -0
  26. data/gemfiles/4.2.gemfile +2 -0
  27. data/gemfiles/5.0.gemfile +2 -0
  28. data/gemfiles/5.1.gemfile +2 -0
  29. data/gemfiles/5.2.gemfile +2 -0
  30. data/gemfiles/6.0.gemfile +4 -1
  31. data/gemfiles/6.1.gemfile +4 -1
  32. data/gemfiles/7.0.gemfile +4 -0
  33. data/gemfiles/7.1.gemfile +3 -0
  34. data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +2 -0
  35. data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +6 -4
  36. data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +2 -0
  37. data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +2 -0
  38. data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +2 -0
  39. data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +2 -0
  40. data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +2 -0
  41. data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +2 -0
  42. data/lib/activerecord-import/active_record/adapters/trilogy_adapter.rb +8 -0
  43. data/lib/activerecord-import/adapters/abstract_adapter.rb +14 -5
  44. data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +2 -0
  45. data/lib/activerecord-import/adapters/mysql2_adapter.rb +2 -0
  46. data/lib/activerecord-import/adapters/mysql_adapter.rb +33 -25
  47. data/lib/activerecord-import/adapters/postgresql_adapter.rb +69 -56
  48. data/lib/activerecord-import/adapters/sqlite3_adapter.rb +39 -39
  49. data/lib/activerecord-import/adapters/trilogy_adapter.rb +7 -0
  50. data/lib/activerecord-import/base.rb +10 -2
  51. data/lib/activerecord-import/import.rb +162 -65
  52. data/lib/activerecord-import/mysql2.rb +2 -0
  53. data/lib/activerecord-import/postgresql.rb +2 -0
  54. data/lib/activerecord-import/sqlite3.rb +2 -0
  55. data/lib/activerecord-import/synchronize.rb +3 -1
  56. data/lib/activerecord-import/value_sets_parser.rb +5 -0
  57. data/lib/activerecord-import/version.rb +3 -1
  58. data/lib/activerecord-import.rb +2 -1
  59. data/test/adapters/jdbcmysql.rb +2 -0
  60. data/test/adapters/jdbcpostgresql.rb +2 -0
  61. data/test/adapters/jdbcsqlite3.rb +2 -0
  62. data/test/adapters/makara_postgis.rb +2 -0
  63. data/test/adapters/mysql2.rb +2 -0
  64. data/test/adapters/mysql2_makara.rb +2 -0
  65. data/test/adapters/mysql2spatial.rb +2 -0
  66. data/test/adapters/postgis.rb +2 -0
  67. data/test/adapters/postgresql.rb +2 -0
  68. data/test/adapters/postgresql_makara.rb +2 -0
  69. data/test/adapters/seamless_database_pool.rb +2 -0
  70. data/test/adapters/spatialite.rb +2 -0
  71. data/test/adapters/sqlite3.rb +2 -0
  72. data/test/adapters/trilogy.rb +9 -0
  73. data/test/database.yml.sample +7 -0
  74. data/test/{travis → github}/database.yml +7 -1
  75. data/test/import_test.rb +93 -2
  76. data/test/jdbcmysql/import_test.rb +5 -3
  77. data/test/jdbcpostgresql/import_test.rb +4 -2
  78. data/test/jdbcsqlite3/import_test.rb +4 -2
  79. data/test/makara_postgis/import_test.rb +4 -2
  80. data/test/models/account.rb +2 -0
  81. data/test/models/alarm.rb +2 -0
  82. data/test/models/animal.rb +8 -0
  83. data/test/models/author.rb +7 -0
  84. data/test/models/bike_maker.rb +3 -0
  85. data/test/models/book.rb +7 -2
  86. data/test/models/car.rb +2 -0
  87. data/test/models/card.rb +5 -0
  88. data/test/models/chapter.rb +2 -0
  89. data/test/models/composite_book.rb +19 -0
  90. data/test/models/composite_chapter.rb +9 -0
  91. data/test/models/customer.rb +18 -0
  92. data/test/models/deck.rb +8 -0
  93. data/test/models/dictionary.rb +2 -0
  94. data/test/models/discount.rb +2 -0
  95. data/test/models/end_note.rb +2 -0
  96. data/test/models/group.rb +2 -0
  97. data/test/models/order.rb +17 -0
  98. data/test/models/playing_card.rb +4 -0
  99. data/test/models/promotion.rb +2 -0
  100. data/test/models/question.rb +2 -0
  101. data/test/models/rule.rb +2 -0
  102. data/test/models/tag.rb +9 -1
  103. data/test/models/tag_alias.rb +11 -0
  104. data/test/models/topic.rb +7 -0
  105. data/test/models/user.rb +2 -0
  106. data/test/models/user_token.rb +2 -0
  107. data/test/models/vendor.rb +2 -0
  108. data/test/models/widget.rb +2 -0
  109. data/test/mysql2/import_test.rb +5 -3
  110. data/test/mysql2_makara/import_test.rb +5 -3
  111. data/test/mysqlspatial2/import_test.rb +5 -3
  112. data/test/postgis/import_test.rb +4 -2
  113. data/test/postgresql/import_test.rb +4 -2
  114. data/test/schema/generic_schema.rb +37 -1
  115. data/test/schema/jdbcpostgresql_schema.rb +3 -1
  116. data/test/schema/mysql2_schema.rb +2 -0
  117. data/test/schema/postgis_schema.rb +3 -1
  118. data/test/schema/postgresql_schema.rb +47 -0
  119. data/test/schema/sqlite3_schema.rb +2 -0
  120. data/test/schema/version.rb +2 -0
  121. data/test/sqlite3/import_test.rb +4 -2
  122. data/test/support/active_support/test_case_extensions.rb +2 -0
  123. data/test/support/assertions.rb +2 -0
  124. data/test/support/factories.rb +2 -0
  125. data/test/support/generate.rb +4 -2
  126. data/test/support/mysql/import_examples.rb +2 -1
  127. data/test/support/postgresql/import_examples.rb +108 -2
  128. data/test/support/shared_examples/on_duplicate_key_ignore.rb +2 -0
  129. data/test/support/shared_examples/on_duplicate_key_update.rb +78 -9
  130. data/test/support/shared_examples/recursive_import.rb +98 -1
  131. data/test/support/sqlite3/import_examples.rb +2 -1
  132. data/test/synchronize_test.rb +2 -0
  133. data/test/test_helper.rb +33 -6
  134. data/test/trilogy/import_test.rb +7 -0
  135. data/test/value_sets_bytes_parser_test.rb +3 -1
  136. data/test/value_sets_records_parser_test.rb +3 -1
  137. metadata +42 -16
  138. data/.travis.yml +0 -70
  139. data/gemfiles/3.2.gemfile +0 -2
  140. data/gemfiles/4.0.gemfile +0 -2
  141. data/gemfiles/4.1.gemfile +0 -2
@@ -1,9 +1,11 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module ActiveRecord::Import::SQLite3Adapter
2
4
  include ActiveRecord::Import::ImportSupport
3
5
  include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
4
6
 
5
- MIN_VERSION_FOR_IMPORT = "3.7.11".freeze
6
- MIN_VERSION_FOR_UPSERT = "3.24.0".freeze
7
+ MIN_VERSION_FOR_IMPORT = "3.7.11"
8
+ MIN_VERSION_FOR_UPSERT = "3.24.0"
7
9
  SQLITE_LIMIT_COMPOUND_SELECT = 500
8
10
 
9
11
  # Override our conformance to ActiveRecord::Import::ImportSupport interface
@@ -22,10 +24,11 @@ module ActiveRecord::Import::SQLite3Adapter
22
24
  def insert_many( sql, values, _options = {}, *args ) # :nodoc:
23
25
  number_of_inserts = 0
24
26
 
25
- base_sql, post_sql = if sql.is_a?( String )
26
- [sql, '']
27
- elsif sql.is_a?( Array )
28
- [sql.shift, sql.join( ' ' )]
27
+ base_sql, post_sql = case sql
28
+ when String
29
+ [sql, '']
30
+ when Array
31
+ [sql.shift, sql.join( ' ' )]
29
32
  end
30
33
 
31
34
  value_sets = ::ActiveRecord::Import::ValueSetsRecordsParser.parse(values,
@@ -54,11 +57,9 @@ module ActiveRecord::Import::SQLite3Adapter
54
57
  def post_sql_statements( table_name, options ) # :nodoc:
55
58
  sql = []
56
59
 
57
- if supports_on_duplicate_key_update?
58
- # Options :recursive and :on_duplicate_key_ignore are mutually exclusive
59
- if (options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:on_duplicate_key_update]
60
- sql << sql_for_on_duplicate_key_ignore( options[:on_duplicate_key_ignore] )
61
- end
60
+ # Options :recursive and :on_duplicate_key_ignore are mutually exclusive
61
+ if supports_on_duplicate_key_update? && ((options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:on_duplicate_key_update])
62
+ sql << sql_for_on_duplicate_key_ignore( options[:on_duplicate_key_ignore] )
62
63
  end
63
64
 
64
65
  sql + super
@@ -71,13 +72,14 @@ module ActiveRecord::Import::SQLite3Adapter
71
72
  # Add a column to be updated on duplicate key update
72
73
  def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
73
74
  arg = options[:on_duplicate_key_update]
74
- if arg.is_a?( Hash )
75
+ case arg
76
+ when Hash
75
77
  columns = arg.fetch( :columns ) { arg[:columns] = [] }
76
78
  case columns
77
79
  when Array then columns << column.to_sym unless columns.include?( column.to_sym )
78
80
  when Hash then columns[column.to_sym] = column.to_sym
79
81
  end
80
- elsif arg.is_a?( Array )
82
+ when Array
81
83
  arg << column.to_sym unless arg.include?( column.to_sym )
82
84
  end
83
85
  end
@@ -92,12 +94,12 @@ module ActiveRecord::Import::SQLite3Adapter
92
94
 
93
95
  # Returns a generated ON CONFLICT DO UPDATE statement given the passed
94
96
  # in +args+.
95
- def sql_for_on_duplicate_key_update( _table_name, *args ) # :nodoc:
96
- arg, primary_key, locking_column = args
97
+ def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
98
+ arg, model, primary_key, locking_column = args
97
99
  arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
98
100
  return unless arg.is_a?( Hash )
99
101
 
100
- sql = ' ON CONFLICT '
102
+ sql = ' ON CONFLICT '.dup
101
103
  conflict_target = sql_for_conflict_target( arg )
102
104
 
103
105
  columns = arg.fetch( :columns, [] )
@@ -112,11 +114,12 @@ module ActiveRecord::Import::SQLite3Adapter
112
114
  end
113
115
 
114
116
  sql << "#{conflict_target}DO UPDATE SET "
115
- if columns.is_a?( Array )
116
- sql << sql_for_on_duplicate_key_update_as_array( locking_column, columns )
117
- elsif columns.is_a?( Hash )
118
- sql << sql_for_on_duplicate_key_update_as_hash( locking_column, columns )
119
- elsif columns.is_a?( String )
117
+ case columns
118
+ when Array
119
+ sql << sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, columns )
120
+ when Hash
121
+ sql << sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, columns )
122
+ when String
120
123
  sql << columns
121
124
  else
122
125
  raise ArgumentError, 'Expected :columns to be an Array or Hash'
@@ -127,22 +130,27 @@ module ActiveRecord::Import::SQLite3Adapter
127
130
  sql
128
131
  end
129
132
 
130
- def sql_for_on_duplicate_key_update_as_array( locking_column, arr ) # :nodoc:
133
+ def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
131
134
  results = arr.map do |column|
132
- qc = quote_column_name( column )
135
+ original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
136
+ qc = quote_column_name( original_column_name )
133
137
  "#{qc}=EXCLUDED.#{qc}"
134
138
  end
135
- increment_locking_column!(results, locking_column)
139
+ increment_locking_column!(table_name, results, locking_column)
136
140
  results.join( ',' )
137
141
  end
138
142
 
139
- def sql_for_on_duplicate_key_update_as_hash( locking_column, hsh ) # :nodoc:
143
+ def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
140
144
  results = hsh.map do |column1, column2|
141
- qc1 = quote_column_name( column1 )
142
- qc2 = quote_column_name( column2 )
145
+ original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
146
+ qc1 = quote_column_name( original_column1_name )
147
+
148
+ original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
149
+ qc2 = quote_column_name( original_column2_name )
150
+
143
151
  "#{qc1}=EXCLUDED.#{qc2}"
144
152
  end
145
- increment_locking_column!(results, locking_column)
153
+ increment_locking_column!(table_name, results, locking_column)
146
154
  results.join( ',' )
147
155
  end
148
156
 
@@ -150,9 +158,9 @@ module ActiveRecord::Import::SQLite3Adapter
150
158
  conflict_target = args[:conflict_target]
151
159
  index_predicate = args[:index_predicate]
152
160
  if conflict_target.present?
153
- '(' << Array( conflict_target ).reject( &:blank? ).join( ', ' ) << ') '.tap do |sql|
154
- sql << "WHERE #{index_predicate} " if index_predicate
155
- end
161
+ sql = "(#{Array( conflict_target ).reject( &:blank? ).join( ', ' )}) "
162
+ sql += "WHERE #{index_predicate} " if index_predicate
163
+ sql
156
164
  end
157
165
  end
158
166
 
@@ -166,14 +174,6 @@ module ActiveRecord::Import::SQLite3Adapter
166
174
  exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key')
167
175
  end
168
176
 
169
- def increment_locking_column!(results, locking_column)
170
- if locking_column.present?
171
- results << "\"#{locking_column}\"=EXCLUDED.\"#{locking_column}\"+1"
172
- end
173
- end
174
-
175
- private
176
-
177
177
  def database_version
178
178
  defined?(sqlite_version) ? sqlite_version : super
179
179
  end
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "activerecord-import/adapters/mysql_adapter"
4
+
5
+ module ActiveRecord::Import::TrilogyAdapter
6
+ include ActiveRecord::Import::MysqlAdapter
7
+ end
@@ -1,9 +1,11 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "pathname"
2
4
  require "active_record"
3
5
  require "active_record/version"
4
6
 
5
7
  module ActiveRecord::Import
6
- ADAPTER_PATH = "activerecord-import/active_record/adapters".freeze
8
+ ADAPTER_PATH = "activerecord-import/active_record/adapters"
7
9
 
8
10
  def self.base_adapter(adapter)
9
11
  case adapter
@@ -27,7 +29,13 @@ module ActiveRecord::Import
27
29
 
28
30
  # Loads the import functionality for the passed in ActiveRecord connection
29
31
  def self.load_from_connection_pool(connection_pool)
30
- require_adapter connection_pool.spec.config[:adapter]
32
+ adapter =
33
+ if connection_pool.respond_to?(:db_config) # ActiveRecord >= 6.1
34
+ connection_pool.db_config.adapter
35
+ else
36
+ connection_pool.spec.config[:adapter]
37
+ end
38
+ require_adapter adapter
31
39
  end
32
40
  end
33
41
 
@@ -1,18 +1,20 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "ostruct"
2
4
 
3
5
  module ActiveRecord::Import::ConnectionAdapters; end
4
6
 
5
- module ActiveRecord::Import #:nodoc:
7
+ module ActiveRecord::Import # :nodoc:
6
8
  Result = Struct.new(:failed_instances, :num_inserts, :ids, :results)
7
9
 
8
- module ImportSupport #:nodoc:
9
- def supports_import? #:nodoc:
10
+ module ImportSupport # :nodoc:
11
+ def supports_import? # :nodoc:
10
12
  true
11
13
  end
12
14
  end
13
15
 
14
- module OnDuplicateKeyUpdateSupport #:nodoc:
15
- def supports_on_duplicate_key_update? #:nodoc:
16
+ module OnDuplicateKeyUpdateSupport # :nodoc:
17
+ def supports_on_duplicate_key_update? # :nodoc:
16
18
  true
17
19
  end
18
20
  end
@@ -34,7 +36,7 @@ module ActiveRecord::Import #:nodoc:
34
36
  @validate_callbacks = klass._validate_callbacks.dup
35
37
 
36
38
  @validate_callbacks.each_with_index do |callback, i|
37
- filter = callback.raw_filter
39
+ filter = callback.respond_to?(:raw_filter) ? callback.raw_filter : callback.filter
38
40
  next unless filter.class.name =~ /Validations::PresenceValidator/ ||
39
41
  (!@options[:validate_uniqueness] &&
40
42
  filter.is_a?(ActiveRecord::Validations::UniquenessValidator))
@@ -49,13 +51,13 @@ module ActiveRecord::Import #:nodoc:
49
51
  associations = klass.reflect_on_all_associations(:belongs_to)
50
52
  associations.each do |assoc|
51
53
  if (index = attrs.index(assoc.name))
52
- key = assoc.foreign_key.to_sym
54
+ key = assoc.foreign_key.is_a?(Array) ? assoc.foreign_key.map(&:to_sym) : assoc.foreign_key.to_sym
53
55
  attrs[index] = key unless attrs.include?(key)
54
56
  end
55
57
  end
56
58
  end
57
59
 
58
- filter.instance_variable_set(:@attributes, attrs)
60
+ filter.instance_variable_set(:@attributes, attrs.flatten)
59
61
 
60
62
  if @validate_callbacks.respond_to?(:chain, true)
61
63
  @validate_callbacks.send(:chain).tap do |chain|
@@ -71,7 +73,7 @@ module ActiveRecord::Import #:nodoc:
71
73
  end
72
74
 
73
75
  def valid_model?(model)
74
- init_validations(model.class) unless model.class == @validator_class
76
+ init_validations(model.class) unless model.instance_of?(@validator_class)
75
77
 
76
78
  validation_context = @options[:validate_with_context]
77
79
  validation_context ||= (model.new_record? ? :create : :update)
@@ -83,7 +85,11 @@ module ActiveRecord::Import #:nodoc:
83
85
 
84
86
  model.run_callbacks(:validation) do
85
87
  if defined?(ActiveSupport::Callbacks::Filters::Environment) # ActiveRecord >= 4.1
86
- runner = @validate_callbacks.compile
88
+ runner = if @validate_callbacks.method(:compile).arity == 0
89
+ @validate_callbacks.compile
90
+ else # ActiveRecord >= 7.1
91
+ @validate_callbacks.compile(nil)
92
+ end
87
93
  env = ActiveSupport::Callbacks::Filters::Environment.new(model, false, nil)
88
94
  if runner.respond_to?(:call) # ActiveRecord < 5.1
89
95
  runner.call(env)
@@ -163,7 +169,7 @@ class ActiveRecord::Associations::CollectionAssociation
163
169
  m.public_send "#{reflection.type}=", owner.class.name if reflection.type
164
170
  end
165
171
 
166
- return model_klass.bulk_import column_names, models, options
172
+ model_klass.bulk_import column_names, models, options
167
173
 
168
174
  # supports array of hash objects
169
175
  elsif args.last.is_a?( Array ) && args.last.first.is_a?(Hash)
@@ -202,11 +208,11 @@ class ActiveRecord::Associations::CollectionAssociation
202
208
  end
203
209
  end
204
210
 
205
- return model_klass.bulk_import column_names, array_of_attributes, options
211
+ model_klass.bulk_import column_names, array_of_attributes, options
206
212
 
207
213
  # supports empty array
208
214
  elsif args.last.is_a?( Array ) && args.last.empty?
209
- return ActiveRecord::Import::Result.new([], 0, [])
215
+ ActiveRecord::Import::Result.new([], 0, [])
210
216
 
211
217
  # supports 2-element array and array
212
218
  elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
@@ -237,7 +243,7 @@ class ActiveRecord::Associations::CollectionAssociation
237
243
  end
238
244
  end
239
245
 
240
- return model_klass.bulk_import column_names, array_of_attributes, options
246
+ model_klass.bulk_import column_names, array_of_attributes, options
241
247
  else
242
248
  raise ArgumentError, "Invalid arguments!"
243
249
  end
@@ -245,16 +251,17 @@ class ActiveRecord::Associations::CollectionAssociation
245
251
  alias import bulk_import unless respond_to? :import
246
252
  end
247
253
 
254
+ module ActiveRecord::Import::Connection
255
+ def establish_connection(args = nil)
256
+ conn = super(args)
257
+ ActiveRecord::Import.load_from_connection_pool connection_pool
258
+ conn
259
+ end
260
+ end
261
+
248
262
  class ActiveRecord::Base
249
263
  class << self
250
- def establish_connection_with_activerecord_import(*args)
251
- conn = establish_connection_without_activerecord_import(*args)
252
- ActiveRecord::Import.load_from_connection_pool connection_pool
253
- conn
254
- end
255
-
256
- alias establish_connection_without_activerecord_import establish_connection
257
- alias establish_connection establish_connection_with_activerecord_import
264
+ prepend ActiveRecord::Import::Connection
258
265
 
259
266
  # Returns true if the current database connection adapter
260
267
  # supports import functionality, otherwise returns false.
@@ -546,11 +553,11 @@ class ActiveRecord::Base
546
553
  alias import! bulk_import! unless ActiveRecord::Base.respond_to? :import!
547
554
 
548
555
  def import_helper( *args )
549
- options = { validate: true, timestamps: true }
556
+ options = { model: self, validate: true, timestamps: true, track_validation_failures: false }
550
557
  options.merge!( args.pop ) if args.last.is_a? Hash
551
558
  # making sure that current model's primary key is used
552
559
  options[:primary_key] = primary_key
553
- options[:locking_column] = locking_column if attribute_names.include?(locking_column)
560
+ options[:locking_column] = locking_column if locking_enabled?
554
561
 
555
562
  is_validating = options[:validate_with_context].present? ? true : options[:validate]
556
563
  validator = ActiveRecord::Import::Validator.new(self, options)
@@ -571,7 +578,7 @@ class ActiveRecord::Base
571
578
 
572
579
  if models.first.id.nil?
573
580
  Array(primary_key).each do |c|
574
- if column_names.include?(c) && columns_hash[c].type == :uuid
581
+ if column_names.include?(c) && schema_columns_hash[c].type == :uuid
575
582
  column_names.delete(c)
576
583
  end
577
584
  end
@@ -581,7 +588,7 @@ class ActiveRecord::Base
581
588
  if respond_to?(:timestamp_attributes_for_update, true)
582
589
  send(:timestamp_attributes_for_update).map(&:to_sym)
583
590
  else
584
- new.send(:timestamp_attributes_for_update_in_model)
591
+ allocate.send(:timestamp_attributes_for_update_in_model)
585
592
  end
586
593
  end
587
594
 
@@ -630,7 +637,7 @@ class ActiveRecord::Base
630
637
  end
631
638
  # supports empty array
632
639
  elsif args.last.is_a?( Array ) && args.last.empty?
633
- return ActiveRecord::Import::Result.new([], 0, [])
640
+ return ActiveRecord::Import::Result.new([], 0, [], [])
634
641
  # supports 2-element array and array
635
642
  elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
636
643
 
@@ -694,7 +701,11 @@ class ActiveRecord::Base
694
701
  return_obj = if is_validating
695
702
  import_with_validations( column_names, array_of_attributes, options ) do |failed_instances|
696
703
  if models
697
- models.each { |m| failed_instances << m if m.errors.any? }
704
+ models.each_with_index do |m, i|
705
+ next unless m.errors.any?
706
+
707
+ failed_instances << (options[:track_validation_failures] ? [i, m] : m)
708
+ end
698
709
  else
699
710
  # create instances for each of our column/value sets
700
711
  arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
@@ -702,14 +713,18 @@ class ActiveRecord::Base
702
713
  # keep track of the instance and the position it is currently at. if this fails
703
714
  # validation we'll use the index to remove it from the array_of_attributes
704
715
  arr.each_with_index do |hsh, i|
705
- model = new
706
- hsh.each_pair { |k, v| model[k] = v }
716
+ # utilize block initializer syntax to prevent failure when 'mass_assignment_sanitizer = :strict'
717
+ model = new do |m|
718
+ hsh.each_pair { |k, v| m[k] = v }
719
+ end
720
+
707
721
  next if validator.valid_model?(model)
708
722
  raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
723
+
709
724
  array_of_attributes[i] = nil
710
725
  failure = model.dup
711
726
  failure.errors.send(:initialize_dup, model.errors)
712
- failed_instances << failure
727
+ failed_instances << (options[:track_validation_failures] ? [i, failure] : failure )
713
728
  end
714
729
  array_of_attributes.compact!
715
730
  end
@@ -729,7 +744,10 @@ class ActiveRecord::Base
729
744
  set_attributes_and_mark_clean(models, return_obj, timestamps, options)
730
745
 
731
746
  # if there are auto-save associations on the models we imported that are new, import them as well
732
- import_associations(models, options.dup) if options[:recursive]
747
+ if options[:recursive]
748
+ options[:on_duplicate_key_update] = on_duplicate_key_update unless on_duplicate_key_update.nil?
749
+ import_associations(models, options.dup.merge(validate: false))
750
+ end
733
751
  end
734
752
 
735
753
  return_obj
@@ -764,27 +782,29 @@ class ActiveRecord::Base
764
782
  def import_without_validations_or_callbacks( column_names, array_of_attributes, options = {} )
765
783
  return ActiveRecord::Import::Result.new([], 0, [], []) if array_of_attributes.empty?
766
784
 
767
- column_names = column_names.map(&:to_sym)
785
+ column_names = column_names.map do |name|
786
+ original_name = attribute_alias?(name) ? attribute_alias(name) : name
787
+ original_name.to_sym
788
+ end
768
789
  scope_columns, scope_values = scope_attributes.to_a.transpose
769
790
 
770
791
  unless scope_columns.blank?
771
792
  scope_columns.zip(scope_values).each do |name, value|
772
793
  name_as_sym = name.to_sym
773
- next if column_names.include?(name_as_sym)
774
-
775
- is_sti = (name_as_sym == inheritance_column.to_sym && self < base_class)
776
- value = Array(value).first if is_sti
777
-
794
+ next if column_names.include?(name_as_sym) || name_as_sym == inheritance_column.to_sym
778
795
  column_names << name_as_sym
779
796
  array_of_attributes.each { |attrs| attrs << value }
780
797
  end
781
798
  end
782
799
 
783
- columns = column_names.each_with_index.map do |name, i|
784
- column = columns_hash[name.to_s]
800
+ if finder_needs_type_condition? && !column_names.include?(inheritance_column.to_sym)
801
+ column_names << inheritance_column.to_sym
802
+ array_of_attributes.each { |attrs| attrs << sti_name }
803
+ end
785
804
 
805
+ columns = column_names.each_with_index.map do |name, i|
806
+ column = schema_columns_hash[name.to_s]
786
807
  raise ActiveRecord::Import::MissingColumnError.new(name.to_s, i) if column.nil?
787
-
788
808
  column
789
809
  end
790
810
 
@@ -800,17 +820,29 @@ class ActiveRecord::Base
800
820
  if supports_import?
801
821
  # generate the sql
802
822
  post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
823
+ import_size = values_sql.size
824
+
825
+ batch_size = options[:batch_size] || import_size
826
+ run_proc = options[:batch_size].to_i.positive? && options[:batch_progress].respond_to?( :call )
827
+ progress_proc = options[:batch_progress]
828
+ current_batch = 0
829
+ batches = (import_size / batch_size.to_f).ceil
803
830
 
804
- batch_size = options[:batch_size] || values_sql.size
805
831
  values_sql.each_slice(batch_size) do |batch_values|
832
+ batch_started_at = Time.now.to_i
833
+
806
834
  # perform the inserts
807
835
  result = connection.insert_many( [insert_sql, post_sql_statements].flatten,
808
836
  batch_values,
809
837
  options,
810
- "#{model_name} Create Many Without Validations Or Callbacks" )
838
+ "#{model_name} Create Many" )
839
+
811
840
  number_inserted += result.num_inserts
812
841
  ids += result.ids
813
842
  results += result.results
843
+ current_batch += 1
844
+
845
+ progress_proc.call(import_size, batches, current_batch, Time.now.to_i - batch_started_at) if run_proc
814
846
  end
815
847
  else
816
848
  transaction(requires_new: true) do
@@ -825,6 +857,15 @@ class ActiveRecord::Base
825
857
 
826
858
  private
827
859
 
860
+ def associated_options(options, associated_class)
861
+ return options unless options.key?(:recursive_on_duplicate_key_update)
862
+
863
+ table_name = associated_class.arel_table.name.to_sym
864
+ options.merge(
865
+ on_duplicate_key_update: options[:recursive_on_duplicate_key_update][table_name]
866
+ )
867
+ end
868
+
828
869
  def set_attributes_and_mark_clean(models, import_result, timestamps, options)
829
870
  return if models.nil?
830
871
  models -= import_result.failed_instances
@@ -836,22 +877,46 @@ class ActiveRecord::Base
836
877
  model.id = id
837
878
 
838
879
  timestamps.each do |attr, value|
839
- model.send(attr + "=", value)
880
+ model.send("#{attr}=", value) if model.send(attr).nil?
840
881
  end
841
882
  end
842
883
  end
843
884
 
844
- if models.size == import_result.results.size
845
- columns = Array(options[:returning])
846
- single_column = "#{columns.first}=" if columns.size == 1
847
- import_result.results.each_with_index do |result, index|
885
+ deserialize_value = lambda do |column, value|
886
+ column = schema_columns_hash[column]
887
+ return value unless column
888
+ if respond_to?(:type_caster)
889
+ type = type_for_attribute(column.name)
890
+ type.deserialize(value)
891
+ elsif column.respond_to?(:type_cast_from_database)
892
+ column.type_cast_from_database(value)
893
+ else
894
+ value
895
+ end
896
+ end
897
+
898
+ set_value = lambda do |model, column, value|
899
+ val = deserialize_value.call(column, value)
900
+ if model.attribute_names.include?(column)
901
+ model.send("#{column}=", val)
902
+ else
903
+ attributes = attributes_builder.build_from_database(model.attributes.merge(column => val))
904
+ model.instance_variable_set(:@attributes, attributes)
905
+ end
906
+ end
907
+
908
+ columns = Array(options[:returning_columns])
909
+ results = Array(import_result.results)
910
+ if models.size == results.size
911
+ single_column = columns.first if columns.size == 1
912
+ results.each_with_index do |result, index|
848
913
  model = models[index]
849
914
 
850
915
  if single_column
851
- model.send(single_column, result)
916
+ set_value.call(model, single_column, result)
852
917
  else
853
918
  columns.each_with_index do |column, col_index|
854
- model.send("#{column}=", result[col_index])
919
+ set_value.call(model, column, result[col_index])
855
920
  end
856
921
  end
857
922
  end
@@ -872,16 +937,22 @@ class ActiveRecord::Base
872
937
 
873
938
  # Sync belongs_to association ids with foreign key field
874
939
  def load_association_ids(model)
940
+ changed_columns = model.changed
875
941
  association_reflections = model.class.reflect_on_all_associations(:belongs_to)
876
942
  association_reflections.each do |association_reflection|
877
- column_name = association_reflection.foreign_key
878
943
  next if association_reflection.options[:polymorphic]
879
- association = model.association(association_reflection.name)
880
- association = association.target
881
- next if association.blank? || model.public_send(column_name).present?
882
944
 
883
- association_primary_key = association_reflection.association_primary_key
884
- model.public_send("#{column_name}=", association.send(association_primary_key))
945
+ column_names = Array(association_reflection.foreign_key).map(&:to_s)
946
+ column_names.each_with_index do |column_name, column_index|
947
+ next if changed_columns.include?(column_name)
948
+
949
+ association = model.association(association_reflection.name)
950
+ association = association.target
951
+ next if association.blank? || model.public_send(column_name).present?
952
+
953
+ association_primary_key = Array(association_reflection.association_primary_key.tr("[]:", "").split(", "))[column_index]
954
+ model.public_send("#{column_name}=", association.send(association_primary_key))
955
+ end
885
956
  end
886
957
  end
887
958
 
@@ -894,17 +965,30 @@ class ActiveRecord::Base
894
965
  associated_objects_by_class = {}
895
966
  models.each { |model| find_associated_objects_for_import(associated_objects_by_class, model) }
896
967
 
897
- # :on_duplicate_key_update and :returning not supported for associations
898
- options.delete(:on_duplicate_key_update)
968
+ # :on_duplicate_key_update only supported for all fields
969
+ options.delete(:on_duplicate_key_update) unless options[:on_duplicate_key_update] == :all
970
+ # :returning not supported for associations
899
971
  options.delete(:returning)
900
972
 
901
973
  associated_objects_by_class.each_value do |associations|
902
974
  associations.each_value do |associated_records|
903
- associated_records.first.class.bulk_import(associated_records, options) unless associated_records.empty?
975
+ next if associated_records.empty?
976
+
977
+ associated_class = associated_records.first.class
978
+ associated_class.bulk_import(associated_records,
979
+ associated_options(options, associated_class))
904
980
  end
905
981
  end
906
982
  end
907
983
 
984
+ def schema_columns_hash
985
+ if respond_to?(:ignored_columns) && ignored_columns.any?
986
+ connection.schema_cache.columns_hash(table_name)
987
+ else
988
+ columns_hash
989
+ end
990
+ end
991
+
908
992
  # We are eventually going to call Class.import <objects> so we build up a hash
909
993
  # of class => objects to import.
910
994
  def find_associated_objects_for_import(associated_objects_by_class, model)
@@ -925,10 +1009,18 @@ class ActiveRecord::Base
925
1009
 
926
1010
  changed_objects = association.select { |a| a.new_record? || a.changed? }
927
1011
  changed_objects.each do |child|
928
- child.public_send("#{association_reflection.foreign_key}=", model.id)
1012
+ Array(association_reflection.inverse_of&.foreign_key || association_reflection.foreign_key).each_with_index do |column, index|
1013
+ child.public_send("#{column}=", Array(model.id)[index])
1014
+ end
1015
+
929
1016
  # For polymorphic associations
1017
+ association_name = if model.class.respond_to?(:polymorphic_name)
1018
+ model.class.polymorphic_name
1019
+ else
1020
+ model.class.base_class
1021
+ end
930
1022
  association_reflection.type.try do |type|
931
- child.public_send("#{type}=", model.class.base_class.name)
1023
+ child.public_send("#{type}=", association_name)
932
1024
  end
933
1025
  end
934
1026
  associated_objects_by_class[model.class.name][association_reflection.name].concat changed_objects
@@ -955,7 +1047,7 @@ class ActiveRecord::Base
955
1047
  elsif column
956
1048
  if respond_to?(:type_caster) # Rails 5.0 and higher
957
1049
  type = type_for_attribute(column.name)
958
- val = type.type == :boolean ? type.cast(val) : type.serialize(val)
1050
+ val = !type.respond_to?(:subtype) && type.type == :boolean ? type.cast(val) : type.serialize(val)
959
1051
  connection_memo.quote(val)
960
1052
  elsif column.respond_to?(:type_cast_from_user) # Rails 4.2
961
1053
  connection_memo.quote(column.type_cast_from_user(val), column)
@@ -964,7 +1056,7 @@ class ActiveRecord::Base
964
1056
  val = serialized_attributes[column.name].dump(val)
965
1057
  end
966
1058
  # Fixes #443 to support binary (i.e. bytea) columns on PG
967
- val = column.type_cast(val) unless column.type.to_sym == :binary
1059
+ val = column.type_cast(val) unless column.type && column.type.to_sym == :binary
968
1060
  connection_memo.quote(val, column)
969
1061
  end
970
1062
  else
@@ -983,13 +1075,18 @@ class ActiveRecord::Base
983
1075
  timestamp_columns[:create] = timestamp_attributes_for_create_in_model
984
1076
  timestamp_columns[:update] = timestamp_attributes_for_update_in_model
985
1077
  else
986
- instance = new
1078
+ instance = allocate
987
1079
  timestamp_columns[:create] = instance.send(:timestamp_attributes_for_create_in_model)
988
1080
  timestamp_columns[:update] = instance.send(:timestamp_attributes_for_update_in_model)
989
1081
  end
990
1082
 
991
1083
  # use tz as set in ActiveRecord::Base
992
- timestamp = ActiveRecord::Base.default_timezone == :utc ? Time.now.utc : Time.now
1084
+ default_timezone = if ActiveRecord.respond_to?(:default_timezone)
1085
+ ActiveRecord.default_timezone
1086
+ else
1087
+ ActiveRecord::Base.default_timezone
1088
+ end
1089
+ timestamp = default_timezone == :utc ? Time.now.utc : Time.now
993
1090
 
994
1091
  [:create, :update].each do |action|
995
1092
  timestamp_columns[action].each do |column|
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  warn <<-MSG
2
4
  [DEPRECATION] loading activerecord-import via 'require "activerecord-import/<adapter-name>"'
3
5
  is deprecated. Update to autorequire using 'require "activerecord-import"'. See