activerecord-import 1.0.2 → 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yaml +151 -0
  3. data/.gitignore +5 -0
  4. data/.rubocop.yml +74 -8
  5. data/.rubocop_todo.yml +10 -16
  6. data/Brewfile +3 -1
  7. data/CHANGELOG.md +138 -3
  8. data/Dockerfile +23 -0
  9. data/Gemfile +24 -14
  10. data/LICENSE +21 -56
  11. data/README.markdown +108 -60
  12. data/Rakefile +3 -0
  13. data/activerecord-import.gemspec +6 -5
  14. data/benchmarks/benchmark.rb +10 -4
  15. data/benchmarks/lib/base.rb +4 -2
  16. data/benchmarks/lib/cli_parser.rb +4 -2
  17. data/benchmarks/lib/float.rb +2 -0
  18. data/benchmarks/lib/mysql2_benchmark.rb +2 -0
  19. data/benchmarks/lib/output_to_csv.rb +2 -0
  20. data/benchmarks/lib/output_to_html.rb +4 -2
  21. data/benchmarks/models/test_innodb.rb +2 -0
  22. data/benchmarks/models/test_memory.rb +2 -0
  23. data/benchmarks/models/test_myisam.rb +2 -0
  24. data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +2 -0
  25. data/docker-compose.yml +34 -0
  26. data/gemfiles/4.2.gemfile +2 -0
  27. data/gemfiles/5.0.gemfile +2 -0
  28. data/gemfiles/5.1.gemfile +2 -0
  29. data/gemfiles/5.2.gemfile +2 -0
  30. data/gemfiles/6.0.gemfile +4 -1
  31. data/gemfiles/6.1.gemfile +4 -1
  32. data/gemfiles/7.0.gemfile +4 -0
  33. data/gemfiles/7.1.gemfile +3 -0
  34. data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +2 -0
  35. data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +6 -4
  36. data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +2 -0
  37. data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +2 -0
  38. data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +2 -0
  39. data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +2 -0
  40. data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +2 -0
  41. data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +2 -0
  42. data/lib/activerecord-import/active_record/adapters/trilogy_adapter.rb +8 -0
  43. data/lib/activerecord-import/adapters/abstract_adapter.rb +14 -5
  44. data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +2 -0
  45. data/lib/activerecord-import/adapters/mysql2_adapter.rb +2 -0
  46. data/lib/activerecord-import/adapters/mysql_adapter.rb +33 -25
  47. data/lib/activerecord-import/adapters/postgresql_adapter.rb +69 -56
  48. data/lib/activerecord-import/adapters/sqlite3_adapter.rb +39 -39
  49. data/lib/activerecord-import/adapters/trilogy_adapter.rb +7 -0
  50. data/lib/activerecord-import/base.rb +10 -2
  51. data/lib/activerecord-import/import.rb +162 -65
  52. data/lib/activerecord-import/mysql2.rb +2 -0
  53. data/lib/activerecord-import/postgresql.rb +2 -0
  54. data/lib/activerecord-import/sqlite3.rb +2 -0
  55. data/lib/activerecord-import/synchronize.rb +3 -1
  56. data/lib/activerecord-import/value_sets_parser.rb +5 -0
  57. data/lib/activerecord-import/version.rb +3 -1
  58. data/lib/activerecord-import.rb +2 -1
  59. data/test/adapters/jdbcmysql.rb +2 -0
  60. data/test/adapters/jdbcpostgresql.rb +2 -0
  61. data/test/adapters/jdbcsqlite3.rb +2 -0
  62. data/test/adapters/makara_postgis.rb +2 -0
  63. data/test/adapters/mysql2.rb +2 -0
  64. data/test/adapters/mysql2_makara.rb +2 -0
  65. data/test/adapters/mysql2spatial.rb +2 -0
  66. data/test/adapters/postgis.rb +2 -0
  67. data/test/adapters/postgresql.rb +2 -0
  68. data/test/adapters/postgresql_makara.rb +2 -0
  69. data/test/adapters/seamless_database_pool.rb +2 -0
  70. data/test/adapters/spatialite.rb +2 -0
  71. data/test/adapters/sqlite3.rb +2 -0
  72. data/test/adapters/trilogy.rb +9 -0
  73. data/test/database.yml.sample +7 -0
  74. data/test/{travis → github}/database.yml +7 -1
  75. data/test/import_test.rb +93 -2
  76. data/test/jdbcmysql/import_test.rb +5 -3
  77. data/test/jdbcpostgresql/import_test.rb +4 -2
  78. data/test/jdbcsqlite3/import_test.rb +4 -2
  79. data/test/makara_postgis/import_test.rb +4 -2
  80. data/test/models/account.rb +2 -0
  81. data/test/models/alarm.rb +2 -0
  82. data/test/models/animal.rb +8 -0
  83. data/test/models/author.rb +7 -0
  84. data/test/models/bike_maker.rb +3 -0
  85. data/test/models/book.rb +7 -2
  86. data/test/models/car.rb +2 -0
  87. data/test/models/card.rb +5 -0
  88. data/test/models/chapter.rb +2 -0
  89. data/test/models/composite_book.rb +19 -0
  90. data/test/models/composite_chapter.rb +9 -0
  91. data/test/models/customer.rb +18 -0
  92. data/test/models/deck.rb +8 -0
  93. data/test/models/dictionary.rb +2 -0
  94. data/test/models/discount.rb +2 -0
  95. data/test/models/end_note.rb +2 -0
  96. data/test/models/group.rb +2 -0
  97. data/test/models/order.rb +17 -0
  98. data/test/models/playing_card.rb +4 -0
  99. data/test/models/promotion.rb +2 -0
  100. data/test/models/question.rb +2 -0
  101. data/test/models/rule.rb +2 -0
  102. data/test/models/tag.rb +9 -1
  103. data/test/models/tag_alias.rb +11 -0
  104. data/test/models/topic.rb +7 -0
  105. data/test/models/user.rb +2 -0
  106. data/test/models/user_token.rb +2 -0
  107. data/test/models/vendor.rb +2 -0
  108. data/test/models/widget.rb +2 -0
  109. data/test/mysql2/import_test.rb +5 -3
  110. data/test/mysql2_makara/import_test.rb +5 -3
  111. data/test/mysqlspatial2/import_test.rb +5 -3
  112. data/test/postgis/import_test.rb +4 -2
  113. data/test/postgresql/import_test.rb +4 -2
  114. data/test/schema/generic_schema.rb +37 -1
  115. data/test/schema/jdbcpostgresql_schema.rb +3 -1
  116. data/test/schema/mysql2_schema.rb +2 -0
  117. data/test/schema/postgis_schema.rb +3 -1
  118. data/test/schema/postgresql_schema.rb +47 -0
  119. data/test/schema/sqlite3_schema.rb +2 -0
  120. data/test/schema/version.rb +2 -0
  121. data/test/sqlite3/import_test.rb +4 -2
  122. data/test/support/active_support/test_case_extensions.rb +2 -0
  123. data/test/support/assertions.rb +2 -0
  124. data/test/support/factories.rb +2 -0
  125. data/test/support/generate.rb +4 -2
  126. data/test/support/mysql/import_examples.rb +2 -1
  127. data/test/support/postgresql/import_examples.rb +108 -2
  128. data/test/support/shared_examples/on_duplicate_key_ignore.rb +2 -0
  129. data/test/support/shared_examples/on_duplicate_key_update.rb +78 -9
  130. data/test/support/shared_examples/recursive_import.rb +98 -1
  131. data/test/support/sqlite3/import_examples.rb +2 -1
  132. data/test/synchronize_test.rb +2 -0
  133. data/test/test_helper.rb +33 -6
  134. data/test/trilogy/import_test.rb +7 -0
  135. data/test/value_sets_bytes_parser_test.rb +3 -1
  136. data/test/value_sets_records_parser_test.rb +3 -1
  137. metadata +42 -16
  138. data/.travis.yml +0 -70
  139. data/gemfiles/3.2.gemfile +0 -2
  140. data/gemfiles/4.0.gemfile +0 -2
  141. data/gemfiles/4.1.gemfile +0 -2
@@ -1,9 +1,11 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module ActiveRecord::Import::SQLite3Adapter
2
4
  include ActiveRecord::Import::ImportSupport
3
5
  include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
4
6
 
5
- MIN_VERSION_FOR_IMPORT = "3.7.11".freeze
6
- MIN_VERSION_FOR_UPSERT = "3.24.0".freeze
7
+ MIN_VERSION_FOR_IMPORT = "3.7.11"
8
+ MIN_VERSION_FOR_UPSERT = "3.24.0"
7
9
  SQLITE_LIMIT_COMPOUND_SELECT = 500
8
10
 
9
11
  # Override our conformance to ActiveRecord::Import::ImportSupport interface
@@ -22,10 +24,11 @@ module ActiveRecord::Import::SQLite3Adapter
22
24
  def insert_many( sql, values, _options = {}, *args ) # :nodoc:
23
25
  number_of_inserts = 0
24
26
 
25
- base_sql, post_sql = if sql.is_a?( String )
26
- [sql, '']
27
- elsif sql.is_a?( Array )
28
- [sql.shift, sql.join( ' ' )]
27
+ base_sql, post_sql = case sql
28
+ when String
29
+ [sql, '']
30
+ when Array
31
+ [sql.shift, sql.join( ' ' )]
29
32
  end
30
33
 
31
34
  value_sets = ::ActiveRecord::Import::ValueSetsRecordsParser.parse(values,
@@ -54,11 +57,9 @@ module ActiveRecord::Import::SQLite3Adapter
54
57
  def post_sql_statements( table_name, options ) # :nodoc:
55
58
  sql = []
56
59
 
57
- if supports_on_duplicate_key_update?
58
- # Options :recursive and :on_duplicate_key_ignore are mutually exclusive
59
- if (options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:on_duplicate_key_update]
60
- sql << sql_for_on_duplicate_key_ignore( options[:on_duplicate_key_ignore] )
61
- end
60
+ # Options :recursive and :on_duplicate_key_ignore are mutually exclusive
61
+ if supports_on_duplicate_key_update? && ((options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:on_duplicate_key_update])
62
+ sql << sql_for_on_duplicate_key_ignore( options[:on_duplicate_key_ignore] )
62
63
  end
63
64
 
64
65
  sql + super
@@ -71,13 +72,14 @@ module ActiveRecord::Import::SQLite3Adapter
71
72
  # Add a column to be updated on duplicate key update
72
73
  def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
73
74
  arg = options[:on_duplicate_key_update]
74
- if arg.is_a?( Hash )
75
+ case arg
76
+ when Hash
75
77
  columns = arg.fetch( :columns ) { arg[:columns] = [] }
76
78
  case columns
77
79
  when Array then columns << column.to_sym unless columns.include?( column.to_sym )
78
80
  when Hash then columns[column.to_sym] = column.to_sym
79
81
  end
80
- elsif arg.is_a?( Array )
82
+ when Array
81
83
  arg << column.to_sym unless arg.include?( column.to_sym )
82
84
  end
83
85
  end
@@ -92,12 +94,12 @@ module ActiveRecord::Import::SQLite3Adapter
92
94
 
93
95
  # Returns a generated ON CONFLICT DO UPDATE statement given the passed
94
96
  # in +args+.
95
- def sql_for_on_duplicate_key_update( _table_name, *args ) # :nodoc:
96
- arg, primary_key, locking_column = args
97
+ def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
98
+ arg, model, primary_key, locking_column = args
97
99
  arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
98
100
  return unless arg.is_a?( Hash )
99
101
 
100
- sql = ' ON CONFLICT '
102
+ sql = ' ON CONFLICT '.dup
101
103
  conflict_target = sql_for_conflict_target( arg )
102
104
 
103
105
  columns = arg.fetch( :columns, [] )
@@ -112,11 +114,12 @@ module ActiveRecord::Import::SQLite3Adapter
112
114
  end
113
115
 
114
116
  sql << "#{conflict_target}DO UPDATE SET "
115
- if columns.is_a?( Array )
116
- sql << sql_for_on_duplicate_key_update_as_array( locking_column, columns )
117
- elsif columns.is_a?( Hash )
118
- sql << sql_for_on_duplicate_key_update_as_hash( locking_column, columns )
119
- elsif columns.is_a?( String )
117
+ case columns
118
+ when Array
119
+ sql << sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, columns )
120
+ when Hash
121
+ sql << sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, columns )
122
+ when String
120
123
  sql << columns
121
124
  else
122
125
  raise ArgumentError, 'Expected :columns to be an Array or Hash'
@@ -127,22 +130,27 @@ module ActiveRecord::Import::SQLite3Adapter
127
130
  sql
128
131
  end
129
132
 
130
- def sql_for_on_duplicate_key_update_as_array( locking_column, arr ) # :nodoc:
133
+ def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
131
134
  results = arr.map do |column|
132
- qc = quote_column_name( column )
135
+ original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
136
+ qc = quote_column_name( original_column_name )
133
137
  "#{qc}=EXCLUDED.#{qc}"
134
138
  end
135
- increment_locking_column!(results, locking_column)
139
+ increment_locking_column!(table_name, results, locking_column)
136
140
  results.join( ',' )
137
141
  end
138
142
 
139
- def sql_for_on_duplicate_key_update_as_hash( locking_column, hsh ) # :nodoc:
143
+ def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
140
144
  results = hsh.map do |column1, column2|
141
- qc1 = quote_column_name( column1 )
142
- qc2 = quote_column_name( column2 )
145
+ original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
146
+ qc1 = quote_column_name( original_column1_name )
147
+
148
+ original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
149
+ qc2 = quote_column_name( original_column2_name )
150
+
143
151
  "#{qc1}=EXCLUDED.#{qc2}"
144
152
  end
145
- increment_locking_column!(results, locking_column)
153
+ increment_locking_column!(table_name, results, locking_column)
146
154
  results.join( ',' )
147
155
  end
148
156
 
@@ -150,9 +158,9 @@ module ActiveRecord::Import::SQLite3Adapter
150
158
  conflict_target = args[:conflict_target]
151
159
  index_predicate = args[:index_predicate]
152
160
  if conflict_target.present?
153
- '(' << Array( conflict_target ).reject( &:blank? ).join( ', ' ) << ') '.tap do |sql|
154
- sql << "WHERE #{index_predicate} " if index_predicate
155
- end
161
+ sql = "(#{Array( conflict_target ).reject( &:blank? ).join( ', ' )}) "
162
+ sql += "WHERE #{index_predicate} " if index_predicate
163
+ sql
156
164
  end
157
165
  end
158
166
 
@@ -166,14 +174,6 @@ module ActiveRecord::Import::SQLite3Adapter
166
174
  exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key')
167
175
  end
168
176
 
169
- def increment_locking_column!(results, locking_column)
170
- if locking_column.present?
171
- results << "\"#{locking_column}\"=EXCLUDED.\"#{locking_column}\"+1"
172
- end
173
- end
174
-
175
- private
176
-
177
177
  def database_version
178
178
  defined?(sqlite_version) ? sqlite_version : super
179
179
  end
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "activerecord-import/adapters/mysql_adapter"
4
+
5
+ module ActiveRecord::Import::TrilogyAdapter
6
+ include ActiveRecord::Import::MysqlAdapter
7
+ end
@@ -1,9 +1,11 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "pathname"
2
4
  require "active_record"
3
5
  require "active_record/version"
4
6
 
5
7
  module ActiveRecord::Import
6
- ADAPTER_PATH = "activerecord-import/active_record/adapters".freeze
8
+ ADAPTER_PATH = "activerecord-import/active_record/adapters"
7
9
 
8
10
  def self.base_adapter(adapter)
9
11
  case adapter
@@ -27,7 +29,13 @@ module ActiveRecord::Import
27
29
 
28
30
  # Loads the import functionality for the passed in ActiveRecord connection
29
31
  def self.load_from_connection_pool(connection_pool)
30
- require_adapter connection_pool.spec.config[:adapter]
32
+ adapter =
33
+ if connection_pool.respond_to?(:db_config) # ActiveRecord >= 6.1
34
+ connection_pool.db_config.adapter
35
+ else
36
+ connection_pool.spec.config[:adapter]
37
+ end
38
+ require_adapter adapter
31
39
  end
32
40
  end
33
41
 
@@ -1,18 +1,20 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "ostruct"
2
4
 
3
5
  module ActiveRecord::Import::ConnectionAdapters; end
4
6
 
5
- module ActiveRecord::Import #:nodoc:
7
+ module ActiveRecord::Import # :nodoc:
6
8
  Result = Struct.new(:failed_instances, :num_inserts, :ids, :results)
7
9
 
8
- module ImportSupport #:nodoc:
9
- def supports_import? #:nodoc:
10
+ module ImportSupport # :nodoc:
11
+ def supports_import? # :nodoc:
10
12
  true
11
13
  end
12
14
  end
13
15
 
14
- module OnDuplicateKeyUpdateSupport #:nodoc:
15
- def supports_on_duplicate_key_update? #:nodoc:
16
+ module OnDuplicateKeyUpdateSupport # :nodoc:
17
+ def supports_on_duplicate_key_update? # :nodoc:
16
18
  true
17
19
  end
18
20
  end
@@ -34,7 +36,7 @@ module ActiveRecord::Import #:nodoc:
34
36
  @validate_callbacks = klass._validate_callbacks.dup
35
37
 
36
38
  @validate_callbacks.each_with_index do |callback, i|
37
- filter = callback.raw_filter
39
+ filter = callback.respond_to?(:raw_filter) ? callback.raw_filter : callback.filter
38
40
  next unless filter.class.name =~ /Validations::PresenceValidator/ ||
39
41
  (!@options[:validate_uniqueness] &&
40
42
  filter.is_a?(ActiveRecord::Validations::UniquenessValidator))
@@ -49,13 +51,13 @@ module ActiveRecord::Import #:nodoc:
49
51
  associations = klass.reflect_on_all_associations(:belongs_to)
50
52
  associations.each do |assoc|
51
53
  if (index = attrs.index(assoc.name))
52
- key = assoc.foreign_key.to_sym
54
+ key = assoc.foreign_key.is_a?(Array) ? assoc.foreign_key.map(&:to_sym) : assoc.foreign_key.to_sym
53
55
  attrs[index] = key unless attrs.include?(key)
54
56
  end
55
57
  end
56
58
  end
57
59
 
58
- filter.instance_variable_set(:@attributes, attrs)
60
+ filter.instance_variable_set(:@attributes, attrs.flatten)
59
61
 
60
62
  if @validate_callbacks.respond_to?(:chain, true)
61
63
  @validate_callbacks.send(:chain).tap do |chain|
@@ -71,7 +73,7 @@ module ActiveRecord::Import #:nodoc:
71
73
  end
72
74
 
73
75
  def valid_model?(model)
74
- init_validations(model.class) unless model.class == @validator_class
76
+ init_validations(model.class) unless model.instance_of?(@validator_class)
75
77
 
76
78
  validation_context = @options[:validate_with_context]
77
79
  validation_context ||= (model.new_record? ? :create : :update)
@@ -83,7 +85,11 @@ module ActiveRecord::Import #:nodoc:
83
85
 
84
86
  model.run_callbacks(:validation) do
85
87
  if defined?(ActiveSupport::Callbacks::Filters::Environment) # ActiveRecord >= 4.1
86
- runner = @validate_callbacks.compile
88
+ runner = if @validate_callbacks.method(:compile).arity == 0
89
+ @validate_callbacks.compile
90
+ else # ActiveRecord >= 7.1
91
+ @validate_callbacks.compile(nil)
92
+ end
87
93
  env = ActiveSupport::Callbacks::Filters::Environment.new(model, false, nil)
88
94
  if runner.respond_to?(:call) # ActiveRecord < 5.1
89
95
  runner.call(env)
@@ -163,7 +169,7 @@ class ActiveRecord::Associations::CollectionAssociation
163
169
  m.public_send "#{reflection.type}=", owner.class.name if reflection.type
164
170
  end
165
171
 
166
- return model_klass.bulk_import column_names, models, options
172
+ model_klass.bulk_import column_names, models, options
167
173
 
168
174
  # supports array of hash objects
169
175
  elsif args.last.is_a?( Array ) && args.last.first.is_a?(Hash)
@@ -202,11 +208,11 @@ class ActiveRecord::Associations::CollectionAssociation
202
208
  end
203
209
  end
204
210
 
205
- return model_klass.bulk_import column_names, array_of_attributes, options
211
+ model_klass.bulk_import column_names, array_of_attributes, options
206
212
 
207
213
  # supports empty array
208
214
  elsif args.last.is_a?( Array ) && args.last.empty?
209
- return ActiveRecord::Import::Result.new([], 0, [])
215
+ ActiveRecord::Import::Result.new([], 0, [])
210
216
 
211
217
  # supports 2-element array and array
212
218
  elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
@@ -237,7 +243,7 @@ class ActiveRecord::Associations::CollectionAssociation
237
243
  end
238
244
  end
239
245
 
240
- return model_klass.bulk_import column_names, array_of_attributes, options
246
+ model_klass.bulk_import column_names, array_of_attributes, options
241
247
  else
242
248
  raise ArgumentError, "Invalid arguments!"
243
249
  end
@@ -245,16 +251,17 @@ class ActiveRecord::Associations::CollectionAssociation
245
251
  alias import bulk_import unless respond_to? :import
246
252
  end
247
253
 
254
+ module ActiveRecord::Import::Connection
255
+ def establish_connection(args = nil)
256
+ conn = super(args)
257
+ ActiveRecord::Import.load_from_connection_pool connection_pool
258
+ conn
259
+ end
260
+ end
261
+
248
262
  class ActiveRecord::Base
249
263
  class << self
250
- def establish_connection_with_activerecord_import(*args)
251
- conn = establish_connection_without_activerecord_import(*args)
252
- ActiveRecord::Import.load_from_connection_pool connection_pool
253
- conn
254
- end
255
-
256
- alias establish_connection_without_activerecord_import establish_connection
257
- alias establish_connection establish_connection_with_activerecord_import
264
+ prepend ActiveRecord::Import::Connection
258
265
 
259
266
  # Returns true if the current database connection adapter
260
267
  # supports import functionality, otherwise returns false.
@@ -546,11 +553,11 @@ class ActiveRecord::Base
546
553
  alias import! bulk_import! unless ActiveRecord::Base.respond_to? :import!
547
554
 
548
555
  def import_helper( *args )
549
- options = { validate: true, timestamps: true }
556
+ options = { model: self, validate: true, timestamps: true, track_validation_failures: false }
550
557
  options.merge!( args.pop ) if args.last.is_a? Hash
551
558
  # making sure that current model's primary key is used
552
559
  options[:primary_key] = primary_key
553
- options[:locking_column] = locking_column if attribute_names.include?(locking_column)
560
+ options[:locking_column] = locking_column if locking_enabled?
554
561
 
555
562
  is_validating = options[:validate_with_context].present? ? true : options[:validate]
556
563
  validator = ActiveRecord::Import::Validator.new(self, options)
@@ -571,7 +578,7 @@ class ActiveRecord::Base
571
578
 
572
579
  if models.first.id.nil?
573
580
  Array(primary_key).each do |c|
574
- if column_names.include?(c) && columns_hash[c].type == :uuid
581
+ if column_names.include?(c) && schema_columns_hash[c].type == :uuid
575
582
  column_names.delete(c)
576
583
  end
577
584
  end
@@ -581,7 +588,7 @@ class ActiveRecord::Base
581
588
  if respond_to?(:timestamp_attributes_for_update, true)
582
589
  send(:timestamp_attributes_for_update).map(&:to_sym)
583
590
  else
584
- new.send(:timestamp_attributes_for_update_in_model)
591
+ allocate.send(:timestamp_attributes_for_update_in_model)
585
592
  end
586
593
  end
587
594
 
@@ -630,7 +637,7 @@ class ActiveRecord::Base
630
637
  end
631
638
  # supports empty array
632
639
  elsif args.last.is_a?( Array ) && args.last.empty?
633
- return ActiveRecord::Import::Result.new([], 0, [])
640
+ return ActiveRecord::Import::Result.new([], 0, [], [])
634
641
  # supports 2-element array and array
635
642
  elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
636
643
 
@@ -694,7 +701,11 @@ class ActiveRecord::Base
694
701
  return_obj = if is_validating
695
702
  import_with_validations( column_names, array_of_attributes, options ) do |failed_instances|
696
703
  if models
697
- models.each { |m| failed_instances << m if m.errors.any? }
704
+ models.each_with_index do |m, i|
705
+ next unless m.errors.any?
706
+
707
+ failed_instances << (options[:track_validation_failures] ? [i, m] : m)
708
+ end
698
709
  else
699
710
  # create instances for each of our column/value sets
700
711
  arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
@@ -702,14 +713,18 @@ class ActiveRecord::Base
702
713
  # keep track of the instance and the position it is currently at. if this fails
703
714
  # validation we'll use the index to remove it from the array_of_attributes
704
715
  arr.each_with_index do |hsh, i|
705
- model = new
706
- hsh.each_pair { |k, v| model[k] = v }
716
+ # utilize block initializer syntax to prevent failure when 'mass_assignment_sanitizer = :strict'
717
+ model = new do |m|
718
+ hsh.each_pair { |k, v| m[k] = v }
719
+ end
720
+
707
721
  next if validator.valid_model?(model)
708
722
  raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
723
+
709
724
  array_of_attributes[i] = nil
710
725
  failure = model.dup
711
726
  failure.errors.send(:initialize_dup, model.errors)
712
- failed_instances << failure
727
+ failed_instances << (options[:track_validation_failures] ? [i, failure] : failure )
713
728
  end
714
729
  array_of_attributes.compact!
715
730
  end
@@ -729,7 +744,10 @@ class ActiveRecord::Base
729
744
  set_attributes_and_mark_clean(models, return_obj, timestamps, options)
730
745
 
731
746
  # if there are auto-save associations on the models we imported that are new, import them as well
732
- import_associations(models, options.dup) if options[:recursive]
747
+ if options[:recursive]
748
+ options[:on_duplicate_key_update] = on_duplicate_key_update unless on_duplicate_key_update.nil?
749
+ import_associations(models, options.dup.merge(validate: false))
750
+ end
733
751
  end
734
752
 
735
753
  return_obj
@@ -764,27 +782,29 @@ class ActiveRecord::Base
764
782
  def import_without_validations_or_callbacks( column_names, array_of_attributes, options = {} )
765
783
  return ActiveRecord::Import::Result.new([], 0, [], []) if array_of_attributes.empty?
766
784
 
767
- column_names = column_names.map(&:to_sym)
785
+ column_names = column_names.map do |name|
786
+ original_name = attribute_alias?(name) ? attribute_alias(name) : name
787
+ original_name.to_sym
788
+ end
768
789
  scope_columns, scope_values = scope_attributes.to_a.transpose
769
790
 
770
791
  unless scope_columns.blank?
771
792
  scope_columns.zip(scope_values).each do |name, value|
772
793
  name_as_sym = name.to_sym
773
- next if column_names.include?(name_as_sym)
774
-
775
- is_sti = (name_as_sym == inheritance_column.to_sym && self < base_class)
776
- value = Array(value).first if is_sti
777
-
794
+ next if column_names.include?(name_as_sym) || name_as_sym == inheritance_column.to_sym
778
795
  column_names << name_as_sym
779
796
  array_of_attributes.each { |attrs| attrs << value }
780
797
  end
781
798
  end
782
799
 
783
- columns = column_names.each_with_index.map do |name, i|
784
- column = columns_hash[name.to_s]
800
+ if finder_needs_type_condition? && !column_names.include?(inheritance_column.to_sym)
801
+ column_names << inheritance_column.to_sym
802
+ array_of_attributes.each { |attrs| attrs << sti_name }
803
+ end
785
804
 
805
+ columns = column_names.each_with_index.map do |name, i|
806
+ column = schema_columns_hash[name.to_s]
786
807
  raise ActiveRecord::Import::MissingColumnError.new(name.to_s, i) if column.nil?
787
-
788
808
  column
789
809
  end
790
810
 
@@ -800,17 +820,29 @@ class ActiveRecord::Base
800
820
  if supports_import?
801
821
  # generate the sql
802
822
  post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
823
+ import_size = values_sql.size
824
+
825
+ batch_size = options[:batch_size] || import_size
826
+ run_proc = options[:batch_size].to_i.positive? && options[:batch_progress].respond_to?( :call )
827
+ progress_proc = options[:batch_progress]
828
+ current_batch = 0
829
+ batches = (import_size / batch_size.to_f).ceil
803
830
 
804
- batch_size = options[:batch_size] || values_sql.size
805
831
  values_sql.each_slice(batch_size) do |batch_values|
832
+ batch_started_at = Time.now.to_i
833
+
806
834
  # perform the inserts
807
835
  result = connection.insert_many( [insert_sql, post_sql_statements].flatten,
808
836
  batch_values,
809
837
  options,
810
- "#{model_name} Create Many Without Validations Or Callbacks" )
838
+ "#{model_name} Create Many" )
839
+
811
840
  number_inserted += result.num_inserts
812
841
  ids += result.ids
813
842
  results += result.results
843
+ current_batch += 1
844
+
845
+ progress_proc.call(import_size, batches, current_batch, Time.now.to_i - batch_started_at) if run_proc
814
846
  end
815
847
  else
816
848
  transaction(requires_new: true) do
@@ -825,6 +857,15 @@ class ActiveRecord::Base
825
857
 
826
858
  private
827
859
 
860
+ def associated_options(options, associated_class)
861
+ return options unless options.key?(:recursive_on_duplicate_key_update)
862
+
863
+ table_name = associated_class.arel_table.name.to_sym
864
+ options.merge(
865
+ on_duplicate_key_update: options[:recursive_on_duplicate_key_update][table_name]
866
+ )
867
+ end
868
+
828
869
  def set_attributes_and_mark_clean(models, import_result, timestamps, options)
829
870
  return if models.nil?
830
871
  models -= import_result.failed_instances
@@ -836,22 +877,46 @@ class ActiveRecord::Base
836
877
  model.id = id
837
878
 
838
879
  timestamps.each do |attr, value|
839
- model.send(attr + "=", value)
880
+ model.send("#{attr}=", value) if model.send(attr).nil?
840
881
  end
841
882
  end
842
883
  end
843
884
 
844
- if models.size == import_result.results.size
845
- columns = Array(options[:returning])
846
- single_column = "#{columns.first}=" if columns.size == 1
847
- import_result.results.each_with_index do |result, index|
885
+ deserialize_value = lambda do |column, value|
886
+ column = schema_columns_hash[column]
887
+ return value unless column
888
+ if respond_to?(:type_caster)
889
+ type = type_for_attribute(column.name)
890
+ type.deserialize(value)
891
+ elsif column.respond_to?(:type_cast_from_database)
892
+ column.type_cast_from_database(value)
893
+ else
894
+ value
895
+ end
896
+ end
897
+
898
+ set_value = lambda do |model, column, value|
899
+ val = deserialize_value.call(column, value)
900
+ if model.attribute_names.include?(column)
901
+ model.send("#{column}=", val)
902
+ else
903
+ attributes = attributes_builder.build_from_database(model.attributes.merge(column => val))
904
+ model.instance_variable_set(:@attributes, attributes)
905
+ end
906
+ end
907
+
908
+ columns = Array(options[:returning_columns])
909
+ results = Array(import_result.results)
910
+ if models.size == results.size
911
+ single_column = columns.first if columns.size == 1
912
+ results.each_with_index do |result, index|
848
913
  model = models[index]
849
914
 
850
915
  if single_column
851
- model.send(single_column, result)
916
+ set_value.call(model, single_column, result)
852
917
  else
853
918
  columns.each_with_index do |column, col_index|
854
- model.send("#{column}=", result[col_index])
919
+ set_value.call(model, column, result[col_index])
855
920
  end
856
921
  end
857
922
  end
@@ -872,16 +937,22 @@ class ActiveRecord::Base
872
937
 
873
938
  # Sync belongs_to association ids with foreign key field
874
939
  def load_association_ids(model)
940
+ changed_columns = model.changed
875
941
  association_reflections = model.class.reflect_on_all_associations(:belongs_to)
876
942
  association_reflections.each do |association_reflection|
877
- column_name = association_reflection.foreign_key
878
943
  next if association_reflection.options[:polymorphic]
879
- association = model.association(association_reflection.name)
880
- association = association.target
881
- next if association.blank? || model.public_send(column_name).present?
882
944
 
883
- association_primary_key = association_reflection.association_primary_key
884
- model.public_send("#{column_name}=", association.send(association_primary_key))
945
+ column_names = Array(association_reflection.foreign_key).map(&:to_s)
946
+ column_names.each_with_index do |column_name, column_index|
947
+ next if changed_columns.include?(column_name)
948
+
949
+ association = model.association(association_reflection.name)
950
+ association = association.target
951
+ next if association.blank? || model.public_send(column_name).present?
952
+
953
+ association_primary_key = Array(association_reflection.association_primary_key.tr("[]:", "").split(", "))[column_index]
954
+ model.public_send("#{column_name}=", association.send(association_primary_key))
955
+ end
885
956
  end
886
957
  end
887
958
 
@@ -894,17 +965,30 @@ class ActiveRecord::Base
894
965
  associated_objects_by_class = {}
895
966
  models.each { |model| find_associated_objects_for_import(associated_objects_by_class, model) }
896
967
 
897
- # :on_duplicate_key_update and :returning not supported for associations
898
- options.delete(:on_duplicate_key_update)
968
+ # :on_duplicate_key_update only supported for all fields
969
+ options.delete(:on_duplicate_key_update) unless options[:on_duplicate_key_update] == :all
970
+ # :returning not supported for associations
899
971
  options.delete(:returning)
900
972
 
901
973
  associated_objects_by_class.each_value do |associations|
902
974
  associations.each_value do |associated_records|
903
- associated_records.first.class.bulk_import(associated_records, options) unless associated_records.empty?
975
+ next if associated_records.empty?
976
+
977
+ associated_class = associated_records.first.class
978
+ associated_class.bulk_import(associated_records,
979
+ associated_options(options, associated_class))
904
980
  end
905
981
  end
906
982
  end
907
983
 
984
+ def schema_columns_hash
985
+ if respond_to?(:ignored_columns) && ignored_columns.any?
986
+ connection.schema_cache.columns_hash(table_name)
987
+ else
988
+ columns_hash
989
+ end
990
+ end
991
+
908
992
  # We are eventually going to call Class.import <objects> so we build up a hash
909
993
  # of class => objects to import.
910
994
  def find_associated_objects_for_import(associated_objects_by_class, model)
@@ -925,10 +1009,18 @@ class ActiveRecord::Base
925
1009
 
926
1010
  changed_objects = association.select { |a| a.new_record? || a.changed? }
927
1011
  changed_objects.each do |child|
928
- child.public_send("#{association_reflection.foreign_key}=", model.id)
1012
+ Array(association_reflection.inverse_of&.foreign_key || association_reflection.foreign_key).each_with_index do |column, index|
1013
+ child.public_send("#{column}=", Array(model.id)[index])
1014
+ end
1015
+
929
1016
  # For polymorphic associations
1017
+ association_name = if model.class.respond_to?(:polymorphic_name)
1018
+ model.class.polymorphic_name
1019
+ else
1020
+ model.class.base_class
1021
+ end
930
1022
  association_reflection.type.try do |type|
931
- child.public_send("#{type}=", model.class.base_class.name)
1023
+ child.public_send("#{type}=", association_name)
932
1024
  end
933
1025
  end
934
1026
  associated_objects_by_class[model.class.name][association_reflection.name].concat changed_objects
@@ -955,7 +1047,7 @@ class ActiveRecord::Base
955
1047
  elsif column
956
1048
  if respond_to?(:type_caster) # Rails 5.0 and higher
957
1049
  type = type_for_attribute(column.name)
958
- val = type.type == :boolean ? type.cast(val) : type.serialize(val)
1050
+ val = !type.respond_to?(:subtype) && type.type == :boolean ? type.cast(val) : type.serialize(val)
959
1051
  connection_memo.quote(val)
960
1052
  elsif column.respond_to?(:type_cast_from_user) # Rails 4.2
961
1053
  connection_memo.quote(column.type_cast_from_user(val), column)
@@ -964,7 +1056,7 @@ class ActiveRecord::Base
964
1056
  val = serialized_attributes[column.name].dump(val)
965
1057
  end
966
1058
  # Fixes #443 to support binary (i.e. bytea) columns on PG
967
- val = column.type_cast(val) unless column.type.to_sym == :binary
1059
+ val = column.type_cast(val) unless column.type && column.type.to_sym == :binary
968
1060
  connection_memo.quote(val, column)
969
1061
  end
970
1062
  else
@@ -983,13 +1075,18 @@ class ActiveRecord::Base
983
1075
  timestamp_columns[:create] = timestamp_attributes_for_create_in_model
984
1076
  timestamp_columns[:update] = timestamp_attributes_for_update_in_model
985
1077
  else
986
- instance = new
1078
+ instance = allocate
987
1079
  timestamp_columns[:create] = instance.send(:timestamp_attributes_for_create_in_model)
988
1080
  timestamp_columns[:update] = instance.send(:timestamp_attributes_for_update_in_model)
989
1081
  end
990
1082
 
991
1083
  # use tz as set in ActiveRecord::Base
992
- timestamp = ActiveRecord::Base.default_timezone == :utc ? Time.now.utc : Time.now
1084
+ default_timezone = if ActiveRecord.respond_to?(:default_timezone)
1085
+ ActiveRecord.default_timezone
1086
+ else
1087
+ ActiveRecord::Base.default_timezone
1088
+ end
1089
+ timestamp = default_timezone == :utc ? Time.now.utc : Time.now
993
1090
 
994
1091
  [:create, :update].each do |action|
995
1092
  timestamp_columns[action].each do |column|
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  warn <<-MSG
2
4
  [DEPRECATION] loading activerecord-import via 'require "activerecord-import/<adapter-name>"'
3
5
  is deprecated. Update to autorequire using 'require "activerecord-import"'. See