activerecord-import 0.17.2 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +1 -0
  3. data/.travis.yml +40 -23
  4. data/CHANGELOG.md +315 -1
  5. data/Gemfile +23 -13
  6. data/LICENSE +21 -56
  7. data/README.markdown +564 -33
  8. data/Rakefile +2 -1
  9. data/activerecord-import.gemspec +3 -3
  10. data/benchmarks/lib/cli_parser.rb +2 -1
  11. data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +0 -0
  12. data/gemfiles/5.1.gemfile +2 -0
  13. data/gemfiles/5.2.gemfile +2 -0
  14. data/gemfiles/6.0.gemfile +2 -0
  15. data/gemfiles/6.1.gemfile +1 -0
  16. data/lib/activerecord-import.rb +2 -15
  17. data/lib/activerecord-import/adapters/abstract_adapter.rb +9 -3
  18. data/lib/activerecord-import/adapters/mysql_adapter.rb +17 -11
  19. data/lib/activerecord-import/adapters/postgresql_adapter.rb +68 -20
  20. data/lib/activerecord-import/adapters/sqlite3_adapter.rb +128 -9
  21. data/lib/activerecord-import/base.rb +12 -7
  22. data/lib/activerecord-import/import.rb +514 -166
  23. data/lib/activerecord-import/synchronize.rb +2 -2
  24. data/lib/activerecord-import/value_sets_parser.rb +16 -0
  25. data/lib/activerecord-import/version.rb +1 -1
  26. data/test/adapters/makara_postgis.rb +1 -0
  27. data/test/import_test.rb +274 -23
  28. data/test/makara_postgis/import_test.rb +8 -0
  29. data/test/models/account.rb +3 -0
  30. data/test/models/animal.rb +6 -0
  31. data/test/models/bike_maker.rb +7 -0
  32. data/test/models/tag.rb +1 -1
  33. data/test/models/topic.rb +14 -0
  34. data/test/models/user.rb +3 -0
  35. data/test/models/user_token.rb +4 -0
  36. data/test/schema/generic_schema.rb +30 -8
  37. data/test/schema/mysql2_schema.rb +19 -0
  38. data/test/schema/postgresql_schema.rb +18 -0
  39. data/test/schema/sqlite3_schema.rb +13 -0
  40. data/test/support/factories.rb +9 -8
  41. data/test/support/generate.rb +6 -6
  42. data/test/support/mysql/import_examples.rb +14 -2
  43. data/test/support/postgresql/import_examples.rb +220 -1
  44. data/test/support/shared_examples/on_duplicate_key_ignore.rb +15 -9
  45. data/test/support/shared_examples/on_duplicate_key_update.rb +271 -8
  46. data/test/support/shared_examples/recursive_import.rb +91 -21
  47. data/test/support/sqlite3/import_examples.rb +189 -25
  48. data/test/synchronize_test.rb +8 -0
  49. data/test/test_helper.rb +24 -3
  50. data/test/value_sets_bytes_parser_test.rb +13 -2
  51. metadata +32 -13
  52. data/test/schema/mysql_schema.rb +0 -16
@@ -31,7 +31,7 @@ module ActiveRecord # :nodoc:
31
31
 
32
32
  klass = instances.first.class
33
33
 
34
- fresh_instances = klass.where(conditions).order(order)
34
+ fresh_instances = klass.unscoped.where(conditions).order(order)
35
35
  instances.each do |instance|
36
36
  matched_instance = fresh_instances.detect do |fresh_instance|
37
37
  keys.all? { |key| fresh_instance.send(key) == instance.send(key) }
@@ -39,8 +39,8 @@ module ActiveRecord # :nodoc:
39
39
 
40
40
  next unless matched_instance
41
41
 
42
- instance.send :clear_aggregation_cache
43
42
  instance.send :clear_association_cache
43
+ instance.send :clear_aggregation_cache if instance.respond_to?(:clear_aggregation_cache, true)
44
44
  instance.instance_variable_set :@attributes, matched_instance.instance_variable_get(:@attributes)
45
45
 
46
46
  if instance.respond_to?(:clear_changes_information)
@@ -1,4 +1,14 @@
1
+ require 'active_support/core_ext/array'
2
+
1
3
  module ActiveRecord::Import
4
+ class ValueSetTooLargeError < StandardError
5
+ attr_reader :size
6
+ def initialize(msg = "Value set exceeds max size", size = 0)
7
+ @size = size
8
+ super(msg)
9
+ end
10
+ end
11
+
2
12
  class ValueSetsBytesParser
3
13
  attr_reader :reserved_bytes, :max_bytes, :values
4
14
 
@@ -18,6 +28,12 @@ module ActiveRecord::Import
18
28
  current_size = 0
19
29
  values.each_with_index do |val, i|
20
30
  comma_bytes = arr.size
31
+ insert_size = reserved_bytes + val.bytesize
32
+
33
+ if insert_size > max_bytes
34
+ raise ValueSetTooLargeError.new("#{insert_size} bytes exceeds the max allowed for an insert [#{@max_bytes}]", insert_size)
35
+ end
36
+
21
37
  bytes_thus_far = reserved_bytes + current_size + val.bytesize + comma_bytes
22
38
  if bytes_thus_far <= max_bytes
23
39
  current_size += val.bytesize
@@ -1,5 +1,5 @@
1
1
  module ActiveRecord
2
2
  module Import
3
- VERSION = "0.17.2".freeze
3
+ VERSION = "1.1.0".freeze
4
4
  end
5
5
  end
@@ -0,0 +1 @@
1
+ ENV["ARE_DB"] = "postgis"
data/test/import_test.rb CHANGED
@@ -17,6 +17,11 @@ describe "#import" do
17
17
  assert_equal error.message, "Last argument should be a two dimensional array '[[]]'. First element in array was a String"
18
18
  end
19
19
 
20
+ it "warns you that you're passing more data than you ought to" do
21
+ error = assert_raise(ArgumentError) { Topic.import %w(title author_name), [['Author #1', 'Book #1', 0]] }
22
+ assert_equal error.message, "Number of values (8) exceeds number of columns (7)"
23
+ end
24
+
20
25
  it "should not produce an error when importing empty arrays" do
21
26
  assert_nothing_raised do
22
27
  Topic.import []
@@ -86,23 +91,73 @@ describe "#import" do
86
91
  assert_nil t.author_email_address
87
92
  end
88
93
  end
89
- end
90
94
 
91
- describe "with composite primary keys" do
92
- it "should import models successfully" do
93
- tags = [Tag.new(tag_id: 1, publisher_id: 1, tag: 'Mystery')]
95
+ context "with extra keys" do
96
+ let(:values) do
97
+ [
98
+ { title: "LDAP", author_name: "Jerry Carter" },
99
+ { title: "Rails Recipes", author_name: "Chad Fowler", author_email_address: "cfowler@test.com" } # author_email_address is unknown
100
+ ]
101
+ end
102
+
103
+ it "should fail when column names are not specified" do
104
+ err = assert_raises ArgumentError do
105
+ Topic.import values, validate: false
106
+ end
107
+
108
+ assert err.message.include? 'Extra keys: [:author_email_address]'
109
+ end
110
+
111
+ it "should succeed when column names are specified" do
112
+ assert_difference "Topic.count", +2 do
113
+ Topic.import columns, values, validate: false
114
+ end
115
+ end
116
+ end
117
+
118
+ context "with missing keys" do
119
+ let(:values) do
120
+ [
121
+ { title: "LDAP", author_name: "Jerry Carter" },
122
+ { title: "Rails Recipes" } # author_name is missing
123
+ ]
124
+ end
125
+
126
+ it "should fail when column names are not specified" do
127
+ err = assert_raises ArgumentError do
128
+ Topic.import values, validate: false
129
+ end
130
+
131
+ assert err.message.include? 'Missing keys: [:author_name]'
132
+ end
133
+
134
+ it "should fail on missing hash key from specified column names" do
135
+ err = assert_raises ArgumentError do
136
+ Topic.import %i(author_name), values, validate: false
137
+ end
94
138
 
95
- assert_difference "Tag.count", +1 do
96
- Tag.import tags
139
+ assert err.message.include? 'Missing keys: [:author_name]'
97
140
  end
98
141
  end
142
+ end
143
+
144
+ unless ENV["SKIP_COMPOSITE_PK"]
145
+ describe "with composite primary keys" do
146
+ it "should import models successfully" do
147
+ tags = [Tag.new(tag_id: 1, publisher_id: 1, tag: 'Mystery')]
148
+
149
+ assert_difference "Tag.count", +1 do
150
+ Tag.import tags
151
+ end
152
+ end
99
153
 
100
- it "should import array of values successfully" do
101
- columns = [:tag_id, :publisher_id, :tag]
102
- values = [[1, 1, 'Mystery'], [2, 1, 'Science']]
154
+ it "should import array of values successfully" do
155
+ columns = [:tag_id, :publisher_id, :tag]
156
+ values = [[1, 1, 'Mystery'], [2, 1, 'Science']]
103
157
 
104
- assert_difference "Tag.count", +2 do
105
- Tag.import columns, values, validate: false
158
+ assert_difference "Tag.count", +2 do
159
+ Tag.import columns, values, validate: false
160
+ end
106
161
  end
107
162
  end
108
163
  end
@@ -119,12 +174,12 @@ describe "#import" do
119
174
  end
120
175
 
121
176
  context "with :validation option" do
122
- let(:columns) { %w(title author_name) }
123
- let(:valid_values) { [["LDAP", "Jerry Carter"], ["Rails Recipes", "Chad Fowler"]] }
124
- let(:valid_values_with_context) { [[1111, "Jerry Carter"], [2222, "Chad Fowler"]] }
125
- let(:invalid_values) { [["The RSpec Book", ""], ["Agile+UX", ""]] }
126
- let(:valid_models) { valid_values.map { |title, author_name| Topic.new(title: title, author_name: author_name) } }
127
- let(:invalid_models) { invalid_values.map { |title, author_name| Topic.new(title: title, author_name: author_name) } }
177
+ let(:columns) { %w(title author_name content) }
178
+ let(:valid_values) { [["LDAP", "Jerry Carter", "Putting Directories to Work."], ["Rails Recipes", "Chad Fowler", "A trusted collection of solutions."]] }
179
+ let(:valid_values_with_context) { [[1111, "Jerry Carter", "1111"], [2222, "Chad Fowler", "2222"]] }
180
+ let(:invalid_values) { [["The RSpec Book", "David Chelimsky", "..."], ["Agile+UX", "", "All about Agile in UX."]] }
181
+ let(:valid_models) { valid_values.map { |title, author_name, content| Topic.new(title: title, author_name: author_name, content: content) } }
182
+ let(:invalid_models) { invalid_values.map { |title, author_name, content| Topic.new(title: title, author_name: author_name, content: content) } }
128
183
 
129
184
  context "with validation checks turned off" do
130
185
  it "should import valid data" do
@@ -159,6 +214,22 @@ describe "#import" do
159
214
  end
160
215
  end
161
216
 
217
+ it "should ignore uniqueness validators" do
218
+ Topic.import columns, valid_values
219
+ assert_difference "Topic.count", +2 do
220
+ Topic.import columns, valid_values
221
+ end
222
+ end
223
+
224
+ it "should not alter the callback chain of the model" do
225
+ attributes = columns.zip(valid_values.first).to_h
226
+ topic = Topic.new attributes
227
+ Topic.import [topic], validate: true
228
+ duplicate_topic = Topic.new attributes
229
+ Topic.import [duplicate_topic], validate: true
230
+ assert duplicate_topic.invalid?
231
+ end
232
+
162
233
  it "should not import invalid data" do
163
234
  assert_no_difference "Topic.count" do
164
235
  Topic.import columns, invalid_values, validate: true
@@ -181,8 +252,18 @@ describe "#import" do
181
252
  end
182
253
  end
183
254
 
255
+ it "should index the failed instances by their poistion in the set if `track_failures` is true" do
256
+ index_offset = valid_values.length
257
+ results = Topic.import columns, valid_values + invalid_values, validate: true, track_validation_failures: true
258
+ assert_equal invalid_values.size, results.failed_instances.size
259
+ invalid_values.each_with_index do |value_set, index|
260
+ assert_equal index + index_offset, results.failed_instances[index].first
261
+ assert_equal value_set.first, results.failed_instances[index].last.title
262
+ end
263
+ end
264
+
184
265
  it "should set ids in valid models if adapter supports setting primary key of imported objects" do
185
- if ActiveRecord::Base.support_setting_primary_key_of_imported_objects?
266
+ if ActiveRecord::Base.supports_setting_primary_key_of_imported_objects?
186
267
  Topic.import (invalid_models + valid_models), validate: true
187
268
  assert_nil invalid_models[0].id
188
269
  assert_nil invalid_models[1].id
@@ -192,7 +273,7 @@ describe "#import" do
192
273
  end
193
274
 
194
275
  it "should set ActiveRecord timestamps in valid models if adapter supports setting primary key of imported objects" do
195
- if ActiveRecord::Base.support_setting_primary_key_of_imported_objects?
276
+ if ActiveRecord::Base.supports_setting_primary_key_of_imported_objects?
196
277
  Timecop.freeze(Time.at(0)) do
197
278
  Topic.import (invalid_models + valid_models), validate: true
198
279
  end
@@ -215,6 +296,48 @@ describe "#import" do
215
296
  end
216
297
  assert_equal 0, Topic.where(title: invalid_values.map(&:first)).count
217
298
  end
299
+
300
+ it "should run callbacks" do
301
+ assert_no_difference "Topic.count" do
302
+ Topic.import columns, [["invalid", "Jerry Carter"]], validate: true
303
+ end
304
+ end
305
+
306
+ it "should call validation methods" do
307
+ assert_no_difference "Topic.count" do
308
+ Topic.import columns, [["validate_failed", "Jerry Carter"]], validate: true
309
+ end
310
+ end
311
+ end
312
+
313
+ context "with uniqueness validators included" do
314
+ it "should not import duplicate records" do
315
+ Topic.import columns, valid_values
316
+ assert_no_difference "Topic.count" do
317
+ Topic.import columns, valid_values, validate_uniqueness: true
318
+ end
319
+ end
320
+ end
321
+
322
+ context "when validatoring presence of belongs_to association" do
323
+ it "should not import records without foreign key" do
324
+ assert_no_difference "UserToken.count" do
325
+ UserToken.import [:token], [['12345abcdef67890']]
326
+ end
327
+ end
328
+
329
+ it "should import records with foreign key" do
330
+ assert_difference "UserToken.count", +1 do
331
+ UserToken.import [:user_name, :token], [%w("Bob", "12345abcdef67890")]
332
+ end
333
+ end
334
+
335
+ it "should not mutate the defined validations" do
336
+ UserToken.import [:user_name, :token], [%w("Bob", "12345abcdef67890")]
337
+ ut = UserToken.new
338
+ ut.valid?
339
+ assert_includes ut.errors.messages, :user
340
+ end
218
341
  end
219
342
  end
220
343
 
@@ -282,6 +405,15 @@ describe "#import" do
282
405
  assert_equal 3, result.num_inserts if Topic.supports_import?
283
406
  end
284
407
  end
408
+
409
+ it "should accept and call an optional callable to run after each batch" do
410
+ lambda_called = 0
411
+
412
+ my_proc = ->(_row_count, _batches, _batch, _duration) { lambda_called += 1 }
413
+ Topic.import Build(10, :topics), batch_size: 4, batch_progress: my_proc
414
+
415
+ assert_equal 3, lambda_called
416
+ end
285
417
  end
286
418
 
287
419
  context "with :synchronize option" do
@@ -290,7 +422,7 @@ describe "#import" do
290
422
 
291
423
  it "doesn't reload any data (doesn't work)" do
292
424
  Topic.import new_topics, synchronize: new_topics
293
- if Topic.support_setting_primary_key_of_imported_objects?
425
+ if Topic.supports_setting_primary_key_of_imported_objects?
294
426
  assert new_topics.all?(&:persisted?), "Records should have been reloaded"
295
427
  else
296
428
  assert new_topics.all?(&:new_record?), "No record should have been reloaded"
@@ -412,11 +544,11 @@ describe "#import" do
412
544
 
413
545
  context "when the timestamps columns are present" do
414
546
  setup do
415
- @existing_book = Book.create(title: "Fell", author_name: "Curry", publisher: "Bayer", created_at: 2.years.ago.utc, created_on: 2.years.ago.utc)
547
+ @existing_book = Book.create(title: "Fell", author_name: "Curry", publisher: "Bayer", created_at: 2.years.ago.utc, created_on: 2.years.ago.utc, updated_at: 2.years.ago.utc, updated_on: 2.years.ago.utc)
416
548
  ActiveRecord::Base.default_timezone = :utc
417
549
  Timecop.freeze(time) do
418
550
  assert_difference "Book.count", +2 do
419
- Book.import %w(title author_name publisher created_at created_on), [["LDAP", "Big Bird", "Del Rey", nil, nil], [@existing_book.title, @existing_book.author_name, @existing_book.publisher, @existing_book.created_at, @existing_book.created_on]]
551
+ Book.import %w(title author_name publisher created_at created_on updated_at updated_on), [["LDAP", "Big Bird", "Del Rey", nil, nil, nil, nil], [@existing_book.title, @existing_book.author_name, @existing_book.publisher, @existing_book.created_at, @existing_book.created_on, @existing_book.updated_at, @existing_book.updated_on]]
420
552
  end
421
553
  end
422
554
  @new_book, @existing_book = Book.last 2
@@ -445,6 +577,23 @@ describe "#import" do
445
577
  it "should set the updated_on column for new records" do
446
578
  assert_in_delta time.to_i, @new_book.updated_on.to_i, 1.second
447
579
  end
580
+
581
+ it "should not set the updated_at column for existing records" do
582
+ assert_equal 2.years.ago.utc.strftime("%Y:%d"), @existing_book.updated_at.strftime("%Y:%d")
583
+ end
584
+
585
+ it "should not set the updated_on column for existing records" do
586
+ assert_equal 2.years.ago.utc.strftime("%Y:%d"), @existing_book.updated_on.strftime("%Y:%d")
587
+ end
588
+
589
+ it "should not set the updated_at column on models if changed" do
590
+ timestamp = Time.now.utc
591
+ books = [
592
+ Book.new(author_name: "Foo", title: "Baz", created_at: timestamp, updated_at: timestamp)
593
+ ]
594
+ Book.import books
595
+ assert_equal timestamp.strftime("%Y:%d"), Book.last.updated_at.strftime("%Y:%d")
596
+ end
448
597
  end
449
598
 
450
599
  context "when a custom time zone is set" do
@@ -489,7 +638,7 @@ describe "#import" do
489
638
 
490
639
  context "importing through an association scope" do
491
640
  { has_many: :chapters, polymorphic: :discounts }.each do |association_type, association|
492
- book = FactoryGirl.create :book
641
+ book = FactoryBot.create :book
493
642
  scope = book.public_send association
494
643
  klass = { chapters: Chapter, discounts: Discount }[association]
495
644
  column = { chapters: :title, discounts: :amount }[association]
@@ -511,10 +660,35 @@ describe "#import" do
511
660
 
512
661
  assert_equal [val1, val2], scope.map(&column).sort
513
662
  end
663
+
664
+ it "works importing array of hashes" do
665
+ scope.import [{ column => val1 }, { column => val2 }]
666
+
667
+ assert_equal [val1, val2], scope.map(&column).sort
668
+ end
669
+ end
670
+
671
+ it "works with a non-standard association primary key" do
672
+ user = User.create(id: 1, name: 'Solomon')
673
+ user.user_tokens.import [:id, :token], [[5, '12345abcdef67890']]
674
+
675
+ token = UserToken.find(5)
676
+ assert_equal 'Solomon', token.user_name
514
677
  end
515
678
  end
516
679
  end
517
680
 
681
+ context "importing model with polymorphic belongs_to" do
682
+ it "works without error" do
683
+ book = FactoryBot.create :book
684
+ discount = Discount.new(discountable: book)
685
+
686
+ Discount.import([discount])
687
+
688
+ assert_equal 1, Discount.count
689
+ end
690
+ end
691
+
518
692
  context 'When importing models with Enum fields' do
519
693
  it 'should be able to import enum fields' do
520
694
  Book.delete_all if Book.count > 0
@@ -589,6 +763,30 @@ describe "#import" do
589
763
  end
590
764
  end
591
765
 
766
+ context 'importing arrays of values with boolean fields' do
767
+ let(:columns) { [:author_name, :title, :for_sale] }
768
+
769
+ it 'should be able to coerce integers as boolean fields' do
770
+ Book.delete_all if Book.count > 0
771
+ values = [['Author #1', 'Book #1', 0], ['Author #2', 'Book #2', 1]]
772
+ assert_difference "Book.count", +2 do
773
+ Book.import columns, values
774
+ end
775
+ assert_equal false, Book.first.for_sale
776
+ assert_equal true, Book.last.for_sale
777
+ end
778
+
779
+ it 'should be able to coerce strings as boolean fields' do
780
+ Book.delete_all if Book.count > 0
781
+ values = [['Author #1', 'Book #1', 'false'], ['Author #2', 'Book #2', 'true']]
782
+ assert_difference "Book.count", +2 do
783
+ Book.import columns, values
784
+ end
785
+ assert_equal false, Book.first.for_sale
786
+ assert_equal true, Book.last.for_sale
787
+ end
788
+ end
789
+
592
790
  describe "importing when model has default_scope" do
593
791
  it "doesn't import the default scope values" do
594
792
  assert_difference "Widget.unscoped.count", +2 do
@@ -638,6 +836,16 @@ describe "#import" do
638
836
  assert_equal(data.as_json, Widget.find_by_w_id(9).json_data)
639
837
  end
640
838
 
839
+ it "imports serialized values from saved records" do
840
+ Widget.import [:w_id, :json_data], [[1, data]]
841
+ assert_equal data.as_json, Widget.last.json_data
842
+
843
+ w = Widget.last
844
+ w.w_id = 2
845
+ Widget.import([w])
846
+ assert_equal data.as_json, Widget.last.json_data
847
+ end
848
+
641
849
  context "with a store" do
642
850
  it "imports serialized attributes set using accessors" do
643
851
  vendors = [Vendor.new(name: 'Vendor 1', color: 'blue')]
@@ -696,5 +904,48 @@ describe "#import" do
696
904
  end
697
905
  end
698
906
  end
907
+
908
+ context "with objects that respond to .to_sql as values" do
909
+ let(:columns) { %w(title author_name) }
910
+ let(:valid_values) { [["LDAP", Book.select("'Jerry Carter'").limit(1)], ["Rails Recipes", Book.select("'Chad Fowler'").limit(1)]] }
911
+
912
+ it "should import data" do
913
+ assert_difference "Topic.count", +2 do
914
+ Topic.import! columns, valid_values
915
+ topics = Topic.all
916
+ assert_equal "Jerry Carter", topics.first.author_name
917
+ assert_equal "Chad Fowler", topics.last.author_name
918
+ end
919
+ end
920
+ end
921
+ end
922
+ describe "importing model with after_initialize callback" do
923
+ let(:columns) { %w(name size) }
924
+ let(:valid_values) { [%w("Deer", "Small"), %w("Monkey", "Medium")] }
925
+ let(:invalid_values) do
926
+ [
927
+ { name: "giraffe", size: "Large" },
928
+ { size: "Medium" } # name is missing
929
+ ]
930
+ end
931
+ context "with validation checks turned off" do
932
+ it "should import valid data" do
933
+ Animal.import(columns, valid_values, validate: false)
934
+ assert_equal 2, Animal.count
935
+ end
936
+ it "should raise ArgumentError" do
937
+ assert_raise(ArgumentError) { Animal.import(invalid_values, validate: false) }
938
+ end
939
+ end
940
+
941
+ context "with validation checks turned on" do
942
+ it "should import valid data" do
943
+ Animal.import(columns, valid_values, validate: true)
944
+ assert_equal 2, Animal.count
945
+ end
946
+ it "should raise ArgumentError" do
947
+ assert_raise(ArgumentError) { Animal.import(invalid_values, validate: true) }
948
+ end
949
+ end
699
950
  end
700
951
  end