activerecord-import 0.17.2 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +1 -0
  3. data/.travis.yml +40 -23
  4. data/CHANGELOG.md +315 -1
  5. data/Gemfile +23 -13
  6. data/LICENSE +21 -56
  7. data/README.markdown +564 -33
  8. data/Rakefile +2 -1
  9. data/activerecord-import.gemspec +3 -3
  10. data/benchmarks/lib/cli_parser.rb +2 -1
  11. data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +0 -0
  12. data/gemfiles/5.1.gemfile +2 -0
  13. data/gemfiles/5.2.gemfile +2 -0
  14. data/gemfiles/6.0.gemfile +2 -0
  15. data/gemfiles/6.1.gemfile +1 -0
  16. data/lib/activerecord-import.rb +2 -15
  17. data/lib/activerecord-import/adapters/abstract_adapter.rb +9 -3
  18. data/lib/activerecord-import/adapters/mysql_adapter.rb +17 -11
  19. data/lib/activerecord-import/adapters/postgresql_adapter.rb +68 -20
  20. data/lib/activerecord-import/adapters/sqlite3_adapter.rb +128 -9
  21. data/lib/activerecord-import/base.rb +12 -7
  22. data/lib/activerecord-import/import.rb +514 -166
  23. data/lib/activerecord-import/synchronize.rb +2 -2
  24. data/lib/activerecord-import/value_sets_parser.rb +16 -0
  25. data/lib/activerecord-import/version.rb +1 -1
  26. data/test/adapters/makara_postgis.rb +1 -0
  27. data/test/import_test.rb +274 -23
  28. data/test/makara_postgis/import_test.rb +8 -0
  29. data/test/models/account.rb +3 -0
  30. data/test/models/animal.rb +6 -0
  31. data/test/models/bike_maker.rb +7 -0
  32. data/test/models/tag.rb +1 -1
  33. data/test/models/topic.rb +14 -0
  34. data/test/models/user.rb +3 -0
  35. data/test/models/user_token.rb +4 -0
  36. data/test/schema/generic_schema.rb +30 -8
  37. data/test/schema/mysql2_schema.rb +19 -0
  38. data/test/schema/postgresql_schema.rb +18 -0
  39. data/test/schema/sqlite3_schema.rb +13 -0
  40. data/test/support/factories.rb +9 -8
  41. data/test/support/generate.rb +6 -6
  42. data/test/support/mysql/import_examples.rb +14 -2
  43. data/test/support/postgresql/import_examples.rb +220 -1
  44. data/test/support/shared_examples/on_duplicate_key_ignore.rb +15 -9
  45. data/test/support/shared_examples/on_duplicate_key_update.rb +271 -8
  46. data/test/support/shared_examples/recursive_import.rb +91 -21
  47. data/test/support/sqlite3/import_examples.rb +189 -25
  48. data/test/synchronize_test.rb +8 -0
  49. data/test/test_helper.rb +24 -3
  50. data/test/value_sets_bytes_parser_test.rb +13 -2
  51. metadata +32 -13
  52. data/test/schema/mysql_schema.rb +0 -16
@@ -8,19 +8,23 @@ def should_support_on_duplicate_key_ignore
8
8
  it "should skip duplicates and continue import" do
9
9
  topics << Topic.new(title: "Book 2", author_name: "Jane Doe")
10
10
  assert_difference "Topic.count", +1 do
11
- Topic.import topics, on_duplicate_key_ignore: true, validate: false
11
+ result = Topic.import topics, on_duplicate_key_ignore: true, validate: false
12
+ assert_not_equal topics.first.id, result.ids.first
13
+ assert_nil topics.last.id
12
14
  end
13
15
  end
14
16
 
15
- context "with composite primary keys" do
16
- it "should import array of values successfully" do
17
- columns = [:tag_id, :publisher_id, :tag]
18
- values = [[1, 1, 'Mystery'], [1, 1, 'Science']]
17
+ unless ENV["SKIP_COMPOSITE_PK"]
18
+ context "with composite primary keys" do
19
+ it "should import array of values successfully" do
20
+ columns = [:tag_id, :publisher_id, :tag]
21
+ values = [[1, 1, 'Mystery'], [1, 1, 'Science']]
19
22
 
20
- assert_difference "Tag.count", +1 do
21
- Tag.import columns, values, on_duplicate_key_ignore: true, validate: false
23
+ assert_difference "Tag.count", +1 do
24
+ Tag.import columns, values, on_duplicate_key_ignore: true, validate: false
25
+ end
26
+ assert_equal 'Mystery', Tag.first.tag
22
27
  end
23
- assert_equal 'Mystery', Tag.first.tag
24
28
  end
25
29
  end
26
30
  end
@@ -29,7 +33,9 @@ def should_support_on_duplicate_key_ignore
29
33
  it "should skip duplicates and continue import" do
30
34
  topics << Topic.new(title: "Book 2", author_name: "Jane Doe")
31
35
  assert_difference "Topic.count", +1 do
32
- Topic.import topics, ignore: true, validate: false
36
+ result = Topic.import topics, ignore: true, validate: false
37
+ assert_not_equal topics.first.id, result.ids.first
38
+ assert_nil topics.last.id
33
39
  end
34
40
  end
35
41
  end
@@ -5,9 +5,250 @@ def should_support_basic_on_duplicate_key_update
5
5
  macro(:perform_import) { raise "supply your own #perform_import in a context below" }
6
6
  macro(:updated_topic) { Topic.find(@topic.id) }
7
7
 
8
+ context "with lock_version upsert" do
9
+ describe 'optimistic lock' do
10
+ it 'lock_version upsert after on_duplcate_key_update by model' do
11
+ users = [
12
+ User.new(name: 'Salomon'),
13
+ User.new(name: 'Nathan')
14
+ ]
15
+ User.import(users)
16
+ assert User.count == users.length
17
+ User.all.each do |user|
18
+ assert_equal 0, user.lock_version
19
+ end
20
+ updated_users = User.all.map do |user|
21
+ user.name += ' Rothschild'
22
+ user
23
+ end
24
+ User.import(updated_users, on_duplicate_key_update: [:name])
25
+ assert User.count == updated_users.length
26
+ User.all.each_with_index do |user, i|
27
+ assert_equal user.name, users[i].name + ' Rothschild'
28
+ assert_equal 1, user.lock_version
29
+ end
30
+ end
31
+
32
+ it 'lock_version upsert after on_duplcate_key_update by array' do
33
+ users = [
34
+ User.new(name: 'Salomon'),
35
+ User.new(name: 'Nathan')
36
+ ]
37
+ User.import(users)
38
+ assert User.count == users.length
39
+ User.all.each do |user|
40
+ assert_equal 0, user.lock_version
41
+ end
42
+
43
+ columns = [:id, :name]
44
+ updated_values = User.all.map do |user|
45
+ user.name += ' Rothschild'
46
+ [user.id, user.name]
47
+ end
48
+ User.import(columns, updated_values, on_duplicate_key_update: [:name])
49
+ assert User.count == updated_values.length
50
+ User.all.each_with_index do |user, i|
51
+ assert_equal user.name, users[i].name + ' Rothschild'
52
+ assert_equal 1, user.lock_version
53
+ end
54
+ end
55
+
56
+ it 'lock_version upsert after on_duplcate_key_update by hash' do
57
+ users = [
58
+ User.new(name: 'Salomon'),
59
+ User.new(name: 'Nathan')
60
+ ]
61
+ User.import(users)
62
+ assert User.count == users.length
63
+ User.all.each do |user|
64
+ assert_equal 0, user.lock_version
65
+ end
66
+ updated_values = User.all.map do |user|
67
+ user.name += ' Rothschild'
68
+ { id: user.id, name: user.name }
69
+ end
70
+ User.import(updated_values, on_duplicate_key_update: [:name])
71
+ assert User.count == updated_values.length
72
+ User.all.each_with_index do |user, i|
73
+ assert_equal user.name, users[i].name + ' Rothschild'
74
+ assert_equal 1, user.lock_version
75
+ end
76
+ updated_values2 = User.all.map do |user|
77
+ user.name += ' jr.'
78
+ { id: user.id, name: user.name }
79
+ end
80
+ User.import(updated_values2, on_duplicate_key_update: [:name])
81
+ assert User.count == updated_values2.length
82
+ User.all.each_with_index do |user, i|
83
+ assert_equal user.name, users[i].name + ' Rothschild jr.'
84
+ assert_equal 2, user.lock_version
85
+ end
86
+ end
87
+
88
+ it 'upsert optimistic lock columns other than lock_version by model' do
89
+ accounts = [
90
+ Account.new(name: 'Salomon'),
91
+ Account.new(name: 'Nathan')
92
+ ]
93
+ Account.import(accounts)
94
+ assert Account.count == accounts.length
95
+ Account.all.each do |user|
96
+ assert_equal 0, user.lock
97
+ end
98
+ updated_accounts = Account.all.map do |user|
99
+ user.name += ' Rothschild'
100
+ user
101
+ end
102
+ Account.import(updated_accounts, on_duplicate_key_update: [:id, :name])
103
+ assert Account.count == updated_accounts.length
104
+ Account.all.each_with_index do |user, i|
105
+ assert_equal user.name, accounts[i].name + ' Rothschild'
106
+ assert_equal 1, user.lock
107
+ end
108
+ end
109
+
110
+ it 'upsert optimistic lock columns other than lock_version by array' do
111
+ accounts = [
112
+ Account.new(name: 'Salomon'),
113
+ Account.new(name: 'Nathan')
114
+ ]
115
+ Account.import(accounts)
116
+ assert Account.count == accounts.length
117
+ Account.all.each do |user|
118
+ assert_equal 0, user.lock
119
+ end
120
+
121
+ columns = [:id, :name]
122
+ updated_values = Account.all.map do |user|
123
+ user.name += ' Rothschild'
124
+ [user.id, user.name]
125
+ end
126
+ Account.import(columns, updated_values, on_duplicate_key_update: [:name])
127
+ assert Account.count == updated_values.length
128
+ Account.all.each_with_index do |user, i|
129
+ assert_equal user.name, accounts[i].name + ' Rothschild'
130
+ assert_equal 1, user.lock
131
+ end
132
+ end
133
+
134
+ it 'upsert optimistic lock columns other than lock_version by hash' do
135
+ accounts = [
136
+ Account.new(name: 'Salomon'),
137
+ Account.new(name: 'Nathan')
138
+ ]
139
+ Account.import(accounts)
140
+ assert Account.count == accounts.length
141
+ Account.all.each do |user|
142
+ assert_equal 0, user.lock
143
+ end
144
+ updated_values = Account.all.map do |user|
145
+ user.name += ' Rothschild'
146
+ { id: user.id, name: user.name }
147
+ end
148
+ Account.import(updated_values, on_duplicate_key_update: [:name])
149
+ assert Account.count == updated_values.length
150
+ Account.all.each_with_index do |user, i|
151
+ assert_equal user.name, accounts[i].name + ' Rothschild'
152
+ assert_equal 1, user.lock
153
+ end
154
+ end
155
+
156
+ it 'update the lock_version of models separated by namespaces by model' do
157
+ makers = [
158
+ Bike::Maker.new(name: 'Yamaha'),
159
+ Bike::Maker.new(name: 'Honda')
160
+ ]
161
+ Bike::Maker.import(makers)
162
+ assert Bike::Maker.count == makers.length
163
+ Bike::Maker.all.each do |maker|
164
+ assert_equal 0, maker.lock_version
165
+ end
166
+ updated_makers = Bike::Maker.all.map do |maker|
167
+ maker.name += ' bikes'
168
+ maker
169
+ end
170
+ Bike::Maker.import(updated_makers, on_duplicate_key_update: [:name])
171
+ assert Bike::Maker.count == updated_makers.length
172
+ Bike::Maker.all.each_with_index do |maker, i|
173
+ assert_equal maker.name, makers[i].name + ' bikes'
174
+ assert_equal 1, maker.lock_version
175
+ end
176
+ end
177
+ it 'update the lock_version of models separated by namespaces by array' do
178
+ makers = [
179
+ Bike::Maker.new(name: 'Yamaha'),
180
+ Bike::Maker.new(name: 'Honda')
181
+ ]
182
+ Bike::Maker.import(makers)
183
+ assert Bike::Maker.count == makers.length
184
+ Bike::Maker.all.each do |maker|
185
+ assert_equal 0, maker.lock_version
186
+ end
187
+
188
+ columns = [:id, :name]
189
+ updated_values = Bike::Maker.all.map do |maker|
190
+ maker.name += ' bikes'
191
+ [maker.id, maker.name]
192
+ end
193
+ Bike::Maker.import(columns, updated_values, on_duplicate_key_update: [:name])
194
+ assert Bike::Maker.count == updated_values.length
195
+ Bike::Maker.all.each_with_index do |maker, i|
196
+ assert_equal maker.name, makers[i].name + ' bikes'
197
+ assert_equal 1, maker.lock_version
198
+ end
199
+ end
200
+
201
+ it 'update the lock_version of models separated by namespaces by hash' do
202
+ makers = [
203
+ Bike::Maker.new(name: 'Yamaha'),
204
+ Bike::Maker.new(name: 'Honda')
205
+ ]
206
+ Bike::Maker.import(makers)
207
+ assert Bike::Maker.count == makers.length
208
+ Bike::Maker.all.each do |maker|
209
+ assert_equal 0, maker.lock_version
210
+ end
211
+ updated_values = Bike::Maker.all.map do |maker|
212
+ maker.name += ' bikes'
213
+ { id: maker.id, name: maker.name }
214
+ end
215
+ Bike::Maker.import(updated_values, on_duplicate_key_update: [:name])
216
+ assert Bike::Maker.count == updated_values.length
217
+ Bike::Maker.all.each_with_index do |maker, i|
218
+ assert_equal maker.name, makers[i].name + ' bikes'
219
+ assert_equal 1, maker.lock_version
220
+ end
221
+ end
222
+ end
223
+ end
224
+
8
225
  context "with :on_duplicate_key_update" do
226
+ describe 'using :all' do
227
+ let(:columns) { %w( id title author_name author_email_address parent_id ) }
228
+ let(:updated_values) { [[99, "Book - 2nd Edition", "Jane Doe", "janedoe@example.com", 57]] }
229
+
230
+ macro(:perform_import) do |*opts|
231
+ Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: :all, validate: false)
232
+ end
233
+
234
+ setup do
235
+ values = [[99, "Book", "John Doe", "john@doe.com", 17, 3]]
236
+ Topic.import columns + ['replies_count'], values, validate: false
237
+ end
238
+
239
+ it 'updates all specified columns' do
240
+ perform_import
241
+ updated_topic = Topic.find(99)
242
+ assert_equal 'Book - 2nd Edition', updated_topic.title
243
+ assert_equal 'Jane Doe', updated_topic.author_name
244
+ assert_equal 'janedoe@example.com', updated_topic.author_email_address
245
+ assert_equal 57, updated_topic.parent_id
246
+ assert_equal 3, updated_topic.replies_count
247
+ end
248
+ end
249
+
9
250
  describe "argument safety" do
10
- it "should not modify the passed in :on_duplicate_key_update columns array" do
251
+ it "should not modify the passed in :on_duplicate_key_update array" do
11
252
  assert_nothing_raised do
12
253
  columns = %w(title author_name).freeze
13
254
  Topic.import columns, [%w(foo, bar)], on_duplicate_key_update: columns
@@ -15,6 +256,26 @@ def should_support_basic_on_duplicate_key_update
15
256
  end
16
257
  end
17
258
 
259
+ context "with timestamps enabled" do
260
+ let(:time) { Chronic.parse("5 minutes from now") }
261
+
262
+ it 'should not overwrite changed updated_at with current timestamp' do
263
+ topic = Topic.create(author_name: "Jane Doe", title: "Book")
264
+ timestamp = Time.now.utc
265
+ topic.updated_at = timestamp
266
+ Topic.import [topic], on_duplicate_key_update: :all, validate: false
267
+ assert_equal timestamp.to_s, Topic.last.updated_at.to_s
268
+ end
269
+
270
+ it 'should update updated_at with current timestamp' do
271
+ topic = Topic.create(author_name: "Jane Doe", title: "Book")
272
+ Timecop.freeze(time) do
273
+ Topic.import [topic], on_duplicate_key_update: [:updated_at], validate: false
274
+ assert_in_delta time.to_i, topic.reload.updated_at.to_i, 1.second
275
+ end
276
+ end
277
+ end
278
+
18
279
  context "with validation checks turned off" do
19
280
  asssertion_group(:should_support_on_duplicate_key_update) do
20
281
  should_not_update_fields_not_mentioned
@@ -77,15 +338,17 @@ def should_support_basic_on_duplicate_key_update
77
338
  end
78
339
  end
79
340
 
80
- context "with composite primary keys" do
81
- it "should import array of values successfully" do
82
- columns = [:tag_id, :publisher_id, :tag]
83
- Tag.import columns, [[1, 1, 'Mystery']], validate: false
341
+ unless ENV["SKIP_COMPOSITE_PK"]
342
+ context "with composite primary keys" do
343
+ it "should import array of values successfully" do
344
+ columns = [:tag_id, :publisher_id, :tag]
345
+ Tag.import columns, [[1, 1, 'Mystery']], validate: false
84
346
 
85
- assert_difference "Tag.count", +0 do
86
- Tag.import columns, [[1, 1, 'Science']], on_duplicate_key_update: [:tag], validate: false
347
+ assert_difference "Tag.count", +0 do
348
+ Tag.import columns, [[1, 1, 'Science']], on_duplicate_key_update: [:tag], validate: false
349
+ end
350
+ assert_equal 'Science', Tag.first.tag
87
351
  end
88
- assert_equal 'Science', Tag.first.tag
89
352
  end
90
353
  end
91
354
  end
@@ -11,7 +11,7 @@ def should_support_recursive_import
11
11
  let(:num_chapters) { 18 }
12
12
  let(:num_endnotes) { 24 }
13
13
 
14
- let(:new_question_with_rule) { FactoryGirl.build :question, :with_rule }
14
+ let(:new_question_with_rule) { FactoryBot.build :question, :with_rule }
15
15
 
16
16
  it 'imports top level' do
17
17
  assert_difference "Topic.count", +num_topics do
@@ -90,6 +90,19 @@ def should_support_recursive_import
90
90
  end
91
91
  end
92
92
 
93
+ # Models are only valid if all associations are valid
94
+ it "only imports models with valid associations" do
95
+ assert_difference "Topic.count", 2 do
96
+ assert_difference "Book.count", 4 do
97
+ assert_difference "Chapter.count", 12 do
98
+ assert_difference "EndNote.count", 16 do
99
+ Topic.import new_topics_with_invalid_chapter, recursive: true
100
+ end
101
+ end
102
+ end
103
+ end
104
+ end
105
+
93
106
  it "skips validation of the associations if requested" do
94
107
  assert_difference "Chapter.count", +num_chapters do
95
108
  Topic.import new_topics_with_invalid_chapter, validate: false, recursive: true
@@ -102,37 +115,65 @@ def should_support_recursive_import
102
115
  end
103
116
  end
104
117
 
105
- describe "with composite primary keys" do
106
- it "should import models and set id" do
107
- tags = []
108
- tags << Tag.new(tag_id: 1, publisher_id: 1, tag: 'Mystery')
109
- tags << Tag.new(tag_id: 2, publisher_id: 1, tag: 'Science')
118
+ it "imports an imported belongs_to association id" do
119
+ first_new_topic = new_topics[0]
120
+ second_new_topic = new_topics[1]
110
121
 
111
- assert_difference "Tag.count", +2 do
112
- Tag.import tags
113
- end
122
+ books = first_new_topic.books.to_a
123
+ Topic.import new_topics, validate: false
124
+
125
+ assert_difference "Book.count", books.size do
126
+ Book.import books, validate: false
127
+ end
128
+
129
+ books.each do |book|
130
+ assert_equal book.topic_id, first_new_topic.id
131
+ end
132
+
133
+ books.each { |book| book.topic_id = second_new_topic.id }
134
+ assert_no_difference "Book.count", books.size do
135
+ Book.import books, validate: false, on_duplicate_key_update: [:topic_id]
136
+ end
137
+
138
+ books.each do |book|
139
+ assert_equal book.topic_id, second_new_topic.id
140
+ end
141
+
142
+ books.each { |book| book.topic_id = nil }
143
+ assert_no_difference "Book.count", books.size do
144
+ Book.import books, validate: false, on_duplicate_key_update: [:topic_id]
145
+ end
146
+
147
+ books.each do |book|
148
+ assert_equal book.topic_id, nil
149
+ end
150
+ end
151
+
152
+ unless ENV["SKIP_COMPOSITE_PK"]
153
+ describe "with composite primary keys" do
154
+ it "should import models and set id" do
155
+ tags = []
156
+ tags << Tag.new(tag_id: 1, publisher_id: 1, tag: 'Mystery')
157
+ tags << Tag.new(tag_id: 2, publisher_id: 1, tag: 'Science')
158
+
159
+ assert_difference "Tag.count", +2 do
160
+ Tag.import tags
161
+ end
114
162
 
115
- assert_equal 1, tags[0].tag_id
116
- assert_equal 2, tags[1].tag_id
163
+ assert_equal 1, tags[0].tag_id
164
+ assert_equal 2, tags[1].tag_id
165
+ end
117
166
  end
118
167
  end
119
168
 
120
- # These models dont validate associated. So we expect that books and topics get inserted, but not chapters
121
- # Putting a transaction around everything wouldn't work, so if you want your chapters to prevent topics from
122
- # being created, you would need to have validates_associated in your models and insert with validation
123
169
  describe "all_or_none" do
124
- [Book, Topic, EndNote].each do |type|
170
+ [Book, Chapter, Topic, EndNote].each do |type|
125
171
  it "creates #{type}" do
126
- assert_difference "#{type}.count", send("num_#{type.to_s.downcase}s") do
172
+ assert_difference "#{type}.count", 0 do
127
173
  Topic.import new_topics_with_invalid_chapter, all_or_none: true, recursive: true
128
174
  end
129
175
  end
130
176
  end
131
- it "doesn't create chapters" do
132
- assert_difference "Chapter.count", 0 do
133
- Topic.import new_topics_with_invalid_chapter, all_or_none: true, recursive: true
134
- end
135
- end
136
177
  end
137
178
 
138
179
  # If adapter supports on_duplicate_key_update, it is only applied to top level models so that SQL with invalid
@@ -151,5 +192,34 @@ def should_support_recursive_import
151
192
  end
152
193
  end
153
194
  end
195
+
196
+ # If returning option is provided, it is only applied to top level models so that SQL with invalid
197
+ # columns, keys, etc isn't generated for child associations when doing recursive import
198
+ describe "returning" do
199
+ let(:new_topics) { Build(1, :topic_with_book) }
200
+
201
+ it "imports objects with associations" do
202
+ assert_difference "Topic.count", +1 do
203
+ Topic.import new_topics, recursive: true, returning: [:content], validate: false
204
+ new_topics.each do |topic|
205
+ assert_not_nil topic.id
206
+ end
207
+ end
208
+ end
209
+ end
210
+
211
+ # If no returning option is provided, it is ignored
212
+ describe "no returning" do
213
+ let(:new_topics) { Build(1, :topic_with_book) }
214
+
215
+ it "is ignored and imports objects with associations" do
216
+ assert_difference "Topic.count", +1 do
217
+ Topic.import new_topics, recursive: true, no_returning: true, validate: false
218
+ new_topics.each do |topic|
219
+ assert_not_nil topic.id
220
+ end
221
+ end
222
+ end
223
+ end
154
224
  end
155
225
  end