activerecord-import 0.17.2 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (52) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +1 -0
  3. data/.travis.yml +40 -23
  4. data/CHANGELOG.md +315 -1
  5. data/Gemfile +23 -13
  6. data/LICENSE +21 -56
  7. data/README.markdown +564 -33
  8. data/Rakefile +2 -1
  9. data/activerecord-import.gemspec +3 -3
  10. data/benchmarks/lib/cli_parser.rb +2 -1
  11. data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +0 -0
  12. data/gemfiles/5.1.gemfile +2 -0
  13. data/gemfiles/5.2.gemfile +2 -0
  14. data/gemfiles/6.0.gemfile +2 -0
  15. data/gemfiles/6.1.gemfile +1 -0
  16. data/lib/activerecord-import.rb +2 -15
  17. data/lib/activerecord-import/adapters/abstract_adapter.rb +9 -3
  18. data/lib/activerecord-import/adapters/mysql_adapter.rb +17 -11
  19. data/lib/activerecord-import/adapters/postgresql_adapter.rb +68 -20
  20. data/lib/activerecord-import/adapters/sqlite3_adapter.rb +128 -9
  21. data/lib/activerecord-import/base.rb +12 -7
  22. data/lib/activerecord-import/import.rb +514 -166
  23. data/lib/activerecord-import/synchronize.rb +2 -2
  24. data/lib/activerecord-import/value_sets_parser.rb +16 -0
  25. data/lib/activerecord-import/version.rb +1 -1
  26. data/test/adapters/makara_postgis.rb +1 -0
  27. data/test/import_test.rb +274 -23
  28. data/test/makara_postgis/import_test.rb +8 -0
  29. data/test/models/account.rb +3 -0
  30. data/test/models/animal.rb +6 -0
  31. data/test/models/bike_maker.rb +7 -0
  32. data/test/models/tag.rb +1 -1
  33. data/test/models/topic.rb +14 -0
  34. data/test/models/user.rb +3 -0
  35. data/test/models/user_token.rb +4 -0
  36. data/test/schema/generic_schema.rb +30 -8
  37. data/test/schema/mysql2_schema.rb +19 -0
  38. data/test/schema/postgresql_schema.rb +18 -0
  39. data/test/schema/sqlite3_schema.rb +13 -0
  40. data/test/support/factories.rb +9 -8
  41. data/test/support/generate.rb +6 -6
  42. data/test/support/mysql/import_examples.rb +14 -2
  43. data/test/support/postgresql/import_examples.rb +220 -1
  44. data/test/support/shared_examples/on_duplicate_key_ignore.rb +15 -9
  45. data/test/support/shared_examples/on_duplicate_key_update.rb +271 -8
  46. data/test/support/shared_examples/recursive_import.rb +91 -21
  47. data/test/support/sqlite3/import_examples.rb +189 -25
  48. data/test/synchronize_test.rb +8 -0
  49. data/test/test_helper.rb +24 -3
  50. data/test/value_sets_bytes_parser_test.rb +13 -2
  51. metadata +32 -13
  52. data/test/schema/mysql_schema.rb +0 -16
@@ -8,19 +8,23 @@ def should_support_on_duplicate_key_ignore
8
8
  it "should skip duplicates and continue import" do
9
9
  topics << Topic.new(title: "Book 2", author_name: "Jane Doe")
10
10
  assert_difference "Topic.count", +1 do
11
- Topic.import topics, on_duplicate_key_ignore: true, validate: false
11
+ result = Topic.import topics, on_duplicate_key_ignore: true, validate: false
12
+ assert_not_equal topics.first.id, result.ids.first
13
+ assert_nil topics.last.id
12
14
  end
13
15
  end
14
16
 
15
- context "with composite primary keys" do
16
- it "should import array of values successfully" do
17
- columns = [:tag_id, :publisher_id, :tag]
18
- values = [[1, 1, 'Mystery'], [1, 1, 'Science']]
17
+ unless ENV["SKIP_COMPOSITE_PK"]
18
+ context "with composite primary keys" do
19
+ it "should import array of values successfully" do
20
+ columns = [:tag_id, :publisher_id, :tag]
21
+ values = [[1, 1, 'Mystery'], [1, 1, 'Science']]
19
22
 
20
- assert_difference "Tag.count", +1 do
21
- Tag.import columns, values, on_duplicate_key_ignore: true, validate: false
23
+ assert_difference "Tag.count", +1 do
24
+ Tag.import columns, values, on_duplicate_key_ignore: true, validate: false
25
+ end
26
+ assert_equal 'Mystery', Tag.first.tag
22
27
  end
23
- assert_equal 'Mystery', Tag.first.tag
24
28
  end
25
29
  end
26
30
  end
@@ -29,7 +33,9 @@ def should_support_on_duplicate_key_ignore
29
33
  it "should skip duplicates and continue import" do
30
34
  topics << Topic.new(title: "Book 2", author_name: "Jane Doe")
31
35
  assert_difference "Topic.count", +1 do
32
- Topic.import topics, ignore: true, validate: false
36
+ result = Topic.import topics, ignore: true, validate: false
37
+ assert_not_equal topics.first.id, result.ids.first
38
+ assert_nil topics.last.id
33
39
  end
34
40
  end
35
41
  end
@@ -5,9 +5,250 @@ def should_support_basic_on_duplicate_key_update
5
5
  macro(:perform_import) { raise "supply your own #perform_import in a context below" }
6
6
  macro(:updated_topic) { Topic.find(@topic.id) }
7
7
 
8
+ context "with lock_version upsert" do
9
+ describe 'optimistic lock' do
10
+ it 'lock_version upsert after on_duplcate_key_update by model' do
11
+ users = [
12
+ User.new(name: 'Salomon'),
13
+ User.new(name: 'Nathan')
14
+ ]
15
+ User.import(users)
16
+ assert User.count == users.length
17
+ User.all.each do |user|
18
+ assert_equal 0, user.lock_version
19
+ end
20
+ updated_users = User.all.map do |user|
21
+ user.name += ' Rothschild'
22
+ user
23
+ end
24
+ User.import(updated_users, on_duplicate_key_update: [:name])
25
+ assert User.count == updated_users.length
26
+ User.all.each_with_index do |user, i|
27
+ assert_equal user.name, users[i].name + ' Rothschild'
28
+ assert_equal 1, user.lock_version
29
+ end
30
+ end
31
+
32
+ it 'lock_version upsert after on_duplcate_key_update by array' do
33
+ users = [
34
+ User.new(name: 'Salomon'),
35
+ User.new(name: 'Nathan')
36
+ ]
37
+ User.import(users)
38
+ assert User.count == users.length
39
+ User.all.each do |user|
40
+ assert_equal 0, user.lock_version
41
+ end
42
+
43
+ columns = [:id, :name]
44
+ updated_values = User.all.map do |user|
45
+ user.name += ' Rothschild'
46
+ [user.id, user.name]
47
+ end
48
+ User.import(columns, updated_values, on_duplicate_key_update: [:name])
49
+ assert User.count == updated_values.length
50
+ User.all.each_with_index do |user, i|
51
+ assert_equal user.name, users[i].name + ' Rothschild'
52
+ assert_equal 1, user.lock_version
53
+ end
54
+ end
55
+
56
+ it 'lock_version upsert after on_duplcate_key_update by hash' do
57
+ users = [
58
+ User.new(name: 'Salomon'),
59
+ User.new(name: 'Nathan')
60
+ ]
61
+ User.import(users)
62
+ assert User.count == users.length
63
+ User.all.each do |user|
64
+ assert_equal 0, user.lock_version
65
+ end
66
+ updated_values = User.all.map do |user|
67
+ user.name += ' Rothschild'
68
+ { id: user.id, name: user.name }
69
+ end
70
+ User.import(updated_values, on_duplicate_key_update: [:name])
71
+ assert User.count == updated_values.length
72
+ User.all.each_with_index do |user, i|
73
+ assert_equal user.name, users[i].name + ' Rothschild'
74
+ assert_equal 1, user.lock_version
75
+ end
76
+ updated_values2 = User.all.map do |user|
77
+ user.name += ' jr.'
78
+ { id: user.id, name: user.name }
79
+ end
80
+ User.import(updated_values2, on_duplicate_key_update: [:name])
81
+ assert User.count == updated_values2.length
82
+ User.all.each_with_index do |user, i|
83
+ assert_equal user.name, users[i].name + ' Rothschild jr.'
84
+ assert_equal 2, user.lock_version
85
+ end
86
+ end
87
+
88
+ it 'upsert optimistic lock columns other than lock_version by model' do
89
+ accounts = [
90
+ Account.new(name: 'Salomon'),
91
+ Account.new(name: 'Nathan')
92
+ ]
93
+ Account.import(accounts)
94
+ assert Account.count == accounts.length
95
+ Account.all.each do |user|
96
+ assert_equal 0, user.lock
97
+ end
98
+ updated_accounts = Account.all.map do |user|
99
+ user.name += ' Rothschild'
100
+ user
101
+ end
102
+ Account.import(updated_accounts, on_duplicate_key_update: [:id, :name])
103
+ assert Account.count == updated_accounts.length
104
+ Account.all.each_with_index do |user, i|
105
+ assert_equal user.name, accounts[i].name + ' Rothschild'
106
+ assert_equal 1, user.lock
107
+ end
108
+ end
109
+
110
+ it 'upsert optimistic lock columns other than lock_version by array' do
111
+ accounts = [
112
+ Account.new(name: 'Salomon'),
113
+ Account.new(name: 'Nathan')
114
+ ]
115
+ Account.import(accounts)
116
+ assert Account.count == accounts.length
117
+ Account.all.each do |user|
118
+ assert_equal 0, user.lock
119
+ end
120
+
121
+ columns = [:id, :name]
122
+ updated_values = Account.all.map do |user|
123
+ user.name += ' Rothschild'
124
+ [user.id, user.name]
125
+ end
126
+ Account.import(columns, updated_values, on_duplicate_key_update: [:name])
127
+ assert Account.count == updated_values.length
128
+ Account.all.each_with_index do |user, i|
129
+ assert_equal user.name, accounts[i].name + ' Rothschild'
130
+ assert_equal 1, user.lock
131
+ end
132
+ end
133
+
134
+ it 'upsert optimistic lock columns other than lock_version by hash' do
135
+ accounts = [
136
+ Account.new(name: 'Salomon'),
137
+ Account.new(name: 'Nathan')
138
+ ]
139
+ Account.import(accounts)
140
+ assert Account.count == accounts.length
141
+ Account.all.each do |user|
142
+ assert_equal 0, user.lock
143
+ end
144
+ updated_values = Account.all.map do |user|
145
+ user.name += ' Rothschild'
146
+ { id: user.id, name: user.name }
147
+ end
148
+ Account.import(updated_values, on_duplicate_key_update: [:name])
149
+ assert Account.count == updated_values.length
150
+ Account.all.each_with_index do |user, i|
151
+ assert_equal user.name, accounts[i].name + ' Rothschild'
152
+ assert_equal 1, user.lock
153
+ end
154
+ end
155
+
156
+ it 'update the lock_version of models separated by namespaces by model' do
157
+ makers = [
158
+ Bike::Maker.new(name: 'Yamaha'),
159
+ Bike::Maker.new(name: 'Honda')
160
+ ]
161
+ Bike::Maker.import(makers)
162
+ assert Bike::Maker.count == makers.length
163
+ Bike::Maker.all.each do |maker|
164
+ assert_equal 0, maker.lock_version
165
+ end
166
+ updated_makers = Bike::Maker.all.map do |maker|
167
+ maker.name += ' bikes'
168
+ maker
169
+ end
170
+ Bike::Maker.import(updated_makers, on_duplicate_key_update: [:name])
171
+ assert Bike::Maker.count == updated_makers.length
172
+ Bike::Maker.all.each_with_index do |maker, i|
173
+ assert_equal maker.name, makers[i].name + ' bikes'
174
+ assert_equal 1, maker.lock_version
175
+ end
176
+ end
177
+ it 'update the lock_version of models separated by namespaces by array' do
178
+ makers = [
179
+ Bike::Maker.new(name: 'Yamaha'),
180
+ Bike::Maker.new(name: 'Honda')
181
+ ]
182
+ Bike::Maker.import(makers)
183
+ assert Bike::Maker.count == makers.length
184
+ Bike::Maker.all.each do |maker|
185
+ assert_equal 0, maker.lock_version
186
+ end
187
+
188
+ columns = [:id, :name]
189
+ updated_values = Bike::Maker.all.map do |maker|
190
+ maker.name += ' bikes'
191
+ [maker.id, maker.name]
192
+ end
193
+ Bike::Maker.import(columns, updated_values, on_duplicate_key_update: [:name])
194
+ assert Bike::Maker.count == updated_values.length
195
+ Bike::Maker.all.each_with_index do |maker, i|
196
+ assert_equal maker.name, makers[i].name + ' bikes'
197
+ assert_equal 1, maker.lock_version
198
+ end
199
+ end
200
+
201
+ it 'update the lock_version of models separated by namespaces by hash' do
202
+ makers = [
203
+ Bike::Maker.new(name: 'Yamaha'),
204
+ Bike::Maker.new(name: 'Honda')
205
+ ]
206
+ Bike::Maker.import(makers)
207
+ assert Bike::Maker.count == makers.length
208
+ Bike::Maker.all.each do |maker|
209
+ assert_equal 0, maker.lock_version
210
+ end
211
+ updated_values = Bike::Maker.all.map do |maker|
212
+ maker.name += ' bikes'
213
+ { id: maker.id, name: maker.name }
214
+ end
215
+ Bike::Maker.import(updated_values, on_duplicate_key_update: [:name])
216
+ assert Bike::Maker.count == updated_values.length
217
+ Bike::Maker.all.each_with_index do |maker, i|
218
+ assert_equal maker.name, makers[i].name + ' bikes'
219
+ assert_equal 1, maker.lock_version
220
+ end
221
+ end
222
+ end
223
+ end
224
+
8
225
  context "with :on_duplicate_key_update" do
226
+ describe 'using :all' do
227
+ let(:columns) { %w( id title author_name author_email_address parent_id ) }
228
+ let(:updated_values) { [[99, "Book - 2nd Edition", "Jane Doe", "janedoe@example.com", 57]] }
229
+
230
+ macro(:perform_import) do |*opts|
231
+ Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: :all, validate: false)
232
+ end
233
+
234
+ setup do
235
+ values = [[99, "Book", "John Doe", "john@doe.com", 17, 3]]
236
+ Topic.import columns + ['replies_count'], values, validate: false
237
+ end
238
+
239
+ it 'updates all specified columns' do
240
+ perform_import
241
+ updated_topic = Topic.find(99)
242
+ assert_equal 'Book - 2nd Edition', updated_topic.title
243
+ assert_equal 'Jane Doe', updated_topic.author_name
244
+ assert_equal 'janedoe@example.com', updated_topic.author_email_address
245
+ assert_equal 57, updated_topic.parent_id
246
+ assert_equal 3, updated_topic.replies_count
247
+ end
248
+ end
249
+
9
250
  describe "argument safety" do
10
- it "should not modify the passed in :on_duplicate_key_update columns array" do
251
+ it "should not modify the passed in :on_duplicate_key_update array" do
11
252
  assert_nothing_raised do
12
253
  columns = %w(title author_name).freeze
13
254
  Topic.import columns, [%w(foo, bar)], on_duplicate_key_update: columns
@@ -15,6 +256,26 @@ def should_support_basic_on_duplicate_key_update
15
256
  end
16
257
  end
17
258
 
259
+ context "with timestamps enabled" do
260
+ let(:time) { Chronic.parse("5 minutes from now") }
261
+
262
+ it 'should not overwrite changed updated_at with current timestamp' do
263
+ topic = Topic.create(author_name: "Jane Doe", title: "Book")
264
+ timestamp = Time.now.utc
265
+ topic.updated_at = timestamp
266
+ Topic.import [topic], on_duplicate_key_update: :all, validate: false
267
+ assert_equal timestamp.to_s, Topic.last.updated_at.to_s
268
+ end
269
+
270
+ it 'should update updated_at with current timestamp' do
271
+ topic = Topic.create(author_name: "Jane Doe", title: "Book")
272
+ Timecop.freeze(time) do
273
+ Topic.import [topic], on_duplicate_key_update: [:updated_at], validate: false
274
+ assert_in_delta time.to_i, topic.reload.updated_at.to_i, 1.second
275
+ end
276
+ end
277
+ end
278
+
18
279
  context "with validation checks turned off" do
19
280
  asssertion_group(:should_support_on_duplicate_key_update) do
20
281
  should_not_update_fields_not_mentioned
@@ -77,15 +338,17 @@ def should_support_basic_on_duplicate_key_update
77
338
  end
78
339
  end
79
340
 
80
- context "with composite primary keys" do
81
- it "should import array of values successfully" do
82
- columns = [:tag_id, :publisher_id, :tag]
83
- Tag.import columns, [[1, 1, 'Mystery']], validate: false
341
+ unless ENV["SKIP_COMPOSITE_PK"]
342
+ context "with composite primary keys" do
343
+ it "should import array of values successfully" do
344
+ columns = [:tag_id, :publisher_id, :tag]
345
+ Tag.import columns, [[1, 1, 'Mystery']], validate: false
84
346
 
85
- assert_difference "Tag.count", +0 do
86
- Tag.import columns, [[1, 1, 'Science']], on_duplicate_key_update: [:tag], validate: false
347
+ assert_difference "Tag.count", +0 do
348
+ Tag.import columns, [[1, 1, 'Science']], on_duplicate_key_update: [:tag], validate: false
349
+ end
350
+ assert_equal 'Science', Tag.first.tag
87
351
  end
88
- assert_equal 'Science', Tag.first.tag
89
352
  end
90
353
  end
91
354
  end
@@ -11,7 +11,7 @@ def should_support_recursive_import
11
11
  let(:num_chapters) { 18 }
12
12
  let(:num_endnotes) { 24 }
13
13
 
14
- let(:new_question_with_rule) { FactoryGirl.build :question, :with_rule }
14
+ let(:new_question_with_rule) { FactoryBot.build :question, :with_rule }
15
15
 
16
16
  it 'imports top level' do
17
17
  assert_difference "Topic.count", +num_topics do
@@ -90,6 +90,19 @@ def should_support_recursive_import
90
90
  end
91
91
  end
92
92
 
93
+ # Models are only valid if all associations are valid
94
+ it "only imports models with valid associations" do
95
+ assert_difference "Topic.count", 2 do
96
+ assert_difference "Book.count", 4 do
97
+ assert_difference "Chapter.count", 12 do
98
+ assert_difference "EndNote.count", 16 do
99
+ Topic.import new_topics_with_invalid_chapter, recursive: true
100
+ end
101
+ end
102
+ end
103
+ end
104
+ end
105
+
93
106
  it "skips validation of the associations if requested" do
94
107
  assert_difference "Chapter.count", +num_chapters do
95
108
  Topic.import new_topics_with_invalid_chapter, validate: false, recursive: true
@@ -102,37 +115,65 @@ def should_support_recursive_import
102
115
  end
103
116
  end
104
117
 
105
- describe "with composite primary keys" do
106
- it "should import models and set id" do
107
- tags = []
108
- tags << Tag.new(tag_id: 1, publisher_id: 1, tag: 'Mystery')
109
- tags << Tag.new(tag_id: 2, publisher_id: 1, tag: 'Science')
118
+ it "imports an imported belongs_to association id" do
119
+ first_new_topic = new_topics[0]
120
+ second_new_topic = new_topics[1]
110
121
 
111
- assert_difference "Tag.count", +2 do
112
- Tag.import tags
113
- end
122
+ books = first_new_topic.books.to_a
123
+ Topic.import new_topics, validate: false
124
+
125
+ assert_difference "Book.count", books.size do
126
+ Book.import books, validate: false
127
+ end
128
+
129
+ books.each do |book|
130
+ assert_equal book.topic_id, first_new_topic.id
131
+ end
132
+
133
+ books.each { |book| book.topic_id = second_new_topic.id }
134
+ assert_no_difference "Book.count", books.size do
135
+ Book.import books, validate: false, on_duplicate_key_update: [:topic_id]
136
+ end
137
+
138
+ books.each do |book|
139
+ assert_equal book.topic_id, second_new_topic.id
140
+ end
141
+
142
+ books.each { |book| book.topic_id = nil }
143
+ assert_no_difference "Book.count", books.size do
144
+ Book.import books, validate: false, on_duplicate_key_update: [:topic_id]
145
+ end
146
+
147
+ books.each do |book|
148
+ assert_equal book.topic_id, nil
149
+ end
150
+ end
151
+
152
+ unless ENV["SKIP_COMPOSITE_PK"]
153
+ describe "with composite primary keys" do
154
+ it "should import models and set id" do
155
+ tags = []
156
+ tags << Tag.new(tag_id: 1, publisher_id: 1, tag: 'Mystery')
157
+ tags << Tag.new(tag_id: 2, publisher_id: 1, tag: 'Science')
158
+
159
+ assert_difference "Tag.count", +2 do
160
+ Tag.import tags
161
+ end
114
162
 
115
- assert_equal 1, tags[0].tag_id
116
- assert_equal 2, tags[1].tag_id
163
+ assert_equal 1, tags[0].tag_id
164
+ assert_equal 2, tags[1].tag_id
165
+ end
117
166
  end
118
167
  end
119
168
 
120
- # These models dont validate associated. So we expect that books and topics get inserted, but not chapters
121
- # Putting a transaction around everything wouldn't work, so if you want your chapters to prevent topics from
122
- # being created, you would need to have validates_associated in your models and insert with validation
123
169
  describe "all_or_none" do
124
- [Book, Topic, EndNote].each do |type|
170
+ [Book, Chapter, Topic, EndNote].each do |type|
125
171
  it "creates #{type}" do
126
- assert_difference "#{type}.count", send("num_#{type.to_s.downcase}s") do
172
+ assert_difference "#{type}.count", 0 do
127
173
  Topic.import new_topics_with_invalid_chapter, all_or_none: true, recursive: true
128
174
  end
129
175
  end
130
176
  end
131
- it "doesn't create chapters" do
132
- assert_difference "Chapter.count", 0 do
133
- Topic.import new_topics_with_invalid_chapter, all_or_none: true, recursive: true
134
- end
135
- end
136
177
  end
137
178
 
138
179
  # If adapter supports on_duplicate_key_update, it is only applied to top level models so that SQL with invalid
@@ -151,5 +192,34 @@ def should_support_recursive_import
151
192
  end
152
193
  end
153
194
  end
195
+
196
+ # If returning option is provided, it is only applied to top level models so that SQL with invalid
197
+ # columns, keys, etc isn't generated for child associations when doing recursive import
198
+ describe "returning" do
199
+ let(:new_topics) { Build(1, :topic_with_book) }
200
+
201
+ it "imports objects with associations" do
202
+ assert_difference "Topic.count", +1 do
203
+ Topic.import new_topics, recursive: true, returning: [:content], validate: false
204
+ new_topics.each do |topic|
205
+ assert_not_nil topic.id
206
+ end
207
+ end
208
+ end
209
+ end
210
+
211
+ # If no returning option is provided, it is ignored
212
+ describe "no returning" do
213
+ let(:new_topics) { Build(1, :topic_with_book) }
214
+
215
+ it "is ignored and imports objects with associations" do
216
+ assert_difference "Topic.count", +1 do
217
+ Topic.import new_topics, recursive: true, no_returning: true, validate: false
218
+ new_topics.each do |topic|
219
+ assert_not_nil topic.id
220
+ end
221
+ end
222
+ end
223
+ end
154
224
  end
155
225
  end