deimos-ruby 1.22.5 → 1.23.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +2 -2
- data/CHANGELOG.md +18 -0
- data/README.md +6 -0
- data/docs/CONFIGURATION.md +4 -0
- data/lib/deimos/active_record_consume/batch_consumption.rb +43 -20
- data/lib/deimos/active_record_consume/batch_record.rb +5 -4
- data/lib/deimos/active_record_consume/batch_record_list.rb +11 -2
- data/lib/deimos/active_record_consume/mass_updater.rb +13 -4
- data/lib/deimos/active_record_consumer.rb +10 -0
- data/lib/deimos/config/configuration.rb +28 -2
- data/lib/deimos/consume/batch_consumption.rb +2 -2
- data/lib/deimos/instrumentation.rb +20 -8
- data/lib/deimos/test_helpers.rb +1 -0
- data/lib/deimos/tracing/datadog.rb +12 -6
- data/lib/deimos/tracing/mock.rb +31 -2
- data/lib/deimos/tracing/provider.rb +6 -0
- data/lib/deimos/utils/db_poller/base.rb +23 -0
- data/lib/deimos/utils/schema_class.rb +10 -2
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +14 -1
- data/spec/active_record_batch_consumer_association_spec.rb +32 -5
- data/spec/active_record_batch_consumer_spec.rb +376 -59
- data/spec/active_record_consume/mass_updater_spec.rb +46 -3
- data/spec/active_record_consumer_spec.rb +74 -1
- data/spec/active_record_producer_spec.rb +4 -1
- data/spec/batch_consumer_spec.rb +4 -1
- data/spec/config/configuration_spec.rb +42 -3
- data/spec/consumer_spec.rb +42 -1
- data/spec/schemas/my_namespace/my_updated_schema.rb +18 -0
- data/spec/utils/db_poller_spec.rb +42 -0
- metadata +5 -3
@@ -96,12 +96,17 @@ module ActiveRecordBatchConsumerTest # rubocop:disable Metrics/ModuleLength
|
|
96
96
|
key_config plain: true
|
97
97
|
record_class Widget
|
98
98
|
|
99
|
-
def should_consume?(record)
|
99
|
+
def should_consume?(record, associations)
|
100
100
|
if self.should_consume_proc
|
101
|
-
|
101
|
+
case self.should_consume_proc.parameters.size
|
102
|
+
when 2
|
103
|
+
self.should_consume_proc.call(record, associations)
|
104
|
+
else
|
105
|
+
self.should_consume_proc.call(record)
|
106
|
+
end
|
107
|
+
else
|
108
|
+
true
|
102
109
|
end
|
103
|
-
|
104
|
-
true
|
105
110
|
end
|
106
111
|
|
107
112
|
def record_attributes(payload, _key)
|
@@ -269,7 +274,7 @@ module ActiveRecordBatchConsumerTest # rubocop:disable Metrics/ModuleLength
|
|
269
274
|
|
270
275
|
context 'with invalid models' do
|
271
276
|
before(:each) do
|
272
|
-
consumer_class.should_consume_proc = proc { |
|
277
|
+
consumer_class.should_consume_proc = proc { |record| record.some_int <= 10 }
|
273
278
|
end
|
274
279
|
|
275
280
|
it 'should only save valid models' do
|
@@ -280,5 +285,27 @@ module ActiveRecordBatchConsumerTest # rubocop:disable Metrics/ModuleLength
|
|
280
285
|
expect(Widget.count).to eq(2)
|
281
286
|
end
|
282
287
|
end
|
288
|
+
|
289
|
+
context 'with invalid associations' do
|
290
|
+
|
291
|
+
before(:each) do
|
292
|
+
consumer_class.should_consume_proc = proc { |record, associations|
|
293
|
+
record.some_int <= 10 && associations['detail']['title'] != 'invalid'
|
294
|
+
}
|
295
|
+
end
|
296
|
+
|
297
|
+
it 'should only save valid associations' do
|
298
|
+
publish_batch([
|
299
|
+
{ key: 2,
|
300
|
+
payload: { test_id: 'xyz', some_int: 5, title: 'valid' } },
|
301
|
+
{ key: 3,
|
302
|
+
payload: { test_id: 'abc', some_int: 15, title: 'valid' } },
|
303
|
+
{ key: 4,
|
304
|
+
payload: { test_id: 'abc', some_int: 9, title: 'invalid' } }
|
305
|
+
])
|
306
|
+
expect(Widget.count).to eq(2)
|
307
|
+
expect(Widget.second.some_int).to eq(5)
|
308
|
+
end
|
309
|
+
end
|
283
310
|
end
|
284
311
|
end
|
@@ -11,6 +11,7 @@ module ActiveRecordBatchConsumerTest
|
|
11
11
|
t.string(:part_two)
|
12
12
|
t.integer(:some_int)
|
13
13
|
t.boolean(:deleted, default: false)
|
14
|
+
t.string(:bulk_import_id)
|
14
15
|
t.timestamps
|
15
16
|
|
16
17
|
t.index(%i(part_one part_two), unique: true)
|
@@ -73,8 +74,12 @@ module ActiveRecordBatchConsumerTest
|
|
73
74
|
describe 'consume_batch' do
|
74
75
|
SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
|
75
76
|
context "with Schema Class consumption #{setting}" do
|
77
|
+
|
76
78
|
before(:each) do
|
77
|
-
Deimos.configure
|
79
|
+
Deimos.configure do |config|
|
80
|
+
config.schema.use_schema_classes = use_schema_classes
|
81
|
+
config.schema.generate_namespace_folders = true
|
82
|
+
end
|
78
83
|
end
|
79
84
|
|
80
85
|
it 'should handle an empty batch' do
|
@@ -205,6 +210,48 @@ module ActiveRecordBatchConsumerTest
|
|
205
210
|
]
|
206
211
|
)
|
207
212
|
end
|
213
|
+
|
214
|
+
it 'should handle deletes with deadlock retries' do
|
215
|
+
allow(Deimos::Utils::DeadlockRetry).to receive(:sleep)
|
216
|
+
allow(instance_double(ActiveRecord::Relation)).to receive(:delete_all).and_raise(
|
217
|
+
ActiveRecord::Deadlocked.new('Lock wait timeout exceeded')
|
218
|
+
).twice.ordered
|
219
|
+
|
220
|
+
Widget.create!(id: 1, test_id: 'abc', some_int: 2)
|
221
|
+
|
222
|
+
publish_batch(
|
223
|
+
[
|
224
|
+
{ key: 1,
|
225
|
+
payload: nil },
|
226
|
+
{ key: 1,
|
227
|
+
payload: nil }
|
228
|
+
]
|
229
|
+
)
|
230
|
+
|
231
|
+
expect(all_widgets).to be_empty
|
232
|
+
end
|
233
|
+
|
234
|
+
it 'should not delete after multiple deadlock retries' do
|
235
|
+
allow(Deimos::Utils::DeadlockRetry).to receive(:sleep)
|
236
|
+
allow(instance_double(ActiveRecord::Relation)).to receive(:delete_all).and_raise(
|
237
|
+
ActiveRecord::Deadlocked.new('Lock wait timeout exceeded')
|
238
|
+
).exactly(3).times
|
239
|
+
|
240
|
+
Widget.create!(id: 1, test_id: 'abc', some_int: 2)
|
241
|
+
|
242
|
+
publish_batch(
|
243
|
+
[
|
244
|
+
{ key: 1,
|
245
|
+
payload: nil },
|
246
|
+
{ key: 1,
|
247
|
+
payload: nil }
|
248
|
+
]
|
249
|
+
)
|
250
|
+
|
251
|
+
expect(Widget.count).to eq(0)
|
252
|
+
|
253
|
+
end
|
254
|
+
|
208
255
|
end
|
209
256
|
end
|
210
257
|
end
|
@@ -254,64 +301,6 @@ module ActiveRecordBatchConsumerTest
|
|
254
301
|
end
|
255
302
|
end
|
256
303
|
|
257
|
-
describe 'batch atomicity' do
|
258
|
-
it 'should roll back if there was an exception while deleting' do
|
259
|
-
Widget.create!(id: 1, test_id: 'abc', some_int: 2)
|
260
|
-
|
261
|
-
travel 1.day
|
262
|
-
|
263
|
-
expect(Widget.connection).to receive(:delete).and_raise('Some error')
|
264
|
-
|
265
|
-
expect {
|
266
|
-
publish_batch(
|
267
|
-
[
|
268
|
-
{ key: 1,
|
269
|
-
payload: { test_id: 'def', some_int: 3 } },
|
270
|
-
{ key: 1,
|
271
|
-
payload: nil }
|
272
|
-
]
|
273
|
-
)
|
274
|
-
}.to raise_error('Some error')
|
275
|
-
|
276
|
-
expect(all_widgets).
|
277
|
-
to match_array(
|
278
|
-
[
|
279
|
-
have_attributes(id: 1, test_id: 'abc', some_int: 2, updated_at: start, created_at: start)
|
280
|
-
]
|
281
|
-
)
|
282
|
-
end
|
283
|
-
|
284
|
-
it 'should roll back if there was an invalid instance while upserting' do
|
285
|
-
Widget.create!(id: 1, test_id: 'abc', some_int: 2) # Updated but rolled back
|
286
|
-
Widget.create!(id: 3, test_id: 'ghi', some_int: 3) # Removed but rolled back
|
287
|
-
|
288
|
-
travel 1.day
|
289
|
-
|
290
|
-
expect {
|
291
|
-
publish_batch(
|
292
|
-
[
|
293
|
-
{ key: 1,
|
294
|
-
payload: { test_id: 'def', some_int: 3 } },
|
295
|
-
{ key: 2,
|
296
|
-
payload: nil },
|
297
|
-
{ key: 2,
|
298
|
-
payload: { test_id: '', some_int: 4 } }, # Empty string is not valid for test_id
|
299
|
-
{ key: 3,
|
300
|
-
payload: nil }
|
301
|
-
]
|
302
|
-
)
|
303
|
-
}.to raise_error(ActiveRecord::RecordInvalid)
|
304
|
-
|
305
|
-
expect(all_widgets).
|
306
|
-
to match_array(
|
307
|
-
[
|
308
|
-
have_attributes(id: 1, test_id: 'abc', some_int: 2, updated_at: start, created_at: start),
|
309
|
-
have_attributes(id: 3, test_id: 'ghi', some_int: 3, updated_at: start, created_at: start)
|
310
|
-
]
|
311
|
-
)
|
312
|
-
end
|
313
|
-
end
|
314
|
-
|
315
304
|
describe 'compound keys' do
|
316
305
|
let(:consumer_class) do
|
317
306
|
Class.new(described_class) do
|
@@ -493,5 +482,333 @@ module ActiveRecordBatchConsumerTest
|
|
493
482
|
end
|
494
483
|
end
|
495
484
|
|
485
|
+
describe 'pre processing' do
|
486
|
+
context 'with uncompacted messages' do
|
487
|
+
let(:consumer_class) do
|
488
|
+
Class.new(described_class) do
|
489
|
+
schema 'MySchema'
|
490
|
+
namespace 'com.my-namespace'
|
491
|
+
key_config plain: true
|
492
|
+
record_class Widget
|
493
|
+
compacted false
|
494
|
+
|
495
|
+
def pre_process(messages)
|
496
|
+
messages.each do |message|
|
497
|
+
message.payload[:some_int] = -message.payload[:some_int]
|
498
|
+
end
|
499
|
+
end
|
500
|
+
|
501
|
+
end
|
502
|
+
end
|
503
|
+
|
504
|
+
it 'should pre-process records' do
|
505
|
+
Widget.create!(id: 1, test_id: 'abc', some_int: 1)
|
506
|
+
Widget.create!(id: 2, test_id: 'def', some_int: 2)
|
507
|
+
|
508
|
+
publish_batch(
|
509
|
+
[
|
510
|
+
{ key: 1,
|
511
|
+
payload: { test_id: 'abc', some_int: 11 } },
|
512
|
+
{ key: 2,
|
513
|
+
payload: { test_id: 'def', some_int: 20 } }
|
514
|
+
]
|
515
|
+
)
|
516
|
+
|
517
|
+
widget_one, widget_two = Widget.all.to_a
|
518
|
+
|
519
|
+
expect(widget_one.some_int).to eq(-11)
|
520
|
+
expect(widget_two.some_int).to eq(-20)
|
521
|
+
end
|
522
|
+
end
|
523
|
+
end
|
524
|
+
|
525
|
+
describe 'global configurations' do
|
526
|
+
|
527
|
+
context 'with a global bulk_import_id_generator' do
|
528
|
+
|
529
|
+
before(:each) do
|
530
|
+
Deimos.configure do
|
531
|
+
consumers.bulk_import_id_generator(proc { 'global' })
|
532
|
+
end
|
533
|
+
end
|
534
|
+
|
535
|
+
it 'should call the default bulk_import_id_generator proc' do
|
536
|
+
publish_batch(
|
537
|
+
[
|
538
|
+
{ key: 1,
|
539
|
+
payload: { test_id: 'abc', some_int: 3 } }
|
540
|
+
]
|
541
|
+
)
|
542
|
+
|
543
|
+
expect(all_widgets).
|
544
|
+
to match_array(
|
545
|
+
[
|
546
|
+
have_attributes(id: 1,
|
547
|
+
test_id: 'abc',
|
548
|
+
some_int: 3,
|
549
|
+
updated_at: start,
|
550
|
+
created_at: start,
|
551
|
+
bulk_import_id: 'global')
|
552
|
+
]
|
553
|
+
)
|
554
|
+
|
555
|
+
end
|
556
|
+
|
557
|
+
end
|
558
|
+
|
559
|
+
context 'with a class defined bulk_import_id_generator' do
|
560
|
+
|
561
|
+
before(:each) do
|
562
|
+
Deimos.configure do
|
563
|
+
consumers.bulk_import_id_generator(proc { 'global' })
|
564
|
+
end
|
565
|
+
consumer_class.config[:bulk_import_id_generator] = proc { 'custom' }
|
566
|
+
end
|
567
|
+
|
568
|
+
it 'should call the default bulk_import_id_generator proc' do
|
569
|
+
|
570
|
+
publish_batch(
|
571
|
+
[
|
572
|
+
{ key: 1,
|
573
|
+
payload: { test_id: 'abc', some_int: 3 } }
|
574
|
+
]
|
575
|
+
)
|
576
|
+
|
577
|
+
expect(all_widgets).
|
578
|
+
to match_array(
|
579
|
+
[
|
580
|
+
have_attributes(id: 1,
|
581
|
+
test_id: 'abc',
|
582
|
+
some_int: 3,
|
583
|
+
updated_at: start,
|
584
|
+
created_at: start,
|
585
|
+
bulk_import_id: 'custom')
|
586
|
+
]
|
587
|
+
)
|
588
|
+
|
589
|
+
end
|
590
|
+
end
|
591
|
+
|
592
|
+
end
|
593
|
+
|
594
|
+
describe 'should_consume?' do
|
595
|
+
|
596
|
+
let(:consumer_class) do
|
597
|
+
Class.new(described_class) do
|
598
|
+
schema 'MySchema'
|
599
|
+
namespace 'com.my-namespace'
|
600
|
+
key_config plain: true
|
601
|
+
record_class Widget
|
602
|
+
compacted false
|
603
|
+
|
604
|
+
def should_consume?(record)
|
605
|
+
record.test_id != 'def'
|
606
|
+
end
|
607
|
+
|
608
|
+
def self.process_invalid_records(_)
|
609
|
+
nil
|
610
|
+
end
|
611
|
+
|
612
|
+
ActiveSupport::Notifications.subscribe('batch_consumption.invalid_records') do |*args|
|
613
|
+
payload = ActiveSupport::Notifications::Event.new(*args).payload
|
614
|
+
payload[:consumer].process_invalid_records(payload[:records])
|
615
|
+
end
|
616
|
+
|
617
|
+
end
|
618
|
+
end
|
619
|
+
|
620
|
+
it "should skip records that shouldn't be consumed" do
|
621
|
+
Widget.create!(id: 1, test_id: 'abc', some_int: 1)
|
622
|
+
Widget.create!(id: 2, test_id: 'def', some_int: 2)
|
623
|
+
publish_batch(
|
624
|
+
[
|
625
|
+
{ key: 1,
|
626
|
+
payload: { test_id: 'abc', some_int: 11 } },
|
627
|
+
{ key: 2,
|
628
|
+
payload: { test_id: 'def', some_int: 20 } }
|
629
|
+
]
|
630
|
+
)
|
631
|
+
|
632
|
+
expect(Widget.count).to eq(2)
|
633
|
+
expect(Widget.all.to_a).to match_array([
|
634
|
+
have_attributes(id: 1,
|
635
|
+
test_id: 'abc',
|
636
|
+
some_int: 11,
|
637
|
+
updated_at: start,
|
638
|
+
created_at: start),
|
639
|
+
have_attributes(id: 2,
|
640
|
+
test_id: 'def',
|
641
|
+
some_int: 2,
|
642
|
+
updated_at: start,
|
643
|
+
created_at: start)
|
644
|
+
])
|
645
|
+
end
|
646
|
+
|
647
|
+
end
|
648
|
+
|
649
|
+
describe 'post processing' do
|
650
|
+
|
651
|
+
context 'with uncompacted messages' do
|
652
|
+
let(:consumer_class) do
|
653
|
+
Class.new(described_class) do
|
654
|
+
schema 'MySchema'
|
655
|
+
namespace 'com.my-namespace'
|
656
|
+
key_config plain: true
|
657
|
+
record_class Widget
|
658
|
+
compacted false
|
659
|
+
|
660
|
+
def should_consume?(record)
|
661
|
+
record.some_int.even?
|
662
|
+
end
|
663
|
+
|
664
|
+
def self.process_valid_records(valid)
|
665
|
+
# Success
|
666
|
+
attrs = valid.first.attributes
|
667
|
+
Widget.find_by(id: attrs['id'], test_id: attrs['test_id']).update!(some_int: 2000)
|
668
|
+
end
|
669
|
+
|
670
|
+
def self.process_invalid_records(invalid)
|
671
|
+
# Invalid
|
672
|
+
attrs = invalid.first.record.attributes
|
673
|
+
Widget.find_by(id: attrs['id'], test_id: attrs['test_id']).update!(some_int: attrs['some_int'])
|
674
|
+
end
|
675
|
+
|
676
|
+
ActiveSupport::Notifications.subscribe('batch_consumption.invalid_records') do |*args|
|
677
|
+
payload = ActiveSupport::Notifications::Event.new(*args).payload
|
678
|
+
payload[:consumer].process_invalid_records(payload[:records])
|
679
|
+
end
|
680
|
+
|
681
|
+
ActiveSupport::Notifications.subscribe('batch_consumption.valid_records') do |*args|
|
682
|
+
payload = ActiveSupport::Notifications::Event.new(*args).payload
|
683
|
+
payload[:consumer].process_valid_records(payload[:records])
|
684
|
+
end
|
685
|
+
|
686
|
+
end
|
687
|
+
end
|
688
|
+
|
689
|
+
it 'should process successful and failed records' do
|
690
|
+
Widget.create!(id: 1, test_id: 'abc', some_int: 1)
|
691
|
+
Widget.create!(id: 2, test_id: 'def', some_int: 2)
|
692
|
+
|
693
|
+
publish_batch(
|
694
|
+
[
|
695
|
+
{ key: 1,
|
696
|
+
payload: { test_id: 'abc', some_int: 11 } },
|
697
|
+
{ key: 2,
|
698
|
+
payload: { test_id: 'def', some_int: 20 } }
|
699
|
+
]
|
700
|
+
)
|
701
|
+
|
702
|
+
widget_one, widget_two = Widget.all.to_a
|
703
|
+
|
704
|
+
expect(widget_one.some_int).to eq(11)
|
705
|
+
expect(widget_two.some_int).to eq(2000)
|
706
|
+
end
|
707
|
+
end
|
708
|
+
|
709
|
+
context 'with compacted messages' do
|
710
|
+
let(:consumer_class) do
|
711
|
+
Class.new(described_class) do
|
712
|
+
schema 'MySchema'
|
713
|
+
namespace 'com.my-namespace'
|
714
|
+
key_config plain: true
|
715
|
+
record_class Widget
|
716
|
+
compacted true
|
717
|
+
|
718
|
+
def should_consume?(record)
|
719
|
+
record.some_int.even?
|
720
|
+
end
|
721
|
+
|
722
|
+
def self.process_valid_records(valid)
|
723
|
+
# Success
|
724
|
+
attrs = valid.first.attributes
|
725
|
+
Widget.find_by(id: attrs['id'], test_id: attrs['test_id']).update!(some_int: 2000)
|
726
|
+
end
|
727
|
+
|
728
|
+
def self.process_invalid_records(invalid)
|
729
|
+
# Invalid
|
730
|
+
attrs = invalid.first.record.attributes
|
731
|
+
Widget.find_by(id: attrs['id'], test_id: attrs['test_id']).update!(some_int: attrs['some_int'])
|
732
|
+
end
|
733
|
+
|
734
|
+
ActiveSupport::Notifications.subscribe('batch_consumption.invalid_records') do |*args|
|
735
|
+
payload = ActiveSupport::Notifications::Event.new(*args).payload
|
736
|
+
payload[:consumer].process_invalid_records(payload[:records])
|
737
|
+
end
|
738
|
+
|
739
|
+
ActiveSupport::Notifications.subscribe('batch_consumption.valid_records') do |*args|
|
740
|
+
payload = ActiveSupport::Notifications::Event.new(*args).payload
|
741
|
+
payload[:consumer].process_valid_records(payload[:records])
|
742
|
+
end
|
743
|
+
|
744
|
+
end
|
745
|
+
end
|
746
|
+
|
747
|
+
it 'should process successful and failed records' do
|
748
|
+
Widget.create!(id: 1, test_id: 'abc', some_int: 1)
|
749
|
+
Widget.create!(id: 2, test_id: 'def', some_int: 2)
|
750
|
+
|
751
|
+
publish_batch(
|
752
|
+
[
|
753
|
+
{ key: 1,
|
754
|
+
payload: { test_id: 'abc', some_int: 11 } },
|
755
|
+
{ key: 2,
|
756
|
+
payload: { test_id: 'def', some_int: 20 } }
|
757
|
+
]
|
758
|
+
)
|
759
|
+
|
760
|
+
widget_one, widget_two = Widget.all.to_a
|
761
|
+
|
762
|
+
expect(widget_one.some_int).to eq(11)
|
763
|
+
expect(widget_two.some_int).to eq(2000)
|
764
|
+
end
|
765
|
+
end
|
766
|
+
|
767
|
+
context 'with post processing errors' do
|
768
|
+
let(:consumer_class) do
|
769
|
+
Class.new(described_class) do
|
770
|
+
schema 'MySchema'
|
771
|
+
namespace 'com.my-namespace'
|
772
|
+
key_config plain: true
|
773
|
+
record_class Widget
|
774
|
+
compacted false
|
775
|
+
|
776
|
+
def self.process_valid_records(_)
|
777
|
+
raise StandardError, 'Something went wrong'
|
778
|
+
end
|
779
|
+
|
780
|
+
ActiveSupport::Notifications.subscribe('batch_consumption.valid_records') do |*args|
|
781
|
+
payload = ActiveSupport::Notifications::Event.new(*args).payload
|
782
|
+
payload[:consumer].process_valid_records(payload[:records])
|
783
|
+
end
|
784
|
+
|
785
|
+
end
|
786
|
+
end
|
787
|
+
|
788
|
+
it 'should save records if an exception occurs in post processing' do
|
789
|
+
Widget.create!(id: 1, test_id: 'abc', some_int: 1)
|
790
|
+
Widget.create!(id: 2, test_id: 'def', some_int: 2)
|
791
|
+
|
792
|
+
expect {
|
793
|
+
publish_batch(
|
794
|
+
[
|
795
|
+
{ key: 1,
|
796
|
+
payload: { test_id: 'abc', some_int: 11 } },
|
797
|
+
{ key: 2,
|
798
|
+
payload: { test_id: 'def', some_int: 20 } }
|
799
|
+
]
|
800
|
+
)
|
801
|
+
}.to raise_error(StandardError, 'Something went wrong')
|
802
|
+
|
803
|
+
widget_one, widget_two = Widget.all.to_a
|
804
|
+
|
805
|
+
expect(widget_one.some_int).to eq(11)
|
806
|
+
expect(widget_two.some_int).to eq(20)
|
807
|
+
|
808
|
+
end
|
809
|
+
end
|
810
|
+
|
811
|
+
end
|
812
|
+
|
496
813
|
end
|
497
814
|
end
|
@@ -39,6 +39,8 @@ RSpec.describe Deimos::ActiveRecordConsume::MassUpdater do
|
|
39
39
|
end
|
40
40
|
end
|
41
41
|
|
42
|
+
let(:bulk_id_generator) { proc { SecureRandom.uuid } }
|
43
|
+
|
42
44
|
before(:each) do
|
43
45
|
stub_const('Widget', widget_class)
|
44
46
|
stub_const('Detail', detail_class)
|
@@ -52,24 +54,65 @@ RSpec.describe Deimos::ActiveRecordConsume::MassUpdater do
|
|
52
54
|
Deimos::ActiveRecordConsume::BatchRecord.new(
|
53
55
|
klass: Widget,
|
54
56
|
attributes: { test_id: 'id1', some_int: 5, detail: { title: 'Title 1' } },
|
55
|
-
bulk_import_column: 'bulk_import_id'
|
57
|
+
bulk_import_column: 'bulk_import_id',
|
58
|
+
bulk_import_id_generator: bulk_id_generator
|
56
59
|
),
|
57
60
|
Deimos::ActiveRecordConsume::BatchRecord.new(
|
58
61
|
klass: Widget,
|
59
62
|
attributes: { test_id: 'id2', some_int: 10, detail: { title: 'Title 2' } },
|
60
|
-
bulk_import_column: 'bulk_import_id'
|
63
|
+
bulk_import_column: 'bulk_import_id',
|
64
|
+
bulk_import_id_generator: bulk_id_generator
|
61
65
|
)
|
62
66
|
]
|
63
67
|
)
|
64
68
|
end
|
65
69
|
|
66
70
|
it 'should mass update the batch' do
|
67
|
-
|
71
|
+
allow(SecureRandom).to receive(:uuid).and_return('1', '2')
|
72
|
+
results = described_class.new(Widget, bulk_import_id_generator: bulk_id_generator).mass_update(batch)
|
73
|
+
expect(results.count).to eq(2)
|
74
|
+
expect(results.map(&:test_id)).to match(%w(id1 id2))
|
68
75
|
expect(Widget.count).to eq(2)
|
76
|
+
expect(Widget.all.to_a.map(&:bulk_import_id)).to match(%w(1 2))
|
69
77
|
expect(Detail.count).to eq(2)
|
70
78
|
expect(Widget.first.detail).not_to be_nil
|
71
79
|
expect(Widget.last.detail).not_to be_nil
|
72
80
|
end
|
73
81
|
|
82
|
+
context 'with deadlock retries' do
|
83
|
+
before(:each) do
|
84
|
+
allow(Deimos::Utils::DeadlockRetry).to receive(:sleep)
|
85
|
+
end
|
86
|
+
|
87
|
+
it 'should upsert rows after deadlocks' do
|
88
|
+
allow(Widget).to receive(:import!).and_raise(
|
89
|
+
ActiveRecord::Deadlocked.new('Lock wait timeout exceeded')
|
90
|
+
).twice.ordered
|
91
|
+
allow(Widget).to receive(:import!).and_raise(
|
92
|
+
ActiveRecord::Deadlocked.new('Lock wait timeout exceeded')
|
93
|
+
).once.and_call_original
|
94
|
+
|
95
|
+
results = described_class.new(Widget, bulk_import_id_generator: bulk_id_generator).mass_update(batch)
|
96
|
+
expect(results.count).to eq(2)
|
97
|
+
expect(results.map(&:test_id)).to match(%w(id1 id2))
|
98
|
+
expect(Widget.count).to eq(2)
|
99
|
+
expect(Detail.count).to eq(2)
|
100
|
+
expect(Widget.first.detail).not_to be_nil
|
101
|
+
expect(Widget.last.detail).not_to be_nil
|
102
|
+
end
|
103
|
+
|
104
|
+
it 'should not upsert after encountering multiple deadlocks' do
|
105
|
+
allow(Widget).to receive(:import!).and_raise(
|
106
|
+
ActiveRecord::Deadlocked.new('Lock wait timeout exceeded')
|
107
|
+
).exactly(3).times
|
108
|
+
expect {
|
109
|
+
described_class.new(Widget, bulk_import_id_generator: bulk_id_generator).mass_update(batch)
|
110
|
+
}.to raise_error(ActiveRecord::Deadlocked)
|
111
|
+
expect(Widget.count).to eq(0)
|
112
|
+
expect(Detail.count).to eq(0)
|
113
|
+
end
|
114
|
+
|
115
|
+
end
|
116
|
+
|
74
117
|
end
|
75
118
|
end
|
@@ -3,6 +3,7 @@
|
|
3
3
|
require 'date'
|
4
4
|
|
5
5
|
# Wrapped in a module to prevent class leakage
|
6
|
+
# rubocop:disable Metrics/ModuleLength
|
6
7
|
module ActiveRecordConsumerTest
|
7
8
|
describe Deimos::ActiveRecordConsumer, 'Message Consumer' do
|
8
9
|
before(:all) do
|
@@ -66,13 +67,84 @@ module ActiveRecordConsumerTest
|
|
66
67
|
stub_const('MyCustomFetchConsumer', consumer_class)
|
67
68
|
|
68
69
|
Time.zone = 'Eastern Time (US & Canada)'
|
70
|
+
|
71
|
+
schema_class = Class.new(Deimos::SchemaClass::Record) do
|
72
|
+
def schema
|
73
|
+
'MySchema'
|
74
|
+
end
|
75
|
+
|
76
|
+
def namespace
|
77
|
+
'com.my-namespace'
|
78
|
+
end
|
79
|
+
|
80
|
+
attr_accessor :test_id
|
81
|
+
attr_accessor :some_int
|
82
|
+
|
83
|
+
def initialize(test_id: nil,
|
84
|
+
some_int: nil)
|
85
|
+
self.test_id = test_id
|
86
|
+
self.some_int = some_int
|
87
|
+
end
|
88
|
+
|
89
|
+
def as_json(_opts={})
|
90
|
+
{
|
91
|
+
'test_id' => @test_id,
|
92
|
+
'some_int' => @some_int,
|
93
|
+
'payload_key' => @payload_key&.as_json
|
94
|
+
}
|
95
|
+
end
|
96
|
+
end
|
97
|
+
stub_const('Schemas::MySchema', schema_class)
|
98
|
+
|
99
|
+
schema_datetime_class = Class.new(Deimos::SchemaClass::Record) do
|
100
|
+
def schema
|
101
|
+
'MySchemaWithDateTimes'
|
102
|
+
end
|
103
|
+
|
104
|
+
def namespace
|
105
|
+
'com.my-namespace'
|
106
|
+
end
|
107
|
+
|
108
|
+
attr_accessor :test_id
|
109
|
+
attr_accessor :some_int
|
110
|
+
attr_accessor :updated_at
|
111
|
+
attr_accessor :some_datetime_int
|
112
|
+
attr_accessor :timestamp
|
113
|
+
|
114
|
+
def initialize(test_id: nil,
|
115
|
+
some_int: nil,
|
116
|
+
updated_at: nil,
|
117
|
+
some_datetime_int: nil,
|
118
|
+
timestamp: nil)
|
119
|
+
self.test_id = test_id
|
120
|
+
self.some_int = some_int
|
121
|
+
self.updated_at = updated_at
|
122
|
+
self.some_datetime_int = some_datetime_int
|
123
|
+
self.timestamp = timestamp
|
124
|
+
end
|
125
|
+
|
126
|
+
def as_json(_opts={})
|
127
|
+
{
|
128
|
+
'test_id' => @test_id,
|
129
|
+
'some_int' => @some_int,
|
130
|
+
'updated_at' => @updated_at,
|
131
|
+
'some_datetime_int' => @some_datetime_int,
|
132
|
+
'timestamp' => @timestamp,
|
133
|
+
'payload_key' => @payload_key&.as_json
|
134
|
+
}
|
135
|
+
end
|
136
|
+
end
|
137
|
+
stub_const('Schemas::MySchemaWithDateTimes', schema_datetime_class)
|
69
138
|
end
|
70
139
|
|
71
140
|
describe 'consume' do
|
72
141
|
SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
|
73
142
|
context "with Schema Class consumption #{setting}" do
|
74
143
|
before(:each) do
|
75
|
-
Deimos.configure
|
144
|
+
Deimos.configure do |config|
|
145
|
+
config.schema.use_schema_classes = use_schema_classes
|
146
|
+
config.schema.generate_namespace_folders = true
|
147
|
+
end
|
76
148
|
end
|
77
149
|
|
78
150
|
it 'should receive events correctly' do
|
@@ -182,3 +254,4 @@ module ActiveRecordConsumerTest
|
|
182
254
|
end
|
183
255
|
end
|
184
256
|
end
|
257
|
+
# rubocop:enable Metrics/ModuleLength
|