deimos-ruby 2.2.0 → 2.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +33 -30
- data/CHANGELOG.md +10 -0
- data/Gemfile +0 -6
- data/deimos-ruby.gemspec +15 -11
- data/karafka.rb +7 -4
- data/lib/deimos/active_record_consume/batch_consumption.rb +7 -7
- data/lib/deimos/active_record_consume/batch_record.rb +2 -2
- data/lib/deimos/active_record_consume/message_consumption.rb +6 -5
- data/lib/deimos/active_record_consume/schema_model_converter.rb +2 -2
- data/lib/deimos/active_record_consumer.rb +1 -0
- data/lib/deimos/active_record_producer.rb +4 -2
- data/lib/deimos/backends/base.rb +1 -3
- data/lib/deimos/backends/outbox.rb +1 -1
- data/lib/deimos/config/configuration.rb +88 -75
- data/lib/deimos/consume/batch_consumption.rb +5 -5
- data/lib/deimos/consume/message_consumption.rb +3 -3
- data/lib/deimos/ext/consumer_route.rb +3 -3
- data/lib/deimos/ext/producer_metrics_listener.rb +2 -2
- data/lib/deimos/ext/producer_middleware.rb +19 -15
- data/lib/deimos/ext/producer_route.rb +4 -2
- data/lib/deimos/ext/routing_defaults.rb +9 -7
- data/lib/deimos/ext/schema_route.rb +22 -15
- data/lib/deimos/kafka_message.rb +1 -1
- data/lib/deimos/kafka_source.rb +36 -31
- data/lib/deimos/kafka_topic_info.rb +1 -1
- data/lib/deimos/logging.rb +20 -19
- data/lib/deimos/message.rb +1 -1
- data/lib/deimos/metrics/minimal_datadog_listener.rb +19 -6
- data/lib/deimos/metrics/provider.rb +4 -4
- data/lib/deimos/producer.rb +3 -1
- data/lib/deimos/railtie.rb +1 -1
- data/lib/deimos/schema_backends/avro_base.rb +1 -1
- data/lib/deimos/schema_backends/avro_schema_coercer.rb +46 -27
- data/lib/deimos/schema_backends/avro_schema_registry.rb +8 -8
- data/lib/deimos/schema_backends/base.rb +9 -9
- data/lib/deimos/schema_backends/plain.rb +1 -1
- data/lib/deimos/schema_backends/proto_base.rb +7 -5
- data/lib/deimos/schema_backends/proto_local.rb +0 -2
- data/lib/deimos/schema_backends/proto_schema_registry.rb +0 -2
- data/lib/deimos/schema_class/base.rb +1 -1
- data/lib/deimos/schema_class/record.rb +3 -3
- data/lib/deimos/test_helpers.rb +31 -26
- data/lib/deimos/tracing/provider.rb +5 -5
- data/lib/deimos/transcoder.rb +6 -2
- data/lib/deimos/utils/db_poller/base.rb +3 -3
- data/lib/deimos/utils/deadlock_retry.rb +2 -2
- data/lib/deimos/utils/outbox_producer.rb +14 -14
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +4 -4
- data/lib/generators/deimos/active_record_generator.rb +2 -1
- data/lib/generators/deimos/db_poller_generator.rb +1 -0
- data/lib/generators/deimos/outbox_backend_generator.rb +1 -0
- data/lib/generators/deimos/schema_class_generator.rb +3 -2
- data/lib/generators/deimos/v2_generator.rb +184 -155
- data/spec/active_record_batch_consumer_association_spec.rb +6 -2
- data/spec/active_record_batch_consumer_spec.rb +83 -106
- data/spec/active_record_consume/batch_consumption_spec.rb +27 -28
- data/spec/active_record_consume/batch_slicer_spec.rb +4 -12
- data/spec/active_record_consume/mass_updater_spec.rb +42 -46
- data/spec/active_record_consume/schema_model_converter_spec.rb +1 -1
- data/spec/active_record_consumer_spec.rb +7 -5
- data/spec/active_record_producer_spec.rb +83 -73
- data/spec/backends/outbox_spec.rb +1 -1
- data/spec/batch_consumer_spec.rb +20 -20
- data/spec/consumer_spec.rb +23 -12
- data/spec/gen/sample/v1/sample_pb.rb +3 -3
- data/spec/generators/active_record_generator_spec.rb +4 -4
- data/spec/generators/schema_class/my_schema_with_circular_reference_spec.rb +2 -1
- data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +9 -2
- data/spec/generators/schema_class_generator_spec.rb +5 -5
- data/spec/kafka_source_spec.rb +13 -6
- data/spec/kafka_topic_info_spec.rb +7 -7
- data/spec/karafka/karafka.rb +6 -5
- data/spec/karafka_config/karafka_spec.rb +22 -19
- data/spec/logging_spec.rb +2 -0
- data/spec/producer_spec.rb +25 -20
- data/spec/schema_backends/avro_base_shared.rb +8 -8
- data/spec/schema_backends/avro_local_spec.rb +5 -6
- data/spec/schema_backends/avro_schema_registry_spec.rb +5 -6
- data/spec/schema_backends/proto_schema_registry_spec.rb +9 -12
- data/spec/schemas/my_namespace/generated.rb +1 -2
- data/spec/schemas/my_namespace/my_schema_with_complex_type.rb +5 -8
- data/spec/schemas/my_namespace/my_schema_with_union_type.rb +22 -23
- data/spec/spec_helper.rb +13 -17
- data/spec/utils/db_poller_spec.rb +5 -5
- data/spec/utils/deadlock_retry_spec.rb +1 -4
- data/spec/utils/outbox_producer_spec.rb +36 -24
- metadata +68 -161
- data/.ruby-version +0 -1
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
# rubocop:disable Lint/ConstantDefinitionInBlock
|
|
3
4
|
# Wrapped in a module to prevent class leakage
|
|
4
5
|
module ActiveRecordBatchConsumerTest
|
|
5
6
|
describe Deimos::ActiveRecordConsumer, 'Batch Consumer' do
|
|
@@ -36,8 +37,8 @@ module ActiveRecordBatchConsumerTest
|
|
|
36
37
|
stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
|
|
37
38
|
register_consumer(MyBatchConsumer,
|
|
38
39
|
'MySchema',
|
|
39
|
-
key_config: {plain: true},
|
|
40
|
-
configs: {bulk_import_id_column: :bulk_import_id})
|
|
40
|
+
key_config: { plain: true },
|
|
41
|
+
configs: { bulk_import_id_column: :bulk_import_id })
|
|
41
42
|
Widget.delete_all
|
|
42
43
|
end
|
|
43
44
|
|
|
@@ -94,11 +95,11 @@ module ActiveRecordBatchConsumerTest
|
|
|
94
95
|
)
|
|
95
96
|
|
|
96
97
|
expect(all_widgets).
|
|
97
|
-
to
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
98
|
+
to contain_exactly(
|
|
99
|
+
have_attributes(id: 1, test_id: 'abc', some_int: 3, updated_at: start,
|
|
100
|
+
created_at: start),
|
|
101
|
+
have_attributes(id: 2, test_id: 'def', some_int: 4, updated_at: start,
|
|
102
|
+
created_at: start)
|
|
102
103
|
)
|
|
103
104
|
end
|
|
104
105
|
|
|
@@ -143,11 +144,9 @@ module ActiveRecordBatchConsumerTest
|
|
|
143
144
|
)
|
|
144
145
|
|
|
145
146
|
expect(all_widgets).
|
|
146
|
-
to
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
]
|
|
150
|
-
)
|
|
147
|
+
to contain_exactly(have_attributes(id: 1, test_id: 'abc', some_int: 3,
|
|
148
|
+
updated_at: Time.zone.now,
|
|
149
|
+
created_at: Time.zone.now))
|
|
151
150
|
end
|
|
152
151
|
|
|
153
152
|
it 'should handle a double update' do
|
|
@@ -165,11 +164,9 @@ module ActiveRecordBatchConsumerTest
|
|
|
165
164
|
)
|
|
166
165
|
|
|
167
166
|
expect(all_widgets).
|
|
168
|
-
to
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
]
|
|
172
|
-
)
|
|
167
|
+
to contain_exactly(have_attributes(id: 1, test_id: 'ghi', some_int: 4,
|
|
168
|
+
updated_at: Time.zone.now,
|
|
169
|
+
created_at: start))
|
|
173
170
|
end
|
|
174
171
|
|
|
175
172
|
it 'should handle a double deletion' do
|
|
@@ -201,11 +198,7 @@ module ActiveRecordBatchConsumerTest
|
|
|
201
198
|
)
|
|
202
199
|
|
|
203
200
|
expect(all_widgets).
|
|
204
|
-
to
|
|
205
|
-
[
|
|
206
|
-
have_attributes(id: 2, test_id: 'def', some_int: 5)
|
|
207
|
-
]
|
|
208
|
-
)
|
|
201
|
+
to contain_exactly(have_attributes(id: 2, test_id: 'def', some_int: 5))
|
|
209
202
|
end
|
|
210
203
|
|
|
211
204
|
it 'should handle deletes with deadlock retries' do
|
|
@@ -257,7 +250,7 @@ module ActiveRecordBatchConsumerTest
|
|
|
257
250
|
before(:each) do
|
|
258
251
|
register_consumer(consumer_class,
|
|
259
252
|
'MySchema',
|
|
260
|
-
key_config: {plain: true})
|
|
253
|
+
key_config: { plain: true })
|
|
261
254
|
|
|
262
255
|
end
|
|
263
256
|
|
|
@@ -292,13 +285,9 @@ module ActiveRecordBatchConsumerTest
|
|
|
292
285
|
)
|
|
293
286
|
|
|
294
287
|
expect(all_widgets).
|
|
295
|
-
to
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
have_attributes(id: 2, test_id: 'def', some_int: 4),
|
|
299
|
-
have_attributes(id: 3, test_id: 'hij', some_int: 9)
|
|
300
|
-
]
|
|
301
|
-
)
|
|
288
|
+
to contain_exactly(have_attributes(id: 1, test_id: 'abc', some_int: 3),
|
|
289
|
+
have_attributes(id: 2, test_id: 'def', some_int: 4),
|
|
290
|
+
have_attributes(id: 3, test_id: 'hij', some_int: 9))
|
|
302
291
|
end
|
|
303
292
|
end
|
|
304
293
|
|
|
@@ -306,7 +295,7 @@ module ActiveRecordBatchConsumerTest
|
|
|
306
295
|
before(:each) do
|
|
307
296
|
register_consumer(consumer_class,
|
|
308
297
|
'MySchema',
|
|
309
|
-
key_config: {schema: 'MySchemaCompound_key'})
|
|
298
|
+
key_config: { schema: 'MySchemaCompound_key' })
|
|
310
299
|
end
|
|
311
300
|
|
|
312
301
|
let(:consumer_class) do
|
|
@@ -347,12 +336,10 @@ module ActiveRecordBatchConsumerTest
|
|
|
347
336
|
)
|
|
348
337
|
|
|
349
338
|
expect(all_widgets).
|
|
350
|
-
to
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
]
|
|
355
|
-
)
|
|
339
|
+
to contain_exactly(have_attributes(test_id: 'aaa', some_int: 3, part_one: 'abc',
|
|
340
|
+
part_two: 'def'),
|
|
341
|
+
have_attributes(test_id: 'bbb', some_int: 4, part_one: 'ghi',
|
|
342
|
+
part_two: 'jkl'))
|
|
356
343
|
end
|
|
357
344
|
end
|
|
358
345
|
|
|
@@ -360,7 +347,7 @@ module ActiveRecordBatchConsumerTest
|
|
|
360
347
|
before(:each) do
|
|
361
348
|
register_consumer(consumer_class,
|
|
362
349
|
'MySchema',
|
|
363
|
-
key_config: {none: true})
|
|
350
|
+
key_config: { none: true })
|
|
364
351
|
end
|
|
365
352
|
|
|
366
353
|
it 'should handle unkeyed topics' do
|
|
@@ -375,13 +362,9 @@ module ActiveRecordBatchConsumerTest
|
|
|
375
362
|
)
|
|
376
363
|
|
|
377
364
|
expect(all_widgets).
|
|
378
|
-
to
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
have_attributes(test_id: 'aaa', some_int: 3),
|
|
382
|
-
have_attributes(test_id: 'bbb', some_int: 4)
|
|
383
|
-
]
|
|
384
|
-
)
|
|
365
|
+
to contain_exactly(have_attributes(test_id: 'xxx', some_int: 2),
|
|
366
|
+
have_attributes(test_id: 'aaa', some_int: 3),
|
|
367
|
+
have_attributes(test_id: 'bbb', some_int: 4))
|
|
385
368
|
end
|
|
386
369
|
end
|
|
387
370
|
|
|
@@ -389,8 +372,9 @@ module ActiveRecordBatchConsumerTest
|
|
|
389
372
|
before(:each) do
|
|
390
373
|
register_consumer(consumer_class,
|
|
391
374
|
'MySchema',
|
|
392
|
-
key_config: {plain: true})
|
|
375
|
+
key_config: { plain: true })
|
|
393
376
|
end
|
|
377
|
+
|
|
394
378
|
let(:consumer_class) do
|
|
395
379
|
Class.new(described_class) do
|
|
396
380
|
record_class Widget
|
|
@@ -438,18 +422,14 @@ module ActiveRecordBatchConsumerTest
|
|
|
438
422
|
)
|
|
439
423
|
|
|
440
424
|
expect(all_widgets).
|
|
441
|
-
to
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
have_attributes(id: 4, test_id: 'uvw', some_int: 5, deleted: false,
|
|
450
|
-
created_at: start, updated_at: Time.zone.now)
|
|
451
|
-
]
|
|
452
|
-
)
|
|
425
|
+
to contain_exactly(have_attributes(id: 1, test_id: 'abc', some_int: 2, deleted: true,
|
|
426
|
+
created_at: start, updated_at: Time.zone.now),
|
|
427
|
+
have_attributes(id: 2, test_id: 'def', some_int: 3, deleted: true,
|
|
428
|
+
created_at: Time.zone.now, updated_at: Time.zone.now),
|
|
429
|
+
have_attributes(id: 3, test_id: 'ghi', some_int: 4, deleted: false,
|
|
430
|
+
created_at: start, updated_at: Time.zone.now),
|
|
431
|
+
have_attributes(id: 4, test_id: 'uvw', some_int: 5, deleted: false,
|
|
432
|
+
created_at: start, updated_at: Time.zone.now))
|
|
453
433
|
end
|
|
454
434
|
end
|
|
455
435
|
|
|
@@ -457,8 +437,9 @@ module ActiveRecordBatchConsumerTest
|
|
|
457
437
|
before(:each) do
|
|
458
438
|
register_consumer(consumer_class,
|
|
459
439
|
'MySchema',
|
|
460
|
-
key_config: {plain: true})
|
|
440
|
+
key_config: { plain: true })
|
|
461
441
|
end
|
|
442
|
+
|
|
462
443
|
let(:consumer_class) do
|
|
463
444
|
Class.new(described_class) do
|
|
464
445
|
record_class Widget
|
|
@@ -483,7 +464,7 @@ module ActiveRecordBatchConsumerTest
|
|
|
483
464
|
)
|
|
484
465
|
|
|
485
466
|
expect(all_widgets).
|
|
486
|
-
to
|
|
467
|
+
to contain_exactly(have_attributes(id: 2, test_id: 'abc123'))
|
|
487
468
|
end
|
|
488
469
|
end
|
|
489
470
|
|
|
@@ -492,8 +473,9 @@ module ActiveRecordBatchConsumerTest
|
|
|
492
473
|
before(:each) do
|
|
493
474
|
register_consumer(consumer_class,
|
|
494
475
|
'MySchema',
|
|
495
|
-
key_config: {plain: true})
|
|
476
|
+
key_config: { plain: true })
|
|
496
477
|
end
|
|
478
|
+
|
|
497
479
|
let(:consumer_class) do
|
|
498
480
|
Class.new(described_class) do
|
|
499
481
|
record_class Widget
|
|
@@ -536,9 +518,8 @@ module ActiveRecordBatchConsumerTest
|
|
|
536
518
|
before(:each) do
|
|
537
519
|
register_consumer(consumer_class,
|
|
538
520
|
'MySchema',
|
|
539
|
-
key_config: {plain: true},
|
|
540
|
-
configs: {bulk_import_id_generator: proc { 'global' }}
|
|
541
|
-
)
|
|
521
|
+
key_config: { plain: true },
|
|
522
|
+
configs: { bulk_import_id_generator: proc { 'global' } })
|
|
542
523
|
end
|
|
543
524
|
|
|
544
525
|
it 'should call the default bulk_import_id_generator proc' do
|
|
@@ -550,16 +531,12 @@ module ActiveRecordBatchConsumerTest
|
|
|
550
531
|
)
|
|
551
532
|
|
|
552
533
|
expect(all_widgets).
|
|
553
|
-
to
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
created_at: start,
|
|
560
|
-
bulk_import_id: 'global')
|
|
561
|
-
]
|
|
562
|
-
)
|
|
534
|
+
to contain_exactly(have_attributes(id: 1,
|
|
535
|
+
test_id: 'abc',
|
|
536
|
+
some_int: 3,
|
|
537
|
+
updated_at: start,
|
|
538
|
+
created_at: start,
|
|
539
|
+
bulk_import_id: 'global'))
|
|
563
540
|
|
|
564
541
|
end
|
|
565
542
|
|
|
@@ -571,14 +548,13 @@ module ActiveRecordBatchConsumerTest
|
|
|
571
548
|
Karafka::App.routes.clear
|
|
572
549
|
Karafka::App.routes.draw do
|
|
573
550
|
defaults do
|
|
574
|
-
bulk_import_id_generator(proc { 'global'})
|
|
551
|
+
bulk_import_id_generator(proc { 'global' })
|
|
575
552
|
end
|
|
576
553
|
end
|
|
577
554
|
register_consumer(consumer_class,
|
|
578
555
|
'MySchema',
|
|
579
|
-
key_config: {plain: true},
|
|
580
|
-
configs: {bulk_import_id_generator: proc { 'custom' }}
|
|
581
|
-
)
|
|
556
|
+
key_config: { plain: true },
|
|
557
|
+
configs: { bulk_import_id_generator: proc { 'custom' } })
|
|
582
558
|
end
|
|
583
559
|
|
|
584
560
|
it 'should call the default bulk_import_id_generator proc' do
|
|
@@ -591,16 +567,12 @@ module ActiveRecordBatchConsumerTest
|
|
|
591
567
|
)
|
|
592
568
|
|
|
593
569
|
expect(all_widgets).
|
|
594
|
-
to
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
created_at: start,
|
|
601
|
-
bulk_import_id: 'custom')
|
|
602
|
-
]
|
|
603
|
-
)
|
|
570
|
+
to contain_exactly(have_attributes(id: 1,
|
|
571
|
+
test_id: 'abc',
|
|
572
|
+
some_int: 3,
|
|
573
|
+
updated_at: start,
|
|
574
|
+
created_at: start,
|
|
575
|
+
bulk_import_id: 'custom'))
|
|
604
576
|
|
|
605
577
|
end
|
|
606
578
|
end
|
|
@@ -612,8 +584,9 @@ module ActiveRecordBatchConsumerTest
|
|
|
612
584
|
before(:each) do
|
|
613
585
|
register_consumer(consumer_class,
|
|
614
586
|
'MySchema',
|
|
615
|
-
key_config: {plain: true})
|
|
587
|
+
key_config: { plain: true })
|
|
616
588
|
end
|
|
589
|
+
|
|
617
590
|
let(:consumer_class) do
|
|
618
591
|
Class.new(described_class) do
|
|
619
592
|
record_class Widget
|
|
@@ -647,18 +620,16 @@ module ActiveRecordBatchConsumerTest
|
|
|
647
620
|
)
|
|
648
621
|
|
|
649
622
|
expect(Widget.count).to eq(2)
|
|
650
|
-
expect(Widget.all.to_a).to
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
created_at: start)
|
|
661
|
-
])
|
|
623
|
+
expect(Widget.all.to_a).to contain_exactly(have_attributes(id: 1,
|
|
624
|
+
test_id: 'abc',
|
|
625
|
+
some_int: 11,
|
|
626
|
+
updated_at: start,
|
|
627
|
+
created_at: start),
|
|
628
|
+
have_attributes(id: 2,
|
|
629
|
+
test_id: 'def',
|
|
630
|
+
some_int: 2,
|
|
631
|
+
updated_at: start,
|
|
632
|
+
created_at: start))
|
|
662
633
|
end
|
|
663
634
|
|
|
664
635
|
end
|
|
@@ -669,8 +640,9 @@ module ActiveRecordBatchConsumerTest
|
|
|
669
640
|
before(:each) do
|
|
670
641
|
register_consumer(consumer_class,
|
|
671
642
|
'MySchema',
|
|
672
|
-
key_config: {plain: true})
|
|
643
|
+
key_config: { plain: true })
|
|
673
644
|
end
|
|
645
|
+
|
|
674
646
|
let(:consumer_class) do
|
|
675
647
|
Class.new(described_class) do
|
|
676
648
|
record_class Widget
|
|
@@ -689,7 +661,8 @@ module ActiveRecordBatchConsumerTest
|
|
|
689
661
|
def self.process_invalid_records(invalid)
|
|
690
662
|
# Invalid
|
|
691
663
|
attrs = invalid.first.record.attributes
|
|
692
|
-
Widget.find_by(id: attrs['id'], test_id: attrs['test_id']).
|
|
664
|
+
Widget.find_by(id: attrs['id'], test_id: attrs['test_id']).
|
|
665
|
+
update!(some_int: attrs['some_int'])
|
|
693
666
|
end
|
|
694
667
|
|
|
695
668
|
Karafka.monitor.subscribe('deimos.batch_consumption.invalid_records') do |payload|
|
|
@@ -727,8 +700,9 @@ module ActiveRecordBatchConsumerTest
|
|
|
727
700
|
before(:each) do
|
|
728
701
|
register_consumer(consumer_class,
|
|
729
702
|
'MySchema',
|
|
730
|
-
key_config: {plain: true})
|
|
703
|
+
key_config: { plain: true })
|
|
731
704
|
end
|
|
705
|
+
|
|
732
706
|
let(:consumer_class) do
|
|
733
707
|
Class.new(described_class) do
|
|
734
708
|
record_class Widget
|
|
@@ -747,7 +721,8 @@ module ActiveRecordBatchConsumerTest
|
|
|
747
721
|
def self.process_invalid_records(invalid)
|
|
748
722
|
# Invalid
|
|
749
723
|
attrs = invalid.first.record.attributes
|
|
750
|
-
Widget.find_by(id: attrs['id'], test_id: attrs['test_id']).
|
|
724
|
+
Widget.find_by(id: attrs['id'], test_id: attrs['test_id']).
|
|
725
|
+
update!(some_int: attrs['some_int'])
|
|
751
726
|
end
|
|
752
727
|
|
|
753
728
|
Karafka.monitor.subscribe('deimos.batch_consumption.invalid_records') do |payload|
|
|
@@ -785,8 +760,9 @@ module ActiveRecordBatchConsumerTest
|
|
|
785
760
|
before(:each) do
|
|
786
761
|
register_consumer(consumer_class,
|
|
787
762
|
'MySchema',
|
|
788
|
-
key_config: {plain: true})
|
|
763
|
+
key_config: { plain: true })
|
|
789
764
|
end
|
|
765
|
+
|
|
790
766
|
let(:consumer_class) do
|
|
791
767
|
Class.new(described_class) do
|
|
792
768
|
record_class Widget
|
|
@@ -831,3 +807,4 @@ module ActiveRecordBatchConsumerTest
|
|
|
831
807
|
|
|
832
808
|
end
|
|
833
809
|
end
|
|
810
|
+
# rubocop:enable Lint/ConstantDefinitionInBlock
|
|
@@ -103,47 +103,46 @@ RSpec.describe Deimos::ActiveRecordConsume::BatchConsumption do
|
|
|
103
103
|
|
|
104
104
|
describe '#compact_messages' do
|
|
105
105
|
it 'for scalars' do
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
result = batch_consumer.send(:compact_messages, [
|
|
106
|
+
message1 = Deimos::Message.new({ v: 'first' }, key: 1234)
|
|
107
|
+
message2 = Deimos::Message.new({ v: 'last' }, key: 1234)
|
|
108
|
+
result = batch_consumer.send(:compact_messages, [message1, message2])
|
|
109
109
|
expect(result.size).to eq(1)
|
|
110
|
-
expect(result.first.equal?(
|
|
110
|
+
expect(result.first.equal?(message2)).to be(true)
|
|
111
111
|
end
|
|
112
112
|
|
|
113
113
|
it 'for hashes' do
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
result = batch_consumer.send(:compact_messages, [
|
|
114
|
+
message1 = Deimos::Message.new({ v: 'first' }, key: { a: 1, b: 2.0, c: 'c' })
|
|
115
|
+
message2 = Deimos::Message.new({ v: 'last' }, key: { a: 1, b: 2.0, c: 'c' })
|
|
116
|
+
result = batch_consumer.send(:compact_messages, [message1, message2])
|
|
117
117
|
expect(result.size).to eq(1)
|
|
118
|
-
expect(result.first.equal?(
|
|
118
|
+
expect(result.first.equal?(message2)).to be(true)
|
|
119
119
|
end
|
|
120
120
|
|
|
121
121
|
it 'for schema classes' do
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
}
|
|
137
|
-
end
|
|
122
|
+
klass = Class.new(Deimos::SchemaClass::Record) do
|
|
123
|
+
attr_accessor :some_name
|
|
124
|
+
|
|
125
|
+
# @override
|
|
126
|
+
def initialize(some_name: '')
|
|
127
|
+
super
|
|
128
|
+
self.some_name = some_name
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# @override
|
|
132
|
+
def as_json(_opts={})
|
|
133
|
+
{
|
|
134
|
+
'some_name' => @some_name
|
|
135
|
+
}
|
|
138
136
|
end
|
|
139
137
|
end
|
|
138
|
+
stub_const('Schemas::Key', klass)
|
|
140
139
|
|
|
141
|
-
|
|
142
|
-
|
|
140
|
+
message1 = Deimos::Message.new({ v: 'first' }, key: Schemas::Key.new(some_name: '2'))
|
|
141
|
+
message2 = Deimos::Message.new({ v: 'last' }, key: Schemas::Key.new(some_name: '2'))
|
|
143
142
|
|
|
144
|
-
result = batch_consumer.send(:compact_messages, [
|
|
143
|
+
result = batch_consumer.send(:compact_messages, [message1, message2])
|
|
145
144
|
expect(result.size).to eq(1)
|
|
146
|
-
expect(result.first.equal?(
|
|
145
|
+
expect(result.first.equal?(message2)).to be(true) # The object is same as the latest one in the batch
|
|
147
146
|
end
|
|
148
147
|
end
|
|
149
148
|
end
|
|
@@ -18,18 +18,10 @@ RSpec.describe Deimos::ActiveRecordConsume::BatchSlicer do
|
|
|
18
18
|
|
|
19
19
|
expect(slices).
|
|
20
20
|
to match([
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
]),
|
|
26
|
-
match_array([
|
|
27
|
-
Deimos::Message.new({ v: 456 }, key: 'A'),
|
|
28
|
-
Deimos::Message.new({ v: 2 }, key: 'C')
|
|
29
|
-
]),
|
|
30
|
-
match_array([
|
|
31
|
-
Deimos::Message.new({ v: 3 }, key: 'C')
|
|
32
|
-
])
|
|
21
|
+
contain_exactly(Deimos::Message.new({ v: 1 }, key: 'C'), Deimos::Message.new({ v: 123 }, key: 'A'),
|
|
22
|
+
Deimos::Message.new({ v: 999 }, key: 'B')),
|
|
23
|
+
contain_exactly(Deimos::Message.new({ v: 456 }, key: 'A'), Deimos::Message.new({ v: 2 }, key: 'C')),
|
|
24
|
+
contain_exactly(Deimos::Message.new({ v: 3 }, key: 'C'))
|
|
33
25
|
])
|
|
34
26
|
end
|
|
35
27
|
|
|
@@ -195,7 +195,7 @@ RSpec.describe Deimos::ActiveRecordConsume::MassUpdater do
|
|
|
195
195
|
WidgetFidget.reset_column_information
|
|
196
196
|
end
|
|
197
197
|
|
|
198
|
-
# rubocop:disable RSpec/
|
|
198
|
+
# rubocop:disable RSpec/ExampleLength
|
|
199
199
|
it 'should backfill the associations when upserting primary records' do
|
|
200
200
|
batch = Deimos::ActiveRecordConsume::BatchRecordList.new(
|
|
201
201
|
[
|
|
@@ -247,14 +247,14 @@ RSpec.describe Deimos::ActiveRecordConsume::MassUpdater do
|
|
|
247
247
|
expect(widget_fidget.note).to eq("Stuff #{ind + 1}")
|
|
248
248
|
end
|
|
249
249
|
end
|
|
250
|
-
# rubocop:enable RSpec/
|
|
250
|
+
# rubocop:enable RSpec/ExampleLength
|
|
251
251
|
|
|
252
252
|
end
|
|
253
253
|
|
|
254
254
|
context 'with recorded primary_keys' do
|
|
255
255
|
before(:all) do
|
|
256
256
|
ActiveRecord::Base.connection.create_table(:fidgets, force: true, id: false) do |t|
|
|
257
|
-
t.string
|
|
257
|
+
t.string(:test_id, primary_key: true)
|
|
258
258
|
t.integer(:some_int)
|
|
259
259
|
t.string(:bulk_import_id)
|
|
260
260
|
t.timestamps
|
|
@@ -281,6 +281,24 @@ RSpec.describe Deimos::ActiveRecordConsume::MassUpdater do
|
|
|
281
281
|
belongs_to :fidget
|
|
282
282
|
end
|
|
283
283
|
end
|
|
284
|
+
let(:batch) do
|
|
285
|
+
Deimos::ActiveRecordConsume::BatchRecordList.new(
|
|
286
|
+
[
|
|
287
|
+
Deimos::ActiveRecordConsume::BatchRecord.new(
|
|
288
|
+
klass: Fidget,
|
|
289
|
+
attributes: { test_id: 'id1', some_int: 5, fidget_detail: { title: 'Title 1' } },
|
|
290
|
+
bulk_import_column: 'bulk_import_id',
|
|
291
|
+
bulk_import_id_generator: bulk_id_generator
|
|
292
|
+
),
|
|
293
|
+
Deimos::ActiveRecordConsume::BatchRecord.new(
|
|
294
|
+
klass: Fidget,
|
|
295
|
+
attributes: { test_id: 'id2', some_int: 10, fidget_detail: { title: 'Title 2' } },
|
|
296
|
+
bulk_import_column: 'bulk_import_id',
|
|
297
|
+
bulk_import_id_generator: bulk_id_generator
|
|
298
|
+
)
|
|
299
|
+
]
|
|
300
|
+
)
|
|
301
|
+
end
|
|
284
302
|
|
|
285
303
|
let(:fidget_class) do
|
|
286
304
|
Class.new(ActiveRecord::Base) do
|
|
@@ -311,26 +329,6 @@ RSpec.describe Deimos::ActiveRecordConsume::MassUpdater do
|
|
|
311
329
|
Fidget.reset_column_information
|
|
312
330
|
end
|
|
313
331
|
|
|
314
|
-
|
|
315
|
-
let(:batch) do
|
|
316
|
-
Deimos::ActiveRecordConsume::BatchRecordList.new(
|
|
317
|
-
[
|
|
318
|
-
Deimos::ActiveRecordConsume::BatchRecord.new(
|
|
319
|
-
klass: Fidget,
|
|
320
|
-
attributes: { test_id: 'id1', some_int: 5, fidget_detail: { title: 'Title 1' } },
|
|
321
|
-
bulk_import_column: 'bulk_import_id',
|
|
322
|
-
bulk_import_id_generator: bulk_id_generator
|
|
323
|
-
),
|
|
324
|
-
Deimos::ActiveRecordConsume::BatchRecord.new(
|
|
325
|
-
klass: Fidget,
|
|
326
|
-
attributes: { test_id: 'id2', some_int: 10, fidget_detail: { title: 'Title 2' } },
|
|
327
|
-
bulk_import_column: 'bulk_import_id',
|
|
328
|
-
bulk_import_id_generator: bulk_id_generator
|
|
329
|
-
)
|
|
330
|
-
]
|
|
331
|
-
)
|
|
332
|
-
end
|
|
333
|
-
|
|
334
332
|
it 'should not backfill the primary key when the primary_key exists' do
|
|
335
333
|
allow(Fidget).to receive(:where).and_call_original
|
|
336
334
|
results = described_class.new(Widget,
|
|
@@ -339,7 +337,7 @@ RSpec.describe Deimos::ActiveRecordConsume::MassUpdater do
|
|
|
339
337
|
key_col_proc: key_proc).mass_update(batch)
|
|
340
338
|
expect(results.count).to eq(2)
|
|
341
339
|
expect(Fidget.count).to eq(2)
|
|
342
|
-
expect(Fidget).not_to have_received(:where).with(:
|
|
340
|
+
expect(Fidget).not_to have_received(:where).with(bulk_import_id: [instance_of(String), instance_of(String)])
|
|
343
341
|
end
|
|
344
342
|
|
|
345
343
|
end
|
|
@@ -374,6 +372,24 @@ RSpec.describe Deimos::ActiveRecordConsume::MassUpdater do
|
|
|
374
372
|
belongs_to :fidget
|
|
375
373
|
end
|
|
376
374
|
end
|
|
375
|
+
let(:batch) do
|
|
376
|
+
Deimos::ActiveRecordConsume::BatchRecordList.new(
|
|
377
|
+
[
|
|
378
|
+
Deimos::ActiveRecordConsume::BatchRecord.new(
|
|
379
|
+
klass: Fidget,
|
|
380
|
+
attributes: { test_id: 'id1', some_int: 5, fidget_detail: { title: 'Title 1' } },
|
|
381
|
+
bulk_import_column: 'bulk_import_id',
|
|
382
|
+
bulk_import_id_generator: bulk_id_generator
|
|
383
|
+
),
|
|
384
|
+
Deimos::ActiveRecordConsume::BatchRecord.new(
|
|
385
|
+
klass: Fidget,
|
|
386
|
+
attributes: { test_id: 'id2', some_int: 10, fidget_detail: { title: 'Title 2' } },
|
|
387
|
+
bulk_import_column: 'bulk_import_id',
|
|
388
|
+
bulk_import_id_generator: bulk_id_generator
|
|
389
|
+
)
|
|
390
|
+
]
|
|
391
|
+
)
|
|
392
|
+
end
|
|
377
393
|
|
|
378
394
|
let(:fidget_class) do
|
|
379
395
|
Class.new(ActiveRecord::Base) do
|
|
@@ -404,26 +420,6 @@ RSpec.describe Deimos::ActiveRecordConsume::MassUpdater do
|
|
|
404
420
|
Fidget.reset_column_information
|
|
405
421
|
end
|
|
406
422
|
|
|
407
|
-
|
|
408
|
-
let(:batch) do
|
|
409
|
-
Deimos::ActiveRecordConsume::BatchRecordList.new(
|
|
410
|
-
[
|
|
411
|
-
Deimos::ActiveRecordConsume::BatchRecord.new(
|
|
412
|
-
klass: Fidget,
|
|
413
|
-
attributes: { test_id: 'id1', some_int: 5, fidget_detail: { title: 'Title 1' } },
|
|
414
|
-
bulk_import_column: 'bulk_import_id',
|
|
415
|
-
bulk_import_id_generator: bulk_id_generator
|
|
416
|
-
),
|
|
417
|
-
Deimos::ActiveRecordConsume::BatchRecord.new(
|
|
418
|
-
klass: Fidget,
|
|
419
|
-
attributes: { test_id: 'id2', some_int: 10, fidget_detail: { title: 'Title 2' } },
|
|
420
|
-
bulk_import_column: 'bulk_import_id',
|
|
421
|
-
bulk_import_id_generator: bulk_id_generator
|
|
422
|
-
)
|
|
423
|
-
]
|
|
424
|
-
)
|
|
425
|
-
end
|
|
426
|
-
|
|
427
423
|
it 'should not backfill the primary key when the primary_key exists' do
|
|
428
424
|
allow(Fidget).to receive(:where).and_call_original
|
|
429
425
|
results = described_class.new(Widget,
|
|
@@ -432,8 +428,8 @@ RSpec.describe Deimos::ActiveRecordConsume::MassUpdater do
|
|
|
432
428
|
key_col_proc: key_proc).mass_update(batch)
|
|
433
429
|
expect(results.count).to eq(2)
|
|
434
430
|
expect(Fidget.count).to eq(2)
|
|
435
|
-
expect(Fidget).to have_received(:where).with(:
|
|
436
|
-
expect(batch.records.map(&:id)).to eq([1,2])
|
|
431
|
+
expect(Fidget).to have_received(:where).with(bulk_import_id: [instance_of(String), instance_of(String)])
|
|
432
|
+
expect(batch.records.map(&:id)).to eq([1, 2])
|
|
437
433
|
end
|
|
438
434
|
|
|
439
435
|
end
|