deimos-ruby 1.6.2 → 1.8.0.pre.beta2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +9 -0
- data/.rubocop.yml +15 -13
- data/.ruby-version +1 -1
- data/CHANGELOG.md +31 -0
- data/Gemfile.lock +43 -36
- data/README.md +141 -16
- data/Rakefile +1 -1
- data/deimos-ruby.gemspec +2 -1
- data/docs/ARCHITECTURE.md +144 -0
- data/docs/CONFIGURATION.md +27 -0
- data/lib/deimos.rb +7 -6
- data/lib/deimos/active_record_consume/batch_consumption.rb +159 -0
- data/lib/deimos/active_record_consume/batch_slicer.rb +27 -0
- data/lib/deimos/active_record_consume/message_consumption.rb +58 -0
- data/lib/deimos/active_record_consume/schema_model_converter.rb +52 -0
- data/lib/deimos/active_record_consumer.rb +33 -75
- data/lib/deimos/active_record_producer.rb +23 -0
- data/lib/deimos/batch_consumer.rb +2 -140
- data/lib/deimos/config/configuration.rb +28 -10
- data/lib/deimos/consume/batch_consumption.rb +150 -0
- data/lib/deimos/consume/message_consumption.rb +94 -0
- data/lib/deimos/consumer.rb +79 -69
- data/lib/deimos/kafka_message.rb +1 -1
- data/lib/deimos/kafka_topic_info.rb +1 -1
- data/lib/deimos/message.rb +6 -1
- data/lib/deimos/metrics/provider.rb +0 -2
- data/lib/deimos/poll_info.rb +9 -0
- data/lib/deimos/tracing/provider.rb +0 -2
- data/lib/deimos/utils/db_poller.rb +149 -0
- data/lib/deimos/utils/db_producer.rb +8 -3
- data/lib/deimos/utils/deadlock_retry.rb +68 -0
- data/lib/deimos/utils/lag_reporter.rb +19 -26
- data/lib/deimos/version.rb +1 -1
- data/lib/generators/deimos/db_poller/templates/migration +11 -0
- data/lib/generators/deimos/db_poller/templates/rails3_migration +16 -0
- data/lib/generators/deimos/db_poller_generator.rb +48 -0
- data/lib/tasks/deimos.rake +7 -0
- data/spec/active_record_batch_consumer_spec.rb +481 -0
- data/spec/active_record_consume/batch_slicer_spec.rb +42 -0
- data/spec/active_record_consume/schema_model_converter_spec.rb +105 -0
- data/spec/active_record_consumer_spec.rb +3 -11
- data/spec/active_record_producer_spec.rb +66 -88
- data/spec/batch_consumer_spec.rb +24 -7
- data/spec/config/configuration_spec.rb +4 -0
- data/spec/consumer_spec.rb +8 -8
- data/spec/deimos_spec.rb +57 -49
- data/spec/handlers/my_batch_consumer.rb +6 -1
- data/spec/handlers/my_consumer.rb +6 -1
- data/spec/message_spec.rb +19 -0
- data/spec/producer_spec.rb +3 -3
- data/spec/rake_spec.rb +1 -1
- data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc +18 -0
- data/spec/schemas/com/my-namespace/Wibble.avsc +43 -0
- data/spec/spec_helper.rb +61 -6
- data/spec/utils/db_poller_spec.rb +320 -0
- data/spec/utils/deadlock_retry_spec.rb +74 -0
- data/spec/utils/lag_reporter_spec.rb +29 -22
- metadata +55 -20
- data/lib/deimos/base_consumer.rb +0 -104
- data/lib/deimos/utils/executor.rb +0 -124
- data/lib/deimos/utils/platform_schema_validation.rb +0 -0
- data/lib/deimos/utils/signal_handler.rb +0 -68
- data/spec/utils/executor_spec.rb +0 -53
- data/spec/utils/signal_handler_spec.rb +0 -16
@@ -0,0 +1,74 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
RSpec.describe Deimos::Utils::DeadlockRetry do
|
4
|
+
include_context 'with widgets'
|
5
|
+
|
6
|
+
before(:each) do
|
7
|
+
allow(described_class).to receive(:sleep)
|
8
|
+
end
|
9
|
+
|
10
|
+
describe 'deadlock handling' do
|
11
|
+
let(:batch) { [{ key: 1, payload: { test_id: 'abc', some_int: 3 } }] }
|
12
|
+
|
13
|
+
it 'should retry deadlocks 3 times' do
|
14
|
+
# Should receive original attempt + 2 retries
|
15
|
+
expect(Widget).
|
16
|
+
to receive(:create).
|
17
|
+
and_raise(ActiveRecord::Deadlocked.new('Lock wait timeout exceeded')).
|
18
|
+
exactly(3).times
|
19
|
+
|
20
|
+
# After 3 tries, should let it bubble up
|
21
|
+
expect {
|
22
|
+
described_class.wrap do
|
23
|
+
Widget.create(test_id: 'abc')
|
24
|
+
end
|
25
|
+
}.to raise_error(ActiveRecord::Deadlocked)
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'should stop retrying deadlocks after success' do
|
29
|
+
allow(Widget).
|
30
|
+
to receive(:create).
|
31
|
+
with(hash_including(test_id: 'first')).
|
32
|
+
and_call_original
|
33
|
+
|
34
|
+
# Fail on first attempt, succeed on second
|
35
|
+
expect(Widget).
|
36
|
+
to receive(:create).
|
37
|
+
with(hash_including(test_id: 'second')).
|
38
|
+
and_raise(ActiveRecord::Deadlocked.new('Deadlock found when trying to get lock')).
|
39
|
+
once.
|
40
|
+
ordered
|
41
|
+
|
42
|
+
expect(Widget).
|
43
|
+
to receive(:create).
|
44
|
+
with(hash_including(test_id: 'second')).
|
45
|
+
once.
|
46
|
+
ordered.
|
47
|
+
and_call_original
|
48
|
+
|
49
|
+
# Should not raise anything
|
50
|
+
described_class.wrap do
|
51
|
+
Widget.create(test_id: 'first')
|
52
|
+
Widget.create(test_id: 'second')
|
53
|
+
end
|
54
|
+
|
55
|
+
expect(Widget.all).to match_array([
|
56
|
+
have_attributes(test_id: 'first'),
|
57
|
+
have_attributes(test_id: 'second')
|
58
|
+
])
|
59
|
+
end
|
60
|
+
|
61
|
+
it 'should not retry non-deadlock exceptions' do
|
62
|
+
expect(Widget).
|
63
|
+
to receive(:create).
|
64
|
+
and_raise(ActiveRecord::StatementInvalid.new('Oops!!')).
|
65
|
+
once
|
66
|
+
|
67
|
+
expect {
|
68
|
+
described_class.wrap do
|
69
|
+
Widget.create(test_id: 'abc')
|
70
|
+
end
|
71
|
+
}.to raise_error(ActiveRecord::StatementInvalid, 'Oops!!')
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -2,8 +2,11 @@
|
|
2
2
|
|
3
3
|
describe Deimos::Utils::LagReporter do
|
4
4
|
|
5
|
+
let(:kafka_client) { instance_double(Kafka::Client) }
|
6
|
+
let(:partition1_tags) { %w(consumer_group:group1 partition:1 topic:my-topic) }
|
7
|
+
let(:partition2_tags) { %w(consumer_group:group1 partition:2 topic:my-topic) }
|
8
|
+
|
5
9
|
before(:each) do
|
6
|
-
kafka_client = instance_double(Kafka::Client)
|
7
10
|
allow(kafka_client).to receive(:last_offset_for).and_return(100)
|
8
11
|
allow(Phobos).to receive(:create_kafka_client).and_return(kafka_client)
|
9
12
|
Deimos.configure { |c| c.consumers.report_lag = true }
|
@@ -20,38 +23,22 @@ describe Deimos::Utils::LagReporter do
|
|
20
23
|
'heartbeat.consumer.kafka',
|
21
24
|
group_id: 'group1', topic_partitions: { 'my-topic': [1] }
|
22
25
|
)
|
23
|
-
|
24
26
|
end
|
25
27
|
|
26
28
|
it 'should report lag' do
|
27
29
|
expect(Deimos.config.metrics).to receive(:gauge).ordered.twice.
|
28
|
-
with('consumer_lag', 95,
|
29
|
-
tags: %w(
|
30
|
-
consumer_group:group1
|
31
|
-
partition:1
|
32
|
-
topic:my-topic
|
33
|
-
))
|
30
|
+
with('consumer_lag', 95, tags: partition1_tags)
|
34
31
|
expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
|
35
|
-
with('consumer_lag', 80,
|
36
|
-
tags: %w(
|
37
|
-
consumer_group:group1
|
38
|
-
partition:2
|
39
|
-
topic:my-topic
|
40
|
-
))
|
32
|
+
with('consumer_lag', 80, tags: partition2_tags)
|
41
33
|
expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
|
42
|
-
with('consumer_lag', 0,
|
43
|
-
tags: %w(
|
44
|
-
consumer_group:group1
|
45
|
-
partition:2
|
46
|
-
topic:my-topic
|
47
|
-
))
|
34
|
+
with('consumer_lag', 0, tags: partition2_tags)
|
48
35
|
ActiveSupport::Notifications.instrument(
|
49
36
|
'seek.consumer.kafka',
|
50
37
|
offset: 5, topic: 'my-topic', group_id: 'group1', partition: 1
|
51
38
|
)
|
52
39
|
ActiveSupport::Notifications.instrument(
|
53
40
|
'start_process_message.consumer.kafka',
|
54
|
-
|
41
|
+
offset: 20, topic: 'my-topic', group_id: 'group1', partition: 2
|
55
42
|
)
|
56
43
|
ActiveSupport::Notifications.instrument(
|
57
44
|
'heartbeat.consumer.kafka',
|
@@ -59,8 +46,28 @@ describe Deimos::Utils::LagReporter do
|
|
59
46
|
)
|
60
47
|
ActiveSupport::Notifications.instrument(
|
61
48
|
'start_process_batch.consumer.kafka',
|
62
|
-
|
49
|
+
last_offset: 100, topic: 'my-topic', group_id: 'group1', partition: 2
|
50
|
+
)
|
51
|
+
ActiveSupport::Notifications.instrument(
|
52
|
+
'heartbeat.consumer.kafka',
|
53
|
+
group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
|
54
|
+
)
|
55
|
+
end
|
56
|
+
|
57
|
+
it 'should update lag after heartbeat' do
|
58
|
+
expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
|
59
|
+
with('consumer_lag', 94, tags: partition2_tags)
|
60
|
+
expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
|
61
|
+
with('consumer_lag', 95, tags: partition2_tags)
|
62
|
+
ActiveSupport::Notifications.instrument(
|
63
|
+
'seek.consumer.kafka',
|
64
|
+
offset: 6, topic: 'my-topic', group_id: 'group1', partition: 2
|
65
|
+
)
|
66
|
+
ActiveSupport::Notifications.instrument(
|
67
|
+
'heartbeat.consumer.kafka',
|
68
|
+
group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
|
63
69
|
)
|
70
|
+
allow(kafka_client).to receive(:last_offset_for).and_return(101)
|
64
71
|
ActiveSupport::Notifications.instrument(
|
65
72
|
'heartbeat.consumer.kafka',
|
66
73
|
group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: deimos-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.8.0.pre.beta2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Orner
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-07-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro_turf
|
@@ -52,6 +52,20 @@ dependencies:
|
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0.7'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: sigurd
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - '='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 0.0.1
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 0.0.1
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
70
|
name: activerecord
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -95,19 +109,19 @@ dependencies:
|
|
95
109
|
- !ruby/object:Gem::Version
|
96
110
|
version: '1.9'
|
97
111
|
- !ruby/object:Gem::Dependency
|
98
|
-
name:
|
112
|
+
name: database_cleaner
|
99
113
|
requirement: !ruby/object:Gem::Requirement
|
100
114
|
requirements:
|
101
115
|
- - "~>"
|
102
116
|
- !ruby/object:Gem::Version
|
103
|
-
version: '1'
|
117
|
+
version: '1.7'
|
104
118
|
type: :development
|
105
119
|
prerelease: false
|
106
120
|
version_requirements: !ruby/object:Gem::Requirement
|
107
121
|
requirements:
|
108
122
|
- - "~>"
|
109
123
|
- !ruby/object:Gem::Version
|
110
|
-
version: '1'
|
124
|
+
version: '1.7'
|
111
125
|
- !ruby/object:Gem::Dependency
|
112
126
|
name: ddtrace
|
113
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -310,7 +324,7 @@ dependencies:
|
|
310
324
|
- - "~>"
|
311
325
|
- !ruby/object:Gem::Version
|
312
326
|
version: '1.3'
|
313
|
-
description:
|
327
|
+
description:
|
314
328
|
email:
|
315
329
|
- daniel.orner@wishabi.com
|
316
330
|
executables:
|
@@ -337,10 +351,15 @@ files:
|
|
337
351
|
- bin/deimos
|
338
352
|
- deimos-ruby.gemspec
|
339
353
|
- docker-compose.yml
|
354
|
+
- docs/ARCHITECTURE.md
|
340
355
|
- docs/CONFIGURATION.md
|
341
356
|
- docs/DATABASE_BACKEND.md
|
342
357
|
- docs/PULL_REQUEST_TEMPLATE.md
|
343
358
|
- lib/deimos.rb
|
359
|
+
- lib/deimos/active_record_consume/batch_consumption.rb
|
360
|
+
- lib/deimos/active_record_consume/batch_slicer.rb
|
361
|
+
- lib/deimos/active_record_consume/message_consumption.rb
|
362
|
+
- lib/deimos/active_record_consume/schema_model_converter.rb
|
344
363
|
- lib/deimos/active_record_consumer.rb
|
345
364
|
- lib/deimos/active_record_producer.rb
|
346
365
|
- lib/deimos/backends/base.rb
|
@@ -348,11 +367,12 @@ files:
|
|
348
367
|
- lib/deimos/backends/kafka.rb
|
349
368
|
- lib/deimos/backends/kafka_async.rb
|
350
369
|
- lib/deimos/backends/test.rb
|
351
|
-
- lib/deimos/base_consumer.rb
|
352
370
|
- lib/deimos/batch_consumer.rb
|
353
371
|
- lib/deimos/config/configurable.rb
|
354
372
|
- lib/deimos/config/configuration.rb
|
355
373
|
- lib/deimos/config/phobos_config.rb
|
374
|
+
- lib/deimos/consume/batch_consumption.rb
|
375
|
+
- lib/deimos/consume/message_consumption.rb
|
356
376
|
- lib/deimos/consumer.rb
|
357
377
|
- lib/deimos/instrumentation.rb
|
358
378
|
- lib/deimos/kafka_message.rb
|
@@ -365,6 +385,7 @@ files:
|
|
365
385
|
- lib/deimos/monkey_patches/phobos_cli.rb
|
366
386
|
- lib/deimos/monkey_patches/phobos_producer.rb
|
367
387
|
- lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb
|
388
|
+
- lib/deimos/poll_info.rb
|
368
389
|
- lib/deimos/producer.rb
|
369
390
|
- lib/deimos/railtie.rb
|
370
391
|
- lib/deimos/schema_backends/avro_base.rb
|
@@ -379,17 +400,22 @@ files:
|
|
379
400
|
- lib/deimos/tracing/datadog.rb
|
380
401
|
- lib/deimos/tracing/mock.rb
|
381
402
|
- lib/deimos/tracing/provider.rb
|
403
|
+
- lib/deimos/utils/db_poller.rb
|
382
404
|
- lib/deimos/utils/db_producer.rb
|
383
|
-
- lib/deimos/utils/
|
405
|
+
- lib/deimos/utils/deadlock_retry.rb
|
384
406
|
- lib/deimos/utils/inline_consumer.rb
|
385
407
|
- lib/deimos/utils/lag_reporter.rb
|
386
|
-
- lib/deimos/utils/platform_schema_validation.rb
|
387
|
-
- lib/deimos/utils/signal_handler.rb
|
388
408
|
- lib/deimos/version.rb
|
389
409
|
- lib/generators/deimos/db_backend/templates/migration
|
390
410
|
- lib/generators/deimos/db_backend/templates/rails3_migration
|
391
411
|
- lib/generators/deimos/db_backend_generator.rb
|
412
|
+
- lib/generators/deimos/db_poller/templates/migration
|
413
|
+
- lib/generators/deimos/db_poller/templates/rails3_migration
|
414
|
+
- lib/generators/deimos/db_poller_generator.rb
|
392
415
|
- lib/tasks/deimos.rake
|
416
|
+
- spec/active_record_batch_consumer_spec.rb
|
417
|
+
- spec/active_record_consume/batch_slicer_spec.rb
|
418
|
+
- spec/active_record_consume/schema_model_converter_spec.rb
|
393
419
|
- spec/active_record_consumer_spec.rb
|
394
420
|
- spec/active_record_producer_spec.rb
|
395
421
|
- spec/backends/base_spec.rb
|
@@ -405,6 +431,7 @@ files:
|
|
405
431
|
- spec/handlers/my_consumer.rb
|
406
432
|
- spec/kafka_source_spec.rb
|
407
433
|
- spec/kafka_topic_info_spec.rb
|
434
|
+
- spec/message_spec.rb
|
408
435
|
- spec/phobos.bad_db.yml
|
409
436
|
- spec/phobos.yml
|
410
437
|
- spec/producer_spec.rb
|
@@ -416,18 +443,20 @@ files:
|
|
416
443
|
- spec/schema_backends/base_spec.rb
|
417
444
|
- spec/schemas/com/my-namespace/MySchema-key.avsc
|
418
445
|
- spec/schemas/com/my-namespace/MySchema.avsc
|
446
|
+
- spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
|
419
447
|
- spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
|
420
448
|
- spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
|
421
449
|
- spec/schemas/com/my-namespace/MySchemaWithId.avsc
|
422
450
|
- spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc
|
451
|
+
- spec/schemas/com/my-namespace/Wibble.avsc
|
423
452
|
- spec/schemas/com/my-namespace/Widget.avsc
|
424
453
|
- spec/schemas/com/my-namespace/WidgetTheSecond.avsc
|
425
454
|
- spec/spec_helper.rb
|
455
|
+
- spec/utils/db_poller_spec.rb
|
426
456
|
- spec/utils/db_producer_spec.rb
|
427
|
-
- spec/utils/
|
457
|
+
- spec/utils/deadlock_retry_spec.rb
|
428
458
|
- spec/utils/lag_reporter_spec.rb
|
429
459
|
- spec/utils/platform_schema_validation_spec.rb
|
430
|
-
- spec/utils/signal_handler_spec.rb
|
431
460
|
- support/deimos-solo.png
|
432
461
|
- support/deimos-with-name-next.png
|
433
462
|
- support/deimos-with-name.png
|
@@ -436,7 +465,7 @@ homepage: ''
|
|
436
465
|
licenses:
|
437
466
|
- Apache-2.0
|
438
467
|
metadata: {}
|
439
|
-
post_install_message:
|
468
|
+
post_install_message:
|
440
469
|
rdoc_options: []
|
441
470
|
require_paths:
|
442
471
|
- lib
|
@@ -447,15 +476,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
447
476
|
version: '0'
|
448
477
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
449
478
|
requirements:
|
450
|
-
- - "
|
479
|
+
- - ">"
|
451
480
|
- !ruby/object:Gem::Version
|
452
|
-
version:
|
481
|
+
version: 1.3.1
|
453
482
|
requirements: []
|
454
|
-
rubygems_version: 3.1.
|
455
|
-
signing_key:
|
483
|
+
rubygems_version: 3.1.3
|
484
|
+
signing_key:
|
456
485
|
specification_version: 4
|
457
486
|
summary: Kafka libraries for Ruby.
|
458
487
|
test_files:
|
488
|
+
- spec/active_record_batch_consumer_spec.rb
|
489
|
+
- spec/active_record_consume/batch_slicer_spec.rb
|
490
|
+
- spec/active_record_consume/schema_model_converter_spec.rb
|
459
491
|
- spec/active_record_consumer_spec.rb
|
460
492
|
- spec/active_record_producer_spec.rb
|
461
493
|
- spec/backends/base_spec.rb
|
@@ -471,6 +503,7 @@ test_files:
|
|
471
503
|
- spec/handlers/my_consumer.rb
|
472
504
|
- spec/kafka_source_spec.rb
|
473
505
|
- spec/kafka_topic_info_spec.rb
|
506
|
+
- spec/message_spec.rb
|
474
507
|
- spec/phobos.bad_db.yml
|
475
508
|
- spec/phobos.yml
|
476
509
|
- spec/producer_spec.rb
|
@@ -482,15 +515,17 @@ test_files:
|
|
482
515
|
- spec/schema_backends/base_spec.rb
|
483
516
|
- spec/schemas/com/my-namespace/MySchema-key.avsc
|
484
517
|
- spec/schemas/com/my-namespace/MySchema.avsc
|
518
|
+
- spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
|
485
519
|
- spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
|
486
520
|
- spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
|
487
521
|
- spec/schemas/com/my-namespace/MySchemaWithId.avsc
|
488
522
|
- spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc
|
523
|
+
- spec/schemas/com/my-namespace/Wibble.avsc
|
489
524
|
- spec/schemas/com/my-namespace/Widget.avsc
|
490
525
|
- spec/schemas/com/my-namespace/WidgetTheSecond.avsc
|
491
526
|
- spec/spec_helper.rb
|
527
|
+
- spec/utils/db_poller_spec.rb
|
492
528
|
- spec/utils/db_producer_spec.rb
|
493
|
-
- spec/utils/
|
529
|
+
- spec/utils/deadlock_retry_spec.rb
|
494
530
|
- spec/utils/lag_reporter_spec.rb
|
495
531
|
- spec/utils/platform_schema_validation_spec.rb
|
496
|
-
- spec/utils/signal_handler_spec.rb
|
data/lib/deimos/base_consumer.rb
DELETED
@@ -1,104 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Deimos
|
4
|
-
# Shared methods for Kafka Consumers
|
5
|
-
class BaseConsumer
|
6
|
-
include SharedConfig
|
7
|
-
|
8
|
-
class << self
|
9
|
-
# @return [Deimos::SchemaBackends::Base]
|
10
|
-
def decoder
|
11
|
-
@decoder ||= Deimos.schema_backend(schema: config[:schema],
|
12
|
-
namespace: config[:namespace])
|
13
|
-
end
|
14
|
-
|
15
|
-
# @return [Deimos::SchemaBackends::Base]
|
16
|
-
def key_decoder
|
17
|
-
@key_decoder ||= Deimos.schema_backend(schema: config[:key_schema],
|
18
|
-
namespace: config[:namespace])
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
# Helper method to decode an encoded key.
|
23
|
-
# @param key [String]
|
24
|
-
# @return [Object] the decoded key.
|
25
|
-
def decode_key(key)
|
26
|
-
return nil if key.nil?
|
27
|
-
|
28
|
-
config = self.class.config
|
29
|
-
unless config[:key_configured]
|
30
|
-
raise 'No key config given - if you are not decoding keys, please use '\
|
31
|
-
'`key_config plain: true`'
|
32
|
-
end
|
33
|
-
|
34
|
-
if config[:key_field]
|
35
|
-
self.class.decoder.decode_key(key, config[:key_field])
|
36
|
-
elsif config[:key_schema]
|
37
|
-
self.class.key_decoder.decode(key, schema: config[:key_schema])
|
38
|
-
else # no encoding
|
39
|
-
key
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
protected
|
44
|
-
|
45
|
-
# @param payload [Hash|String]
|
46
|
-
# @param metadata [Hash]
|
47
|
-
def _with_error_span(payload, metadata)
|
48
|
-
@span = Deimos.config.tracer&.start(
|
49
|
-
'deimos-consumer',
|
50
|
-
resource: self.class.name.gsub('::', '-')
|
51
|
-
)
|
52
|
-
yield
|
53
|
-
rescue StandardError => e
|
54
|
-
_handle_error(e, payload, metadata)
|
55
|
-
ensure
|
56
|
-
Deimos.config.tracer&.finish(@span)
|
57
|
-
end
|
58
|
-
|
59
|
-
def _report_time_delayed(payload, metadata)
|
60
|
-
return if payload.nil? || payload['timestamp'].blank?
|
61
|
-
|
62
|
-
begin
|
63
|
-
time_delayed = Time.now.in_time_zone - payload['timestamp'].to_datetime
|
64
|
-
rescue ArgumentError
|
65
|
-
Deimos.config.logger.info(
|
66
|
-
message: "Error parsing timestamp! #{payload['timestamp']}"
|
67
|
-
)
|
68
|
-
return
|
69
|
-
end
|
70
|
-
Deimos.config.metrics&.histogram('handler', time_delayed, tags: %W(
|
71
|
-
time:time_delayed
|
72
|
-
topic:#{metadata[:topic]}
|
73
|
-
))
|
74
|
-
end
|
75
|
-
|
76
|
-
# Overrideable method to determine if a given error should be considered
|
77
|
-
# "fatal" and always be reraised.
|
78
|
-
# @param error [Exception]
|
79
|
-
# @param payload [Hash]
|
80
|
-
# @param metadata [Hash]
|
81
|
-
# @return [Boolean]
|
82
|
-
def fatal_error?(_error, _payload, _metadata)
|
83
|
-
false
|
84
|
-
end
|
85
|
-
|
86
|
-
# @param exception [Exception]
|
87
|
-
# @param payload [Hash]
|
88
|
-
# @param metadata [Hash]
|
89
|
-
def _handle_error(exception, payload, metadata)
|
90
|
-
Deimos.config.tracer&.set_error(@span, exception)
|
91
|
-
|
92
|
-
raise if Deimos.config.consumers.reraise_errors ||
|
93
|
-
Deimos.config.consumers.fatal_error&.call(exception, payload, metadata) ||
|
94
|
-
fatal_error?(exception, payload, metadata)
|
95
|
-
end
|
96
|
-
|
97
|
-
# @param _time_taken [Float]
|
98
|
-
# @param _payload [Hash]
|
99
|
-
# @param _metadata [Hash]
|
100
|
-
def _handle_success(_time_taken, _payload, _metadata)
|
101
|
-
raise NotImplementedError
|
102
|
-
end
|
103
|
-
end
|
104
|
-
end
|