deimos-ruby 1.6.3 → 1.8.1.pre.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +9 -0
- data/.rubocop.yml +22 -16
- data/.ruby-version +1 -1
- data/CHANGELOG.md +42 -0
- data/Gemfile.lock +125 -98
- data/README.md +164 -16
- data/Rakefile +1 -1
- data/deimos-ruby.gemspec +4 -3
- data/docs/ARCHITECTURE.md +144 -0
- data/docs/CONFIGURATION.md +27 -0
- data/lib/deimos.rb +8 -7
- data/lib/deimos/active_record_consume/batch_consumption.rb +159 -0
- data/lib/deimos/active_record_consume/batch_slicer.rb +27 -0
- data/lib/deimos/active_record_consume/message_consumption.rb +58 -0
- data/lib/deimos/active_record_consume/schema_model_converter.rb +52 -0
- data/lib/deimos/active_record_consumer.rb +33 -75
- data/lib/deimos/active_record_producer.rb +23 -0
- data/lib/deimos/batch_consumer.rb +2 -140
- data/lib/deimos/config/configuration.rb +28 -10
- data/lib/deimos/consume/batch_consumption.rb +150 -0
- data/lib/deimos/consume/message_consumption.rb +94 -0
- data/lib/deimos/consumer.rb +79 -70
- data/lib/deimos/kafka_message.rb +1 -1
- data/lib/deimos/kafka_topic_info.rb +22 -3
- data/lib/deimos/message.rb +6 -1
- data/lib/deimos/metrics/provider.rb +0 -2
- data/lib/deimos/poll_info.rb +9 -0
- data/lib/deimos/schema_backends/avro_base.rb +28 -1
- data/lib/deimos/schema_backends/base.rb +15 -2
- data/lib/deimos/tracing/provider.rb +0 -2
- data/lib/deimos/utils/db_poller.rb +149 -0
- data/lib/deimos/utils/db_producer.rb +59 -16
- data/lib/deimos/utils/deadlock_retry.rb +68 -0
- data/lib/deimos/utils/lag_reporter.rb +19 -26
- data/lib/deimos/version.rb +1 -1
- data/lib/generators/deimos/active_record/templates/migration.rb.tt +28 -0
- data/lib/generators/deimos/active_record/templates/model.rb.tt +5 -0
- data/lib/generators/deimos/active_record_generator.rb +79 -0
- data/lib/generators/deimos/db_backend/templates/migration +1 -0
- data/lib/generators/deimos/db_backend/templates/rails3_migration +1 -0
- data/lib/generators/deimos/db_poller/templates/migration +11 -0
- data/lib/generators/deimos/db_poller/templates/rails3_migration +16 -0
- data/lib/generators/deimos/db_poller_generator.rb +48 -0
- data/lib/tasks/deimos.rake +7 -0
- data/spec/active_record_batch_consumer_spec.rb +481 -0
- data/spec/active_record_consume/batch_slicer_spec.rb +42 -0
- data/spec/active_record_consume/schema_model_converter_spec.rb +105 -0
- data/spec/active_record_consumer_spec.rb +3 -11
- data/spec/active_record_producer_spec.rb +66 -88
- data/spec/batch_consumer_spec.rb +24 -7
- data/spec/config/configuration_spec.rb +4 -0
- data/spec/consumer_spec.rb +8 -8
- data/spec/deimos_spec.rb +57 -49
- data/spec/generators/active_record_generator_spec.rb +56 -0
- data/spec/handlers/my_batch_consumer.rb +6 -1
- data/spec/handlers/my_consumer.rb +6 -1
- data/spec/kafka_topic_info_spec.rb +39 -16
- data/spec/message_spec.rb +19 -0
- data/spec/producer_spec.rb +3 -3
- data/spec/rake_spec.rb +1 -1
- data/spec/schemas/com/my-namespace/Generated.avsc +71 -0
- data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc +18 -0
- data/spec/schemas/com/my-namespace/Wibble.avsc +43 -0
- data/spec/spec_helper.rb +62 -6
- data/spec/utils/db_poller_spec.rb +320 -0
- data/spec/utils/db_producer_spec.rb +84 -10
- data/spec/utils/deadlock_retry_spec.rb +74 -0
- data/spec/utils/lag_reporter_spec.rb +29 -22
- metadata +66 -30
- data/lib/deimos/base_consumer.rb +0 -104
- data/lib/deimos/utils/executor.rb +0 -124
- data/lib/deimos/utils/platform_schema_validation.rb +0 -0
- data/lib/deimos/utils/signal_handler.rb +0 -68
- data/spec/utils/executor_spec.rb +0 -53
- data/spec/utils/signal_handler_spec.rb +0 -16
@@ -0,0 +1,74 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
RSpec.describe Deimos::Utils::DeadlockRetry do
|
4
|
+
include_context 'with widgets'
|
5
|
+
|
6
|
+
before(:each) do
|
7
|
+
allow(described_class).to receive(:sleep)
|
8
|
+
end
|
9
|
+
|
10
|
+
describe 'deadlock handling' do
|
11
|
+
let(:batch) { [{ key: 1, payload: { test_id: 'abc', some_int: 3 } }] }
|
12
|
+
|
13
|
+
it 'should retry deadlocks 3 times' do
|
14
|
+
# Should receive original attempt + 2 retries
|
15
|
+
expect(Widget).
|
16
|
+
to receive(:create).
|
17
|
+
and_raise(ActiveRecord::Deadlocked.new('Lock wait timeout exceeded')).
|
18
|
+
exactly(3).times
|
19
|
+
|
20
|
+
# After 3 tries, should let it bubble up
|
21
|
+
expect {
|
22
|
+
described_class.wrap do
|
23
|
+
Widget.create(test_id: 'abc')
|
24
|
+
end
|
25
|
+
}.to raise_error(ActiveRecord::Deadlocked)
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'should stop retrying deadlocks after success' do
|
29
|
+
allow(Widget).
|
30
|
+
to receive(:create).
|
31
|
+
with(hash_including(test_id: 'first')).
|
32
|
+
and_call_original
|
33
|
+
|
34
|
+
# Fail on first attempt, succeed on second
|
35
|
+
expect(Widget).
|
36
|
+
to receive(:create).
|
37
|
+
with(hash_including(test_id: 'second')).
|
38
|
+
and_raise(ActiveRecord::Deadlocked.new('Deadlock found when trying to get lock')).
|
39
|
+
once.
|
40
|
+
ordered
|
41
|
+
|
42
|
+
expect(Widget).
|
43
|
+
to receive(:create).
|
44
|
+
with(hash_including(test_id: 'second')).
|
45
|
+
once.
|
46
|
+
ordered.
|
47
|
+
and_call_original
|
48
|
+
|
49
|
+
# Should not raise anything
|
50
|
+
described_class.wrap do
|
51
|
+
Widget.create(test_id: 'first')
|
52
|
+
Widget.create(test_id: 'second')
|
53
|
+
end
|
54
|
+
|
55
|
+
expect(Widget.all).to match_array([
|
56
|
+
have_attributes(test_id: 'first'),
|
57
|
+
have_attributes(test_id: 'second')
|
58
|
+
])
|
59
|
+
end
|
60
|
+
|
61
|
+
it 'should not retry non-deadlock exceptions' do
|
62
|
+
expect(Widget).
|
63
|
+
to receive(:create).
|
64
|
+
and_raise(ActiveRecord::StatementInvalid.new('Oops!!')).
|
65
|
+
once
|
66
|
+
|
67
|
+
expect {
|
68
|
+
described_class.wrap do
|
69
|
+
Widget.create(test_id: 'abc')
|
70
|
+
end
|
71
|
+
}.to raise_error(ActiveRecord::StatementInvalid, 'Oops!!')
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -2,8 +2,11 @@
|
|
2
2
|
|
3
3
|
describe Deimos::Utils::LagReporter do
|
4
4
|
|
5
|
+
let(:kafka_client) { instance_double(Kafka::Client) }
|
6
|
+
let(:partition1_tags) { %w(consumer_group:group1 partition:1 topic:my-topic) }
|
7
|
+
let(:partition2_tags) { %w(consumer_group:group1 partition:2 topic:my-topic) }
|
8
|
+
|
5
9
|
before(:each) do
|
6
|
-
kafka_client = instance_double(Kafka::Client)
|
7
10
|
allow(kafka_client).to receive(:last_offset_for).and_return(100)
|
8
11
|
allow(Phobos).to receive(:create_kafka_client).and_return(kafka_client)
|
9
12
|
Deimos.configure { |c| c.consumers.report_lag = true }
|
@@ -20,38 +23,22 @@ describe Deimos::Utils::LagReporter do
|
|
20
23
|
'heartbeat.consumer.kafka',
|
21
24
|
group_id: 'group1', topic_partitions: { 'my-topic': [1] }
|
22
25
|
)
|
23
|
-
|
24
26
|
end
|
25
27
|
|
26
28
|
it 'should report lag' do
|
27
29
|
expect(Deimos.config.metrics).to receive(:gauge).ordered.twice.
|
28
|
-
with('consumer_lag', 95,
|
29
|
-
tags: %w(
|
30
|
-
consumer_group:group1
|
31
|
-
partition:1
|
32
|
-
topic:my-topic
|
33
|
-
))
|
30
|
+
with('consumer_lag', 95, tags: partition1_tags)
|
34
31
|
expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
|
35
|
-
with('consumer_lag', 80,
|
36
|
-
tags: %w(
|
37
|
-
consumer_group:group1
|
38
|
-
partition:2
|
39
|
-
topic:my-topic
|
40
|
-
))
|
32
|
+
with('consumer_lag', 80, tags: partition2_tags)
|
41
33
|
expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
|
42
|
-
with('consumer_lag', 0,
|
43
|
-
tags: %w(
|
44
|
-
consumer_group:group1
|
45
|
-
partition:2
|
46
|
-
topic:my-topic
|
47
|
-
))
|
34
|
+
with('consumer_lag', 0, tags: partition2_tags)
|
48
35
|
ActiveSupport::Notifications.instrument(
|
49
36
|
'seek.consumer.kafka',
|
50
37
|
offset: 5, topic: 'my-topic', group_id: 'group1', partition: 1
|
51
38
|
)
|
52
39
|
ActiveSupport::Notifications.instrument(
|
53
40
|
'start_process_message.consumer.kafka',
|
54
|
-
|
41
|
+
offset: 20, topic: 'my-topic', group_id: 'group1', partition: 2
|
55
42
|
)
|
56
43
|
ActiveSupport::Notifications.instrument(
|
57
44
|
'heartbeat.consumer.kafka',
|
@@ -59,8 +46,28 @@ describe Deimos::Utils::LagReporter do
|
|
59
46
|
)
|
60
47
|
ActiveSupport::Notifications.instrument(
|
61
48
|
'start_process_batch.consumer.kafka',
|
62
|
-
|
49
|
+
last_offset: 100, topic: 'my-topic', group_id: 'group1', partition: 2
|
50
|
+
)
|
51
|
+
ActiveSupport::Notifications.instrument(
|
52
|
+
'heartbeat.consumer.kafka',
|
53
|
+
group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
|
54
|
+
)
|
55
|
+
end
|
56
|
+
|
57
|
+
it 'should update lag after heartbeat' do
|
58
|
+
expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
|
59
|
+
with('consumer_lag', 94, tags: partition2_tags)
|
60
|
+
expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
|
61
|
+
with('consumer_lag', 95, tags: partition2_tags)
|
62
|
+
ActiveSupport::Notifications.instrument(
|
63
|
+
'seek.consumer.kafka',
|
64
|
+
offset: 6, topic: 'my-topic', group_id: 'group1', partition: 2
|
65
|
+
)
|
66
|
+
ActiveSupport::Notifications.instrument(
|
67
|
+
'heartbeat.consumer.kafka',
|
68
|
+
group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
|
63
69
|
)
|
70
|
+
allow(kafka_client).to receive(:last_offset_for).and_return(101)
|
64
71
|
ActiveSupport::Notifications.instrument(
|
65
72
|
'heartbeat.consumer.kafka',
|
66
73
|
group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: deimos-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.8.1.pre.beta1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Orner
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-07-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro_turf
|
@@ -52,20 +52,34 @@ dependencies:
|
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0.7'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: sigurd
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - '='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 0.0.1
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 0.0.1
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
70
|
name: activerecord
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
58
72
|
requirements:
|
59
73
|
- - "~>"
|
60
74
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
75
|
+
version: '6'
|
62
76
|
type: :development
|
63
77
|
prerelease: false
|
64
78
|
version_requirements: !ruby/object:Gem::Requirement
|
65
79
|
requirements:
|
66
80
|
- - "~>"
|
67
81
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
82
|
+
version: '6'
|
69
83
|
- !ruby/object:Gem::Dependency
|
70
84
|
name: activerecord-import
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -95,19 +109,19 @@ dependencies:
|
|
95
109
|
- !ruby/object:Gem::Version
|
96
110
|
version: '1.9'
|
97
111
|
- !ruby/object:Gem::Dependency
|
98
|
-
name:
|
112
|
+
name: database_cleaner
|
99
113
|
requirement: !ruby/object:Gem::Requirement
|
100
114
|
requirements:
|
101
115
|
- - "~>"
|
102
116
|
- !ruby/object:Gem::Version
|
103
|
-
version: '1'
|
117
|
+
version: '1.7'
|
104
118
|
type: :development
|
105
119
|
prerelease: false
|
106
120
|
version_requirements: !ruby/object:Gem::Requirement
|
107
121
|
requirements:
|
108
122
|
- - "~>"
|
109
123
|
- !ruby/object:Gem::Version
|
110
|
-
version: '1'
|
124
|
+
version: '1.7'
|
111
125
|
- !ruby/object:Gem::Dependency
|
112
126
|
name: ddtrace
|
113
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -212,20 +226,14 @@ dependencies:
|
|
212
226
|
requirements:
|
213
227
|
- - "~>"
|
214
228
|
- !ruby/object:Gem::Version
|
215
|
-
version: '
|
216
|
-
- - ">="
|
217
|
-
- !ruby/object:Gem::Version
|
218
|
-
version: 5.2.4.2
|
229
|
+
version: '6'
|
219
230
|
type: :development
|
220
231
|
prerelease: false
|
221
232
|
version_requirements: !ruby/object:Gem::Requirement
|
222
233
|
requirements:
|
223
234
|
- - "~>"
|
224
235
|
- !ruby/object:Gem::Version
|
225
|
-
version: '
|
226
|
-
- - ">="
|
227
|
-
- !ruby/object:Gem::Version
|
228
|
-
version: 5.2.4.2
|
236
|
+
version: '6'
|
229
237
|
- !ruby/object:Gem::Dependency
|
230
238
|
name: rake
|
231
239
|
requirement: !ruby/object:Gem::Requirement
|
@@ -310,7 +318,7 @@ dependencies:
|
|
310
318
|
- - "~>"
|
311
319
|
- !ruby/object:Gem::Version
|
312
320
|
version: '1.3'
|
313
|
-
description:
|
321
|
+
description:
|
314
322
|
email:
|
315
323
|
- daniel.orner@wishabi.com
|
316
324
|
executables:
|
@@ -337,10 +345,15 @@ files:
|
|
337
345
|
- bin/deimos
|
338
346
|
- deimos-ruby.gemspec
|
339
347
|
- docker-compose.yml
|
348
|
+
- docs/ARCHITECTURE.md
|
340
349
|
- docs/CONFIGURATION.md
|
341
350
|
- docs/DATABASE_BACKEND.md
|
342
351
|
- docs/PULL_REQUEST_TEMPLATE.md
|
343
352
|
- lib/deimos.rb
|
353
|
+
- lib/deimos/active_record_consume/batch_consumption.rb
|
354
|
+
- lib/deimos/active_record_consume/batch_slicer.rb
|
355
|
+
- lib/deimos/active_record_consume/message_consumption.rb
|
356
|
+
- lib/deimos/active_record_consume/schema_model_converter.rb
|
344
357
|
- lib/deimos/active_record_consumer.rb
|
345
358
|
- lib/deimos/active_record_producer.rb
|
346
359
|
- lib/deimos/backends/base.rb
|
@@ -348,11 +361,12 @@ files:
|
|
348
361
|
- lib/deimos/backends/kafka.rb
|
349
362
|
- lib/deimos/backends/kafka_async.rb
|
350
363
|
- lib/deimos/backends/test.rb
|
351
|
-
- lib/deimos/base_consumer.rb
|
352
364
|
- lib/deimos/batch_consumer.rb
|
353
365
|
- lib/deimos/config/configurable.rb
|
354
366
|
- lib/deimos/config/configuration.rb
|
355
367
|
- lib/deimos/config/phobos_config.rb
|
368
|
+
- lib/deimos/consume/batch_consumption.rb
|
369
|
+
- lib/deimos/consume/message_consumption.rb
|
356
370
|
- lib/deimos/consumer.rb
|
357
371
|
- lib/deimos/instrumentation.rb
|
358
372
|
- lib/deimos/kafka_message.rb
|
@@ -365,6 +379,7 @@ files:
|
|
365
379
|
- lib/deimos/monkey_patches/phobos_cli.rb
|
366
380
|
- lib/deimos/monkey_patches/phobos_producer.rb
|
367
381
|
- lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb
|
382
|
+
- lib/deimos/poll_info.rb
|
368
383
|
- lib/deimos/producer.rb
|
369
384
|
- lib/deimos/railtie.rb
|
370
385
|
- lib/deimos/schema_backends/avro_base.rb
|
@@ -379,17 +394,25 @@ files:
|
|
379
394
|
- lib/deimos/tracing/datadog.rb
|
380
395
|
- lib/deimos/tracing/mock.rb
|
381
396
|
- lib/deimos/tracing/provider.rb
|
397
|
+
- lib/deimos/utils/db_poller.rb
|
382
398
|
- lib/deimos/utils/db_producer.rb
|
383
|
-
- lib/deimos/utils/
|
399
|
+
- lib/deimos/utils/deadlock_retry.rb
|
384
400
|
- lib/deimos/utils/inline_consumer.rb
|
385
401
|
- lib/deimos/utils/lag_reporter.rb
|
386
|
-
- lib/deimos/utils/platform_schema_validation.rb
|
387
|
-
- lib/deimos/utils/signal_handler.rb
|
388
402
|
- lib/deimos/version.rb
|
403
|
+
- lib/generators/deimos/active_record/templates/migration.rb.tt
|
404
|
+
- lib/generators/deimos/active_record/templates/model.rb.tt
|
405
|
+
- lib/generators/deimos/active_record_generator.rb
|
389
406
|
- lib/generators/deimos/db_backend/templates/migration
|
390
407
|
- lib/generators/deimos/db_backend/templates/rails3_migration
|
391
408
|
- lib/generators/deimos/db_backend_generator.rb
|
409
|
+
- lib/generators/deimos/db_poller/templates/migration
|
410
|
+
- lib/generators/deimos/db_poller/templates/rails3_migration
|
411
|
+
- lib/generators/deimos/db_poller_generator.rb
|
392
412
|
- lib/tasks/deimos.rake
|
413
|
+
- spec/active_record_batch_consumer_spec.rb
|
414
|
+
- spec/active_record_consume/batch_slicer_spec.rb
|
415
|
+
- spec/active_record_consume/schema_model_converter_spec.rb
|
393
416
|
- spec/active_record_consumer_spec.rb
|
394
417
|
- spec/active_record_producer_spec.rb
|
395
418
|
- spec/backends/base_spec.rb
|
@@ -401,10 +424,12 @@ files:
|
|
401
424
|
- spec/config/configuration_spec.rb
|
402
425
|
- spec/consumer_spec.rb
|
403
426
|
- spec/deimos_spec.rb
|
427
|
+
- spec/generators/active_record_generator_spec.rb
|
404
428
|
- spec/handlers/my_batch_consumer.rb
|
405
429
|
- spec/handlers/my_consumer.rb
|
406
430
|
- spec/kafka_source_spec.rb
|
407
431
|
- spec/kafka_topic_info_spec.rb
|
432
|
+
- spec/message_spec.rb
|
408
433
|
- spec/phobos.bad_db.yml
|
409
434
|
- spec/phobos.yml
|
410
435
|
- spec/producer_spec.rb
|
@@ -414,20 +439,23 @@ files:
|
|
414
439
|
- spec/schema_backends/avro_schema_registry_spec.rb
|
415
440
|
- spec/schema_backends/avro_validation_spec.rb
|
416
441
|
- spec/schema_backends/base_spec.rb
|
442
|
+
- spec/schemas/com/my-namespace/Generated.avsc
|
417
443
|
- spec/schemas/com/my-namespace/MySchema-key.avsc
|
418
444
|
- spec/schemas/com/my-namespace/MySchema.avsc
|
445
|
+
- spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
|
419
446
|
- spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
|
420
447
|
- spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
|
421
448
|
- spec/schemas/com/my-namespace/MySchemaWithId.avsc
|
422
449
|
- spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc
|
450
|
+
- spec/schemas/com/my-namespace/Wibble.avsc
|
423
451
|
- spec/schemas/com/my-namespace/Widget.avsc
|
424
452
|
- spec/schemas/com/my-namespace/WidgetTheSecond.avsc
|
425
453
|
- spec/spec_helper.rb
|
454
|
+
- spec/utils/db_poller_spec.rb
|
426
455
|
- spec/utils/db_producer_spec.rb
|
427
|
-
- spec/utils/
|
456
|
+
- spec/utils/deadlock_retry_spec.rb
|
428
457
|
- spec/utils/lag_reporter_spec.rb
|
429
458
|
- spec/utils/platform_schema_validation_spec.rb
|
430
|
-
- spec/utils/signal_handler_spec.rb
|
431
459
|
- support/deimos-solo.png
|
432
460
|
- support/deimos-with-name-next.png
|
433
461
|
- support/deimos-with-name.png
|
@@ -436,7 +464,7 @@ homepage: ''
|
|
436
464
|
licenses:
|
437
465
|
- Apache-2.0
|
438
466
|
metadata: {}
|
439
|
-
post_install_message:
|
467
|
+
post_install_message:
|
440
468
|
rdoc_options: []
|
441
469
|
require_paths:
|
442
470
|
- lib
|
@@ -447,15 +475,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
447
475
|
version: '0'
|
448
476
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
449
477
|
requirements:
|
450
|
-
- - "
|
478
|
+
- - ">"
|
451
479
|
- !ruby/object:Gem::Version
|
452
|
-
version:
|
480
|
+
version: 1.3.1
|
453
481
|
requirements: []
|
454
|
-
rubygems_version: 3.1.
|
455
|
-
signing_key:
|
482
|
+
rubygems_version: 3.1.3
|
483
|
+
signing_key:
|
456
484
|
specification_version: 4
|
457
485
|
summary: Kafka libraries for Ruby.
|
458
486
|
test_files:
|
487
|
+
- spec/active_record_batch_consumer_spec.rb
|
488
|
+
- spec/active_record_consume/batch_slicer_spec.rb
|
489
|
+
- spec/active_record_consume/schema_model_converter_spec.rb
|
459
490
|
- spec/active_record_consumer_spec.rb
|
460
491
|
- spec/active_record_producer_spec.rb
|
461
492
|
- spec/backends/base_spec.rb
|
@@ -467,10 +498,12 @@ test_files:
|
|
467
498
|
- spec/config/configuration_spec.rb
|
468
499
|
- spec/consumer_spec.rb
|
469
500
|
- spec/deimos_spec.rb
|
501
|
+
- spec/generators/active_record_generator_spec.rb
|
470
502
|
- spec/handlers/my_batch_consumer.rb
|
471
503
|
- spec/handlers/my_consumer.rb
|
472
504
|
- spec/kafka_source_spec.rb
|
473
505
|
- spec/kafka_topic_info_spec.rb
|
506
|
+
- spec/message_spec.rb
|
474
507
|
- spec/phobos.bad_db.yml
|
475
508
|
- spec/phobos.yml
|
476
509
|
- spec/producer_spec.rb
|
@@ -480,17 +513,20 @@ test_files:
|
|
480
513
|
- spec/schema_backends/avro_schema_registry_spec.rb
|
481
514
|
- spec/schema_backends/avro_validation_spec.rb
|
482
515
|
- spec/schema_backends/base_spec.rb
|
516
|
+
- spec/schemas/com/my-namespace/Generated.avsc
|
483
517
|
- spec/schemas/com/my-namespace/MySchema-key.avsc
|
484
518
|
- spec/schemas/com/my-namespace/MySchema.avsc
|
519
|
+
- spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
|
485
520
|
- spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
|
486
521
|
- spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
|
487
522
|
- spec/schemas/com/my-namespace/MySchemaWithId.avsc
|
488
523
|
- spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc
|
524
|
+
- spec/schemas/com/my-namespace/Wibble.avsc
|
489
525
|
- spec/schemas/com/my-namespace/Widget.avsc
|
490
526
|
- spec/schemas/com/my-namespace/WidgetTheSecond.avsc
|
491
527
|
- spec/spec_helper.rb
|
528
|
+
- spec/utils/db_poller_spec.rb
|
492
529
|
- spec/utils/db_producer_spec.rb
|
493
|
-
- spec/utils/
|
530
|
+
- spec/utils/deadlock_retry_spec.rb
|
494
531
|
- spec/utils/lag_reporter_spec.rb
|
495
532
|
- spec/utils/platform_schema_validation_spec.rb
|
496
|
-
- spec/utils/signal_handler_spec.rb
|
data/lib/deimos/base_consumer.rb
DELETED
@@ -1,104 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Deimos
|
4
|
-
# Shared methods for Kafka Consumers
|
5
|
-
class BaseConsumer
|
6
|
-
include SharedConfig
|
7
|
-
|
8
|
-
class << self
|
9
|
-
# @return [Deimos::SchemaBackends::Base]
|
10
|
-
def decoder
|
11
|
-
@decoder ||= Deimos.schema_backend(schema: config[:schema],
|
12
|
-
namespace: config[:namespace])
|
13
|
-
end
|
14
|
-
|
15
|
-
# @return [Deimos::SchemaBackends::Base]
|
16
|
-
def key_decoder
|
17
|
-
@key_decoder ||= Deimos.schema_backend(schema: config[:key_schema],
|
18
|
-
namespace: config[:namespace])
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
# Helper method to decode an encoded key.
|
23
|
-
# @param key [String]
|
24
|
-
# @return [Object] the decoded key.
|
25
|
-
def decode_key(key)
|
26
|
-
return nil if key.nil?
|
27
|
-
|
28
|
-
config = self.class.config
|
29
|
-
unless config[:key_configured]
|
30
|
-
raise 'No key config given - if you are not decoding keys, please use '\
|
31
|
-
'`key_config plain: true`'
|
32
|
-
end
|
33
|
-
|
34
|
-
if config[:key_field]
|
35
|
-
self.class.decoder.decode_key(key, config[:key_field])
|
36
|
-
elsif config[:key_schema]
|
37
|
-
self.class.key_decoder.decode(key, schema: config[:key_schema])
|
38
|
-
else # no encoding
|
39
|
-
key
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
protected
|
44
|
-
|
45
|
-
# @param payload [Hash|String]
|
46
|
-
# @param metadata [Hash]
|
47
|
-
def _with_error_span(payload, metadata)
|
48
|
-
@span = Deimos.config.tracer&.start(
|
49
|
-
'deimos-consumer',
|
50
|
-
resource: self.class.name.gsub('::', '-')
|
51
|
-
)
|
52
|
-
yield
|
53
|
-
rescue StandardError => e
|
54
|
-
_handle_error(e, payload, metadata)
|
55
|
-
ensure
|
56
|
-
Deimos.config.tracer&.finish(@span)
|
57
|
-
end
|
58
|
-
|
59
|
-
def _report_time_delayed(payload, metadata)
|
60
|
-
return if payload.nil? || payload['timestamp'].blank?
|
61
|
-
|
62
|
-
begin
|
63
|
-
time_delayed = Time.now.in_time_zone - payload['timestamp'].to_datetime
|
64
|
-
rescue ArgumentError
|
65
|
-
Deimos.config.logger.info(
|
66
|
-
message: "Error parsing timestamp! #{payload['timestamp']}"
|
67
|
-
)
|
68
|
-
return
|
69
|
-
end
|
70
|
-
Deimos.config.metrics&.histogram('handler', time_delayed, tags: %W(
|
71
|
-
time:time_delayed
|
72
|
-
topic:#{metadata[:topic]}
|
73
|
-
))
|
74
|
-
end
|
75
|
-
|
76
|
-
# Overrideable method to determine if a given error should be considered
|
77
|
-
# "fatal" and always be reraised.
|
78
|
-
# @param error [Exception]
|
79
|
-
# @param payload [Hash]
|
80
|
-
# @param metadata [Hash]
|
81
|
-
# @return [Boolean]
|
82
|
-
def fatal_error?(_error, _payload, _metadata)
|
83
|
-
false
|
84
|
-
end
|
85
|
-
|
86
|
-
# @param exception [Exception]
|
87
|
-
# @param payload [Hash]
|
88
|
-
# @param metadata [Hash]
|
89
|
-
def _handle_error(exception, payload, metadata)
|
90
|
-
Deimos.config.tracer&.set_error(@span, exception)
|
91
|
-
|
92
|
-
raise if Deimos.config.consumers.reraise_errors ||
|
93
|
-
Deimos.config.consumers.fatal_error&.call(exception, payload, metadata) ||
|
94
|
-
fatal_error?(exception, payload, metadata)
|
95
|
-
end
|
96
|
-
|
97
|
-
# @param _time_taken [Float]
|
98
|
-
# @param _payload [Hash]
|
99
|
-
# @param _metadata [Hash]
|
100
|
-
def _handle_success(_time_taken, _payload, _metadata)
|
101
|
-
raise NotImplementedError
|
102
|
-
end
|
103
|
-
end
|
104
|
-
end
|