deimos-ruby 1.6.3 → 1.8.1.pre.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +9 -0
  3. data/.rubocop.yml +22 -16
  4. data/.ruby-version +1 -1
  5. data/CHANGELOG.md +42 -0
  6. data/Gemfile.lock +125 -98
  7. data/README.md +164 -16
  8. data/Rakefile +1 -1
  9. data/deimos-ruby.gemspec +4 -3
  10. data/docs/ARCHITECTURE.md +144 -0
  11. data/docs/CONFIGURATION.md +27 -0
  12. data/lib/deimos.rb +8 -7
  13. data/lib/deimos/active_record_consume/batch_consumption.rb +159 -0
  14. data/lib/deimos/active_record_consume/batch_slicer.rb +27 -0
  15. data/lib/deimos/active_record_consume/message_consumption.rb +58 -0
  16. data/lib/deimos/active_record_consume/schema_model_converter.rb +52 -0
  17. data/lib/deimos/active_record_consumer.rb +33 -75
  18. data/lib/deimos/active_record_producer.rb +23 -0
  19. data/lib/deimos/batch_consumer.rb +2 -140
  20. data/lib/deimos/config/configuration.rb +28 -10
  21. data/lib/deimos/consume/batch_consumption.rb +150 -0
  22. data/lib/deimos/consume/message_consumption.rb +94 -0
  23. data/lib/deimos/consumer.rb +79 -70
  24. data/lib/deimos/kafka_message.rb +1 -1
  25. data/lib/deimos/kafka_topic_info.rb +22 -3
  26. data/lib/deimos/message.rb +6 -1
  27. data/lib/deimos/metrics/provider.rb +0 -2
  28. data/lib/deimos/poll_info.rb +9 -0
  29. data/lib/deimos/schema_backends/avro_base.rb +28 -1
  30. data/lib/deimos/schema_backends/base.rb +15 -2
  31. data/lib/deimos/tracing/provider.rb +0 -2
  32. data/lib/deimos/utils/db_poller.rb +149 -0
  33. data/lib/deimos/utils/db_producer.rb +59 -16
  34. data/lib/deimos/utils/deadlock_retry.rb +68 -0
  35. data/lib/deimos/utils/lag_reporter.rb +19 -26
  36. data/lib/deimos/version.rb +1 -1
  37. data/lib/generators/deimos/active_record/templates/migration.rb.tt +28 -0
  38. data/lib/generators/deimos/active_record/templates/model.rb.tt +5 -0
  39. data/lib/generators/deimos/active_record_generator.rb +79 -0
  40. data/lib/generators/deimos/db_backend/templates/migration +1 -0
  41. data/lib/generators/deimos/db_backend/templates/rails3_migration +1 -0
  42. data/lib/generators/deimos/db_poller/templates/migration +11 -0
  43. data/lib/generators/deimos/db_poller/templates/rails3_migration +16 -0
  44. data/lib/generators/deimos/db_poller_generator.rb +48 -0
  45. data/lib/tasks/deimos.rake +7 -0
  46. data/spec/active_record_batch_consumer_spec.rb +481 -0
  47. data/spec/active_record_consume/batch_slicer_spec.rb +42 -0
  48. data/spec/active_record_consume/schema_model_converter_spec.rb +105 -0
  49. data/spec/active_record_consumer_spec.rb +3 -11
  50. data/spec/active_record_producer_spec.rb +66 -88
  51. data/spec/batch_consumer_spec.rb +24 -7
  52. data/spec/config/configuration_spec.rb +4 -0
  53. data/spec/consumer_spec.rb +8 -8
  54. data/spec/deimos_spec.rb +57 -49
  55. data/spec/generators/active_record_generator_spec.rb +56 -0
  56. data/spec/handlers/my_batch_consumer.rb +6 -1
  57. data/spec/handlers/my_consumer.rb +6 -1
  58. data/spec/kafka_topic_info_spec.rb +39 -16
  59. data/spec/message_spec.rb +19 -0
  60. data/spec/producer_spec.rb +3 -3
  61. data/spec/rake_spec.rb +1 -1
  62. data/spec/schemas/com/my-namespace/Generated.avsc +71 -0
  63. data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc +18 -0
  64. data/spec/schemas/com/my-namespace/Wibble.avsc +43 -0
  65. data/spec/spec_helper.rb +62 -6
  66. data/spec/utils/db_poller_spec.rb +320 -0
  67. data/spec/utils/db_producer_spec.rb +84 -10
  68. data/spec/utils/deadlock_retry_spec.rb +74 -0
  69. data/spec/utils/lag_reporter_spec.rb +29 -22
  70. metadata +66 -30
  71. data/lib/deimos/base_consumer.rb +0 -104
  72. data/lib/deimos/utils/executor.rb +0 -124
  73. data/lib/deimos/utils/platform_schema_validation.rb +0 -0
  74. data/lib/deimos/utils/signal_handler.rb +0 -68
  75. data/spec/utils/executor_spec.rb +0 -53
  76. data/spec/utils/signal_handler_spec.rb +0 -16
@@ -0,0 +1,74 @@
1
+ # frozen_string_literal: true
2
+
3
+ RSpec.describe Deimos::Utils::DeadlockRetry do
4
+ include_context 'with widgets'
5
+
6
+ before(:each) do
7
+ allow(described_class).to receive(:sleep)
8
+ end
9
+
10
+ describe 'deadlock handling' do
11
+ let(:batch) { [{ key: 1, payload: { test_id: 'abc', some_int: 3 } }] }
12
+
13
+ it 'should retry deadlocks 3 times' do
14
+ # Should receive original attempt + 2 retries
15
+ expect(Widget).
16
+ to receive(:create).
17
+ and_raise(ActiveRecord::Deadlocked.new('Lock wait timeout exceeded')).
18
+ exactly(3).times
19
+
20
+ # After 3 tries, should let it bubble up
21
+ expect {
22
+ described_class.wrap do
23
+ Widget.create(test_id: 'abc')
24
+ end
25
+ }.to raise_error(ActiveRecord::Deadlocked)
26
+ end
27
+
28
+ it 'should stop retrying deadlocks after success' do
29
+ allow(Widget).
30
+ to receive(:create).
31
+ with(hash_including(test_id: 'first')).
32
+ and_call_original
33
+
34
+ # Fail on first attempt, succeed on second
35
+ expect(Widget).
36
+ to receive(:create).
37
+ with(hash_including(test_id: 'second')).
38
+ and_raise(ActiveRecord::Deadlocked.new('Deadlock found when trying to get lock')).
39
+ once.
40
+ ordered
41
+
42
+ expect(Widget).
43
+ to receive(:create).
44
+ with(hash_including(test_id: 'second')).
45
+ once.
46
+ ordered.
47
+ and_call_original
48
+
49
+ # Should not raise anything
50
+ described_class.wrap do
51
+ Widget.create(test_id: 'first')
52
+ Widget.create(test_id: 'second')
53
+ end
54
+
55
+ expect(Widget.all).to match_array([
56
+ have_attributes(test_id: 'first'),
57
+ have_attributes(test_id: 'second')
58
+ ])
59
+ end
60
+
61
+ it 'should not retry non-deadlock exceptions' do
62
+ expect(Widget).
63
+ to receive(:create).
64
+ and_raise(ActiveRecord::StatementInvalid.new('Oops!!')).
65
+ once
66
+
67
+ expect {
68
+ described_class.wrap do
69
+ Widget.create(test_id: 'abc')
70
+ end
71
+ }.to raise_error(ActiveRecord::StatementInvalid, 'Oops!!')
72
+ end
73
+ end
74
+ end
@@ -2,8 +2,11 @@
2
2
 
3
3
  describe Deimos::Utils::LagReporter do
4
4
 
5
+ let(:kafka_client) { instance_double(Kafka::Client) }
6
+ let(:partition1_tags) { %w(consumer_group:group1 partition:1 topic:my-topic) }
7
+ let(:partition2_tags) { %w(consumer_group:group1 partition:2 topic:my-topic) }
8
+
5
9
  before(:each) do
6
- kafka_client = instance_double(Kafka::Client)
7
10
  allow(kafka_client).to receive(:last_offset_for).and_return(100)
8
11
  allow(Phobos).to receive(:create_kafka_client).and_return(kafka_client)
9
12
  Deimos.configure { |c| c.consumers.report_lag = true }
@@ -20,38 +23,22 @@ describe Deimos::Utils::LagReporter do
20
23
  'heartbeat.consumer.kafka',
21
24
  group_id: 'group1', topic_partitions: { 'my-topic': [1] }
22
25
  )
23
-
24
26
  end
25
27
 
26
28
  it 'should report lag' do
27
29
  expect(Deimos.config.metrics).to receive(:gauge).ordered.twice.
28
- with('consumer_lag', 95,
29
- tags: %w(
30
- consumer_group:group1
31
- partition:1
32
- topic:my-topic
33
- ))
30
+ with('consumer_lag', 95, tags: partition1_tags)
34
31
  expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
35
- with('consumer_lag', 80,
36
- tags: %w(
37
- consumer_group:group1
38
- partition:2
39
- topic:my-topic
40
- ))
32
+ with('consumer_lag', 80, tags: partition2_tags)
41
33
  expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
42
- with('consumer_lag', 0,
43
- tags: %w(
44
- consumer_group:group1
45
- partition:2
46
- topic:my-topic
47
- ))
34
+ with('consumer_lag', 0, tags: partition2_tags)
48
35
  ActiveSupport::Notifications.instrument(
49
36
  'seek.consumer.kafka',
50
37
  offset: 5, topic: 'my-topic', group_id: 'group1', partition: 1
51
38
  )
52
39
  ActiveSupport::Notifications.instrument(
53
40
  'start_process_message.consumer.kafka',
54
- offset_lag: 80, topic: 'my-topic', group_id: 'group1', partition: 2
41
+ offset: 20, topic: 'my-topic', group_id: 'group1', partition: 2
55
42
  )
56
43
  ActiveSupport::Notifications.instrument(
57
44
  'heartbeat.consumer.kafka',
@@ -59,8 +46,28 @@ describe Deimos::Utils::LagReporter do
59
46
  )
60
47
  ActiveSupport::Notifications.instrument(
61
48
  'start_process_batch.consumer.kafka',
62
- offset_lag: 0, topic: 'my-topic', group_id: 'group1', partition: 2
49
+ last_offset: 100, topic: 'my-topic', group_id: 'group1', partition: 2
50
+ )
51
+ ActiveSupport::Notifications.instrument(
52
+ 'heartbeat.consumer.kafka',
53
+ group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
54
+ )
55
+ end
56
+
57
+ it 'should update lag after heartbeat' do
58
+ expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
59
+ with('consumer_lag', 94, tags: partition2_tags)
60
+ expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
61
+ with('consumer_lag', 95, tags: partition2_tags)
62
+ ActiveSupport::Notifications.instrument(
63
+ 'seek.consumer.kafka',
64
+ offset: 6, topic: 'my-topic', group_id: 'group1', partition: 2
65
+ )
66
+ ActiveSupport::Notifications.instrument(
67
+ 'heartbeat.consumer.kafka',
68
+ group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
63
69
  )
70
+ allow(kafka_client).to receive(:last_offset_for).and_return(101)
64
71
  ActiveSupport::Notifications.instrument(
65
72
  'heartbeat.consumer.kafka',
66
73
  group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: deimos-ruby
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.6.3
4
+ version: 1.8.1.pre.beta1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Daniel Orner
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-05-04 00:00:00.000000000 Z
11
+ date: 2020-07-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: avro_turf
@@ -52,20 +52,34 @@ dependencies:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0.7'
55
+ - !ruby/object:Gem::Dependency
56
+ name: sigurd
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - '='
60
+ - !ruby/object:Gem::Version
61
+ version: 0.0.1
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - '='
67
+ - !ruby/object:Gem::Version
68
+ version: 0.0.1
55
69
  - !ruby/object:Gem::Dependency
56
70
  name: activerecord
57
71
  requirement: !ruby/object:Gem::Requirement
58
72
  requirements:
59
73
  - - "~>"
60
74
  - !ruby/object:Gem::Version
61
- version: '5.2'
75
+ version: '6'
62
76
  type: :development
63
77
  prerelease: false
64
78
  version_requirements: !ruby/object:Gem::Requirement
65
79
  requirements:
66
80
  - - "~>"
67
81
  - !ruby/object:Gem::Version
68
- version: '5.2'
82
+ version: '6'
69
83
  - !ruby/object:Gem::Dependency
70
84
  name: activerecord-import
71
85
  requirement: !ruby/object:Gem::Requirement
@@ -95,19 +109,19 @@ dependencies:
95
109
  - !ruby/object:Gem::Version
96
110
  version: '1.9'
97
111
  - !ruby/object:Gem::Dependency
98
- name: bundler
112
+ name: database_cleaner
99
113
  requirement: !ruby/object:Gem::Requirement
100
114
  requirements:
101
115
  - - "~>"
102
116
  - !ruby/object:Gem::Version
103
- version: '1'
117
+ version: '1.7'
104
118
  type: :development
105
119
  prerelease: false
106
120
  version_requirements: !ruby/object:Gem::Requirement
107
121
  requirements:
108
122
  - - "~>"
109
123
  - !ruby/object:Gem::Version
110
- version: '1'
124
+ version: '1.7'
111
125
  - !ruby/object:Gem::Dependency
112
126
  name: ddtrace
113
127
  requirement: !ruby/object:Gem::Requirement
@@ -212,20 +226,14 @@ dependencies:
212
226
  requirements:
213
227
  - - "~>"
214
228
  - !ruby/object:Gem::Version
215
- version: '5.2'
216
- - - ">="
217
- - !ruby/object:Gem::Version
218
- version: 5.2.4.2
229
+ version: '6'
219
230
  type: :development
220
231
  prerelease: false
221
232
  version_requirements: !ruby/object:Gem::Requirement
222
233
  requirements:
223
234
  - - "~>"
224
235
  - !ruby/object:Gem::Version
225
- version: '5.2'
226
- - - ">="
227
- - !ruby/object:Gem::Version
228
- version: 5.2.4.2
236
+ version: '6'
229
237
  - !ruby/object:Gem::Dependency
230
238
  name: rake
231
239
  requirement: !ruby/object:Gem::Requirement
@@ -310,7 +318,7 @@ dependencies:
310
318
  - - "~>"
311
319
  - !ruby/object:Gem::Version
312
320
  version: '1.3'
313
- description:
321
+ description:
314
322
  email:
315
323
  - daniel.orner@wishabi.com
316
324
  executables:
@@ -337,10 +345,15 @@ files:
337
345
  - bin/deimos
338
346
  - deimos-ruby.gemspec
339
347
  - docker-compose.yml
348
+ - docs/ARCHITECTURE.md
340
349
  - docs/CONFIGURATION.md
341
350
  - docs/DATABASE_BACKEND.md
342
351
  - docs/PULL_REQUEST_TEMPLATE.md
343
352
  - lib/deimos.rb
353
+ - lib/deimos/active_record_consume/batch_consumption.rb
354
+ - lib/deimos/active_record_consume/batch_slicer.rb
355
+ - lib/deimos/active_record_consume/message_consumption.rb
356
+ - lib/deimos/active_record_consume/schema_model_converter.rb
344
357
  - lib/deimos/active_record_consumer.rb
345
358
  - lib/deimos/active_record_producer.rb
346
359
  - lib/deimos/backends/base.rb
@@ -348,11 +361,12 @@ files:
348
361
  - lib/deimos/backends/kafka.rb
349
362
  - lib/deimos/backends/kafka_async.rb
350
363
  - lib/deimos/backends/test.rb
351
- - lib/deimos/base_consumer.rb
352
364
  - lib/deimos/batch_consumer.rb
353
365
  - lib/deimos/config/configurable.rb
354
366
  - lib/deimos/config/configuration.rb
355
367
  - lib/deimos/config/phobos_config.rb
368
+ - lib/deimos/consume/batch_consumption.rb
369
+ - lib/deimos/consume/message_consumption.rb
356
370
  - lib/deimos/consumer.rb
357
371
  - lib/deimos/instrumentation.rb
358
372
  - lib/deimos/kafka_message.rb
@@ -365,6 +379,7 @@ files:
365
379
  - lib/deimos/monkey_patches/phobos_cli.rb
366
380
  - lib/deimos/monkey_patches/phobos_producer.rb
367
381
  - lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb
382
+ - lib/deimos/poll_info.rb
368
383
  - lib/deimos/producer.rb
369
384
  - lib/deimos/railtie.rb
370
385
  - lib/deimos/schema_backends/avro_base.rb
@@ -379,17 +394,25 @@ files:
379
394
  - lib/deimos/tracing/datadog.rb
380
395
  - lib/deimos/tracing/mock.rb
381
396
  - lib/deimos/tracing/provider.rb
397
+ - lib/deimos/utils/db_poller.rb
382
398
  - lib/deimos/utils/db_producer.rb
383
- - lib/deimos/utils/executor.rb
399
+ - lib/deimos/utils/deadlock_retry.rb
384
400
  - lib/deimos/utils/inline_consumer.rb
385
401
  - lib/deimos/utils/lag_reporter.rb
386
- - lib/deimos/utils/platform_schema_validation.rb
387
- - lib/deimos/utils/signal_handler.rb
388
402
  - lib/deimos/version.rb
403
+ - lib/generators/deimos/active_record/templates/migration.rb.tt
404
+ - lib/generators/deimos/active_record/templates/model.rb.tt
405
+ - lib/generators/deimos/active_record_generator.rb
389
406
  - lib/generators/deimos/db_backend/templates/migration
390
407
  - lib/generators/deimos/db_backend/templates/rails3_migration
391
408
  - lib/generators/deimos/db_backend_generator.rb
409
+ - lib/generators/deimos/db_poller/templates/migration
410
+ - lib/generators/deimos/db_poller/templates/rails3_migration
411
+ - lib/generators/deimos/db_poller_generator.rb
392
412
  - lib/tasks/deimos.rake
413
+ - spec/active_record_batch_consumer_spec.rb
414
+ - spec/active_record_consume/batch_slicer_spec.rb
415
+ - spec/active_record_consume/schema_model_converter_spec.rb
393
416
  - spec/active_record_consumer_spec.rb
394
417
  - spec/active_record_producer_spec.rb
395
418
  - spec/backends/base_spec.rb
@@ -401,10 +424,12 @@ files:
401
424
  - spec/config/configuration_spec.rb
402
425
  - spec/consumer_spec.rb
403
426
  - spec/deimos_spec.rb
427
+ - spec/generators/active_record_generator_spec.rb
404
428
  - spec/handlers/my_batch_consumer.rb
405
429
  - spec/handlers/my_consumer.rb
406
430
  - spec/kafka_source_spec.rb
407
431
  - spec/kafka_topic_info_spec.rb
432
+ - spec/message_spec.rb
408
433
  - spec/phobos.bad_db.yml
409
434
  - spec/phobos.yml
410
435
  - spec/producer_spec.rb
@@ -414,20 +439,23 @@ files:
414
439
  - spec/schema_backends/avro_schema_registry_spec.rb
415
440
  - spec/schema_backends/avro_validation_spec.rb
416
441
  - spec/schema_backends/base_spec.rb
442
+ - spec/schemas/com/my-namespace/Generated.avsc
417
443
  - spec/schemas/com/my-namespace/MySchema-key.avsc
418
444
  - spec/schemas/com/my-namespace/MySchema.avsc
445
+ - spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
419
446
  - spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
420
447
  - spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
421
448
  - spec/schemas/com/my-namespace/MySchemaWithId.avsc
422
449
  - spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc
450
+ - spec/schemas/com/my-namespace/Wibble.avsc
423
451
  - spec/schemas/com/my-namespace/Widget.avsc
424
452
  - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
425
453
  - spec/spec_helper.rb
454
+ - spec/utils/db_poller_spec.rb
426
455
  - spec/utils/db_producer_spec.rb
427
- - spec/utils/executor_spec.rb
456
+ - spec/utils/deadlock_retry_spec.rb
428
457
  - spec/utils/lag_reporter_spec.rb
429
458
  - spec/utils/platform_schema_validation_spec.rb
430
- - spec/utils/signal_handler_spec.rb
431
459
  - support/deimos-solo.png
432
460
  - support/deimos-with-name-next.png
433
461
  - support/deimos-with-name.png
@@ -436,7 +464,7 @@ homepage: ''
436
464
  licenses:
437
465
  - Apache-2.0
438
466
  metadata: {}
439
- post_install_message:
467
+ post_install_message:
440
468
  rdoc_options: []
441
469
  require_paths:
442
470
  - lib
@@ -447,15 +475,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
447
475
  version: '0'
448
476
  required_rubygems_version: !ruby/object:Gem::Requirement
449
477
  requirements:
450
- - - ">="
478
+ - - ">"
451
479
  - !ruby/object:Gem::Version
452
- version: '0'
480
+ version: 1.3.1
453
481
  requirements: []
454
- rubygems_version: 3.1.2
455
- signing_key:
482
+ rubygems_version: 3.1.3
483
+ signing_key:
456
484
  specification_version: 4
457
485
  summary: Kafka libraries for Ruby.
458
486
  test_files:
487
+ - spec/active_record_batch_consumer_spec.rb
488
+ - spec/active_record_consume/batch_slicer_spec.rb
489
+ - spec/active_record_consume/schema_model_converter_spec.rb
459
490
  - spec/active_record_consumer_spec.rb
460
491
  - spec/active_record_producer_spec.rb
461
492
  - spec/backends/base_spec.rb
@@ -467,10 +498,12 @@ test_files:
467
498
  - spec/config/configuration_spec.rb
468
499
  - spec/consumer_spec.rb
469
500
  - spec/deimos_spec.rb
501
+ - spec/generators/active_record_generator_spec.rb
470
502
  - spec/handlers/my_batch_consumer.rb
471
503
  - spec/handlers/my_consumer.rb
472
504
  - spec/kafka_source_spec.rb
473
505
  - spec/kafka_topic_info_spec.rb
506
+ - spec/message_spec.rb
474
507
  - spec/phobos.bad_db.yml
475
508
  - spec/phobos.yml
476
509
  - spec/producer_spec.rb
@@ -480,17 +513,20 @@ test_files:
480
513
  - spec/schema_backends/avro_schema_registry_spec.rb
481
514
  - spec/schema_backends/avro_validation_spec.rb
482
515
  - spec/schema_backends/base_spec.rb
516
+ - spec/schemas/com/my-namespace/Generated.avsc
483
517
  - spec/schemas/com/my-namespace/MySchema-key.avsc
484
518
  - spec/schemas/com/my-namespace/MySchema.avsc
519
+ - spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
485
520
  - spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
486
521
  - spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
487
522
  - spec/schemas/com/my-namespace/MySchemaWithId.avsc
488
523
  - spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc
524
+ - spec/schemas/com/my-namespace/Wibble.avsc
489
525
  - spec/schemas/com/my-namespace/Widget.avsc
490
526
  - spec/schemas/com/my-namespace/WidgetTheSecond.avsc
491
527
  - spec/spec_helper.rb
528
+ - spec/utils/db_poller_spec.rb
492
529
  - spec/utils/db_producer_spec.rb
493
- - spec/utils/executor_spec.rb
530
+ - spec/utils/deadlock_retry_spec.rb
494
531
  - spec/utils/lag_reporter_spec.rb
495
532
  - spec/utils/platform_schema_validation_spec.rb
496
- - spec/utils/signal_handler_spec.rb
@@ -1,104 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Deimos
4
- # Shared methods for Kafka Consumers
5
- class BaseConsumer
6
- include SharedConfig
7
-
8
- class << self
9
- # @return [Deimos::SchemaBackends::Base]
10
- def decoder
11
- @decoder ||= Deimos.schema_backend(schema: config[:schema],
12
- namespace: config[:namespace])
13
- end
14
-
15
- # @return [Deimos::SchemaBackends::Base]
16
- def key_decoder
17
- @key_decoder ||= Deimos.schema_backend(schema: config[:key_schema],
18
- namespace: config[:namespace])
19
- end
20
- end
21
-
22
- # Helper method to decode an encoded key.
23
- # @param key [String]
24
- # @return [Object] the decoded key.
25
- def decode_key(key)
26
- return nil if key.nil?
27
-
28
- config = self.class.config
29
- unless config[:key_configured]
30
- raise 'No key config given - if you are not decoding keys, please use '\
31
- '`key_config plain: true`'
32
- end
33
-
34
- if config[:key_field]
35
- self.class.decoder.decode_key(key, config[:key_field])
36
- elsif config[:key_schema]
37
- self.class.key_decoder.decode(key, schema: config[:key_schema])
38
- else # no encoding
39
- key
40
- end
41
- end
42
-
43
- protected
44
-
45
- # @param payload [Hash|String]
46
- # @param metadata [Hash]
47
- def _with_error_span(payload, metadata)
48
- @span = Deimos.config.tracer&.start(
49
- 'deimos-consumer',
50
- resource: self.class.name.gsub('::', '-')
51
- )
52
- yield
53
- rescue StandardError => e
54
- _handle_error(e, payload, metadata)
55
- ensure
56
- Deimos.config.tracer&.finish(@span)
57
- end
58
-
59
- def _report_time_delayed(payload, metadata)
60
- return if payload.nil? || payload['timestamp'].blank?
61
-
62
- begin
63
- time_delayed = Time.now.in_time_zone - payload['timestamp'].to_datetime
64
- rescue ArgumentError
65
- Deimos.config.logger.info(
66
- message: "Error parsing timestamp! #{payload['timestamp']}"
67
- )
68
- return
69
- end
70
- Deimos.config.metrics&.histogram('handler', time_delayed, tags: %W(
71
- time:time_delayed
72
- topic:#{metadata[:topic]}
73
- ))
74
- end
75
-
76
- # Overrideable method to determine if a given error should be considered
77
- # "fatal" and always be reraised.
78
- # @param error [Exception]
79
- # @param payload [Hash]
80
- # @param metadata [Hash]
81
- # @return [Boolean]
82
- def fatal_error?(_error, _payload, _metadata)
83
- false
84
- end
85
-
86
- # @param exception [Exception]
87
- # @param payload [Hash]
88
- # @param metadata [Hash]
89
- def _handle_error(exception, payload, metadata)
90
- Deimos.config.tracer&.set_error(@span, exception)
91
-
92
- raise if Deimos.config.consumers.reraise_errors ||
93
- Deimos.config.consumers.fatal_error&.call(exception, payload, metadata) ||
94
- fatal_error?(exception, payload, metadata)
95
- end
96
-
97
- # @param _time_taken [Float]
98
- # @param _payload [Hash]
99
- # @param _metadata [Hash]
100
- def _handle_success(_time_taken, _payload, _metadata)
101
- raise NotImplementedError
102
- end
103
- end
104
- end