deimos-ruby 1.16.3 → 1.16.5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (84) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yml +3 -3
  3. data/.gitignore +1 -0
  4. data/.rubocop.yml +20 -14
  5. data/.rubocop_todo.yml +364 -0
  6. data/.ruby-version +2 -1
  7. data/CHANGELOG.md +9 -0
  8. data/Gemfile +6 -0
  9. data/README.md +7 -1
  10. data/Steepfile +6 -0
  11. data/deimos-ruby.gemspec +3 -2
  12. data/lib/deimos/active_record_consume/batch_consumption.rb +7 -2
  13. data/lib/deimos/active_record_consume/batch_slicer.rb +2 -0
  14. data/lib/deimos/active_record_consume/message_consumption.rb +8 -4
  15. data/lib/deimos/active_record_consumer.rb +7 -4
  16. data/lib/deimos/active_record_producer.rb +3 -0
  17. data/lib/deimos/backends/base.rb +4 -2
  18. data/lib/deimos/backends/kafka.rb +1 -0
  19. data/lib/deimos/backends/kafka_async.rb +1 -0
  20. data/lib/deimos/config/configuration.rb +4 -0
  21. data/lib/deimos/config/phobos_config.rb +2 -1
  22. data/lib/deimos/consume/batch_consumption.rb +8 -1
  23. data/lib/deimos/consume/message_consumption.rb +4 -1
  24. data/lib/deimos/instrumentation.rb +11 -4
  25. data/lib/deimos/kafka_message.rb +1 -0
  26. data/lib/deimos/kafka_source.rb +5 -0
  27. data/lib/deimos/kafka_topic_info.rb +4 -0
  28. data/lib/deimos/message.rb +19 -2
  29. data/lib/deimos/metrics/datadog.rb +2 -1
  30. data/lib/deimos/metrics/mock.rb +2 -2
  31. data/lib/deimos/metrics/provider.rb +6 -0
  32. data/lib/deimos/monkey_patches/phobos_cli.rb +1 -1
  33. data/lib/deimos/monkey_patches/phobos_producer.rb +1 -0
  34. data/lib/deimos/producer.rb +12 -6
  35. data/lib/deimos/schema_backends/base.rb +31 -17
  36. data/lib/deimos/schema_backends/mock.rb +2 -2
  37. data/lib/deimos/schema_class/base.rb +9 -5
  38. data/lib/deimos/schema_class/enum.rb +4 -2
  39. data/lib/deimos/schema_class/record.rb +5 -5
  40. data/lib/deimos/shared_config.rb +6 -2
  41. data/lib/deimos/test_helpers.rb +21 -4
  42. data/lib/deimos/tracing/datadog.rb +1 -1
  43. data/lib/deimos/tracing/mock.rb +4 -3
  44. data/lib/deimos/tracing/provider.rb +5 -0
  45. data/lib/deimos/utils/db_poller.rb +9 -1
  46. data/lib/deimos/utils/db_producer.rb +14 -2
  47. data/lib/deimos/utils/deadlock_retry.rb +3 -0
  48. data/lib/deimos/utils/inline_consumer.rb +14 -6
  49. data/lib/deimos/utils/lag_reporter.rb +11 -0
  50. data/lib/deimos/utils/schema_controller_mixin.rb +8 -0
  51. data/lib/deimos/version.rb +1 -1
  52. data/lib/deimos.rb +3 -2
  53. data/lib/generators/deimos/active_record_generator.rb +1 -1
  54. data/lib/generators/deimos/db_backend_generator.rb +1 -0
  55. data/lib/generators/deimos/db_poller_generator.rb +1 -0
  56. data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt +1 -1
  57. data/lib/generators/deimos/schema_class_generator.rb +13 -4
  58. data/rbs_collection.lock.yaml +176 -0
  59. data/rbs_collection.yaml +15 -0
  60. data/sig/avro.rbs +14 -0
  61. data/sig/defs.rbs +1867 -0
  62. data/sig/fig_tree.rbs +2 -0
  63. data/spec/consumer_spec.rb +14 -14
  64. data/spec/generators/schema_class/my_schema_spec.rb +3 -3
  65. data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +1 -1
  66. data/spec/producer_spec.rb +1 -1
  67. data/spec/schemas/my_namespace/my_schema_with_complex_type.rb +3 -3
  68. data/spec/snapshots/consumers-no-nest.snap +1 -1
  69. data/spec/snapshots/consumers.snap +1 -1
  70. data/spec/snapshots/consumers_and_producers-no-nest.snap +3 -3
  71. data/spec/snapshots/consumers_and_producers.snap +3 -3
  72. data/spec/snapshots/consumers_circular-no-nest.snap +1 -1
  73. data/spec/snapshots/consumers_circular.snap +1 -1
  74. data/spec/snapshots/consumers_complex_types-no-nest.snap +1 -1
  75. data/spec/snapshots/consumers_complex_types.snap +1 -1
  76. data/spec/snapshots/consumers_nested-no-nest.snap +1 -1
  77. data/spec/snapshots/consumers_nested.snap +1 -1
  78. data/spec/snapshots/namespace_folders.snap +3 -3
  79. data/spec/snapshots/producers_with_key-no-nest.snap +1 -1
  80. data/spec/snapshots/producers_with_key.snap +1 -1
  81. metadata +39 -21
  82. data/.gemfiles/avro_turf-0.gemfile +0 -3
  83. data/.gemfiles/avro_turf-1.gemfile +0 -3
  84. data/.ruby-gemset +0 -1
data/sig/fig_tree.rbs ADDED
@@ -0,0 +1,2 @@
1
+ module FigTree
2
+ end
@@ -32,8 +32,8 @@ module ConsumerTest
32
32
 
33
33
  it 'should consume a message' do
34
34
  test_consume_message(MyConsumer,
35
- {'test_id' => 'foo',
36
- 'some_int' => 123}) do |payload, _metadata|
35
+ { 'test_id' => 'foo',
36
+ 'some_int' => 123 }) do |payload, _metadata|
37
37
  expect(payload['test_id']).to eq('foo')
38
38
  expect(payload['some_int']).to eq(123)
39
39
  end
@@ -64,15 +64,15 @@ module ConsumerTest
64
64
 
65
65
  it 'should consume a message on a topic' do
66
66
  test_consume_message('my_consume_topic',
67
- {'test_id' => 'foo',
68
- 'some_int' => 123}) do |payload, _metadata|
67
+ { 'test_id' => 'foo',
68
+ 'some_int' => 123 }) do |payload, _metadata|
69
69
  expect(payload['test_id']).to eq('foo')
70
70
  expect(payload['some_int']).to eq(123)
71
71
  end
72
72
  end
73
73
 
74
74
  it 'should fail on invalid message' do
75
- test_consume_invalid_message(MyConsumer, {'invalid' => 'key'})
75
+ test_consume_invalid_message(MyConsumer, { 'invalid' => 'key' })
76
76
  end
77
77
 
78
78
  it 'should fail if reraise is false but fatal_error is true' do
@@ -85,14 +85,14 @@ module ConsumerTest
85
85
  config.consumers.fatal_error = proc { true }
86
86
  config.consumers.reraise_errors = false
87
87
  end
88
- test_consume_invalid_message(MyConsumer, {'invalid' => 'key'})
88
+ test_consume_invalid_message(MyConsumer, { 'invalid' => 'key' })
89
89
  end
90
90
 
91
91
  it 'should fail on message with extra fields' do
92
92
  test_consume_invalid_message(MyConsumer,
93
- {'test_id' => 'foo',
93
+ { 'test_id' => 'foo',
94
94
  'some_int' => 123,
95
- 'extra_field' => 'field name'})
95
+ 'extra_field' => 'field name' })
96
96
  end
97
97
 
98
98
  it 'should not fail when before_consume fails without reraising errors' do
@@ -189,10 +189,10 @@ module ConsumerTest
189
189
  it 'should consume a message' do
190
190
  expect(Deimos.config.metrics).to receive(:histogram).twice
191
191
  test_consume_message('my_consume_topic',
192
- {'test_id' => 'foo',
192
+ { 'test_id' => 'foo',
193
193
  'some_int' => 123,
194
194
  'updated_at' => Time.now.to_i,
195
- 'timestamp' => 2.minutes.ago.to_s}) do |payload, _metadata|
195
+ 'timestamp' => 2.minutes.ago.to_s }) do |payload, _metadata|
196
196
  expect(payload['test_id']).to eq('foo')
197
197
  end
198
198
  end
@@ -200,17 +200,17 @@ module ConsumerTest
200
200
  it 'should fail nicely when timestamp wrong format' do
201
201
  expect(Deimos.config.metrics).to receive(:histogram).twice
202
202
  test_consume_message('my_consume_topic',
203
- {'test_id' => 'foo',
203
+ { 'test_id' => 'foo',
204
204
  'some_int' => 123,
205
205
  'updated_at' => Time.now.to_i,
206
- 'timestamp' => 'dffdf'}) do |payload, _metadata|
206
+ 'timestamp' => 'dffdf' }) do |payload, _metadata|
207
207
  expect(payload['test_id']).to eq('foo')
208
208
  end
209
209
  test_consume_message('my_consume_topic',
210
- {'test_id' => 'foo',
210
+ { 'test_id' => 'foo',
211
211
  'some_int' => 123,
212
212
  'updated_at' => Time.now.to_i,
213
- 'timestamp' => ''}) do |payload, _metadata|
213
+ 'timestamp' => '' }) do |payload, _metadata|
214
214
  expect(payload['test_id']).to eq('foo')
215
215
  end
216
216
  end
@@ -2,15 +2,15 @@ RSpec.describe Schemas::MyNamespace::MySchema do
2
2
  let(:key) { Schemas::MyNamespace::MySchemaKey.new(test_id: 123) }
3
3
 
4
4
  it 'should produce a tombstone with a hash' do
5
- result = described_class.tombstone({test_id: 123})
5
+ result = described_class.tombstone({ test_id: 123 })
6
6
  expect(result.payload_key).to eq(key)
7
- expect(result.to_h).to eq({ payload_key: { 'test_id' => 123}})
7
+ expect(result.to_h).to eq({ payload_key: { 'test_id' => 123 } })
8
8
  end
9
9
 
10
10
  it 'should work with a record' do
11
11
  key = Schemas::MyNamespace::MySchemaKey.new(test_id: 123)
12
12
  result = described_class.tombstone(key)
13
13
  expect(result.payload_key).to eq(key)
14
- expect(result.to_h).to eq({ payload_key: { 'test_id' => 123}})
14
+ expect(result.to_h).to eq({ payload_key: { 'test_id' => 123 } })
15
15
  end
16
16
  end
@@ -55,7 +55,7 @@ RSpec.describe Schemas::MyNamespace::MySchemaWithComplexType do
55
55
  it 'should return a tombstone' do
56
56
  record = described_class.tombstone('foo')
57
57
  expect(record.tombstone_key).to eq('foo')
58
- expect(record.to_h).to eq({payload_key: 'foo'})
58
+ expect(record.to_h).to eq({ payload_key: 'foo' })
59
59
  end
60
60
  end
61
61
 
@@ -79,7 +79,7 @@ module ProducerTest
79
79
  expect(event.payload[:payloads]).to eq([{ 'invalid' => 'key' }])
80
80
  end
81
81
  expect(MyProducer.encoder).to receive(:validate).and_raise('OH NOES')
82
- expect { MyProducer.publish({'invalid' => 'key', :payload_key => 'key'}) }.
82
+ expect { MyProducer.publish({ 'invalid' => 'key', :payload_key => 'key' }) }.
83
83
  to raise_error('OH NOES')
84
84
  Deimos.unsubscribe(subscriber)
85
85
  end
@@ -9,7 +9,7 @@ module Schemas; module MyNamespace
9
9
  ### Secondary Schema Classes ###
10
10
  # Autogenerated Schema for Record at com.my-namespace.ARecord
11
11
  class ARecord < Deimos::SchemaClass::Record
12
-
12
+
13
13
  ### Attribute Accessors ###
14
14
  # @return [String]
15
15
  attr_accessor :a_record_field
@@ -150,8 +150,8 @@ module Schemas; module MyNamespace
150
150
  test_string_array: ["test"],
151
151
  test_int_array: [123],
152
152
  test_optional_int: 123,
153
- some_integer_map: {"abc"=>123},
154
- some_record: {"a_record_field"=>"Test String"},
153
+ some_integer_map: { "abc"=>123 },
154
+ some_record: { "a_record_field"=>"Test String" },
155
155
  some_optional_record: nil,
156
156
  some_record_array: nil,
157
157
  some_record_map: nil,
@@ -201,7 +201,7 @@ module Schemas
201
201
  end
202
202
 
203
203
  def self.tombstone(key)
204
- record = self.new
204
+ record = self.allocate
205
205
  record.tombstone_key = key
206
206
  record.a_string = key
207
207
  record
@@ -168,7 +168,7 @@ module Schemas
168
168
  end
169
169
 
170
170
  def self.tombstone(key)
171
- record = self.new
171
+ record = self.allocate
172
172
  record.tombstone_key = key
173
173
  record.a_string = key
174
174
  record
@@ -201,7 +201,7 @@ module Schemas
201
201
  end
202
202
 
203
203
  def self.tombstone(key)
204
- record = self.new
204
+ record = self.allocate
205
205
  record.tombstone_key = key
206
206
  record.a_string = key
207
207
  record
@@ -380,7 +380,7 @@ module Schemas
380
380
  end
381
381
 
382
382
  def self.tombstone(key)
383
- record = self.new
383
+ record = self.allocate
384
384
  record.tombstone_key = key
385
385
  record.test_id = key
386
386
  record
@@ -446,7 +446,7 @@ module Schemas
446
446
  end
447
447
 
448
448
  def self.tombstone(key)
449
- record = self.new
449
+ record = self.allocate
450
450
  record.tombstone_key = MySchemaKey.initialize_from_value(key)
451
451
  record.payload_key = key
452
452
  record
@@ -168,7 +168,7 @@ module Schemas
168
168
  end
169
169
 
170
170
  def self.tombstone(key)
171
- record = self.new
171
+ record = self.allocate
172
172
  record.tombstone_key = key
173
173
  record.a_string = key
174
174
  record
@@ -341,7 +341,7 @@ module Schemas
341
341
  end
342
342
 
343
343
  def self.tombstone(key)
344
- record = self.new
344
+ record = self.allocate
345
345
  record.tombstone_key = key
346
346
  record.test_id = key
347
347
  record
@@ -407,7 +407,7 @@ module Schemas
407
407
  end
408
408
 
409
409
  def self.tombstone(key)
410
- record = self.new
410
+ record = self.allocate
411
411
  record.tombstone_key = MySchemaKey.initialize_from_value(key)
412
412
  record.payload_key = key
413
413
  record
@@ -634,7 +634,7 @@ module Schemas
634
634
  end
635
635
 
636
636
  def self.tombstone(key)
637
- record = self.new
637
+ record = self.allocate
638
638
  record.tombstone_key = key
639
639
  record.a_string = key
640
640
  record
@@ -628,7 +628,7 @@ module Schemas
628
628
  end
629
629
 
630
630
  def self.tombstone(key)
631
- record = self.new
631
+ record = self.allocate
632
632
  record.tombstone_key = key
633
633
  record.a_string = key
634
634
  record
@@ -765,7 +765,7 @@ module Schemas
765
765
  end
766
766
 
767
767
  def self.tombstone(key)
768
- record = self.new
768
+ record = self.allocate
769
769
  record.tombstone_key = key
770
770
  record.a_string = key
771
771
  record
@@ -825,7 +825,7 @@ module Schemas
825
825
  end
826
826
 
827
827
  def self.tombstone(key)
828
- record = self.new
828
+ record = self.allocate
829
829
  record.tombstone_key = key
830
830
  record.a_string = key
831
831
  record
@@ -373,7 +373,7 @@ module Schemas
373
373
  end
374
374
 
375
375
  def self.tombstone(key)
376
- record = self.new
376
+ record = self.allocate
377
377
  record.tombstone_key = key
378
378
  record.test_id = key
379
379
  record
@@ -334,7 +334,7 @@ module Schemas
334
334
  end
335
335
 
336
336
  def self.tombstone(key)
337
- record = self.new
337
+ record = self.allocate
338
338
  record.tombstone_key = key
339
339
  record.test_id = key
340
340
  record
@@ -129,7 +129,7 @@ module Schemas; module MyNamespace
129
129
  end
130
130
 
131
131
  def self.tombstone(key)
132
- record = self.new
132
+ record = self.allocate
133
133
  record.tombstone_key = key
134
134
  record.a_string = key
135
135
  record
@@ -263,7 +263,7 @@ module Schemas; module MyNamespace
263
263
  end
264
264
 
265
265
  def self.tombstone(key)
266
- record = self.new
266
+ record = self.allocate
267
267
  record.tombstone_key = key
268
268
  record.test_id = key
269
269
  record
@@ -329,7 +329,7 @@ module Schemas; module MyNamespace
329
329
  end
330
330
 
331
331
  def self.tombstone(key)
332
- record = self.new
332
+ record = self.allocate
333
333
  record.tombstone_key = MySchemaKey.initialize_from_value(key)
334
334
  record.payload_key = key
335
335
  record
@@ -432,7 +432,7 @@ module Schemas
432
432
  end
433
433
 
434
434
  def self.tombstone(key)
435
- record = self.new
435
+ record = self.allocate
436
436
  record.tombstone_key = MySchemaKey.initialize_from_value(key)
437
437
  record.payload_key = key
438
438
  record
@@ -393,7 +393,7 @@ module Schemas
393
393
  end
394
394
 
395
395
  def self.tombstone(key)
396
- record = self.new
396
+ record = self.allocate
397
397
  record.tombstone_key = MySchemaKey.initialize_from_value(key)
398
398
  record.payload_key = key
399
399
  record
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: deimos-ruby
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.16.3
4
+ version: 1.16.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Daniel Orner
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-09-08 00:00:00.000000000 Z
11
+ date: 2022-09-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: avro_turf
@@ -16,7 +16,7 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: '0.11'
19
+ version: '1.4'
20
20
  - - "<"
21
21
  - !ruby/object:Gem::Version
22
22
  version: '2'
@@ -26,10 +26,24 @@ dependencies:
26
26
  requirements:
27
27
  - - ">="
28
28
  - !ruby/object:Gem::Version
29
- version: '0.11'
29
+ version: '1.4'
30
30
  - - "<"
31
31
  - !ruby/object:Gem::Version
32
32
  version: '2'
33
+ - !ruby/object:Gem::Dependency
34
+ name: fig_tree
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: 0.0.2
40
+ type: :runtime
41
+ prerelease: false
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - "~>"
45
+ - !ruby/object:Gem::Version
46
+ version: 0.0.2
33
47
  - !ruby/object:Gem::Dependency
34
48
  name: phobos
35
49
  requirement: !ruby/object:Gem::Requirement
@@ -84,20 +98,6 @@ dependencies:
84
98
  - - "<"
85
99
  - !ruby/object:Gem::Version
86
100
  version: '1.0'
87
- - !ruby/object:Gem::Dependency
88
- name: fig_tree
89
- requirement: !ruby/object:Gem::Requirement
90
- requirements:
91
- - - "~>"
92
- - !ruby/object:Gem::Version
93
- version: 0.0.2
94
- type: :runtime
95
- prerelease: false
96
- version_requirements: !ruby/object:Gem::Requirement
97
- requirements:
98
- - - "~>"
99
- - !ruby/object:Gem::Version
100
- version: 0.0.2
101
101
  - !ruby/object:Gem::Dependency
102
102
  name: activerecord-import
103
103
  requirement: !ruby/object:Gem::Requirement
@@ -364,6 +364,20 @@ dependencies:
364
364
  - - "~>"
365
365
  - !ruby/object:Gem::Version
366
366
  version: '1.3'
367
+ - !ruby/object:Gem::Dependency
368
+ name: steep
369
+ requirement: !ruby/object:Gem::Requirement
370
+ requirements:
371
+ - - "~>"
372
+ - !ruby/object:Gem::Version
373
+ version: '1.0'
374
+ type: :development
375
+ prerelease: false
376
+ version_requirements: !ruby/object:Gem::Requirement
377
+ requirements:
378
+ - - "~>"
379
+ - !ruby/object:Gem::Version
380
+ version: '1.0'
367
381
  description:
368
382
  email:
369
383
  - daniel.orner@wishabi.com
@@ -374,14 +388,12 @@ extensions: []
374
388
  extra_rdoc_files: []
375
389
  files:
376
390
  - ".circleci/config.yml"
377
- - ".gemfiles/avro_turf-0.gemfile"
378
- - ".gemfiles/avro_turf-1.gemfile"
379
391
  - ".github/workflows/ci.yml"
380
392
  - ".gitignore"
381
393
  - ".gitmodules"
382
394
  - ".rspec"
383
395
  - ".rubocop.yml"
384
- - ".ruby-gemset"
396
+ - ".rubocop_todo.yml"
385
397
  - ".ruby-version"
386
398
  - CHANGELOG.md
387
399
  - CODE_OF_CONDUCT.md
@@ -391,6 +403,7 @@ files:
391
403
  - LICENSE.md
392
404
  - README.md
393
405
  - Rakefile
406
+ - Steepfile
394
407
  - bin/console
395
408
  - bin/deimos
396
409
  - deimos-ruby.gemspec
@@ -469,7 +482,12 @@ files:
469
482
  - lib/generators/deimos/schema_class/templates/schema_record.rb.tt
470
483
  - lib/generators/deimos/schema_class_generator.rb
471
484
  - lib/tasks/deimos.rake
485
+ - rbs_collection.lock.yaml
486
+ - rbs_collection.yaml
472
487
  - regenerate_test_schema_classes.rb
488
+ - sig/avro.rbs
489
+ - sig/defs.rbs
490
+ - sig/fig_tree.rbs
473
491
  - spec/active_record_batch_consumer_spec.rb
474
492
  - spec/active_record_consume/batch_slicer_spec.rb
475
493
  - spec/active_record_consume/schema_model_converter_spec.rb
@@ -1,3 +0,0 @@
1
- eval_gemfile '../Gemfile'
2
-
3
- gem 'avro_turf', '~> 0.11'
@@ -1,3 +0,0 @@
1
- eval_gemfile '../Gemfile'
2
-
3
- gem 'avro_turf', '~> 1.0'
data/.ruby-gemset DELETED
@@ -1 +0,0 @@
1
- deimos