deimos-ruby 1.16.3 → 1.16.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +3 -3
- data/.gitignore +1 -0
- data/.rubocop.yml +20 -14
- data/.rubocop_todo.yml +364 -0
- data/.ruby-version +2 -1
- data/CHANGELOG.md +9 -0
- data/Gemfile +6 -0
- data/README.md +7 -1
- data/Steepfile +6 -0
- data/deimos-ruby.gemspec +3 -2
- data/lib/deimos/active_record_consume/batch_consumption.rb +7 -2
- data/lib/deimos/active_record_consume/batch_slicer.rb +2 -0
- data/lib/deimos/active_record_consume/message_consumption.rb +8 -4
- data/lib/deimos/active_record_consumer.rb +7 -4
- data/lib/deimos/active_record_producer.rb +3 -0
- data/lib/deimos/backends/base.rb +4 -2
- data/lib/deimos/backends/kafka.rb +1 -0
- data/lib/deimos/backends/kafka_async.rb +1 -0
- data/lib/deimos/config/configuration.rb +4 -0
- data/lib/deimos/config/phobos_config.rb +2 -1
- data/lib/deimos/consume/batch_consumption.rb +8 -1
- data/lib/deimos/consume/message_consumption.rb +4 -1
- data/lib/deimos/instrumentation.rb +11 -4
- data/lib/deimos/kafka_message.rb +1 -0
- data/lib/deimos/kafka_source.rb +5 -0
- data/lib/deimos/kafka_topic_info.rb +4 -0
- data/lib/deimos/message.rb +19 -2
- data/lib/deimos/metrics/datadog.rb +2 -1
- data/lib/deimos/metrics/mock.rb +2 -2
- data/lib/deimos/metrics/provider.rb +6 -0
- data/lib/deimos/monkey_patches/phobos_cli.rb +1 -1
- data/lib/deimos/monkey_patches/phobos_producer.rb +1 -0
- data/lib/deimos/producer.rb +12 -6
- data/lib/deimos/schema_backends/base.rb +31 -17
- data/lib/deimos/schema_backends/mock.rb +2 -2
- data/lib/deimos/schema_class/base.rb +9 -5
- data/lib/deimos/schema_class/enum.rb +4 -2
- data/lib/deimos/schema_class/record.rb +5 -5
- data/lib/deimos/shared_config.rb +6 -2
- data/lib/deimos/test_helpers.rb +21 -4
- data/lib/deimos/tracing/datadog.rb +1 -1
- data/lib/deimos/tracing/mock.rb +4 -3
- data/lib/deimos/tracing/provider.rb +5 -0
- data/lib/deimos/utils/db_poller.rb +9 -1
- data/lib/deimos/utils/db_producer.rb +14 -2
- data/lib/deimos/utils/deadlock_retry.rb +3 -0
- data/lib/deimos/utils/inline_consumer.rb +14 -6
- data/lib/deimos/utils/lag_reporter.rb +11 -0
- data/lib/deimos/utils/schema_controller_mixin.rb +8 -0
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +3 -2
- data/lib/generators/deimos/active_record_generator.rb +1 -1
- data/lib/generators/deimos/db_backend_generator.rb +1 -0
- data/lib/generators/deimos/db_poller_generator.rb +1 -0
- data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt +1 -1
- data/lib/generators/deimos/schema_class_generator.rb +13 -4
- data/rbs_collection.lock.yaml +176 -0
- data/rbs_collection.yaml +15 -0
- data/sig/avro.rbs +14 -0
- data/sig/defs.rbs +1867 -0
- data/sig/fig_tree.rbs +2 -0
- data/spec/consumer_spec.rb +14 -14
- data/spec/generators/schema_class/my_schema_spec.rb +3 -3
- data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +1 -1
- data/spec/producer_spec.rb +1 -1
- data/spec/schemas/my_namespace/my_schema_with_complex_type.rb +3 -3
- data/spec/snapshots/consumers-no-nest.snap +1 -1
- data/spec/snapshots/consumers.snap +1 -1
- data/spec/snapshots/consumers_and_producers-no-nest.snap +3 -3
- data/spec/snapshots/consumers_and_producers.snap +3 -3
- data/spec/snapshots/consumers_circular-no-nest.snap +1 -1
- data/spec/snapshots/consumers_circular.snap +1 -1
- data/spec/snapshots/consumers_complex_types-no-nest.snap +1 -1
- data/spec/snapshots/consumers_complex_types.snap +1 -1
- data/spec/snapshots/consumers_nested-no-nest.snap +1 -1
- data/spec/snapshots/consumers_nested.snap +1 -1
- data/spec/snapshots/namespace_folders.snap +3 -3
- data/spec/snapshots/producers_with_key-no-nest.snap +1 -1
- data/spec/snapshots/producers_with_key.snap +1 -1
- metadata +39 -21
- data/.gemfiles/avro_turf-0.gemfile +0 -3
- data/.gemfiles/avro_turf-1.gemfile +0 -3
- data/.ruby-gemset +0 -1
data/sig/fig_tree.rbs
ADDED
data/spec/consumer_spec.rb
CHANGED
@@ -32,8 +32,8 @@ module ConsumerTest
|
|
32
32
|
|
33
33
|
it 'should consume a message' do
|
34
34
|
test_consume_message(MyConsumer,
|
35
|
-
{'test_id' => 'foo',
|
36
|
-
'some_int' => 123}) do |payload, _metadata|
|
35
|
+
{ 'test_id' => 'foo',
|
36
|
+
'some_int' => 123 }) do |payload, _metadata|
|
37
37
|
expect(payload['test_id']).to eq('foo')
|
38
38
|
expect(payload['some_int']).to eq(123)
|
39
39
|
end
|
@@ -64,15 +64,15 @@ module ConsumerTest
|
|
64
64
|
|
65
65
|
it 'should consume a message on a topic' do
|
66
66
|
test_consume_message('my_consume_topic',
|
67
|
-
{'test_id' => 'foo',
|
68
|
-
'some_int' => 123}) do |payload, _metadata|
|
67
|
+
{ 'test_id' => 'foo',
|
68
|
+
'some_int' => 123 }) do |payload, _metadata|
|
69
69
|
expect(payload['test_id']).to eq('foo')
|
70
70
|
expect(payload['some_int']).to eq(123)
|
71
71
|
end
|
72
72
|
end
|
73
73
|
|
74
74
|
it 'should fail on invalid message' do
|
75
|
-
test_consume_invalid_message(MyConsumer, {'invalid' => 'key'})
|
75
|
+
test_consume_invalid_message(MyConsumer, { 'invalid' => 'key' })
|
76
76
|
end
|
77
77
|
|
78
78
|
it 'should fail if reraise is false but fatal_error is true' do
|
@@ -85,14 +85,14 @@ module ConsumerTest
|
|
85
85
|
config.consumers.fatal_error = proc { true }
|
86
86
|
config.consumers.reraise_errors = false
|
87
87
|
end
|
88
|
-
test_consume_invalid_message(MyConsumer, {'invalid' => 'key'})
|
88
|
+
test_consume_invalid_message(MyConsumer, { 'invalid' => 'key' })
|
89
89
|
end
|
90
90
|
|
91
91
|
it 'should fail on message with extra fields' do
|
92
92
|
test_consume_invalid_message(MyConsumer,
|
93
|
-
{'test_id' => 'foo',
|
93
|
+
{ 'test_id' => 'foo',
|
94
94
|
'some_int' => 123,
|
95
|
-
'extra_field' => 'field name'})
|
95
|
+
'extra_field' => 'field name' })
|
96
96
|
end
|
97
97
|
|
98
98
|
it 'should not fail when before_consume fails without reraising errors' do
|
@@ -189,10 +189,10 @@ module ConsumerTest
|
|
189
189
|
it 'should consume a message' do
|
190
190
|
expect(Deimos.config.metrics).to receive(:histogram).twice
|
191
191
|
test_consume_message('my_consume_topic',
|
192
|
-
{'test_id' => 'foo',
|
192
|
+
{ 'test_id' => 'foo',
|
193
193
|
'some_int' => 123,
|
194
194
|
'updated_at' => Time.now.to_i,
|
195
|
-
'timestamp' => 2.minutes.ago.to_s}) do |payload, _metadata|
|
195
|
+
'timestamp' => 2.minutes.ago.to_s }) do |payload, _metadata|
|
196
196
|
expect(payload['test_id']).to eq('foo')
|
197
197
|
end
|
198
198
|
end
|
@@ -200,17 +200,17 @@ module ConsumerTest
|
|
200
200
|
it 'should fail nicely when timestamp wrong format' do
|
201
201
|
expect(Deimos.config.metrics).to receive(:histogram).twice
|
202
202
|
test_consume_message('my_consume_topic',
|
203
|
-
{'test_id' => 'foo',
|
203
|
+
{ 'test_id' => 'foo',
|
204
204
|
'some_int' => 123,
|
205
205
|
'updated_at' => Time.now.to_i,
|
206
|
-
'timestamp' => 'dffdf'}) do |payload, _metadata|
|
206
|
+
'timestamp' => 'dffdf' }) do |payload, _metadata|
|
207
207
|
expect(payload['test_id']).to eq('foo')
|
208
208
|
end
|
209
209
|
test_consume_message('my_consume_topic',
|
210
|
-
{'test_id' => 'foo',
|
210
|
+
{ 'test_id' => 'foo',
|
211
211
|
'some_int' => 123,
|
212
212
|
'updated_at' => Time.now.to_i,
|
213
|
-
'timestamp' => ''}) do |payload, _metadata|
|
213
|
+
'timestamp' => '' }) do |payload, _metadata|
|
214
214
|
expect(payload['test_id']).to eq('foo')
|
215
215
|
end
|
216
216
|
end
|
@@ -2,15 +2,15 @@ RSpec.describe Schemas::MyNamespace::MySchema do
|
|
2
2
|
let(:key) { Schemas::MyNamespace::MySchemaKey.new(test_id: 123) }
|
3
3
|
|
4
4
|
it 'should produce a tombstone with a hash' do
|
5
|
-
result = described_class.tombstone({test_id: 123})
|
5
|
+
result = described_class.tombstone({ test_id: 123 })
|
6
6
|
expect(result.payload_key).to eq(key)
|
7
|
-
expect(result.to_h).to eq({ payload_key: { 'test_id' => 123}})
|
7
|
+
expect(result.to_h).to eq({ payload_key: { 'test_id' => 123 } })
|
8
8
|
end
|
9
9
|
|
10
10
|
it 'should work with a record' do
|
11
11
|
key = Schemas::MyNamespace::MySchemaKey.new(test_id: 123)
|
12
12
|
result = described_class.tombstone(key)
|
13
13
|
expect(result.payload_key).to eq(key)
|
14
|
-
expect(result.to_h).to eq({ payload_key: { 'test_id' => 123}})
|
14
|
+
expect(result.to_h).to eq({ payload_key: { 'test_id' => 123 } })
|
15
15
|
end
|
16
16
|
end
|
@@ -55,7 +55,7 @@ RSpec.describe Schemas::MyNamespace::MySchemaWithComplexType do
|
|
55
55
|
it 'should return a tombstone' do
|
56
56
|
record = described_class.tombstone('foo')
|
57
57
|
expect(record.tombstone_key).to eq('foo')
|
58
|
-
expect(record.to_h).to eq({payload_key: 'foo'})
|
58
|
+
expect(record.to_h).to eq({ payload_key: 'foo' })
|
59
59
|
end
|
60
60
|
end
|
61
61
|
|
data/spec/producer_spec.rb
CHANGED
@@ -79,7 +79,7 @@ module ProducerTest
|
|
79
79
|
expect(event.payload[:payloads]).to eq([{ 'invalid' => 'key' }])
|
80
80
|
end
|
81
81
|
expect(MyProducer.encoder).to receive(:validate).and_raise('OH NOES')
|
82
|
-
expect { MyProducer.publish({'invalid' => 'key', :payload_key => 'key'}) }.
|
82
|
+
expect { MyProducer.publish({ 'invalid' => 'key', :payload_key => 'key' }) }.
|
83
83
|
to raise_error('OH NOES')
|
84
84
|
Deimos.unsubscribe(subscriber)
|
85
85
|
end
|
@@ -9,7 +9,7 @@ module Schemas; module MyNamespace
|
|
9
9
|
### Secondary Schema Classes ###
|
10
10
|
# Autogenerated Schema for Record at com.my-namespace.ARecord
|
11
11
|
class ARecord < Deimos::SchemaClass::Record
|
12
|
-
|
12
|
+
|
13
13
|
### Attribute Accessors ###
|
14
14
|
# @return [String]
|
15
15
|
attr_accessor :a_record_field
|
@@ -150,8 +150,8 @@ module Schemas; module MyNamespace
|
|
150
150
|
test_string_array: ["test"],
|
151
151
|
test_int_array: [123],
|
152
152
|
test_optional_int: 123,
|
153
|
-
some_integer_map: {"abc"=>123},
|
154
|
-
some_record: {"a_record_field"=>"Test String"},
|
153
|
+
some_integer_map: { "abc"=>123 },
|
154
|
+
some_record: { "a_record_field"=>"Test String" },
|
155
155
|
some_optional_record: nil,
|
156
156
|
some_record_array: nil,
|
157
157
|
some_record_map: nil,
|
@@ -201,7 +201,7 @@ module Schemas
|
|
201
201
|
end
|
202
202
|
|
203
203
|
def self.tombstone(key)
|
204
|
-
record = self.
|
204
|
+
record = self.allocate
|
205
205
|
record.tombstone_key = key
|
206
206
|
record.a_string = key
|
207
207
|
record
|
@@ -380,7 +380,7 @@ module Schemas
|
|
380
380
|
end
|
381
381
|
|
382
382
|
def self.tombstone(key)
|
383
|
-
record = self.
|
383
|
+
record = self.allocate
|
384
384
|
record.tombstone_key = key
|
385
385
|
record.test_id = key
|
386
386
|
record
|
@@ -446,7 +446,7 @@ module Schemas
|
|
446
446
|
end
|
447
447
|
|
448
448
|
def self.tombstone(key)
|
449
|
-
record = self.
|
449
|
+
record = self.allocate
|
450
450
|
record.tombstone_key = MySchemaKey.initialize_from_value(key)
|
451
451
|
record.payload_key = key
|
452
452
|
record
|
@@ -168,7 +168,7 @@ module Schemas
|
|
168
168
|
end
|
169
169
|
|
170
170
|
def self.tombstone(key)
|
171
|
-
record = self.
|
171
|
+
record = self.allocate
|
172
172
|
record.tombstone_key = key
|
173
173
|
record.a_string = key
|
174
174
|
record
|
@@ -341,7 +341,7 @@ module Schemas
|
|
341
341
|
end
|
342
342
|
|
343
343
|
def self.tombstone(key)
|
344
|
-
record = self.
|
344
|
+
record = self.allocate
|
345
345
|
record.tombstone_key = key
|
346
346
|
record.test_id = key
|
347
347
|
record
|
@@ -407,7 +407,7 @@ module Schemas
|
|
407
407
|
end
|
408
408
|
|
409
409
|
def self.tombstone(key)
|
410
|
-
record = self.
|
410
|
+
record = self.allocate
|
411
411
|
record.tombstone_key = MySchemaKey.initialize_from_value(key)
|
412
412
|
record.payload_key = key
|
413
413
|
record
|
@@ -129,7 +129,7 @@ module Schemas; module MyNamespace
|
|
129
129
|
end
|
130
130
|
|
131
131
|
def self.tombstone(key)
|
132
|
-
record = self.
|
132
|
+
record = self.allocate
|
133
133
|
record.tombstone_key = key
|
134
134
|
record.a_string = key
|
135
135
|
record
|
@@ -263,7 +263,7 @@ module Schemas; module MyNamespace
|
|
263
263
|
end
|
264
264
|
|
265
265
|
def self.tombstone(key)
|
266
|
-
record = self.
|
266
|
+
record = self.allocate
|
267
267
|
record.tombstone_key = key
|
268
268
|
record.test_id = key
|
269
269
|
record
|
@@ -329,7 +329,7 @@ module Schemas; module MyNamespace
|
|
329
329
|
end
|
330
330
|
|
331
331
|
def self.tombstone(key)
|
332
|
-
record = self.
|
332
|
+
record = self.allocate
|
333
333
|
record.tombstone_key = MySchemaKey.initialize_from_value(key)
|
334
334
|
record.payload_key = key
|
335
335
|
record
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: deimos-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.16.
|
4
|
+
version: 1.16.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Orner
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-09-
|
11
|
+
date: 2022-09-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro_turf
|
@@ -16,7 +16,7 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '1.4'
|
20
20
|
- - "<"
|
21
21
|
- !ruby/object:Gem::Version
|
22
22
|
version: '2'
|
@@ -26,10 +26,24 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version: '
|
29
|
+
version: '1.4'
|
30
30
|
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: fig_tree
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - "~>"
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: 0.0.2
|
40
|
+
type: :runtime
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - "~>"
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: 0.0.2
|
33
47
|
- !ruby/object:Gem::Dependency
|
34
48
|
name: phobos
|
35
49
|
requirement: !ruby/object:Gem::Requirement
|
@@ -84,20 +98,6 @@ dependencies:
|
|
84
98
|
- - "<"
|
85
99
|
- !ruby/object:Gem::Version
|
86
100
|
version: '1.0'
|
87
|
-
- !ruby/object:Gem::Dependency
|
88
|
-
name: fig_tree
|
89
|
-
requirement: !ruby/object:Gem::Requirement
|
90
|
-
requirements:
|
91
|
-
- - "~>"
|
92
|
-
- !ruby/object:Gem::Version
|
93
|
-
version: 0.0.2
|
94
|
-
type: :runtime
|
95
|
-
prerelease: false
|
96
|
-
version_requirements: !ruby/object:Gem::Requirement
|
97
|
-
requirements:
|
98
|
-
- - "~>"
|
99
|
-
- !ruby/object:Gem::Version
|
100
|
-
version: 0.0.2
|
101
101
|
- !ruby/object:Gem::Dependency
|
102
102
|
name: activerecord-import
|
103
103
|
requirement: !ruby/object:Gem::Requirement
|
@@ -364,6 +364,20 @@ dependencies:
|
|
364
364
|
- - "~>"
|
365
365
|
- !ruby/object:Gem::Version
|
366
366
|
version: '1.3'
|
367
|
+
- !ruby/object:Gem::Dependency
|
368
|
+
name: steep
|
369
|
+
requirement: !ruby/object:Gem::Requirement
|
370
|
+
requirements:
|
371
|
+
- - "~>"
|
372
|
+
- !ruby/object:Gem::Version
|
373
|
+
version: '1.0'
|
374
|
+
type: :development
|
375
|
+
prerelease: false
|
376
|
+
version_requirements: !ruby/object:Gem::Requirement
|
377
|
+
requirements:
|
378
|
+
- - "~>"
|
379
|
+
- !ruby/object:Gem::Version
|
380
|
+
version: '1.0'
|
367
381
|
description:
|
368
382
|
email:
|
369
383
|
- daniel.orner@wishabi.com
|
@@ -374,14 +388,12 @@ extensions: []
|
|
374
388
|
extra_rdoc_files: []
|
375
389
|
files:
|
376
390
|
- ".circleci/config.yml"
|
377
|
-
- ".gemfiles/avro_turf-0.gemfile"
|
378
|
-
- ".gemfiles/avro_turf-1.gemfile"
|
379
391
|
- ".github/workflows/ci.yml"
|
380
392
|
- ".gitignore"
|
381
393
|
- ".gitmodules"
|
382
394
|
- ".rspec"
|
383
395
|
- ".rubocop.yml"
|
384
|
-
- ".
|
396
|
+
- ".rubocop_todo.yml"
|
385
397
|
- ".ruby-version"
|
386
398
|
- CHANGELOG.md
|
387
399
|
- CODE_OF_CONDUCT.md
|
@@ -391,6 +403,7 @@ files:
|
|
391
403
|
- LICENSE.md
|
392
404
|
- README.md
|
393
405
|
- Rakefile
|
406
|
+
- Steepfile
|
394
407
|
- bin/console
|
395
408
|
- bin/deimos
|
396
409
|
- deimos-ruby.gemspec
|
@@ -469,7 +482,12 @@ files:
|
|
469
482
|
- lib/generators/deimos/schema_class/templates/schema_record.rb.tt
|
470
483
|
- lib/generators/deimos/schema_class_generator.rb
|
471
484
|
- lib/tasks/deimos.rake
|
485
|
+
- rbs_collection.lock.yaml
|
486
|
+
- rbs_collection.yaml
|
472
487
|
- regenerate_test_schema_classes.rb
|
488
|
+
- sig/avro.rbs
|
489
|
+
- sig/defs.rbs
|
490
|
+
- sig/fig_tree.rbs
|
473
491
|
- spec/active_record_batch_consumer_spec.rb
|
474
492
|
- spec/active_record_consume/batch_slicer_spec.rb
|
475
493
|
- spec/active_record_consume/schema_model_converter_spec.rb
|
data/.ruby-gemset
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
deimos
|