deimos-ruby 1.16.3 → 1.16.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +3 -3
- data/.gitignore +1 -0
- data/.rubocop.yml +20 -14
- data/.rubocop_todo.yml +364 -0
- data/.ruby-version +2 -1
- data/CHANGELOG.md +9 -0
- data/Gemfile +6 -0
- data/README.md +7 -1
- data/Steepfile +6 -0
- data/deimos-ruby.gemspec +3 -2
- data/lib/deimos/active_record_consume/batch_consumption.rb +7 -2
- data/lib/deimos/active_record_consume/batch_slicer.rb +2 -0
- data/lib/deimos/active_record_consume/message_consumption.rb +8 -4
- data/lib/deimos/active_record_consumer.rb +7 -4
- data/lib/deimos/active_record_producer.rb +3 -0
- data/lib/deimos/backends/base.rb +4 -2
- data/lib/deimos/backends/kafka.rb +1 -0
- data/lib/deimos/backends/kafka_async.rb +1 -0
- data/lib/deimos/config/configuration.rb +4 -0
- data/lib/deimos/config/phobos_config.rb +2 -1
- data/lib/deimos/consume/batch_consumption.rb +8 -1
- data/lib/deimos/consume/message_consumption.rb +4 -1
- data/lib/deimos/instrumentation.rb +11 -4
- data/lib/deimos/kafka_message.rb +1 -0
- data/lib/deimos/kafka_source.rb +5 -0
- data/lib/deimos/kafka_topic_info.rb +4 -0
- data/lib/deimos/message.rb +19 -2
- data/lib/deimos/metrics/datadog.rb +2 -1
- data/lib/deimos/metrics/mock.rb +2 -2
- data/lib/deimos/metrics/provider.rb +6 -0
- data/lib/deimos/monkey_patches/phobos_cli.rb +1 -1
- data/lib/deimos/monkey_patches/phobos_producer.rb +1 -0
- data/lib/deimos/producer.rb +12 -6
- data/lib/deimos/schema_backends/base.rb +31 -17
- data/lib/deimos/schema_backends/mock.rb +2 -2
- data/lib/deimos/schema_class/base.rb +9 -5
- data/lib/deimos/schema_class/enum.rb +4 -2
- data/lib/deimos/schema_class/record.rb +5 -5
- data/lib/deimos/shared_config.rb +6 -2
- data/lib/deimos/test_helpers.rb +21 -4
- data/lib/deimos/tracing/datadog.rb +1 -1
- data/lib/deimos/tracing/mock.rb +4 -3
- data/lib/deimos/tracing/provider.rb +5 -0
- data/lib/deimos/utils/db_poller.rb +9 -1
- data/lib/deimos/utils/db_producer.rb +14 -2
- data/lib/deimos/utils/deadlock_retry.rb +3 -0
- data/lib/deimos/utils/inline_consumer.rb +14 -6
- data/lib/deimos/utils/lag_reporter.rb +11 -0
- data/lib/deimos/utils/schema_controller_mixin.rb +8 -0
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +3 -2
- data/lib/generators/deimos/active_record_generator.rb +1 -1
- data/lib/generators/deimos/db_backend_generator.rb +1 -0
- data/lib/generators/deimos/db_poller_generator.rb +1 -0
- data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt +1 -1
- data/lib/generators/deimos/schema_class_generator.rb +13 -4
- data/rbs_collection.lock.yaml +176 -0
- data/rbs_collection.yaml +15 -0
- data/sig/avro.rbs +14 -0
- data/sig/defs.rbs +1867 -0
- data/sig/fig_tree.rbs +2 -0
- data/spec/consumer_spec.rb +14 -14
- data/spec/generators/schema_class/my_schema_spec.rb +3 -3
- data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +1 -1
- data/spec/producer_spec.rb +1 -1
- data/spec/schemas/my_namespace/my_schema_with_complex_type.rb +3 -3
- data/spec/snapshots/consumers-no-nest.snap +1 -1
- data/spec/snapshots/consumers.snap +1 -1
- data/spec/snapshots/consumers_and_producers-no-nest.snap +3 -3
- data/spec/snapshots/consumers_and_producers.snap +3 -3
- data/spec/snapshots/consumers_circular-no-nest.snap +1 -1
- data/spec/snapshots/consumers_circular.snap +1 -1
- data/spec/snapshots/consumers_complex_types-no-nest.snap +1 -1
- data/spec/snapshots/consumers_complex_types.snap +1 -1
- data/spec/snapshots/consumers_nested-no-nest.snap +1 -1
- data/spec/snapshots/consumers_nested.snap +1 -1
- data/spec/snapshots/namespace_folders.snap +3 -3
- data/spec/snapshots/producers_with_key-no-nest.snap +1 -1
- data/spec/snapshots/producers_with_key.snap +1 -1
- metadata +39 -21
- data/.gemfiles/avro_turf-0.gemfile +0 -3
- data/.gemfiles/avro_turf-1.gemfile +0 -3
- data/.ruby-gemset +0 -1
data/sig/fig_tree.rbs
ADDED
data/spec/consumer_spec.rb
CHANGED
@@ -32,8 +32,8 @@ module ConsumerTest
|
|
32
32
|
|
33
33
|
it 'should consume a message' do
|
34
34
|
test_consume_message(MyConsumer,
|
35
|
-
{'test_id' => 'foo',
|
36
|
-
'some_int' => 123}) do |payload, _metadata|
|
35
|
+
{ 'test_id' => 'foo',
|
36
|
+
'some_int' => 123 }) do |payload, _metadata|
|
37
37
|
expect(payload['test_id']).to eq('foo')
|
38
38
|
expect(payload['some_int']).to eq(123)
|
39
39
|
end
|
@@ -64,15 +64,15 @@ module ConsumerTest
|
|
64
64
|
|
65
65
|
it 'should consume a message on a topic' do
|
66
66
|
test_consume_message('my_consume_topic',
|
67
|
-
{'test_id' => 'foo',
|
68
|
-
'some_int' => 123}) do |payload, _metadata|
|
67
|
+
{ 'test_id' => 'foo',
|
68
|
+
'some_int' => 123 }) do |payload, _metadata|
|
69
69
|
expect(payload['test_id']).to eq('foo')
|
70
70
|
expect(payload['some_int']).to eq(123)
|
71
71
|
end
|
72
72
|
end
|
73
73
|
|
74
74
|
it 'should fail on invalid message' do
|
75
|
-
test_consume_invalid_message(MyConsumer, {'invalid' => 'key'})
|
75
|
+
test_consume_invalid_message(MyConsumer, { 'invalid' => 'key' })
|
76
76
|
end
|
77
77
|
|
78
78
|
it 'should fail if reraise is false but fatal_error is true' do
|
@@ -85,14 +85,14 @@ module ConsumerTest
|
|
85
85
|
config.consumers.fatal_error = proc { true }
|
86
86
|
config.consumers.reraise_errors = false
|
87
87
|
end
|
88
|
-
test_consume_invalid_message(MyConsumer, {'invalid' => 'key'})
|
88
|
+
test_consume_invalid_message(MyConsumer, { 'invalid' => 'key' })
|
89
89
|
end
|
90
90
|
|
91
91
|
it 'should fail on message with extra fields' do
|
92
92
|
test_consume_invalid_message(MyConsumer,
|
93
|
-
{'test_id' => 'foo',
|
93
|
+
{ 'test_id' => 'foo',
|
94
94
|
'some_int' => 123,
|
95
|
-
'extra_field' => 'field name'})
|
95
|
+
'extra_field' => 'field name' })
|
96
96
|
end
|
97
97
|
|
98
98
|
it 'should not fail when before_consume fails without reraising errors' do
|
@@ -189,10 +189,10 @@ module ConsumerTest
|
|
189
189
|
it 'should consume a message' do
|
190
190
|
expect(Deimos.config.metrics).to receive(:histogram).twice
|
191
191
|
test_consume_message('my_consume_topic',
|
192
|
-
{'test_id' => 'foo',
|
192
|
+
{ 'test_id' => 'foo',
|
193
193
|
'some_int' => 123,
|
194
194
|
'updated_at' => Time.now.to_i,
|
195
|
-
'timestamp' => 2.minutes.ago.to_s}) do |payload, _metadata|
|
195
|
+
'timestamp' => 2.minutes.ago.to_s }) do |payload, _metadata|
|
196
196
|
expect(payload['test_id']).to eq('foo')
|
197
197
|
end
|
198
198
|
end
|
@@ -200,17 +200,17 @@ module ConsumerTest
|
|
200
200
|
it 'should fail nicely when timestamp wrong format' do
|
201
201
|
expect(Deimos.config.metrics).to receive(:histogram).twice
|
202
202
|
test_consume_message('my_consume_topic',
|
203
|
-
{'test_id' => 'foo',
|
203
|
+
{ 'test_id' => 'foo',
|
204
204
|
'some_int' => 123,
|
205
205
|
'updated_at' => Time.now.to_i,
|
206
|
-
'timestamp' => 'dffdf'}) do |payload, _metadata|
|
206
|
+
'timestamp' => 'dffdf' }) do |payload, _metadata|
|
207
207
|
expect(payload['test_id']).to eq('foo')
|
208
208
|
end
|
209
209
|
test_consume_message('my_consume_topic',
|
210
|
-
{'test_id' => 'foo',
|
210
|
+
{ 'test_id' => 'foo',
|
211
211
|
'some_int' => 123,
|
212
212
|
'updated_at' => Time.now.to_i,
|
213
|
-
'timestamp' => ''}) do |payload, _metadata|
|
213
|
+
'timestamp' => '' }) do |payload, _metadata|
|
214
214
|
expect(payload['test_id']).to eq('foo')
|
215
215
|
end
|
216
216
|
end
|
@@ -2,15 +2,15 @@ RSpec.describe Schemas::MyNamespace::MySchema do
|
|
2
2
|
let(:key) { Schemas::MyNamespace::MySchemaKey.new(test_id: 123) }
|
3
3
|
|
4
4
|
it 'should produce a tombstone with a hash' do
|
5
|
-
result = described_class.tombstone({test_id: 123})
|
5
|
+
result = described_class.tombstone({ test_id: 123 })
|
6
6
|
expect(result.payload_key).to eq(key)
|
7
|
-
expect(result.to_h).to eq({ payload_key: { 'test_id' => 123}})
|
7
|
+
expect(result.to_h).to eq({ payload_key: { 'test_id' => 123 } })
|
8
8
|
end
|
9
9
|
|
10
10
|
it 'should work with a record' do
|
11
11
|
key = Schemas::MyNamespace::MySchemaKey.new(test_id: 123)
|
12
12
|
result = described_class.tombstone(key)
|
13
13
|
expect(result.payload_key).to eq(key)
|
14
|
-
expect(result.to_h).to eq({ payload_key: { 'test_id' => 123}})
|
14
|
+
expect(result.to_h).to eq({ payload_key: { 'test_id' => 123 } })
|
15
15
|
end
|
16
16
|
end
|
@@ -55,7 +55,7 @@ RSpec.describe Schemas::MyNamespace::MySchemaWithComplexType do
|
|
55
55
|
it 'should return a tombstone' do
|
56
56
|
record = described_class.tombstone('foo')
|
57
57
|
expect(record.tombstone_key).to eq('foo')
|
58
|
-
expect(record.to_h).to eq({payload_key: 'foo'})
|
58
|
+
expect(record.to_h).to eq({ payload_key: 'foo' })
|
59
59
|
end
|
60
60
|
end
|
61
61
|
|
data/spec/producer_spec.rb
CHANGED
@@ -79,7 +79,7 @@ module ProducerTest
|
|
79
79
|
expect(event.payload[:payloads]).to eq([{ 'invalid' => 'key' }])
|
80
80
|
end
|
81
81
|
expect(MyProducer.encoder).to receive(:validate).and_raise('OH NOES')
|
82
|
-
expect { MyProducer.publish({'invalid' => 'key', :payload_key => 'key'}) }.
|
82
|
+
expect { MyProducer.publish({ 'invalid' => 'key', :payload_key => 'key' }) }.
|
83
83
|
to raise_error('OH NOES')
|
84
84
|
Deimos.unsubscribe(subscriber)
|
85
85
|
end
|
@@ -9,7 +9,7 @@ module Schemas; module MyNamespace
|
|
9
9
|
### Secondary Schema Classes ###
|
10
10
|
# Autogenerated Schema for Record at com.my-namespace.ARecord
|
11
11
|
class ARecord < Deimos::SchemaClass::Record
|
12
|
-
|
12
|
+
|
13
13
|
### Attribute Accessors ###
|
14
14
|
# @return [String]
|
15
15
|
attr_accessor :a_record_field
|
@@ -150,8 +150,8 @@ module Schemas; module MyNamespace
|
|
150
150
|
test_string_array: ["test"],
|
151
151
|
test_int_array: [123],
|
152
152
|
test_optional_int: 123,
|
153
|
-
some_integer_map: {"abc"=>123},
|
154
|
-
some_record: {"a_record_field"=>"Test String"},
|
153
|
+
some_integer_map: { "abc"=>123 },
|
154
|
+
some_record: { "a_record_field"=>"Test String" },
|
155
155
|
some_optional_record: nil,
|
156
156
|
some_record_array: nil,
|
157
157
|
some_record_map: nil,
|
@@ -201,7 +201,7 @@ module Schemas
|
|
201
201
|
end
|
202
202
|
|
203
203
|
def self.tombstone(key)
|
204
|
-
record = self.
|
204
|
+
record = self.allocate
|
205
205
|
record.tombstone_key = key
|
206
206
|
record.a_string = key
|
207
207
|
record
|
@@ -380,7 +380,7 @@ module Schemas
|
|
380
380
|
end
|
381
381
|
|
382
382
|
def self.tombstone(key)
|
383
|
-
record = self.
|
383
|
+
record = self.allocate
|
384
384
|
record.tombstone_key = key
|
385
385
|
record.test_id = key
|
386
386
|
record
|
@@ -446,7 +446,7 @@ module Schemas
|
|
446
446
|
end
|
447
447
|
|
448
448
|
def self.tombstone(key)
|
449
|
-
record = self.
|
449
|
+
record = self.allocate
|
450
450
|
record.tombstone_key = MySchemaKey.initialize_from_value(key)
|
451
451
|
record.payload_key = key
|
452
452
|
record
|
@@ -168,7 +168,7 @@ module Schemas
|
|
168
168
|
end
|
169
169
|
|
170
170
|
def self.tombstone(key)
|
171
|
-
record = self.
|
171
|
+
record = self.allocate
|
172
172
|
record.tombstone_key = key
|
173
173
|
record.a_string = key
|
174
174
|
record
|
@@ -341,7 +341,7 @@ module Schemas
|
|
341
341
|
end
|
342
342
|
|
343
343
|
def self.tombstone(key)
|
344
|
-
record = self.
|
344
|
+
record = self.allocate
|
345
345
|
record.tombstone_key = key
|
346
346
|
record.test_id = key
|
347
347
|
record
|
@@ -407,7 +407,7 @@ module Schemas
|
|
407
407
|
end
|
408
408
|
|
409
409
|
def self.tombstone(key)
|
410
|
-
record = self.
|
410
|
+
record = self.allocate
|
411
411
|
record.tombstone_key = MySchemaKey.initialize_from_value(key)
|
412
412
|
record.payload_key = key
|
413
413
|
record
|
@@ -129,7 +129,7 @@ module Schemas; module MyNamespace
|
|
129
129
|
end
|
130
130
|
|
131
131
|
def self.tombstone(key)
|
132
|
-
record = self.
|
132
|
+
record = self.allocate
|
133
133
|
record.tombstone_key = key
|
134
134
|
record.a_string = key
|
135
135
|
record
|
@@ -263,7 +263,7 @@ module Schemas; module MyNamespace
|
|
263
263
|
end
|
264
264
|
|
265
265
|
def self.tombstone(key)
|
266
|
-
record = self.
|
266
|
+
record = self.allocate
|
267
267
|
record.tombstone_key = key
|
268
268
|
record.test_id = key
|
269
269
|
record
|
@@ -329,7 +329,7 @@ module Schemas; module MyNamespace
|
|
329
329
|
end
|
330
330
|
|
331
331
|
def self.tombstone(key)
|
332
|
-
record = self.
|
332
|
+
record = self.allocate
|
333
333
|
record.tombstone_key = MySchemaKey.initialize_from_value(key)
|
334
334
|
record.payload_key = key
|
335
335
|
record
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: deimos-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.16.
|
4
|
+
version: 1.16.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Orner
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-09-
|
11
|
+
date: 2022-09-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro_turf
|
@@ -16,7 +16,7 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '1.4'
|
20
20
|
- - "<"
|
21
21
|
- !ruby/object:Gem::Version
|
22
22
|
version: '2'
|
@@ -26,10 +26,24 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version: '
|
29
|
+
version: '1.4'
|
30
30
|
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: fig_tree
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - "~>"
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: 0.0.2
|
40
|
+
type: :runtime
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - "~>"
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: 0.0.2
|
33
47
|
- !ruby/object:Gem::Dependency
|
34
48
|
name: phobos
|
35
49
|
requirement: !ruby/object:Gem::Requirement
|
@@ -84,20 +98,6 @@ dependencies:
|
|
84
98
|
- - "<"
|
85
99
|
- !ruby/object:Gem::Version
|
86
100
|
version: '1.0'
|
87
|
-
- !ruby/object:Gem::Dependency
|
88
|
-
name: fig_tree
|
89
|
-
requirement: !ruby/object:Gem::Requirement
|
90
|
-
requirements:
|
91
|
-
- - "~>"
|
92
|
-
- !ruby/object:Gem::Version
|
93
|
-
version: 0.0.2
|
94
|
-
type: :runtime
|
95
|
-
prerelease: false
|
96
|
-
version_requirements: !ruby/object:Gem::Requirement
|
97
|
-
requirements:
|
98
|
-
- - "~>"
|
99
|
-
- !ruby/object:Gem::Version
|
100
|
-
version: 0.0.2
|
101
101
|
- !ruby/object:Gem::Dependency
|
102
102
|
name: activerecord-import
|
103
103
|
requirement: !ruby/object:Gem::Requirement
|
@@ -364,6 +364,20 @@ dependencies:
|
|
364
364
|
- - "~>"
|
365
365
|
- !ruby/object:Gem::Version
|
366
366
|
version: '1.3'
|
367
|
+
- !ruby/object:Gem::Dependency
|
368
|
+
name: steep
|
369
|
+
requirement: !ruby/object:Gem::Requirement
|
370
|
+
requirements:
|
371
|
+
- - "~>"
|
372
|
+
- !ruby/object:Gem::Version
|
373
|
+
version: '1.0'
|
374
|
+
type: :development
|
375
|
+
prerelease: false
|
376
|
+
version_requirements: !ruby/object:Gem::Requirement
|
377
|
+
requirements:
|
378
|
+
- - "~>"
|
379
|
+
- !ruby/object:Gem::Version
|
380
|
+
version: '1.0'
|
367
381
|
description:
|
368
382
|
email:
|
369
383
|
- daniel.orner@wishabi.com
|
@@ -374,14 +388,12 @@ extensions: []
|
|
374
388
|
extra_rdoc_files: []
|
375
389
|
files:
|
376
390
|
- ".circleci/config.yml"
|
377
|
-
- ".gemfiles/avro_turf-0.gemfile"
|
378
|
-
- ".gemfiles/avro_turf-1.gemfile"
|
379
391
|
- ".github/workflows/ci.yml"
|
380
392
|
- ".gitignore"
|
381
393
|
- ".gitmodules"
|
382
394
|
- ".rspec"
|
383
395
|
- ".rubocop.yml"
|
384
|
-
- ".
|
396
|
+
- ".rubocop_todo.yml"
|
385
397
|
- ".ruby-version"
|
386
398
|
- CHANGELOG.md
|
387
399
|
- CODE_OF_CONDUCT.md
|
@@ -391,6 +403,7 @@ files:
|
|
391
403
|
- LICENSE.md
|
392
404
|
- README.md
|
393
405
|
- Rakefile
|
406
|
+
- Steepfile
|
394
407
|
- bin/console
|
395
408
|
- bin/deimos
|
396
409
|
- deimos-ruby.gemspec
|
@@ -469,7 +482,12 @@ files:
|
|
469
482
|
- lib/generators/deimos/schema_class/templates/schema_record.rb.tt
|
470
483
|
- lib/generators/deimos/schema_class_generator.rb
|
471
484
|
- lib/tasks/deimos.rake
|
485
|
+
- rbs_collection.lock.yaml
|
486
|
+
- rbs_collection.yaml
|
472
487
|
- regenerate_test_schema_classes.rb
|
488
|
+
- sig/avro.rbs
|
489
|
+
- sig/defs.rbs
|
490
|
+
- sig/fig_tree.rbs
|
473
491
|
- spec/active_record_batch_consumer_spec.rb
|
474
492
|
- spec/active_record_consume/batch_slicer_spec.rb
|
475
493
|
- spec/active_record_consume/schema_model_converter_spec.rb
|
data/.ruby-gemset
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
deimos
|