deimos-ruby 1.22.5 → 1.23.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +2 -2
- data/CHANGELOG.md +18 -0
- data/README.md +6 -0
- data/docs/CONFIGURATION.md +4 -0
- data/lib/deimos/active_record_consume/batch_consumption.rb +43 -20
- data/lib/deimos/active_record_consume/batch_record.rb +5 -4
- data/lib/deimos/active_record_consume/batch_record_list.rb +11 -2
- data/lib/deimos/active_record_consume/mass_updater.rb +13 -4
- data/lib/deimos/active_record_consumer.rb +10 -0
- data/lib/deimos/config/configuration.rb +28 -2
- data/lib/deimos/consume/batch_consumption.rb +2 -2
- data/lib/deimos/instrumentation.rb +20 -8
- data/lib/deimos/test_helpers.rb +1 -0
- data/lib/deimos/tracing/datadog.rb +12 -6
- data/lib/deimos/tracing/mock.rb +31 -2
- data/lib/deimos/tracing/provider.rb +6 -0
- data/lib/deimos/utils/db_poller/base.rb +23 -0
- data/lib/deimos/utils/schema_class.rb +10 -2
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +14 -1
- data/spec/active_record_batch_consumer_association_spec.rb +32 -5
- data/spec/active_record_batch_consumer_spec.rb +376 -59
- data/spec/active_record_consume/mass_updater_spec.rb +46 -3
- data/spec/active_record_consumer_spec.rb +74 -1
- data/spec/active_record_producer_spec.rb +4 -1
- data/spec/batch_consumer_spec.rb +4 -1
- data/spec/config/configuration_spec.rb +42 -3
- data/spec/consumer_spec.rb +42 -1
- data/spec/schemas/my_namespace/my_updated_schema.rb +18 -0
- data/spec/utils/db_poller_spec.rb +42 -0
- metadata +5 -3
@@ -69,7 +69,10 @@ describe Deimos::ActiveRecordProducer do
|
|
69
69
|
SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
|
70
70
|
context "with Schema Class consumption #{setting}" do
|
71
71
|
before(:each) do
|
72
|
-
Deimos.configure
|
72
|
+
Deimos.configure do |config|
|
73
|
+
config.schema.use_schema_classes = use_schema_classes
|
74
|
+
config.schema.generate_namespace_folders = true
|
75
|
+
end
|
73
76
|
end
|
74
77
|
|
75
78
|
it 'should send events correctly' do
|
data/spec/batch_consumer_spec.rb
CHANGED
@@ -40,7 +40,10 @@ module ConsumerTest
|
|
40
40
|
end
|
41
41
|
|
42
42
|
before(:each) do
|
43
|
-
Deimos.configure
|
43
|
+
Deimos.configure do |config|
|
44
|
+
config.schema.use_schema_classes = use_schema_classes
|
45
|
+
config.schema.generate_namespace_folders = true
|
46
|
+
end
|
44
47
|
end
|
45
48
|
|
46
49
|
it 'should provide backwards compatibility for BatchConsumer class' do
|
@@ -91,7 +91,8 @@ describe Deimos, 'configuration' do
|
|
91
91
|
heartbeat_interval: 10,
|
92
92
|
handler: 'ConsumerTest::MyConsumer',
|
93
93
|
use_schema_classes: nil,
|
94
|
-
max_db_batch_size: nil
|
94
|
+
max_db_batch_size: nil,
|
95
|
+
bulk_import_id_generator: nil
|
95
96
|
}, {
|
96
97
|
topic: 'my_batch_consume_topic',
|
97
98
|
group_id: 'my_batch_group_id',
|
@@ -109,7 +110,8 @@ describe Deimos, 'configuration' do
|
|
109
110
|
heartbeat_interval: 10,
|
110
111
|
handler: 'ConsumerTest::MyBatchConsumer',
|
111
112
|
use_schema_classes: nil,
|
112
|
-
max_db_batch_size: nil
|
113
|
+
max_db_batch_size: nil,
|
114
|
+
bulk_import_id_generator: nil
|
113
115
|
}
|
114
116
|
],
|
115
117
|
producer: {
|
@@ -261,7 +263,8 @@ describe Deimos, 'configuration' do
|
|
261
263
|
heartbeat_interval: 13,
|
262
264
|
handler: 'MyConfigConsumer',
|
263
265
|
use_schema_classes: false,
|
264
|
-
max_db_batch_size: nil
|
266
|
+
max_db_batch_size: nil,
|
267
|
+
bulk_import_id_generator: nil
|
265
268
|
}
|
266
269
|
],
|
267
270
|
producer: {
|
@@ -279,4 +282,40 @@ describe Deimos, 'configuration' do
|
|
279
282
|
}
|
280
283
|
)
|
281
284
|
end
|
285
|
+
|
286
|
+
it 'should override global configurations' do
|
287
|
+
described_class.configure do
|
288
|
+
consumers.bulk_import_id_generator(-> { 'global' })
|
289
|
+
consumers.replace_associations true
|
290
|
+
|
291
|
+
consumer do
|
292
|
+
class_name 'MyConfigConsumer'
|
293
|
+
schema 'blah'
|
294
|
+
topic 'blah'
|
295
|
+
group_id 'myconsumerid'
|
296
|
+
bulk_import_id_generator(-> { 'consumer' })
|
297
|
+
replace_associations false
|
298
|
+
end
|
299
|
+
|
300
|
+
consumer do
|
301
|
+
class_name 'MyConfigConsumer2'
|
302
|
+
schema 'blah'
|
303
|
+
topic 'blah'
|
304
|
+
group_id 'myconsumerid'
|
305
|
+
end
|
306
|
+
end
|
307
|
+
|
308
|
+
consumers = described_class.config.consumers
|
309
|
+
expect(consumers.replace_associations).to eq(true)
|
310
|
+
expect(consumers.bulk_import_id_generator.call).to eq('global')
|
311
|
+
|
312
|
+
custom = MyConfigConsumer.config
|
313
|
+
expect(custom[:replace_associations]).to eq(false)
|
314
|
+
expect(custom[:bulk_import_id_generator].call).to eq('consumer')
|
315
|
+
|
316
|
+
default = MyConfigConsumer2.config
|
317
|
+
expect(default[:replace_associations]).to eq(true)
|
318
|
+
expect(default[:bulk_import_id_generator].call).to eq('global')
|
319
|
+
|
320
|
+
end
|
282
321
|
end
|
data/spec/consumer_spec.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
# :nodoc:
|
4
|
+
# rubocop:disable Metrics/ModuleLength
|
4
5
|
module ConsumerTest
|
5
6
|
describe Deimos::Consumer, 'Message Consumer' do
|
6
7
|
prepend_before(:each) do
|
@@ -26,8 +27,12 @@ module ConsumerTest
|
|
26
27
|
describe 'consume' do
|
27
28
|
SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
|
28
29
|
context "with Schema Class consumption #{setting}" do
|
30
|
+
|
29
31
|
before(:each) do
|
30
|
-
Deimos.configure
|
32
|
+
Deimos.configure do |config|
|
33
|
+
config.schema.use_schema_classes = use_schema_classes
|
34
|
+
config.schema.generate_namespace_folders = true
|
35
|
+
end
|
31
36
|
end
|
32
37
|
|
33
38
|
it 'should consume a message' do
|
@@ -127,6 +132,41 @@ module ConsumerTest
|
|
127
132
|
end
|
128
133
|
end
|
129
134
|
end
|
135
|
+
|
136
|
+
context 'with overriden schema classes' do
|
137
|
+
|
138
|
+
before(:each) do
|
139
|
+
Deimos.configure do |config|
|
140
|
+
config.schema.use_schema_classes = true
|
141
|
+
config.schema.generate_namespace_folders = true
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
prepend_before(:each) do
|
146
|
+
consumer_class = Class.new(described_class) do
|
147
|
+
schema 'MyUpdatedSchema'
|
148
|
+
namespace 'com.my-namespace'
|
149
|
+
key_config field: 'test_id'
|
150
|
+
|
151
|
+
# :nodoc:
|
152
|
+
def consume(_payload, _metadata)
|
153
|
+
raise 'This should not be called unless call_original is set'
|
154
|
+
end
|
155
|
+
end
|
156
|
+
stub_const('ConsumerTest::MyConsumer', consumer_class)
|
157
|
+
end
|
158
|
+
|
159
|
+
it 'should consume messages' do
|
160
|
+
test_consume_message('my_consume_topic',
|
161
|
+
{ 'test_id' => 'foo',
|
162
|
+
'some_int' => 1 }) do |payload, _metadata|
|
163
|
+
expect(payload['test_id']).to eq('foo')
|
164
|
+
expect(payload['some_int']).to eq(1)
|
165
|
+
expect(payload['super_int']).to eq(9000)
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
end
|
130
170
|
end
|
131
171
|
|
132
172
|
describe 'decode_key' do
|
@@ -218,3 +258,4 @@ module ConsumerTest
|
|
218
258
|
end
|
219
259
|
end
|
220
260
|
end
|
261
|
+
# rubocop:enable Metrics/ModuleLength
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This file is autogenerated by Deimos, Do NOT modify
|
4
|
+
module Schemas; module MyNamespace
|
5
|
+
### Primary Schema Class ###
|
6
|
+
# Autogenerated Schema for Record at com.my-namespace.MySchema
|
7
|
+
class MyUpdatedSchema < Schemas::MyNamespace::MySchema
|
8
|
+
|
9
|
+
attr_accessor :super_int
|
10
|
+
|
11
|
+
def initialize(test_id: nil,
|
12
|
+
some_int: nil)
|
13
|
+
super
|
14
|
+
self.super_int = some_int.nil? ? 10 : some_int * 9000
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -191,6 +191,48 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
|
|
191
191
|
expect(Deimos.config.tracer).to have_received(:finish).with('a span')
|
192
192
|
end
|
193
193
|
|
194
|
+
context 'with skip_too_large_messages on' do
|
195
|
+
before(:each) { config.skip_too_large_messages = true }
|
196
|
+
|
197
|
+
it 'should skip and move on' do
|
198
|
+
error = Kafka::MessageSizeTooLarge.new('OH NOES')
|
199
|
+
allow(poller).to receive(:sleep)
|
200
|
+
allow(poller).to receive(:process_batch) do
|
201
|
+
raise error
|
202
|
+
end
|
203
|
+
poller.retrieve_poll_info
|
204
|
+
poller.process_batch_with_span(widgets, status)
|
205
|
+
expect(poller).not_to have_received(:sleep)
|
206
|
+
expect(Deimos.config.tracer).to have_received(:set_error).with('a span', error)
|
207
|
+
expect(status.batches_errored).to eq(1)
|
208
|
+
expect(status.batches_processed).to eq(0)
|
209
|
+
expect(status.messages_processed).to eq(3)
|
210
|
+
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
context 'with skip_too_large_messages off' do
|
215
|
+
it 'should retry forever' do
|
216
|
+
called_once = false
|
217
|
+
allow(poller).to receive(:sleep)
|
218
|
+
allow(poller).to receive(:process_batch) do
|
219
|
+
unless called_once
|
220
|
+
called_once = true
|
221
|
+
raise Kafka::MessageSizeTooLarge, 'OH NOES'
|
222
|
+
end
|
223
|
+
end
|
224
|
+
poller.retrieve_poll_info
|
225
|
+
poller.process_batch_with_span(widgets, status)
|
226
|
+
expect(poller).to have_received(:sleep).once.with(0.5)
|
227
|
+
expect(Deimos.config.tracer).to have_received(:finish).with('a span')
|
228
|
+
expect(status.batches_errored).to eq(0)
|
229
|
+
expect(status.batches_processed).to eq(1)
|
230
|
+
expect(status.messages_processed).to eq(3)
|
231
|
+
|
232
|
+
end
|
233
|
+
|
234
|
+
end
|
235
|
+
|
194
236
|
it 'should retry on Kafka error' do
|
195
237
|
called_once = false
|
196
238
|
allow(poller).to receive(:sleep)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: deimos-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.23.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Orner
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-01-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro_turf
|
@@ -579,6 +579,7 @@ files:
|
|
579
579
|
- spec/schemas/my_namespace/my_schema_with_date_time.rb
|
580
580
|
- spec/schemas/my_namespace/my_schema_with_id.rb
|
581
581
|
- spec/schemas/my_namespace/my_schema_with_unique_id.rb
|
582
|
+
- spec/schemas/my_namespace/my_updated_schema.rb
|
582
583
|
- spec/schemas/my_namespace/wibble.rb
|
583
584
|
- spec/schemas/my_namespace/widget.rb
|
584
585
|
- spec/schemas/my_namespace/widget_the_second.rb
|
@@ -632,7 +633,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
632
633
|
- !ruby/object:Gem::Version
|
633
634
|
version: '0'
|
634
635
|
requirements: []
|
635
|
-
rubygems_version: 3.
|
636
|
+
rubygems_version: 3.1.2
|
636
637
|
signing_key:
|
637
638
|
specification_version: 4
|
638
639
|
summary: Kafka libraries for Ruby.
|
@@ -706,6 +707,7 @@ test_files:
|
|
706
707
|
- spec/schemas/my_namespace/my_schema_with_date_time.rb
|
707
708
|
- spec/schemas/my_namespace/my_schema_with_id.rb
|
708
709
|
- spec/schemas/my_namespace/my_schema_with_unique_id.rb
|
710
|
+
- spec/schemas/my_namespace/my_updated_schema.rb
|
709
711
|
- spec/schemas/my_namespace/wibble.rb
|
710
712
|
- spec/schemas/my_namespace/widget.rb
|
711
713
|
- spec/schemas/my_namespace/widget_the_second.rb
|