deimos-ruby 1.22.5 → 1.23.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +2 -2
- data/CHANGELOG.md +18 -0
- data/README.md +6 -0
- data/docs/CONFIGURATION.md +4 -0
- data/lib/deimos/active_record_consume/batch_consumption.rb +43 -20
- data/lib/deimos/active_record_consume/batch_record.rb +5 -4
- data/lib/deimos/active_record_consume/batch_record_list.rb +11 -2
- data/lib/deimos/active_record_consume/mass_updater.rb +13 -4
- data/lib/deimos/active_record_consumer.rb +10 -0
- data/lib/deimos/config/configuration.rb +28 -2
- data/lib/deimos/consume/batch_consumption.rb +2 -2
- data/lib/deimos/instrumentation.rb +20 -8
- data/lib/deimos/test_helpers.rb +1 -0
- data/lib/deimos/tracing/datadog.rb +12 -6
- data/lib/deimos/tracing/mock.rb +31 -2
- data/lib/deimos/tracing/provider.rb +6 -0
- data/lib/deimos/utils/db_poller/base.rb +23 -0
- data/lib/deimos/utils/schema_class.rb +10 -2
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +14 -1
- data/spec/active_record_batch_consumer_association_spec.rb +32 -5
- data/spec/active_record_batch_consumer_spec.rb +376 -59
- data/spec/active_record_consume/mass_updater_spec.rb +46 -3
- data/spec/active_record_consumer_spec.rb +74 -1
- data/spec/active_record_producer_spec.rb +4 -1
- data/spec/batch_consumer_spec.rb +4 -1
- data/spec/config/configuration_spec.rb +42 -3
- data/spec/consumer_spec.rb +42 -1
- data/spec/schemas/my_namespace/my_updated_schema.rb +18 -0
- data/spec/utils/db_poller_spec.rb +42 -0
- metadata +5 -3
@@ -69,7 +69,10 @@ describe Deimos::ActiveRecordProducer do
|
|
69
69
|
SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
|
70
70
|
context "with Schema Class consumption #{setting}" do
|
71
71
|
before(:each) do
|
72
|
-
Deimos.configure
|
72
|
+
Deimos.configure do |config|
|
73
|
+
config.schema.use_schema_classes = use_schema_classes
|
74
|
+
config.schema.generate_namespace_folders = true
|
75
|
+
end
|
73
76
|
end
|
74
77
|
|
75
78
|
it 'should send events correctly' do
|
data/spec/batch_consumer_spec.rb
CHANGED
@@ -40,7 +40,10 @@ module ConsumerTest
|
|
40
40
|
end
|
41
41
|
|
42
42
|
before(:each) do
|
43
|
-
Deimos.configure
|
43
|
+
Deimos.configure do |config|
|
44
|
+
config.schema.use_schema_classes = use_schema_classes
|
45
|
+
config.schema.generate_namespace_folders = true
|
46
|
+
end
|
44
47
|
end
|
45
48
|
|
46
49
|
it 'should provide backwards compatibility for BatchConsumer class' do
|
@@ -91,7 +91,8 @@ describe Deimos, 'configuration' do
|
|
91
91
|
heartbeat_interval: 10,
|
92
92
|
handler: 'ConsumerTest::MyConsumer',
|
93
93
|
use_schema_classes: nil,
|
94
|
-
max_db_batch_size: nil
|
94
|
+
max_db_batch_size: nil,
|
95
|
+
bulk_import_id_generator: nil
|
95
96
|
}, {
|
96
97
|
topic: 'my_batch_consume_topic',
|
97
98
|
group_id: 'my_batch_group_id',
|
@@ -109,7 +110,8 @@ describe Deimos, 'configuration' do
|
|
109
110
|
heartbeat_interval: 10,
|
110
111
|
handler: 'ConsumerTest::MyBatchConsumer',
|
111
112
|
use_schema_classes: nil,
|
112
|
-
max_db_batch_size: nil
|
113
|
+
max_db_batch_size: nil,
|
114
|
+
bulk_import_id_generator: nil
|
113
115
|
}
|
114
116
|
],
|
115
117
|
producer: {
|
@@ -261,7 +263,8 @@ describe Deimos, 'configuration' do
|
|
261
263
|
heartbeat_interval: 13,
|
262
264
|
handler: 'MyConfigConsumer',
|
263
265
|
use_schema_classes: false,
|
264
|
-
max_db_batch_size: nil
|
266
|
+
max_db_batch_size: nil,
|
267
|
+
bulk_import_id_generator: nil
|
265
268
|
}
|
266
269
|
],
|
267
270
|
producer: {
|
@@ -279,4 +282,40 @@ describe Deimos, 'configuration' do
|
|
279
282
|
}
|
280
283
|
)
|
281
284
|
end
|
285
|
+
|
286
|
+
it 'should override global configurations' do
|
287
|
+
described_class.configure do
|
288
|
+
consumers.bulk_import_id_generator(-> { 'global' })
|
289
|
+
consumers.replace_associations true
|
290
|
+
|
291
|
+
consumer do
|
292
|
+
class_name 'MyConfigConsumer'
|
293
|
+
schema 'blah'
|
294
|
+
topic 'blah'
|
295
|
+
group_id 'myconsumerid'
|
296
|
+
bulk_import_id_generator(-> { 'consumer' })
|
297
|
+
replace_associations false
|
298
|
+
end
|
299
|
+
|
300
|
+
consumer do
|
301
|
+
class_name 'MyConfigConsumer2'
|
302
|
+
schema 'blah'
|
303
|
+
topic 'blah'
|
304
|
+
group_id 'myconsumerid'
|
305
|
+
end
|
306
|
+
end
|
307
|
+
|
308
|
+
consumers = described_class.config.consumers
|
309
|
+
expect(consumers.replace_associations).to eq(true)
|
310
|
+
expect(consumers.bulk_import_id_generator.call).to eq('global')
|
311
|
+
|
312
|
+
custom = MyConfigConsumer.config
|
313
|
+
expect(custom[:replace_associations]).to eq(false)
|
314
|
+
expect(custom[:bulk_import_id_generator].call).to eq('consumer')
|
315
|
+
|
316
|
+
default = MyConfigConsumer2.config
|
317
|
+
expect(default[:replace_associations]).to eq(true)
|
318
|
+
expect(default[:bulk_import_id_generator].call).to eq('global')
|
319
|
+
|
320
|
+
end
|
282
321
|
end
|
data/spec/consumer_spec.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
# :nodoc:
|
4
|
+
# rubocop:disable Metrics/ModuleLength
|
4
5
|
module ConsumerTest
|
5
6
|
describe Deimos::Consumer, 'Message Consumer' do
|
6
7
|
prepend_before(:each) do
|
@@ -26,8 +27,12 @@ module ConsumerTest
|
|
26
27
|
describe 'consume' do
|
27
28
|
SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
|
28
29
|
context "with Schema Class consumption #{setting}" do
|
30
|
+
|
29
31
|
before(:each) do
|
30
|
-
Deimos.configure
|
32
|
+
Deimos.configure do |config|
|
33
|
+
config.schema.use_schema_classes = use_schema_classes
|
34
|
+
config.schema.generate_namespace_folders = true
|
35
|
+
end
|
31
36
|
end
|
32
37
|
|
33
38
|
it 'should consume a message' do
|
@@ -127,6 +132,41 @@ module ConsumerTest
|
|
127
132
|
end
|
128
133
|
end
|
129
134
|
end
|
135
|
+
|
136
|
+
context 'with overriden schema classes' do
|
137
|
+
|
138
|
+
before(:each) do
|
139
|
+
Deimos.configure do |config|
|
140
|
+
config.schema.use_schema_classes = true
|
141
|
+
config.schema.generate_namespace_folders = true
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
prepend_before(:each) do
|
146
|
+
consumer_class = Class.new(described_class) do
|
147
|
+
schema 'MyUpdatedSchema'
|
148
|
+
namespace 'com.my-namespace'
|
149
|
+
key_config field: 'test_id'
|
150
|
+
|
151
|
+
# :nodoc:
|
152
|
+
def consume(_payload, _metadata)
|
153
|
+
raise 'This should not be called unless call_original is set'
|
154
|
+
end
|
155
|
+
end
|
156
|
+
stub_const('ConsumerTest::MyConsumer', consumer_class)
|
157
|
+
end
|
158
|
+
|
159
|
+
it 'should consume messages' do
|
160
|
+
test_consume_message('my_consume_topic',
|
161
|
+
{ 'test_id' => 'foo',
|
162
|
+
'some_int' => 1 }) do |payload, _metadata|
|
163
|
+
expect(payload['test_id']).to eq('foo')
|
164
|
+
expect(payload['some_int']).to eq(1)
|
165
|
+
expect(payload['super_int']).to eq(9000)
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
end
|
130
170
|
end
|
131
171
|
|
132
172
|
describe 'decode_key' do
|
@@ -218,3 +258,4 @@ module ConsumerTest
|
|
218
258
|
end
|
219
259
|
end
|
220
260
|
end
|
261
|
+
# rubocop:enable Metrics/ModuleLength
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This file is autogenerated by Deimos, Do NOT modify
|
4
|
+
module Schemas; module MyNamespace
|
5
|
+
### Primary Schema Class ###
|
6
|
+
# Autogenerated Schema for Record at com.my-namespace.MySchema
|
7
|
+
class MyUpdatedSchema < Schemas::MyNamespace::MySchema
|
8
|
+
|
9
|
+
attr_accessor :super_int
|
10
|
+
|
11
|
+
def initialize(test_id: nil,
|
12
|
+
some_int: nil)
|
13
|
+
super
|
14
|
+
self.super_int = some_int.nil? ? 10 : some_int * 9000
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -191,6 +191,48 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
|
|
191
191
|
expect(Deimos.config.tracer).to have_received(:finish).with('a span')
|
192
192
|
end
|
193
193
|
|
194
|
+
context 'with skip_too_large_messages on' do
|
195
|
+
before(:each) { config.skip_too_large_messages = true }
|
196
|
+
|
197
|
+
it 'should skip and move on' do
|
198
|
+
error = Kafka::MessageSizeTooLarge.new('OH NOES')
|
199
|
+
allow(poller).to receive(:sleep)
|
200
|
+
allow(poller).to receive(:process_batch) do
|
201
|
+
raise error
|
202
|
+
end
|
203
|
+
poller.retrieve_poll_info
|
204
|
+
poller.process_batch_with_span(widgets, status)
|
205
|
+
expect(poller).not_to have_received(:sleep)
|
206
|
+
expect(Deimos.config.tracer).to have_received(:set_error).with('a span', error)
|
207
|
+
expect(status.batches_errored).to eq(1)
|
208
|
+
expect(status.batches_processed).to eq(0)
|
209
|
+
expect(status.messages_processed).to eq(3)
|
210
|
+
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
context 'with skip_too_large_messages off' do
|
215
|
+
it 'should retry forever' do
|
216
|
+
called_once = false
|
217
|
+
allow(poller).to receive(:sleep)
|
218
|
+
allow(poller).to receive(:process_batch) do
|
219
|
+
unless called_once
|
220
|
+
called_once = true
|
221
|
+
raise Kafka::MessageSizeTooLarge, 'OH NOES'
|
222
|
+
end
|
223
|
+
end
|
224
|
+
poller.retrieve_poll_info
|
225
|
+
poller.process_batch_with_span(widgets, status)
|
226
|
+
expect(poller).to have_received(:sleep).once.with(0.5)
|
227
|
+
expect(Deimos.config.tracer).to have_received(:finish).with('a span')
|
228
|
+
expect(status.batches_errored).to eq(0)
|
229
|
+
expect(status.batches_processed).to eq(1)
|
230
|
+
expect(status.messages_processed).to eq(3)
|
231
|
+
|
232
|
+
end
|
233
|
+
|
234
|
+
end
|
235
|
+
|
194
236
|
it 'should retry on Kafka error' do
|
195
237
|
called_once = false
|
196
238
|
allow(poller).to receive(:sleep)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: deimos-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.23.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Orner
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-01-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro_turf
|
@@ -579,6 +579,7 @@ files:
|
|
579
579
|
- spec/schemas/my_namespace/my_schema_with_date_time.rb
|
580
580
|
- spec/schemas/my_namespace/my_schema_with_id.rb
|
581
581
|
- spec/schemas/my_namespace/my_schema_with_unique_id.rb
|
582
|
+
- spec/schemas/my_namespace/my_updated_schema.rb
|
582
583
|
- spec/schemas/my_namespace/wibble.rb
|
583
584
|
- spec/schemas/my_namespace/widget.rb
|
584
585
|
- spec/schemas/my_namespace/widget_the_second.rb
|
@@ -632,7 +633,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
632
633
|
- !ruby/object:Gem::Version
|
633
634
|
version: '0'
|
634
635
|
requirements: []
|
635
|
-
rubygems_version: 3.
|
636
|
+
rubygems_version: 3.1.2
|
636
637
|
signing_key:
|
637
638
|
specification_version: 4
|
638
639
|
summary: Kafka libraries for Ruby.
|
@@ -706,6 +707,7 @@ test_files:
|
|
706
707
|
- spec/schemas/my_namespace/my_schema_with_date_time.rb
|
707
708
|
- spec/schemas/my_namespace/my_schema_with_id.rb
|
708
709
|
- spec/schemas/my_namespace/my_schema_with_unique_id.rb
|
710
|
+
- spec/schemas/my_namespace/my_updated_schema.rb
|
709
711
|
- spec/schemas/my_namespace/wibble.rb
|
710
712
|
- spec/schemas/my_namespace/widget.rb
|
711
713
|
- spec/schemas/my_namespace/widget_the_second.rb
|