deimos-ruby 1.17.1 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -1
- data/CHANGELOG.md +9 -0
- data/README.md +19 -0
- data/docs/CONFIGURATION.md +8 -3
- data/lib/deimos/config/configuration.rb +19 -7
- data/lib/deimos/schema_backends/mock.rb +1 -1
- data/lib/deimos/utils/db_poller/base.rb +139 -0
- data/lib/deimos/utils/db_poller/state_based.rb +57 -0
- data/lib/deimos/utils/db_poller/time_based.rb +82 -0
- data/lib/deimos/utils/db_poller.rb +22 -170
- data/lib/deimos/version.rb +1 -1
- data/rbs_collection.lock.yaml +43 -19
- data/sig/defs.rbs +234 -173
- data/spec/utils/db_poller_spec.rb +48 -35
- metadata +5 -2
@@ -1,14 +1,17 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'deimos/utils/db_poller'
|
4
|
+
|
3
5
|
# rubocop:disable Layout/LineLength
|
4
6
|
|
5
|
-
# @param
|
7
|
+
# @param secs [Integer]
|
8
|
+
# @param mins [Integer]
|
6
9
|
# @return [Time]
|
7
10
|
def time_value(secs: 0, mins: 0)
|
8
11
|
Time.local(2015, 5, 5, 1, 0, 0) + (secs + (mins * 60))
|
9
12
|
end
|
10
13
|
|
11
|
-
each_db_config(Deimos::Utils::DbPoller) do
|
14
|
+
each_db_config(Deimos::Utils::DbPoller::Base) do
|
12
15
|
|
13
16
|
before(:each) do
|
14
17
|
Deimos::PollInfo.delete_all
|
@@ -36,7 +39,7 @@ each_db_config(Deimos::Utils::DbPoller) do
|
|
36
39
|
|
37
40
|
it 'should raise an error if no pollers configured' do
|
38
41
|
Deimos.configure {}
|
39
|
-
expect {
|
42
|
+
expect { Deimos::Utils::DbPoller.start! }.to raise_error('No pollers configured!')
|
40
43
|
end
|
41
44
|
|
42
45
|
it 'should start pollers as configured' do
|
@@ -49,14 +52,14 @@ each_db_config(Deimos::Utils::DbPoller) do
|
|
49
52
|
end
|
50
53
|
end
|
51
54
|
|
52
|
-
allow(Deimos::Utils::DbPoller).to receive(:new)
|
55
|
+
allow(Deimos::Utils::DbPoller::TimeBased).to receive(:new)
|
53
56
|
signal_double = instance_double(Sigurd::SignalHandler, run!: nil)
|
54
57
|
allow(Sigurd::SignalHandler).to receive(:new).and_return(signal_double)
|
55
|
-
|
56
|
-
expect(Deimos::Utils::DbPoller).to have_received(:new).twice
|
57
|
-
expect(Deimos::Utils::DbPoller).to have_received(:new).
|
58
|
+
Deimos::Utils::DbPoller.start!
|
59
|
+
expect(Deimos::Utils::DbPoller::TimeBased).to have_received(:new).twice
|
60
|
+
expect(Deimos::Utils::DbPoller::TimeBased).to have_received(:new).
|
58
61
|
with(Deimos.config.db_poller_objects[0])
|
59
|
-
expect(Deimos::Utils::DbPoller).to have_received(:new).
|
62
|
+
expect(Deimos::Utils::DbPoller::TimeBased).to have_received(:new).
|
60
63
|
with(Deimos.config.db_poller_objects[1])
|
61
64
|
end
|
62
65
|
end
|
@@ -65,7 +68,7 @@ each_db_config(Deimos::Utils::DbPoller) do
|
|
65
68
|
include_context 'with widgets'
|
66
69
|
|
67
70
|
let(:poller) do
|
68
|
-
poller =
|
71
|
+
poller = Deimos::Utils::DbPoller.class_for_config(config.mode).new(config)
|
69
72
|
allow(poller).to receive(:sleep)
|
70
73
|
poller
|
71
74
|
end
|
@@ -160,6 +163,7 @@ each_db_config(Deimos::Utils::DbPoller) do
|
|
160
163
|
|
161
164
|
describe '#process_batch' do
|
162
165
|
let(:widgets) { (1..3).map { Widget.create!(test_id: 'some_id', some_int: 4) } }
|
166
|
+
let(:status) { Deimos::Utils::DbPoller::PollStatus.new(0, 0, 0) }
|
163
167
|
|
164
168
|
before(:each) do
|
165
169
|
allow(Deimos.config.tracer).to receive(:start).and_return('a span')
|
@@ -172,7 +176,7 @@ each_db_config(Deimos::Utils::DbPoller) do
|
|
172
176
|
widgets.last.update_attribute(:updated_at, time_value(mins: -30))
|
173
177
|
expect(MyProducer).to receive(:send_events).with(widgets)
|
174
178
|
poller.retrieve_poll_info
|
175
|
-
poller.
|
179
|
+
poller.process_and_touch_info(widgets, status)
|
176
180
|
info = Deimos::PollInfo.last
|
177
181
|
expect(info.last_sent.in_time_zone).to eq(time_value(mins: -30))
|
178
182
|
expect(info.last_sent_id).to eq(widgets.last.id)
|
@@ -180,7 +184,10 @@ each_db_config(Deimos::Utils::DbPoller) do
|
|
180
184
|
|
181
185
|
it 'should create a span' do
|
182
186
|
poller.retrieve_poll_info
|
183
|
-
poller.process_batch_with_span(widgets)
|
187
|
+
poller.process_batch_with_span(widgets, status)
|
188
|
+
expect(status.batches_errored).to eq(0)
|
189
|
+
expect(status.batches_processed).to eq(1)
|
190
|
+
expect(status.messages_processed).to eq(3)
|
184
191
|
expect(Deimos.config.tracer).to have_received(:finish).with('a span')
|
185
192
|
end
|
186
193
|
|
@@ -194,9 +201,12 @@ each_db_config(Deimos::Utils::DbPoller) do
|
|
194
201
|
end
|
195
202
|
end
|
196
203
|
poller.retrieve_poll_info
|
197
|
-
poller.process_batch_with_span(widgets)
|
204
|
+
poller.process_batch_with_span(widgets, status)
|
198
205
|
expect(poller).to have_received(:sleep).once.with(0.5)
|
199
206
|
expect(Deimos.config.tracer).to have_received(:finish).with('a span')
|
207
|
+
expect(status.batches_errored).to eq(0)
|
208
|
+
expect(status.batches_processed).to eq(1)
|
209
|
+
expect(status.messages_processed).to eq(3)
|
200
210
|
end
|
201
211
|
|
202
212
|
it 'should retry only once on other errors' do
|
@@ -204,9 +214,12 @@ each_db_config(Deimos::Utils::DbPoller) do
|
|
204
214
|
allow(poller).to receive(:sleep)
|
205
215
|
allow(poller).to receive(:process_batch).and_raise(error)
|
206
216
|
poller.retrieve_poll_info
|
207
|
-
poller.process_batch_with_span(widgets)
|
217
|
+
poller.process_batch_with_span(widgets, status)
|
208
218
|
expect(poller).to have_received(:sleep).once.with(0.5)
|
209
219
|
expect(Deimos.config.tracer).to have_received(:set_error).with('a span', error)
|
220
|
+
expect(status.batches_errored).to eq(1)
|
221
|
+
expect(status.batches_processed).to eq(0)
|
222
|
+
expect(status.messages_processed).to eq(3)
|
210
223
|
end
|
211
224
|
end
|
212
225
|
|
@@ -243,16 +256,16 @@ each_db_config(Deimos::Utils::DbPoller) do
|
|
243
256
|
info = Deimos::PollInfo.last
|
244
257
|
config.full_table = true
|
245
258
|
expect(MyProducer).to receive(:poll_query).at_least(:once).and_call_original
|
246
|
-
expect(poller).to receive(:
|
247
|
-
with([old_widget, widgets[0], widgets[1]]).and_wrap_original do |m, *args|
|
259
|
+
expect(poller).to receive(:process_and_touch_info).ordered.
|
260
|
+
with([old_widget, widgets[0], widgets[1]], anything).and_wrap_original do |m, *args|
|
248
261
|
m.call(*args)
|
249
262
|
expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 32))
|
250
263
|
expect(info.last_sent_id).to eq(widgets[1].id)
|
251
264
|
end
|
252
|
-
expect(poller).to receive(:
|
253
|
-
with([widgets[2], widgets[3], widgets[4]]).and_call_original
|
254
|
-
expect(poller).to receive(:
|
255
|
-
with([widgets[5], widgets[6]]).and_call_original
|
265
|
+
expect(poller).to receive(:process_and_touch_info).ordered.
|
266
|
+
with([widgets[2], widgets[3], widgets[4]], anything).and_call_original
|
267
|
+
expect(poller).to receive(:process_and_touch_info).ordered.
|
268
|
+
with([widgets[5], widgets[6]], anything).and_call_original
|
256
269
|
poller.process_updates
|
257
270
|
|
258
271
|
# this is the updated_at of widgets[6]
|
@@ -263,12 +276,12 @@ each_db_config(Deimos::Utils::DbPoller) do
|
|
263
276
|
|
264
277
|
travel 61.seconds
|
265
278
|
# should reprocess the table
|
266
|
-
expect(poller).to receive(:
|
267
|
-
with([last_widget, old_widget, widgets[0]]).and_call_original
|
268
|
-
expect(poller).to receive(:
|
269
|
-
with([widgets[1], widgets[2], widgets[3]]).and_call_original
|
270
|
-
expect(poller).to receive(:
|
271
|
-
with([widgets[4], widgets[5], widgets[6]]).and_call_original
|
279
|
+
expect(poller).to receive(:process_and_touch_info).ordered.
|
280
|
+
with([last_widget, old_widget, widgets[0]], anything).and_call_original
|
281
|
+
expect(poller).to receive(:process_and_touch_info).ordered.
|
282
|
+
with([widgets[1], widgets[2], widgets[3]], anything).and_call_original
|
283
|
+
expect(poller).to receive(:process_and_touch_info).ordered.
|
284
|
+
with([widgets[4], widgets[5], widgets[6]], anything).and_call_original
|
272
285
|
poller.process_updates
|
273
286
|
|
274
287
|
expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 37))
|
@@ -278,12 +291,12 @@ each_db_config(Deimos::Utils::DbPoller) do
|
|
278
291
|
it 'should send events across multiple batches' do
|
279
292
|
allow(Deimos.config.logger).to receive(:info)
|
280
293
|
allow(MyProducer).to receive(:poll_query).and_call_original
|
281
|
-
expect(poller).to receive(:
|
282
|
-
with([widgets[0], widgets[1], widgets[2]]).and_call_original
|
283
|
-
expect(poller).to receive(:
|
284
|
-
with([widgets[3], widgets[4], widgets[5]]).and_call_original
|
285
|
-
expect(poller).to receive(:
|
286
|
-
with([widgets[6]]).and_call_original
|
294
|
+
expect(poller).to receive(:process_and_touch_info).ordered.
|
295
|
+
with([widgets[0], widgets[1], widgets[2]], anything).and_call_original
|
296
|
+
expect(poller).to receive(:process_and_touch_info).ordered.
|
297
|
+
with([widgets[3], widgets[4], widgets[5]], anything).and_call_original
|
298
|
+
expect(poller).to receive(:process_and_touch_info).ordered.
|
299
|
+
with([widgets[6]], anything).and_call_original
|
287
300
|
poller.process_updates
|
288
301
|
|
289
302
|
expect(MyProducer).to have_received(:poll_query).
|
@@ -294,7 +307,7 @@ each_db_config(Deimos::Utils::DbPoller) do
|
|
294
307
|
|
295
308
|
travel 61.seconds
|
296
309
|
# process the last widget which came in during the delay
|
297
|
-
expect(poller).to receive(:
|
310
|
+
expect(poller).to receive(:process_and_touch_info).with([last_widget], anything).
|
298
311
|
and_call_original
|
299
312
|
poller.process_updates
|
300
313
|
|
@@ -307,7 +320,7 @@ each_db_config(Deimos::Utils::DbPoller) do
|
|
307
320
|
|
308
321
|
travel 61.seconds
|
309
322
|
# nothing else to process
|
310
|
-
expect(poller).not_to receive(:
|
323
|
+
expect(poller).not_to receive(:process_and_touch_info)
|
311
324
|
poller.process_updates
|
312
325
|
poller.process_updates
|
313
326
|
|
@@ -317,7 +330,7 @@ each_db_config(Deimos::Utils::DbPoller) do
|
|
317
330
|
column_name: :updated_at,
|
318
331
|
min_id: last_widget.id)
|
319
332
|
expect(Deimos.config.logger).to have_received(:info).
|
320
|
-
with('Poll my-topic-with-id complete at 2015-05-05 00:59:58 -0400 (
|
333
|
+
with('Poll my-topic-with-id complete at 2015-05-05 00:59:58 -0400 (3 batches, 0 errored batches, 7 processed messages)')
|
321
334
|
end
|
322
335
|
|
323
336
|
describe 'errors' do
|
@@ -354,7 +367,7 @@ each_db_config(Deimos::Utils::DbPoller) do
|
|
354
367
|
expect(info.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 30))
|
355
368
|
expect(info.last_sent_id).to eq(widgets[6].id)
|
356
369
|
expect(Deimos.config.logger).to have_received(:info).
|
357
|
-
with('Poll my-topic-with-id complete at 2015-05-05 00:59:58 -0400 (
|
370
|
+
with('Poll my-topic-with-id complete at 2015-05-05 00:59:58 -0400 (2 batches, 1 errored batches, 7 processed messages)')
|
358
371
|
end
|
359
372
|
end
|
360
373
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: deimos-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.18.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Orner
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-11-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro_turf
|
@@ -475,6 +475,9 @@ files:
|
|
475
475
|
- lib/deimos/tracing/mock.rb
|
476
476
|
- lib/deimos/tracing/provider.rb
|
477
477
|
- lib/deimos/utils/db_poller.rb
|
478
|
+
- lib/deimos/utils/db_poller/base.rb
|
479
|
+
- lib/deimos/utils/db_poller/state_based.rb
|
480
|
+
- lib/deimos/utils/db_poller/time_based.rb
|
478
481
|
- lib/deimos/utils/db_producer.rb
|
479
482
|
- lib/deimos/utils/deadlock_retry.rb
|
480
483
|
- lib/deimos/utils/inline_consumer.rb
|