deimos-ruby 1.17.1 → 1.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,14 +1,17 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'deimos/utils/db_poller'
4
+
3
5
  # rubocop:disable Layout/LineLength
4
6
 
5
- # @param seconds [Integer]
7
+ # @param secs [Integer]
8
+ # @param mins [Integer]
6
9
  # @return [Time]
7
10
  def time_value(secs: 0, mins: 0)
8
11
  Time.local(2015, 5, 5, 1, 0, 0) + (secs + (mins * 60))
9
12
  end
10
13
 
11
- each_db_config(Deimos::Utils::DbPoller) do
14
+ each_db_config(Deimos::Utils::DbPoller::Base) do
12
15
 
13
16
  before(:each) do
14
17
  Deimos::PollInfo.delete_all
@@ -36,7 +39,7 @@ each_db_config(Deimos::Utils::DbPoller) do
36
39
 
37
40
  it 'should raise an error if no pollers configured' do
38
41
  Deimos.configure {}
39
- expect { described_class.start! }.to raise_error('No pollers configured!')
42
+ expect { Deimos::Utils::DbPoller.start! }.to raise_error('No pollers configured!')
40
43
  end
41
44
 
42
45
  it 'should start pollers as configured' do
@@ -49,14 +52,14 @@ each_db_config(Deimos::Utils::DbPoller) do
49
52
  end
50
53
  end
51
54
 
52
- allow(Deimos::Utils::DbPoller).to receive(:new)
55
+ allow(Deimos::Utils::DbPoller::TimeBased).to receive(:new)
53
56
  signal_double = instance_double(Sigurd::SignalHandler, run!: nil)
54
57
  allow(Sigurd::SignalHandler).to receive(:new).and_return(signal_double)
55
- described_class.start!
56
- expect(Deimos::Utils::DbPoller).to have_received(:new).twice
57
- expect(Deimos::Utils::DbPoller).to have_received(:new).
58
+ Deimos::Utils::DbPoller.start!
59
+ expect(Deimos::Utils::DbPoller::TimeBased).to have_received(:new).twice
60
+ expect(Deimos::Utils::DbPoller::TimeBased).to have_received(:new).
58
61
  with(Deimos.config.db_poller_objects[0])
59
- expect(Deimos::Utils::DbPoller).to have_received(:new).
62
+ expect(Deimos::Utils::DbPoller::TimeBased).to have_received(:new).
60
63
  with(Deimos.config.db_poller_objects[1])
61
64
  end
62
65
  end
@@ -65,7 +68,7 @@ each_db_config(Deimos::Utils::DbPoller) do
65
68
  include_context 'with widgets'
66
69
 
67
70
  let(:poller) do
68
- poller = described_class.new(config)
71
+ poller = Deimos::Utils::DbPoller.class_for_config(config.mode).new(config)
69
72
  allow(poller).to receive(:sleep)
70
73
  poller
71
74
  end
@@ -160,6 +163,7 @@ each_db_config(Deimos::Utils::DbPoller) do
160
163
 
161
164
  describe '#process_batch' do
162
165
  let(:widgets) { (1..3).map { Widget.create!(test_id: 'some_id', some_int: 4) } }
166
+ let(:status) { Deimos::Utils::DbPoller::PollStatus.new(0, 0, 0) }
163
167
 
164
168
  before(:each) do
165
169
  allow(Deimos.config.tracer).to receive(:start).and_return('a span')
@@ -172,7 +176,7 @@ each_db_config(Deimos::Utils::DbPoller) do
172
176
  widgets.last.update_attribute(:updated_at, time_value(mins: -30))
173
177
  expect(MyProducer).to receive(:send_events).with(widgets)
174
178
  poller.retrieve_poll_info
175
- poller.process_batch(widgets)
179
+ poller.process_and_touch_info(widgets, status)
176
180
  info = Deimos::PollInfo.last
177
181
  expect(info.last_sent.in_time_zone).to eq(time_value(mins: -30))
178
182
  expect(info.last_sent_id).to eq(widgets.last.id)
@@ -180,7 +184,10 @@ each_db_config(Deimos::Utils::DbPoller) do
180
184
 
181
185
  it 'should create a span' do
182
186
  poller.retrieve_poll_info
183
- poller.process_batch_with_span(widgets)
187
+ poller.process_batch_with_span(widgets, status)
188
+ expect(status.batches_errored).to eq(0)
189
+ expect(status.batches_processed).to eq(1)
190
+ expect(status.messages_processed).to eq(3)
184
191
  expect(Deimos.config.tracer).to have_received(:finish).with('a span')
185
192
  end
186
193
 
@@ -194,9 +201,12 @@ each_db_config(Deimos::Utils::DbPoller) do
194
201
  end
195
202
  end
196
203
  poller.retrieve_poll_info
197
- poller.process_batch_with_span(widgets)
204
+ poller.process_batch_with_span(widgets, status)
198
205
  expect(poller).to have_received(:sleep).once.with(0.5)
199
206
  expect(Deimos.config.tracer).to have_received(:finish).with('a span')
207
+ expect(status.batches_errored).to eq(0)
208
+ expect(status.batches_processed).to eq(1)
209
+ expect(status.messages_processed).to eq(3)
200
210
  end
201
211
 
202
212
  it 'should retry only once on other errors' do
@@ -204,9 +214,12 @@ each_db_config(Deimos::Utils::DbPoller) do
204
214
  allow(poller).to receive(:sleep)
205
215
  allow(poller).to receive(:process_batch).and_raise(error)
206
216
  poller.retrieve_poll_info
207
- poller.process_batch_with_span(widgets)
217
+ poller.process_batch_with_span(widgets, status)
208
218
  expect(poller).to have_received(:sleep).once.with(0.5)
209
219
  expect(Deimos.config.tracer).to have_received(:set_error).with('a span', error)
220
+ expect(status.batches_errored).to eq(1)
221
+ expect(status.batches_processed).to eq(0)
222
+ expect(status.messages_processed).to eq(3)
210
223
  end
211
224
  end
212
225
 
@@ -243,16 +256,16 @@ each_db_config(Deimos::Utils::DbPoller) do
243
256
  info = Deimos::PollInfo.last
244
257
  config.full_table = true
245
258
  expect(MyProducer).to receive(:poll_query).at_least(:once).and_call_original
246
- expect(poller).to receive(:process_batch).ordered.
247
- with([old_widget, widgets[0], widgets[1]]).and_wrap_original do |m, *args|
259
+ expect(poller).to receive(:process_and_touch_info).ordered.
260
+ with([old_widget, widgets[0], widgets[1]], anything).and_wrap_original do |m, *args|
248
261
  m.call(*args)
249
262
  expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 32))
250
263
  expect(info.last_sent_id).to eq(widgets[1].id)
251
264
  end
252
- expect(poller).to receive(:process_batch).ordered.
253
- with([widgets[2], widgets[3], widgets[4]]).and_call_original
254
- expect(poller).to receive(:process_batch).ordered.
255
- with([widgets[5], widgets[6]]).and_call_original
265
+ expect(poller).to receive(:process_and_touch_info).ordered.
266
+ with([widgets[2], widgets[3], widgets[4]], anything).and_call_original
267
+ expect(poller).to receive(:process_and_touch_info).ordered.
268
+ with([widgets[5], widgets[6]], anything).and_call_original
256
269
  poller.process_updates
257
270
 
258
271
  # this is the updated_at of widgets[6]
@@ -263,12 +276,12 @@ each_db_config(Deimos::Utils::DbPoller) do
263
276
 
264
277
  travel 61.seconds
265
278
  # should reprocess the table
266
- expect(poller).to receive(:process_batch).ordered.
267
- with([last_widget, old_widget, widgets[0]]).and_call_original
268
- expect(poller).to receive(:process_batch).ordered.
269
- with([widgets[1], widgets[2], widgets[3]]).and_call_original
270
- expect(poller).to receive(:process_batch).ordered.
271
- with([widgets[4], widgets[5], widgets[6]]).and_call_original
279
+ expect(poller).to receive(:process_and_touch_info).ordered.
280
+ with([last_widget, old_widget, widgets[0]], anything).and_call_original
281
+ expect(poller).to receive(:process_and_touch_info).ordered.
282
+ with([widgets[1], widgets[2], widgets[3]], anything).and_call_original
283
+ expect(poller).to receive(:process_and_touch_info).ordered.
284
+ with([widgets[4], widgets[5], widgets[6]], anything).and_call_original
272
285
  poller.process_updates
273
286
 
274
287
  expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 37))
@@ -278,12 +291,12 @@ each_db_config(Deimos::Utils::DbPoller) do
278
291
  it 'should send events across multiple batches' do
279
292
  allow(Deimos.config.logger).to receive(:info)
280
293
  allow(MyProducer).to receive(:poll_query).and_call_original
281
- expect(poller).to receive(:process_batch).ordered.
282
- with([widgets[0], widgets[1], widgets[2]]).and_call_original
283
- expect(poller).to receive(:process_batch).ordered.
284
- with([widgets[3], widgets[4], widgets[5]]).and_call_original
285
- expect(poller).to receive(:process_batch).ordered.
286
- with([widgets[6]]).and_call_original
294
+ expect(poller).to receive(:process_and_touch_info).ordered.
295
+ with([widgets[0], widgets[1], widgets[2]], anything).and_call_original
296
+ expect(poller).to receive(:process_and_touch_info).ordered.
297
+ with([widgets[3], widgets[4], widgets[5]], anything).and_call_original
298
+ expect(poller).to receive(:process_and_touch_info).ordered.
299
+ with([widgets[6]], anything).and_call_original
287
300
  poller.process_updates
288
301
 
289
302
  expect(MyProducer).to have_received(:poll_query).
@@ -294,7 +307,7 @@ each_db_config(Deimos::Utils::DbPoller) do
294
307
 
295
308
  travel 61.seconds
296
309
  # process the last widget which came in during the delay
297
- expect(poller).to receive(:process_batch).with([last_widget]).
310
+ expect(poller).to receive(:process_and_touch_info).with([last_widget], anything).
298
311
  and_call_original
299
312
  poller.process_updates
300
313
 
@@ -307,7 +320,7 @@ each_db_config(Deimos::Utils::DbPoller) do
307
320
 
308
321
  travel 61.seconds
309
322
  # nothing else to process
310
- expect(poller).not_to receive(:process_batch)
323
+ expect(poller).not_to receive(:process_and_touch_info)
311
324
  poller.process_updates
312
325
  poller.process_updates
313
326
 
@@ -317,7 +330,7 @@ each_db_config(Deimos::Utils::DbPoller) do
317
330
  column_name: :updated_at,
318
331
  min_id: last_widget.id)
319
332
  expect(Deimos.config.logger).to have_received(:info).
320
- with('Poll my-topic-with-id complete at 2015-05-05 00:59:58 -0400 (7 messages, 3 successful batches, 0 batches errored}')
333
+ with('Poll my-topic-with-id complete at 2015-05-05 00:59:58 -0400 (3 batches, 0 errored batches, 7 processed messages)')
321
334
  end
322
335
 
323
336
  describe 'errors' do
@@ -354,7 +367,7 @@ each_db_config(Deimos::Utils::DbPoller) do
354
367
  expect(info.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 30))
355
368
  expect(info.last_sent_id).to eq(widgets[6].id)
356
369
  expect(Deimos.config.logger).to have_received(:info).
357
- with('Poll my-topic-with-id complete at 2015-05-05 00:59:58 -0400 (7 messages, 2 successful batches, 1 batches errored}')
370
+ with('Poll my-topic-with-id complete at 2015-05-05 00:59:58 -0400 (2 batches, 1 errored batches, 7 processed messages)')
358
371
  end
359
372
  end
360
373
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: deimos-ruby
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.17.1
4
+ version: 1.18.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Daniel Orner
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-10-20 00:00:00.000000000 Z
11
+ date: 2022-11-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: avro_turf
@@ -475,6 +475,9 @@ files:
475
475
  - lib/deimos/tracing/mock.rb
476
476
  - lib/deimos/tracing/provider.rb
477
477
  - lib/deimos/utils/db_poller.rb
478
+ - lib/deimos/utils/db_poller/base.rb
479
+ - lib/deimos/utils/db_poller/state_based.rb
480
+ - lib/deimos/utils/db_poller/time_based.rb
478
481
  - lib/deimos/utils/db_producer.rb
479
482
  - lib/deimos/utils/deadlock_retry.rb
480
483
  - lib/deimos/utils/inline_consumer.rb