deimos-ruby 1.17.1 → 1.18.0

data/spec/utils/db_poller_spec.rb

@@ -1,14 +1,17 @@
 # frozen_string_literal: true
 
+require 'deimos/utils/db_poller'
+
 # rubocop:disable Layout/LineLength
 
-# @param seconds [Integer]
+# @param secs [Integer]
+# @param mins [Integer]
 # @return [Time]
 def time_value(secs: 0, mins: 0)
   Time.local(2015, 5, 5, 1, 0, 0) + (secs + (mins * 60))
 end
 
-each_db_config(Deimos::Utils::DbPoller) do
+each_db_config(Deimos::Utils::DbPoller::Base) do
 
   before(:each) do
     Deimos::PollInfo.delete_all
@@ -36,7 +39,7 @@ each_db_config(Deimos::Utils::DbPoller) do
 
     it 'should raise an error if no pollers configured' do
       Deimos.configure {}
-      expect { described_class.start! }.to raise_error('No pollers configured!')
+      expect { Deimos::Utils::DbPoller.start! }.to raise_error('No pollers configured!')
     end
 
     it 'should start pollers as configured' do
@@ -49,14 +52,14 @@ each_db_config(Deimos::Utils::DbPoller) do
         end
       end
 
-      allow(Deimos::Utils::DbPoller).to receive(:new)
+      allow(Deimos::Utils::DbPoller::TimeBased).to receive(:new)
       signal_double = instance_double(Sigurd::SignalHandler, run!: nil)
       allow(Sigurd::SignalHandler).to receive(:new).and_return(signal_double)
-      described_class.start!
-      expect(Deimos::Utils::DbPoller).to have_received(:new).twice
-      expect(Deimos::Utils::DbPoller).to have_received(:new).
+      Deimos::Utils::DbPoller.start!
+      expect(Deimos::Utils::DbPoller::TimeBased).to have_received(:new).twice
+      expect(Deimos::Utils::DbPoller::TimeBased).to have_received(:new).
         with(Deimos.config.db_poller_objects[0])
-      expect(Deimos::Utils::DbPoller).to have_received(:new).
+      expect(Deimos::Utils::DbPoller::TimeBased).to have_received(:new).
         with(Deimos.config.db_poller_objects[1])
     end
   end
@@ -65,7 +68,7 @@ each_db_config(Deimos::Utils::DbPoller) do
     include_context 'with widgets'
 
     let(:poller) do
-      poller = described_class.new(config)
+      poller = Deimos::Utils::DbPoller.class_for_config(config.mode).new(config)
       allow(poller).to receive(:sleep)
       poller
     end
@@ -160,6 +163,7 @@ each_db_config(Deimos::Utils::DbPoller) do
 
     describe '#process_batch' do
       let(:widgets) { (1..3).map { Widget.create!(test_id: 'some_id', some_int: 4) } }
+      let(:status) { Deimos::Utils::DbPoller::PollStatus.new(0, 0, 0) }
 
       before(:each) do
         allow(Deimos.config.tracer).to receive(:start).and_return('a span')
@@ -172,7 +176,7 @@ each_db_config(Deimos::Utils::DbPoller) do
         widgets.last.update_attribute(:updated_at, time_value(mins: -30))
         expect(MyProducer).to receive(:send_events).with(widgets)
         poller.retrieve_poll_info
-        poller.process_batch(widgets)
+        poller.process_and_touch_info(widgets, status)
         info = Deimos::PollInfo.last
         expect(info.last_sent.in_time_zone).to eq(time_value(mins: -30))
         expect(info.last_sent_id).to eq(widgets.last.id)
@@ -180,7 +184,10 @@ each_db_config(Deimos::Utils::DbPoller) do
 
       it 'should create a span' do
         poller.retrieve_poll_info
-        poller.process_batch_with_span(widgets)
+        poller.process_batch_with_span(widgets, status)
+        expect(status.batches_errored).to eq(0)
+        expect(status.batches_processed).to eq(1)
+        expect(status.messages_processed).to eq(3)
         expect(Deimos.config.tracer).to have_received(:finish).with('a span')
       end
 
@@ -194,9 +201,12 @@ each_db_config(Deimos::Utils::DbPoller) do
           end
         end
         poller.retrieve_poll_info
-        poller.process_batch_with_span(widgets)
+        poller.process_batch_with_span(widgets, status)
         expect(poller).to have_received(:sleep).once.with(0.5)
         expect(Deimos.config.tracer).to have_received(:finish).with('a span')
+        expect(status.batches_errored).to eq(0)
+        expect(status.batches_processed).to eq(1)
+        expect(status.messages_processed).to eq(3)
       end
 
       it 'should retry only once on other errors' do
@@ -204,9 +214,12 @@ each_db_config(Deimos::Utils::DbPoller) do
         allow(poller).to receive(:sleep)
         allow(poller).to receive(:process_batch).and_raise(error)
         poller.retrieve_poll_info
-        poller.process_batch_with_span(widgets)
+        poller.process_batch_with_span(widgets, status)
         expect(poller).to have_received(:sleep).once.with(0.5)
         expect(Deimos.config.tracer).to have_received(:set_error).with('a span', error)
+        expect(status.batches_errored).to eq(1)
+        expect(status.batches_processed).to eq(0)
+        expect(status.messages_processed).to eq(3)
       end
     end
 
@@ -243,16 +256,16 @@ each_db_config(Deimos::Utils::DbPoller) do
         info = Deimos::PollInfo.last
         config.full_table = true
         expect(MyProducer).to receive(:poll_query).at_least(:once).and_call_original
-        expect(poller).to receive(:process_batch).ordered.
-          with([old_widget, widgets[0], widgets[1]]).and_wrap_original do |m, *args|
+        expect(poller).to receive(:process_and_touch_info).ordered.
+          with([old_widget, widgets[0], widgets[1]], anything).and_wrap_original do |m, *args|
             m.call(*args)
             expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 32))
             expect(info.last_sent_id).to eq(widgets[1].id)
           end
-        expect(poller).to receive(:process_batch).ordered.
-          with([widgets[2], widgets[3], widgets[4]]).and_call_original
-        expect(poller).to receive(:process_batch).ordered.
-          with([widgets[5], widgets[6]]).and_call_original
+        expect(poller).to receive(:process_and_touch_info).ordered.
+          with([widgets[2], widgets[3], widgets[4]], anything).and_call_original
+        expect(poller).to receive(:process_and_touch_info).ordered.
+          with([widgets[5], widgets[6]], anything).and_call_original
         poller.process_updates
 
         # this is the updated_at of widgets[6]
@@ -263,12 +276,12 @@ each_db_config(Deimos::Utils::DbPoller) do
 
         travel 61.seconds
         # should reprocess the table
-        expect(poller).to receive(:process_batch).ordered.
-          with([last_widget, old_widget, widgets[0]]).and_call_original
-        expect(poller).to receive(:process_batch).ordered.
-          with([widgets[1], widgets[2], widgets[3]]).and_call_original
-        expect(poller).to receive(:process_batch).ordered.
-          with([widgets[4], widgets[5], widgets[6]]).and_call_original
+        expect(poller).to receive(:process_and_touch_info).ordered.
+          with([last_widget, old_widget, widgets[0]], anything).and_call_original
+        expect(poller).to receive(:process_and_touch_info).ordered.
+          with([widgets[1], widgets[2], widgets[3]], anything).and_call_original
+        expect(poller).to receive(:process_and_touch_info).ordered.
+          with([widgets[4], widgets[5], widgets[6]], anything).and_call_original
         poller.process_updates
 
         expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 37))
@@ -278,12 +291,12 @@ each_db_config(Deimos::Utils::DbPoller) do
       it 'should send events across multiple batches' do
         allow(Deimos.config.logger).to receive(:info)
         allow(MyProducer).to receive(:poll_query).and_call_original
-        expect(poller).to receive(:process_batch).ordered.
-          with([widgets[0], widgets[1], widgets[2]]).and_call_original
-        expect(poller).to receive(:process_batch).ordered.
-          with([widgets[3], widgets[4], widgets[5]]).and_call_original
-        expect(poller).to receive(:process_batch).ordered.
-          with([widgets[6]]).and_call_original
+        expect(poller).to receive(:process_and_touch_info).ordered.
+          with([widgets[0], widgets[1], widgets[2]], anything).and_call_original
+        expect(poller).to receive(:process_and_touch_info).ordered.
+          with([widgets[3], widgets[4], widgets[5]], anything).and_call_original
+        expect(poller).to receive(:process_and_touch_info).ordered.
+          with([widgets[6]], anything).and_call_original
         poller.process_updates
 
         expect(MyProducer).to have_received(:poll_query).
@@ -294,7 +307,7 @@ each_db_config(Deimos::Utils::DbPoller) do
 
         travel 61.seconds
         # process the last widget which came in during the delay
-        expect(poller).to receive(:process_batch).with([last_widget]).
+        expect(poller).to receive(:process_and_touch_info).with([last_widget], anything).
           and_call_original
         poller.process_updates
 
@@ -307,7 +320,7 @@ each_db_config(Deimos::Utils::DbPoller) do
 
         travel 61.seconds
         # nothing else to process
-        expect(poller).not_to receive(:process_batch)
+        expect(poller).not_to receive(:process_and_touch_info)
         poller.process_updates
         poller.process_updates
 
@@ -317,7 +330,7 @@ each_db_config(Deimos::Utils::DbPoller) do
                column_name: :updated_at,
                min_id: last_widget.id)
         expect(Deimos.config.logger).to have_received(:info).
-          with('Poll my-topic-with-id complete at 2015-05-05 00:59:58 -0400 (7 messages, 3 successful batches, 0 batches errored}')
+          with('Poll my-topic-with-id complete at 2015-05-05 00:59:58 -0400 (3 batches, 0 errored batches, 7 processed messages)')
       end
 
       describe 'errors' do
@@ -354,7 +367,7 @@ each_db_config(Deimos::Utils::DbPoller) do
           expect(info.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 30))
           expect(info.last_sent_id).to eq(widgets[6].id)
           expect(Deimos.config.logger).to have_received(:info).
-            with('Poll my-topic-with-id complete at 2015-05-05 00:59:58 -0400 (7 messages, 2 successful batches, 1 batches errored}')
+            with('Poll my-topic-with-id complete at 2015-05-05 00:59:58 -0400 (2 batches, 1 errored batches, 7 processed messages)')
         end
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.17.1
+  version: 1.18.0
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-20 00:00:00.000000000 Z
+date: 2022-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -475,6 +475,9 @@ files:
 - lib/deimos/tracing/mock.rb
 - lib/deimos/tracing/provider.rb
 - lib/deimos/utils/db_poller.rb
+- lib/deimos/utils/db_poller/base.rb
+- lib/deimos/utils/db_poller/state_based.rb
+- lib/deimos/utils/db_poller/time_based.rb
 - lib/deimos/utils/db_producer.rb
 - lib/deimos/utils/deadlock_retry.rb
 - lib/deimos/utils/inline_consumer.rb