deimos-ruby 1.22.2 → 1.22.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5aa472d5bdd8c86fe0502451f7554598d189fa0d5004603d8970561feed38d9f
-  data.tar.gz: 669d564cb23fccc41a9b5a3dd37c49a17787adf7fcecd486746f9d91031d8367
+  metadata.gz: 014d4e4a2682f57a22a065431a0de6ae0ca76fd64cb8e754f391f8f40edd8e3f
+  data.tar.gz: b7bec2abc1e5394ec435dc0a0ef86dbb9b26e379cca0db63f448a89c5b4717ad
 SHA512:
-  metadata.gz: 3f49565845478349949d7dd5909b7d276590f91201d0848123c42a0b433114ce2a31cda8a0d43058f6f3f76114d3dfc1d32b8ee53a2a291588865ed5aaa8a8ec
-  data.tar.gz: 295802ba93bbf1e8a9c6ca95ac1944b85d3676881c3cb37854f883d42630f68f11a8971527611f1bf1c4681e5b8fab4aa202c1bcd2b1863296236b9a175bd253
+  metadata.gz: 6ac915ad65b58c77d08bf61c70f978f1ffd37ef58ae39d0e03156b778d5a36ba00de227b855b5ccbd0251d6eda1f4ea7e288b7328961d6fd790cc6a4423b9210
+  data.tar.gz: ca1d551a581a1a968746858a3d99aacc9ace8e7f342cfe4526cc9d27aa158360c2f858b1bf9cc1018c1848b26689b239a0b74f0ccbc2d00b085776ed34a52525

data/CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 1.22.3 - 2023-06-13
+
+- Fix: Don't update last_sent to current time on every poll.
+- Feature: Allow for infinite retries in DB poller.
+
 # 1.22.2 - 2023-05-10
 - Feature: Add `DEIMOS_TASK_NAME` env variable when running a task (consumer, DB poller, DB producer).
 

data/lib/deimos/config/configuration.rb

@@ -474,7 +474,8 @@ module Deimos # rubocop:disable Metrics/ModuleLength
       # time, it will run again immediately and the timeout
       # will be pushed to the next e.g. 1 minute.
       setting :run_every, 60
-      # The number of times to retry production when encountering a *non-Kafka* error.
+      # The number of times to retry production when encountering a *non-Kafka* error. Set to nil
+      # for infinite retries.
       setting :retries, 1
       # Amount of time, in seconds, to wait before catching updates, to allow transactions
       # to complete but still pick up the right records. Should only be set for time-based mode.

data/lib/deimos/utils/db_poller/base.rb

@@ -124,7 +124,7 @@ module Deimos
             retry
           rescue StandardError => e
             Deimos.config.logger.error("Error publishing through DB poller: #{e.message}}")
-            if retries < @config.retries
+            if @config.retries.nil? || retries < @config.retries
               retries += 1
               sleep(0.5)
               retry

data/lib/deimos/utils/db_poller/state_based.rb

@@ -12,20 +12,23 @@ module Deimos
         def process_updates
           Deimos.config.logger.info("Polling #{log_identifier}")
           status = PollStatus.new(0, 0, 0)
+          first_batch = true
 
           # poll_query gets all the relevant data from the database, as defined
           # by the producer itself.
           loop do
             Deimos.config.logger.debug("Polling #{log_identifier}, batch #{status.current_batch}")
             batch = fetch_results.to_a
-            if batch.empty?
-              @info.touch(:last_sent)
-              break
-            end
+            break if batch.empty?
 
+            first_batch = false
             success = process_batch_with_span(batch, status)
             finalize_batch(batch, success)
           end
+
+          # If there were no results at all, we update last_sent so that we still get a wait
+          # before the next poll.
+          @info.touch(:last_sent) if first_batch
           Deimos.config.logger.info("Poll #{log_identifier} complete (#{status.report}")
         end
 

data/lib/deimos/utils/db_poller/time_based.rb

@@ -30,20 +30,23 @@ module Deimos
           time_to = Time.zone.now - @config.delay_time
           Deimos.config.logger.info("Polling #{log_identifier} from #{time_from} to #{time_to}")
           status = PollStatus.new(0, 0, 0)
+          first_batch = true
 
           # poll_query gets all the relevant data from the database, as defined
           # by the producer itself.
           loop do
             Deimos.config.logger.debug("Polling #{log_identifier}, batch #{status.current_batch}")
             batch = fetch_results(time_from, time_to).to_a
-            if batch.empty?
-              @info.touch(:last_sent)
-              break
-            end
+            break if batch.empty?
 
+            first_batch = false
             process_and_touch_info(batch, status)
             time_from = last_updated(batch.last)
           end
+
+          # If there were no results at all, we update last_sent so that we still get a wait
+          # before the next poll.
+          @info.touch(:last_sent) if first_batch
           Deimos.config.logger.info("Poll #{log_identifier} complete at #{time_to} (#{status.report})")
         end
 

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.22.2'
+  VERSION = '1.22.3'
 end

data/spec/utils/db_poller_spec.rb

@@ -324,13 +324,18 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
         poller.process_updates
         poller.process_updates
 
-        expect(MyProducer).to have_received(:poll_query).twice.
+        expect(MyProducer).to have_received(:poll_query).
           with(time_from: time_value(secs: -1),
                time_to: time_value(secs: 120), # plus 122 seconds minus 2 seconds
                column_name: :updated_at,
                min_id: last_widget.id)
+        expect(MyProducer).to have_received(:poll_query).
+          with(time_from: time_value(secs: 122),
+               time_to: time_value(secs: 120), # yes this is weird but it's because of travel_to
+               column_name: :updated_at,
+               min_id: last_widget.id)
         expect(Deimos.config.logger).to have_received(:info).
-          with('Poll my-topic-with-id complete at 2015-05-05 00:59:58 -0400 (3 batches, 0 errored batches, 7 processed messages)')
+          with('Poll MyProducer: ["my-topic-with-id"] complete at 2015-05-05 00:59:58 -0400 (3 batches, 0 errored batches, 7 processed messages)')
       end
 
       it 'should update PollInfo timestamp after processing' do
@@ -382,7 +387,7 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
           expect(info.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 30))
           expect(info.last_sent_id).to eq(widgets[6].id)
           expect(Deimos.config.logger).to have_received(:info).
-            with('Poll my-topic-with-id complete at 2015-05-05 00:59:58 -0400 (2 batches, 1 errored batches, 7 processed messages)')
+            with('Poll MyProducer: ["my-topic-with-id"] complete at 2015-05-05 00:59:58 -0400 (2 batches, 1 errored batches, 7 processed messages)')
         end
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.22.2
+  version: 1.22.3
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-10 00:00:00.000000000 Z
+date: 2023-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf