deimos-ruby 1.22.1 → 1.22.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b1819694e6ec5bc9cad69f0a3d7edaad53d4daae80ee0af95312044d503e40d1
-  data.tar.gz: 1577ab483b8fc8a16dc614c12156e2161a3ab2d372b45ee7825f30837fdc36e4
+  metadata.gz: 014d4e4a2682f57a22a065431a0de6ae0ca76fd64cb8e754f391f8f40edd8e3f
+  data.tar.gz: b7bec2abc1e5394ec435dc0a0ef86dbb9b26e379cca0db63f448a89c5b4717ad
 SHA512:
-  metadata.gz: 1fc162b425d00a308679662d4a48753dbe1fac0ab252d329ad10622f894301de2e35dcc2ca3c0d9366b516ebbce7e8d008fd37ee1c2526d9f5a5be3188d6380f
-  data.tar.gz: b09c3c0a0c7980b9bd2431754001c8b3f18d6a21981dbb7dd5b1336f60a3411e26c5c938b3d55e48039c2306ffac54bc2f95c4beac4700b994fdcf4168900ddc
+  metadata.gz: 6ac915ad65b58c77d08bf61c70f978f1ffd37ef58ae39d0e03156b778d5a36ba00de227b855b5ccbd0251d6eda1f4ea7e288b7328961d6fd790cc6a4423b9210
+  data.tar.gz: ca1d551a581a1a968746858a3d99aacc9ace8e7f342cfe4526cc9d27aa158360c2f858b1bf9cc1018c1848b26689b239a0b74f0ccbc2d00b085776ed34a52525

data/CHANGELOG.md

@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 1.22.3 - 2023-06-13
+
+- Fix: Don't update last_sent to current time on every poll.
+- Feature: Allow for infinite retries in DB poller.
+
+# 1.22.2 - 2023-05-10
+- Feature: Add `DEIMOS_TASK_NAME` env variable when running a task (consumer, DB poller, DB producer).
+
 # 1.22.1 - 2023-05-01
 
 - Fix: Bug introduced with 1.22 when associations are not used

data/lib/deimos/config/configuration.rb

@@ -474,7 +474,8 @@ module Deimos # rubocop:disable Metrics/ModuleLength
       # time, it will run again immediately and the timeout
       # will be pushed to the next e.g. 1 minute.
       setting :run_every, 60
-      # The number of times to retry production when encountering a *non-Kafka* error.
+      # The number of times to retry production when encountering a *non-Kafka* error. Set to nil
+      # for infinite retries.
       setting :retries, 1
       # Amount of time, in seconds, to wait before catching updates, to allow transactions
       # to complete but still pick up the right records. Should only be set for time-based mode.

data/lib/deimos/utils/db_poller/base.rb

@@ -124,7 +124,7 @@ module Deimos
             retry
           rescue StandardError => e
             Deimos.config.logger.error("Error publishing through DB poller: #{e.message}}")
-            if retries < @config.retries
+            if @config.retries.nil? || retries < @config.retries
               retries += 1
               sleep(0.5)
               retry

data/lib/deimos/utils/db_poller/state_based.rb

@@ -12,20 +12,23 @@ module Deimos
         def process_updates
           Deimos.config.logger.info("Polling #{log_identifier}")
           status = PollStatus.new(0, 0, 0)
+          first_batch = true
 
           # poll_query gets all the relevant data from the database, as defined
           # by the producer itself.
           loop do
             Deimos.config.logger.debug("Polling #{log_identifier}, batch #{status.current_batch}")
             batch = fetch_results.to_a
-            if batch.empty?
-              @info.touch(:last_sent)
-              break
-            end
+            break if batch.empty?
 
+            first_batch = false
             success = process_batch_with_span(batch, status)
             finalize_batch(batch, success)
           end
+
+          # If there were no results at all, we update last_sent so that we still get a wait
+          # before the next poll.
+          @info.touch(:last_sent) if first_batch
           Deimos.config.logger.info("Poll #{log_identifier} complete (#{status.report}")
         end
 

data/lib/deimos/utils/db_poller/time_based.rb

@@ -30,20 +30,23 @@ module Deimos
           time_to = Time.zone.now - @config.delay_time
           Deimos.config.logger.info("Polling #{log_identifier} from #{time_from} to #{time_to}")
           status = PollStatus.new(0, 0, 0)
+          first_batch = true
 
           # poll_query gets all the relevant data from the database, as defined
           # by the producer itself.
           loop do
             Deimos.config.logger.debug("Polling #{log_identifier}, batch #{status.current_batch}")
             batch = fetch_results(time_from, time_to).to_a
-            if batch.empty?
-              @info.touch(:last_sent)
-              break
-            end
+            break if batch.empty?
 
+            first_batch = false
             process_and_touch_info(batch, status)
             time_from = last_updated(batch.last)
           end
+
+          # If there were no results at all, we update last_sent so that we still get a wait
+          # before the next poll.
+          @info.touch(:last_sent) if first_batch
           Deimos.config.logger.info("Poll #{log_identifier} complete at #{time_to} (#{status.report})")
         end
 

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.22.1'
+  VERSION = '1.22.3'
 end

data/lib/tasks/deimos.rake

@@ -12,6 +12,7 @@ namespace :deimos do
       config.producers.backend = :kafka if config.producers.backend == :kafka_async
     end
     ENV['DEIMOS_RAKE_TASK'] = 'true'
+    ENV['DEIMOS_TASK_NAME'] = 'consumer'
     STDOUT.sync = true
     Rails.logger.info('Running deimos:start rake task.')
     Phobos::CLI::Commands.start(%w(start --skip_config))
@@ -20,6 +21,7 @@ namespace :deimos do
   desc 'Starts the Deimos database producer'
   task db_producer: :environment do
     ENV['DEIMOS_RAKE_TASK'] = 'true'
+    ENV['DEIMOS_TASK_NAME'] = 'db_producer'
     STDOUT.sync = true
     Rails.logger.info('Running deimos:db_producer rake task.')
     thread_count = ENV['THREAD_COUNT'].to_i.zero? ? 1 : ENV['THREAD_COUNT'].to_i
@@ -28,6 +30,7 @@ namespace :deimos do
 
   task db_poller: :environment do
     ENV['DEIMOS_RAKE_TASK'] = 'true'
+    ENV['DEIMOS_TASK_NAME'] = 'db_poller'
     STDOUT.sync = true
     Rails.logger.info('Running deimos:db_poller rake task.')
     Deimos::Utils::DbPoller.start!

data/spec/utils/db_poller_spec.rb

@@ -324,13 +324,18 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
         poller.process_updates
         poller.process_updates
 
-        expect(MyProducer).to have_received(:poll_query).twice.
+        expect(MyProducer).to have_received(:poll_query).
           with(time_from: time_value(secs: -1),
                time_to: time_value(secs: 120), # plus 122 seconds minus 2 seconds
                column_name: :updated_at,
                min_id: last_widget.id)
+        expect(MyProducer).to have_received(:poll_query).
+          with(time_from: time_value(secs: 122),
+               time_to: time_value(secs: 120), # yes this is weird but it's because of travel_to
+               column_name: :updated_at,
+               min_id: last_widget.id)
         expect(Deimos.config.logger).to have_received(:info).
-          with('Poll my-topic-with-id complete at 2015-05-05 00:59:58 -0400 (3 batches, 0 errored batches, 7 processed messages)')
+          with('Poll MyProducer: ["my-topic-with-id"] complete at 2015-05-05 00:59:58 -0400 (3 batches, 0 errored batches, 7 processed messages)')
       end
 
       it 'should update PollInfo timestamp after processing' do
@@ -382,7 +387,7 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
           expect(info.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 30))
           expect(info.last_sent_id).to eq(widgets[6].id)
           expect(Deimos.config.logger).to have_received(:info).
-            with('Poll my-topic-with-id complete at 2015-05-05 00:59:58 -0400 (2 batches, 1 errored batches, 7 processed messages)')
+            with('Poll MyProducer: ["my-topic-with-id"] complete at 2015-05-05 00:59:58 -0400 (2 batches, 1 errored batches, 7 processed messages)')
         end
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.22.1
+  version: 1.22.3
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-04 00:00:00.000000000 Z
+date: 2023-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf