deimos-ruby 1.16.5 → 1.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 021fca59477fbfa890fa77ae867cd3a006b65e2c0235d1bb47ccdd1780e51d9c
-  data.tar.gz: 8330ba3d8f90567c7abbfd91298dd04d80f9507f702a0d4ac45300f68e5d224a
+  metadata.gz: 8b4270b2fc345a60377ead079f053ea92281bd606df2d24da782c583bf25e3b9
+  data.tar.gz: 6c824200ea66395439dfbc00a0478a30f1410d10032915fb063f94e0d7efbeb0
 SHA512:
-  metadata.gz: ab861eb37e8590664f47e0e11082ed5aac344a730c009c5261edcbdcbfcd00a39bc8e1f44bf2be5c05b1eb587ba15ad32e77edfd80ca0789da05049c91dd20e9
-  data.tar.gz: 851b9b844d11be5d90d3b4e652a648e7ce1e8e90e1ddd96f25a34d690cb89e2503302a80ad9cb891cbaa30949a0394e68337b34de86cf848a6815730d1548ca0
+  metadata.gz: db33a95a08af5e4d5a6e42ea7d8d7193432647afe1c18186d8d4dbf58dca3fa7994cf06c67abda80c841a918746fe4059e72386c6ab2d98080bd1e9aa727e25e
+  data.tar.gz: 8d091dfb4478e66dc898949bc87304dc82a48fc29e1c89e2b1235ea199d338a59c62cc8b29667a03eb45e45db846187feaa28f8f490db3beb17c8e70ddfb7cf0

data/.ruby-version

@@ -1,2 +1,2 @@
-3.1.0
+2.7.0
 

data/CHANGELOG.md

@@ -7,9 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
-# 1.16.5 - 2022-09-30
-
+# 1.17.0 - 2022-10-19
 - Fix the linting step in the CI
+- CHANGE: Add retries to DB poller and bypass "bad batches".
+- Add tracing spans to DB poller production.
 
 # 1.16.4 - 2022-09-09
 

data/Gemfile

@@ -9,4 +9,3 @@ if !ENV['CI'] || ENV['CI'] == ''
   # TODO: once all PRs are merged, add this to gemspec as a development dependency
   gem 'sord', git: 'git@github.com:dorner/sord.git', ref: 'local-develop'
 end
-

data/README.md

@@ -800,6 +800,9 @@ class MyProducer < Deimos::ActiveRecordProducer
 end
 ```
 
+Note that the poller will retry infinitely if it encounters a Kafka-related error such
+as a communication failure. For all other errors, it will retry once by default.
+
 ## Running consumers
 
 Deimos includes a rake task. Once it's in your gemfile, just run
@@ -1207,7 +1210,7 @@ You can/should re-generate RBS types when methods or classes change by running t
 ### Linting
 
 Deimos uses Rubocop to lint the code. Please run Rubocop on your code 
-before submitting a PR. The GitHub CI will also run rubocop on your pull request.
+before submitting a PR. The GitHub CI will also run rubocop on your pull request. 
 
 ---
 <p align="center">

data/Steepfile

@@ -1,6 +1,8 @@
+# frozen_string_literal: true
+
 target :app do
-  check "lib"
-  signature "sig"
+  check 'lib'
+  signature 'sig'
 
-  library "set", "pathname"
+  library 'set', 'pathname'
 end

data/docs/CONFIGURATION.md

@@ -15,7 +15,7 @@ Config name|Default|Description
 logger|`Logger.new(STDOUT)`|The logger that Deimos will use.
 phobos_logger|`Deimos.config.logger`|The logger passed to Phobos.
 metrics|`Deimos::Metrics::Mock.new`|The metrics backend use for reporting.
-tracer|`Deimos::Tracer::Mock.new`|The tracer backend used for debugging.
+tracer|`Deimos::Tracing::Mock.new`|The tracer backend used for debugging.
 
 ## Defining Producers
 
@@ -119,6 +119,7 @@ timestamp_column|`:updated_at`|Name of the column to query. Remember to add an i
 delay_time|2|Amount of time in seconds to wait before picking up records, to allow for transactions to finish.
 full_table|false|If set to true, do a full table dump to Kafka each run. Good for very small tables.
 start_from_beginning|true|If false, start from the current time instead of the beginning of time if this is the first time running the poller.
+retries|1|The number of times to retry for a *non-Kafka* error.
 
 ## Kafka Configuration
 

data/lib/deimos/config/configuration.rb

@@ -461,6 +461,8 @@ module Deimos
       # If false, start from the current time instead of the beginning of time
       # if this is the first time running the poller.
       setting :start_from_beginning, true
+      # The number of times to retry production when encountering a *non-Kafka* error.
+      setting :retries, 1
     end
 
     deprecate 'kafka_logger', 'kafka.logger'

data/lib/deimos/metrics/mock.rb

@@ -7,7 +7,7 @@ module Deimos
     # A mock Metrics wrapper which just logs the metrics
     class Mock < Provider
       # @param logger [Logger,nil]
-      def initialize(logger=nil)
+      def initialize(logger=nil) # rubocop:disable Lint/MissingSuper
         @logger = logger || Logger.new(STDOUT)
         @logger.info('MockMetricsProvider initialized')
       end

data/lib/deimos/schema_backends/mock.rb

@@ -24,7 +24,7 @@ module Deimos
       end
 
       # @override
-      def coerce_field(field, value)
+      def coerce_field(_field, value)
         value
       end
 

data/lib/deimos/utils/db_poller.rb

@@ -14,6 +14,9 @@ module Deimos
       # @return [Integer]
       attr_reader :id
 
+      # @return [Hash]
+      attr_reader :config
+
       # Begin the DB Poller process.
       # @return [void]
       def self.start!
@@ -110,6 +113,7 @@ module Deimos
         Deimos.config.logger.info("Polling #{@producer.topic} from #{time_from} to #{time_to}")
         message_count = 0
         batch_count = 0
+        error_count = 0
 
         # poll_query gets all the relevant data from the database, as defined
         # by the producer itself.
@@ -118,12 +122,15 @@ module Deimos
           batch = fetch_results(time_from, time_to).to_a
           break if batch.empty?
 
-          batch_count += 1
-          process_batch(batch)
+          if process_batch_with_span(batch)
+            batch_count += 1
+          else
+            error_count += 1
+          end
           message_count += batch.size
           time_from = last_updated(batch.last)
         end
-        Deimos.config.logger.info("Poll #{@producer.topic} complete at #{time_to} (#{message_count} messages, #{batch_count} batches}")
+        Deimos.config.logger.info("Poll #{@producer.topic} complete at #{time_to} (#{message_count} messages, #{batch_count} successful batches, #{error_count} batches errored}")
       end
 
       # @param time_from [ActiveSupport::TimeWithZone]
@@ -143,15 +150,52 @@ module Deimos
 
       # @param batch [Array<ActiveRecord::Base>]
       # @return [void]
-      def process_batch(batch)
+      def process_batch_with_span(batch)
+        retries = 0
+        begin
+          span = Deimos.config.tracer&.start(
+            'deimos-db-poller',
+            resource: @producer.class.name.gsub('::', '-')
+          )
+          process_batch(batch)
+          Deimos.config.tracer&.finish(span)
+        rescue Kafka::Error => e # keep trying till it fixes itself
+          Deimos.config.logger.error("Error publishing through DB Poller: #{e.message}")
+          sleep(0.5)
+          retry
+        rescue StandardError => e
+          Deimos.config.logger.error("Error publishing through DB poller: #{e.message}}")
+          if retries < @config.retries
+            retries += 1
+            sleep(0.5)
+            retry
+          else
+            Deimos.config.logger.error('Retries exceeded, moving on to next batch')
+            Deimos.config.tracer&.set_error(span, e)
+            self.touch_info(batch)
+            return false
+          end
+        end
+        true
+      end
+
+      # @param batch [Array<ActiveRecord::Base>]
+      # @return [void]
+      def touch_info(batch)
         record = batch.last
         id_method = record.class.primary_key
         last_id = record.public_send(id_method)
         last_updated_at = last_updated(record)
-        @producer.send_events(batch)
         @info.attributes = { last_sent: last_updated_at, last_sent_id: last_id }
         @info.save!
       end
+
+      # @param batch [Array<ActiveRecord::Base>]
+      # @return [void]
+      def process_batch(batch)
+        @producer.send_events(batch)
+        self.touch_info(batch)
+      end
     end
   end
 end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.16.5'
+  VERSION = '1.17.0'
 end

data/spec/utils/db_poller_spec.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+# rubocop:disable Layout/LineLength
+
 # @param seconds [Integer]
 # @return [Time]
 def time_value(secs: 0, mins: 0)
@@ -156,16 +158,56 @@ each_db_config(Deimos::Utils::DbPoller) do
       expect(poller.should_run?).to eq(false)
     end
 
-    specify '#process_batch' do
-      travel_to time_value
-      widgets = (1..3).map { Widget.create!(test_id: 'some_id', some_int: 4) }
-      widgets.last.update_attribute(:updated_at, time_value(mins: -30))
-      expect(MyProducer).to receive(:send_events).with(widgets)
-      poller.retrieve_poll_info
-      poller.process_batch(widgets)
-      info = Deimos::PollInfo.last
-      expect(info.last_sent.in_time_zone).to eq(time_value(mins: -30))
-      expect(info.last_sent_id).to eq(widgets.last.id)
+    describe '#process_batch' do
+      let(:widgets) { (1..3).map { Widget.create!(test_id: 'some_id', some_int: 4) } }
+
+      before(:each) do
+        allow(Deimos.config.tracer).to receive(:start).and_return('a span')
+        allow(Deimos.config.tracer).to receive(:set_error)
+        allow(Deimos.config.tracer).to receive(:finish)
+      end
+
+      it 'should process the batch' do
+        travel_to time_value
+        widgets.last.update_attribute(:updated_at, time_value(mins: -30))
+        expect(MyProducer).to receive(:send_events).with(widgets)
+        poller.retrieve_poll_info
+        poller.process_batch(widgets)
+        info = Deimos::PollInfo.last
+        expect(info.last_sent.in_time_zone).to eq(time_value(mins: -30))
+        expect(info.last_sent_id).to eq(widgets.last.id)
+      end
+
+      it 'should create a span' do
+        poller.retrieve_poll_info
+        poller.process_batch_with_span(widgets)
+        expect(Deimos.config.tracer).to have_received(:finish).with('a span')
+      end
+
+      it 'should retry on Kafka error' do
+        called_once = false
+        allow(poller).to receive(:sleep)
+        allow(poller).to receive(:process_batch) do
+          unless called_once
+            called_once = true
+            raise Kafka::Error, 'OH NOES'
+          end
+        end
+        poller.retrieve_poll_info
+        poller.process_batch_with_span(widgets)
+        expect(poller).to have_received(:sleep).once.with(0.5)
+        expect(Deimos.config.tracer).to have_received(:finish).with('a span')
+      end
+
+      it 'should retry only once on other errors' do
+        error = RuntimeError.new('OH NOES')
+        allow(poller).to receive(:sleep)
+        allow(poller).to receive(:process_batch).and_raise(error)
+        poller.retrieve_poll_info
+        poller.process_batch_with_span(widgets)
+        expect(poller).to have_received(:sleep).once.with(0.5)
+        expect(Deimos.config.tracer).to have_received(:set_error).with('a span', error)
+      end
     end
 
     describe '#process_updates' do
@@ -234,6 +276,7 @@ each_db_config(Deimos::Utils::DbPoller) do
       end
 
       it 'should send events across multiple batches' do
+        allow(Deimos.config.logger).to receive(:info)
         allow(MyProducer).to receive(:poll_query).and_call_original
         expect(poller).to receive(:process_batch).ordered.
           with([widgets[0], widgets[1], widgets[2]]).and_call_original
@@ -273,48 +316,48 @@ each_db_config(Deimos::Utils::DbPoller) do
                time_to: time_value(secs: 120), # plus 122 seconds minus 2 seconds
                column_name: :updated_at,
                min_id: last_widget.id)
+        expect(Deimos.config.logger).to have_received(:info).
+          with('Poll my-topic-with-id complete at 2015-05-05 00:59:58 -0400 (7 messages, 3 successful batches, 0 batches errored}')
       end
 
-      it 'should recover correctly with errors and save the right ID' do
-        widgets.each do |w|
-          w.update_attribute(:updated_at, time_value(mins: -61, secs: 30))
+      describe 'errors' do
+        before(:each) do
+          poller.config.retries = 0
+          allow(Deimos.config.logger).to receive(:info)
         end
-        allow(MyProducer).to receive(:poll_query).and_call_original
-        expect(poller).to receive(:process_batch).ordered.
-          with([widgets[0], widgets[1], widgets[2]]).and_call_original
-        expect(poller).to receive(:process_batch).ordered.
-          with([widgets[3], widgets[4], widgets[5]]).and_raise('OH NOES')
-
-        expect { poller.process_updates }.to raise_exception('OH NOES')
-
-        expect(MyProducer).to have_received(:poll_query).
-          with(time_from: time_value(mins: -61),
-               time_to: time_value(secs: -2),
-               column_name: :updated_at,
-               min_id: 0)
-
-        info = Deimos::PollInfo.last
-        expect(info.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 30))
-        expect(info.last_sent_id).to eq(widgets[2].id)
 
-        travel 61.seconds
-        # process the last widget which came in during the delay
-        expect(poller).to receive(:process_batch).ordered.
-          with([widgets[3], widgets[4], widgets[5]]).and_call_original
-        expect(poller).to receive(:process_batch).with([widgets[6], last_widget]).
-          and_call_original
-        poller.process_updates
-        expect(MyProducer).to have_received(:poll_query).
-          with(time_from: time_value(mins: -61, secs: 30),
-               time_to: time_value(secs: 59),
-               column_name: :updated_at,
-               min_id: widgets[2].id)
+        after(:each) do
+          poller.config.retries = 1
+        end
 
-        expect(info.reload.last_sent.in_time_zone).to eq(time_value(secs: -1))
-        expect(info.last_sent_id).to eq(last_widget.id)
+        it 'should recover correctly with errors and save the right ID' do
+          widgets.each do |w|
+            w.update_attribute(:updated_at, time_value(mins: -61, secs: 30))
+          end
+          allow(MyProducer).to receive(:poll_query).and_call_original
+          expect(poller).to receive(:process_batch).ordered.
+            with([widgets[0], widgets[1], widgets[2]]).and_call_original
+          expect(poller).to receive(:process_batch).ordered.
+            with([widgets[3], widgets[4], widgets[5]]).and_raise('OH NOES')
+          expect(poller).to receive(:process_batch).ordered.
+            with([widgets[6]]).and_call_original
+
+          poller.process_updates
+
+          expect(MyProducer).to have_received(:poll_query).
+            with(time_from: time_value(mins: -61),
+                 time_to: time_value(secs: -2),
+                 column_name: :updated_at,
+                 min_id: 0)
+
+          info = Deimos::PollInfo.last
+          expect(info.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 30))
+          expect(info.last_sent_id).to eq(widgets[6].id)
+          expect(Deimos.config.logger).to have_received(:info).
+            with('Poll my-topic-with-id complete at 2015-05-05 00:59:58 -0400 (7 messages, 2 successful batches, 1 batches errored}')
+        end
       end
-
     end
-
   end
 end
+# rubocop:enable Layout/LineLength

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.16.5
+  version: 1.17.0
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-09-30 00:00:00.000000000 Z
+date: 2022-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -607,7 +607,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.21
+rubygems_version: 3.3.20
 signing_key:
 specification_version: 4
 summary: Kafka libraries for Ruby.