deimos-ruby 1.23.2 → 1.23.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b0c3a1e7084ccdf781b9a826de555c98661f564c3ecaaf8daf80427d59f91c6e
-  data.tar.gz: 995449f489eb5bc76e870e2666c53a48866d54202b98d508c278f393c604d412
+  metadata.gz: 42d51126a6bf86b0aa338798c9b9a4b479154eda722394940df0bbc8a23e3947
+  data.tar.gz: 8d6c27244277078ea3b5d2a23440071aba43db3938c43cbbfe4fef27db207556
 SHA512:
-  metadata.gz: a203d1c8d44f90f512ea4a4c3fa03d524a6fde8afad3e1683a5a1c7c544e21e159bdd61cde92f0a256465b06bdb30ae6472ae415afe0d988afa9a8eb217bd1b8
-  data.tar.gz: c510ed78c61d2ed42de36ba631d69070f9d9a9406cedb15bb7c0182cb984c0a13c25887b633467c819432809bf1aa73b48e3ce81589f1b9f82d6519588e1c043
+  metadata.gz: 86d98c4e4cc84a380b69196c4d04c51a6a8aaaa05601e2d7deac536495d7c689c987555e23e1bcafb895dc78e24492c9c0b174ee273c0859c9d76795a510d154
+  data.tar.gz: 8e7875b92bb043c95721a9a12747f6f6727bc225f1802f5d6a6ba34ee4757884347aaa42b3050d9b77f668b7ebd2654aa322cd46dea3c426d140a191c19f7fb0

data/CHANGELOG.md

@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 1.23.3 - 2024-01-25
+- Feature: Add configuration to skip messages that are too large to publish via DB poller.
+
 # 1.23.2 - 2024-01-22
 - Fix: Send a `publish_error` metric for errors other than `DeliveryFailed`.
 

data/lib/deimos/config/configuration.rb

@@ -500,6 +500,9 @@ module Deimos # rubocop:disable Metrics/ModuleLength
       # The number of times to retry production when encountering a *non-Kafka* error. Set to nil
       # for infinite retries.
       setting :retries, 1
+      # If true, rather than shutting down when finding a message that is too large, log an
+      # error and skip it.
+      setting :skip_too_large_messages, false
       # Amount of time, in seconds, to wait before catching updates, to allow transactions
       # to complete but still pick up the right records. Should only be set for time-based mode.
       setting :delay_time, 2

data/lib/deimos/instrumentation.rb

@@ -43,6 +43,32 @@ module Deimos
 
   # This module listens to events published by RubyKafka.
   module KafkaListener
+    # @param exception [Exception]
+    def self.handle_exception_with_messages(exception)
+      messages = exception.failed_messages
+      messages.group_by(&:topic).each do |topic, batch|
+        producer = Deimos::Producer.descendants.find { |c| c.topic == topic }
+        next if batch.empty? || !producer
+
+        decoder = Deimos.schema_backend(schema: producer.config[:schema],
+                                        namespace: producer.config[:namespace])
+        payloads = batch.map { |m| decoder.decode(m.value) }
+
+        Deimos.config.metrics&.increment(
+          'publish_error',
+          tags: %W(topic:#{topic}),
+          by: payloads.size
+        )
+        Deimos.instrument(
+          'produce_error',
+          producer: producer,
+          topic: topic,
+          exception_object: exception,
+          payloads: payloads
+        )
+      end
+    end
+
     # Listens for any exceptions that happen during publishing and re-publishes
     # as a Deimos event.
     # @param event [ActiveSupport::Notifications::Event]
@@ -52,28 +78,7 @@ module Deimos
       return unless exception
 
       if exception.respond_to?(:failed_messages)
-        messages = exception.failed_messages
-        messages.group_by(&:topic).each do |topic, batch|
-          producer = Deimos::Producer.descendants.find { |c| c.topic == topic }
-          next if batch.empty? || !producer
-
-          decoder = Deimos.schema_backend(schema: producer.config[:schema],
-                                          namespace: producer.config[:namespace])
-          payloads = batch.map { |m| decoder.decode(m.value) }
-
-          Deimos.config.metrics&.increment(
-            'publish_error',
-            tags: %W(topic:#{topic}),
-            by: payloads.size
-          )
-          Deimos.instrument(
-            'produce_error',
-            producer: producer,
-            topic: topic,
-            exception_object: exception,
-            payloads: payloads
-          )
-      end
+        handle_exception_with_messages(exception)
       else
         Deimos.config.metrics&.increment(
           'publish_error',

data/lib/deimos/utils/db_poller/base.rb

@@ -105,6 +105,25 @@ module Deimos
           raise Deimos::MissingImplementationError
         end
 
+        # @param exception [Exception]
+        # @param batch [Array<ActiveRecord::Base>]
+        # @param status [PollStatus]
+        # @param span [Object]
+        # @return [Boolean]
+        def handle_message_too_large(exception, batch, status, span)
+          Deimos.config.logger.error("Error publishing through DB Poller: #{exception.message}")
+          if @config.skip_too_large_messages
+            Deimos.config.logger.error("Skipping messages #{batch.map(&:id).join(', ')} since they are too large")
+            Deimos.config.tracer&.set_error(span, exception)
+            status.batches_errored += 1
+            true
+          else # do the same thing as regular Kafka::Error
+            sleep(0.5)
+            false
+          end
+        end
+
+        # rubocop:disable Metrics/AbcSize
         # @param batch [Array<ActiveRecord::Base>]
         # @param status [PollStatus]
         # @return [Boolean]
@@ -118,6 +137,9 @@ module Deimos
             process_batch(batch)
             Deimos.config.tracer&.finish(span)
             status.batches_processed += 1
+          rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge,
+                 Kafka::RecordListTooLarge => e
+            retry unless handle_message_too_large(e, batch, status, span)
           rescue Kafka::Error => e # keep trying till it fixes itself
             Deimos.config.logger.error("Error publishing through DB Poller: #{e.message}")
             sleep(0.5)
@@ -139,6 +161,7 @@ module Deimos
           end
           true
         end
+        # rubocop:enable Metrics/AbcSize
 
         # Publish batch using the configured producers
         # @param batch [Array<ActiveRecord::Base>]

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.23.2'
+  VERSION = '1.23.3'
 end

data/spec/utils/db_poller_spec.rb

@@ -191,6 +191,48 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
         expect(Deimos.config.tracer).to have_received(:finish).with('a span')
       end
 
+      context 'with skip_too_large_messages on' do
+        before(:each) { config.skip_too_large_messages = true }
+
+        it 'should skip and move on' do
+          error = Kafka::MessageSizeTooLarge.new('OH NOES')
+          allow(poller).to receive(:sleep)
+          allow(poller).to receive(:process_batch) do
+            raise error
+          end
+          poller.retrieve_poll_info
+          poller.process_batch_with_span(widgets, status)
+          expect(poller).not_to have_received(:sleep)
+          expect(Deimos.config.tracer).to have_received(:set_error).with('a span', error)
+          expect(status.batches_errored).to eq(1)
+          expect(status.batches_processed).to eq(0)
+          expect(status.messages_processed).to eq(3)
+
+        end
+      end
+
+      context 'with skip_too_large_messages off' do
+        it 'should retry forever' do
+          called_once = false
+          allow(poller).to receive(:sleep)
+          allow(poller).to receive(:process_batch) do
+            unless called_once
+              called_once = true
+              raise Kafka::MessageSizeTooLarge, 'OH NOES'
+            end
+          end
+          poller.retrieve_poll_info
+          poller.process_batch_with_span(widgets, status)
+          expect(poller).to have_received(:sleep).once.with(0.5)
+          expect(Deimos.config.tracer).to have_received(:finish).with('a span')
+          expect(status.batches_errored).to eq(0)
+          expect(status.batches_processed).to eq(1)
+          expect(status.messages_processed).to eq(3)
+
+        end
+
+      end
+
       it 'should retry on Kafka error' do
         called_once = false
         allow(poller).to receive(:sleep)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.23.2
+  version: 1.23.3
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-22 00:00:00.000000000 Z
+date: 2024-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf