karafka 2.3.1 → 2.3.2

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb

@@ -51,14 +51,12 @@ module Karafka
                   else
                     resume
                   end
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
-                  pause(coordinator.seek_offset, nil, false)
+                  apply_dlq_flow do
+                    skippable_message, = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                    mark_as_consumed(skippable_message)
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb

@@ -57,14 +57,12 @@ module Karafka
                   else
                     resume
                   end
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
-                  pause(coordinator.seek_offset, nil, false)
+                  apply_dlq_flow do
+                    skippable_message, = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                    mark_as_consumed(skippable_message)
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb

@@ -44,8 +44,6 @@ module Karafka
                   return if coordinator.manual_pause?
 
                   handle_post_filtering
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 # If we've reached number of retries that we could, we need to skip the first
                 # message that was not marked as consumed, pause and continue, while also moving
                 # this message to the dead topic.
@@ -53,14 +51,14 @@ module Karafka
                 # For a Mom setup, this means, that user has to manage the checkpointing by
                 # himself. If no checkpointing is ever done, we end up with an endless loop.
                 else
-                  coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  # We can commit the offset here because we know that we skip it "forever" and
-                  # since AJ consumer commits the offset after each job, we also know that the
-                  # previous job was successful
-                  mark_as_consumed(skippable_message)
-                  pause(coordinator.seek_offset, nil, false)
+                  apply_dlq_flow do
+                    skippable_message, = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                    # We can commit the offset here because we know that we skip it "forever" and
+                    # since AJ consumer commits the offset after each job, we also know that the
+                    # previous job was successful
+                    mark_as_consumed(skippable_message)
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb

@@ -48,23 +48,15 @@ module Karafka
                   mark_as_consumed(last_group_message)
 
                   handle_post_filtering
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
-                # If we've reached number of retries that we could, we need to skip the first
-                # message that was not marked as consumed, pause and continue, while also moving
-                # this message to the dead topic.
-                #
-                # For a Mom setup, this means, that user has to manage the checkpointing by
-                # himself. If no checkpointing is ever done, we end up with an endless loop.
                 else
-                  coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  # We can commit the offset here because we know that we skip it "forever" and
-                  # since AJ consumer commits the offset after each job, we also know that the
-                  # previous job was successful
-                  mark_as_consumed(skippable_message)
-                  pause(coordinator.seek_offset, nil, false)
+                  apply_dlq_flow do
+                    skippable_message, = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                    # We can commit the offset here because we know that we skip it "forever" and
+                    # since AJ consumer commits the offset after each job, we also know that the
+                    # previous job was successful
+                    mark_as_consumed(skippable_message)
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb

@@ -47,14 +47,12 @@ module Karafka
                   seek(coordinator.seek_offset, false) unless revoked?
 
                   resume
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
-                  pause(coordinator.seek_offset, nil, false)
+                  apply_dlq_flow do
+                    skippable_message, = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                    mark_as_consumed(skippable_message)
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb

@@ -51,14 +51,12 @@ module Karafka
                   seek(coordinator.seek_offset, false) unless revoked?
 
                   resume
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
-                  pause(coordinator.seek_offset, nil, false)
+                  apply_dlq_flow do
+                    skippable_message, = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                    mark_as_consumed(skippable_message)
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb

@@ -42,17 +42,15 @@ module Karafka
                 if coordinator.success?
                   # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
                   coordinator.pause_tracker.reset
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  # We can commit the offset here because we know that we skip it "forever" and
-                  # since AJ consumer commits the offset after each job, we also know that the
-                  # previous job was successful
-                  mark_as_consumed(skippable_message)
-                  pause(coordinator.seek_offset, nil, false)
+                  apply_dlq_flow do
+                    skippable_message, = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                    # We can commit the offset here because we know that we skip it "forever" and
+                    # since AJ consumer commits the offset after each job, we also know that the
+                    # previous job was successful
+                    mark_as_consumed(skippable_message)
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb

@@ -48,16 +48,14 @@ module Karafka
                   return if revoked?
 
                   mark_as_consumed(last_group_message)
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  # Here we are in a collapsed state, hence we can apply the same logic as
-                  # Aj::DlqMom
-                  coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
-                  pause(coordinator.seek_offset, nil, false)
+                  apply_dlq_flow do
+                    # Here we are in a collapsed state, hence we can apply the same logic as
+                    # Aj::DlqMom
+                    skippable_message, = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                    mark_as_consumed(skippable_message)
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -76,16 +76,10 @@ module Karafka
                   return if coordinator.manual_pause?
 
                   mark_as_consumed(last_group_message)
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
-                # If we've reached number of retries that we could, we need to skip the first
-                # message that was not marked as consumed, pause and continue, while also moving
-                # this message to the dead topic
                 else
-                  # We reset the pause to indicate we will now consider it as "ok".
-                  coordinator.pause_tracker.reset
-                  dispatch_if_needed_and_mark_as_consumed
-                  pause(coordinator.seek_offset, nil, false)
+                  apply_dlq_flow do
+                    dispatch_if_needed_and_mark_as_consumed
+                  end
                 end
               end
             end
@@ -183,7 +177,10 @@ module Karafka
             #   topic is set to false, we will skip the dispatch, effectively ignoring the broken
             #   message without taking any action.
             def dispatch_to_dlq?
-              topic.dead_letter_queue.topic
+              return false unless topic.dead_letter_queue.topic
+              return false unless @_dispatch_to_dlq
+
+              true
             end
 
             # @return [Boolean] should we use a transaction to move the data to the DLQ.
@@ -192,6 +189,35 @@ module Karafka
             def dispatch_in_a_transaction?
               producer.transactional? && topic.dead_letter_queue.transactional?
             end
+
+            # Runs the DLQ strategy and based on it it performs certain operations
+            #
+            # In case of `:skip` and `:dispatch` will run the exact flow provided in a block
+            # In case of `:retry` always `#retry_after_pause` is applied
+            def apply_dlq_flow
+              flow = topic.dead_letter_queue.strategy.call(errors_tracker, attempt)
+
+              case flow
+              when :retry
+                retry_after_pause
+
+                return
+              when :skip
+                @_dispatch_to_dlq = false
+              when :dispatch
+                @_dispatch_to_dlq = true
+              else
+                raise Karafka::UnsupportedCaseError, flow
+              end
+
+              # We reset the pause to indicate we will now consider it as "ok".
+              coordinator.pause_tracker.reset
+
+              yield
+
+              # Always backoff after DLQ dispatch even on skip to prevent overloads on errors
+              pause(coordinator.seek_offset, nil, false)
+            end
           end
         end
       end

data/lib/karafka/pro/processing/strategies/dlq/ftr.rb

@@ -42,14 +42,10 @@ module Karafka
                   mark_as_consumed(last_group_message)
 
                   handle_post_filtering
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  coordinator.pause_tracker.reset
-
-                  dispatch_if_needed_and_mark_as_consumed
-
-                  pause(coordinator.seek_offset, nil, false)
+                  apply_dlq_flow do
+                    dispatch_if_needed_and_mark_as_consumed
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb

@@ -53,16 +53,12 @@ module Karafka
                   else
                     resume
                   end
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  coordinator.pause_tracker.reset
-
-                  return resume if revoked?
+                  apply_dlq_flow do
+                    return resume if revoked?
 
-                  dispatch_if_needed_and_mark_as_consumed
-
-                  pause(coordinator.seek_offset, nil, false)
+                    dispatch_if_needed_and_mark_as_consumed
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb

@@ -48,18 +48,15 @@ module Karafka
                   else
                     resume
                   end
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  coordinator.pause_tracker.reset
-
-                  return resume if revoked?
+                  apply_dlq_flow do
+                    return resume if revoked?
 
-                  skippable_message, _marked = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                    skippable_message, _marked = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                  coordinator.seek_offset = skippable_message.offset + 1
-                  pause(coordinator.seek_offset, nil, false)
+                    coordinator.seek_offset = skippable_message.offset + 1
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb

@@ -41,23 +41,13 @@ module Karafka
                   return if coordinator.manual_pause?
 
                   handle_post_filtering
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
-                # If we've reached number of retries that we could, we need to skip the first
-                # message that was not marked as consumed, pause and continue, while also moving
-                # this message to the dead topic.
-                #
-                # For a Mom setup, this means, that user has to manage the checkpointing by
-                # himself. If no checkpointing is ever done, we end up with an endless loop.
                 else
-                  # We reset the pause to indicate we will now consider it as "ok".
-                  coordinator.pause_tracker.reset
-
-                  skippable_message, _marked = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  apply_dlq_flow do
+                    skippable_message, _marked = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                  coordinator.seek_offset = skippable_message.offset + 1
-                  pause(coordinator.seek_offset, nil, false)
+                    coordinator.seek_offset = skippable_message.offset + 1
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/lrj.rb

@@ -42,16 +42,12 @@ module Karafka
                   seek(coordinator.seek_offset, false) unless revoked? || coordinator.manual_seek?
 
                   resume
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  coordinator.pause_tracker.reset
-
-                  return resume if revoked?
-
-                  dispatch_if_needed_and_mark_as_consumed
+                  apply_dlq_flow do
+                    return resume if revoked?
 
-                  pause(coordinator.seek_offset, nil, false)
+                    dispatch_if_needed_and_mark_as_consumed
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb

@@ -42,18 +42,15 @@ module Karafka
                   end
 
                   resume
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
                 else
-                  coordinator.pause_tracker.reset
-
-                  return resume if revoked?
+                  apply_dlq_flow do
+                    return resume if revoked?
 
-                  skippable_message, _marked = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                    skippable_message, _marked = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                  coordinator.seek_offset = skippable_message.offset + 1
-                  pause(coordinator.seek_offset, nil, false)
+                    coordinator.seek_offset = skippable_message.offset + 1
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/dlq/mom.rb

@@ -35,28 +35,18 @@ module Karafka
 
                 if coordinator.success?
                   coordinator.pause_tracker.reset
-                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                  retry_after_pause
-                # If we've reached number of retries that we could, we need to skip the first
-                # message that was not marked as consumed, pause and continue, while also moving
-                # this message to the dead topic.
-                #
-                # For a Mom setup, this means, that user has to manage the checkpointing by
-                # himself. If no checkpointing is ever done, we end up with an endless loop.
                 else
-                  # We reset the pause to indicate we will now consider it as "ok".
-                  coordinator.pause_tracker.reset
-
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  apply_dlq_flow do
+                    skippable_message, = find_skippable_message
+                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                  # Save the next offset we want to go with after moving given message to DLQ
-                  # Without this, we would not be able to move forward and we would end up
-                  # in an infinite loop trying to un-pause from the message we've already processed
-                  # Of course, since it's a MoM a rebalance or kill, will move it back as no
-                  # offsets are being committed
-                  coordinator.seek_offset = skippable_message.offset + 1
-                  pause(coordinator.seek_offset, nil, false)
+                    # Save the next offset we want to go with after moving given message to DLQ
+                    # Without this, we would not be able to move forward and we would end up
+                    # in an infinite loop trying to un-pause from the message we've already
+                    # processed. Of course, since it's a MoM a rebalance or kill, will move it back
+                    # as no offsets are being committed
+                    coordinator.seek_offset = skippable_message.offset + 1
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/strategies/vp/default.rb

@@ -155,6 +155,13 @@ module Karafka
             def handle_before_schedule_consume
               super
 
+              # We should not register offsets in virtual manager when in collapse as virtual
+              # manager is not used then for offsets materialization.
+              #
+              # If we would do so, it would cause increased storage in cases of endless errors
+              # that are being retried in collapse without a DLQ.
+              return if collapsed?
+
               coordinator.virtual_offset_manager.register(
                 messages.map(&:offset)
               )

data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb

@@ -28,6 +28,12 @@ module Karafka
                 ).fetch('en').fetch('validations').fetch('topic')
               end
 
+              nested(:dead_letter_queue) do
+                # We use strategy based DLQ for every case in Pro
+                # For default (when no strategy) a default `max_retries` based strategy is used
+                required(:strategy) { |val| val.respond_to?(:call) }
+              end
+
               # Make sure that when we use virtual partitions with DLQ, at least one retry is set
               # We cannot use VP with DLQ without retries as we in order to provide ordering
               # warranties on errors with VP, we need to collapse the VPs concurrency and retry

data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class DeadLetterQueue < Base
+          # Expansions to the topic API in DLQ
+          module Topic
+            # @param strategy [#call, nil] Strategy we want to use or nil if a default strategy
+            # (same as in OSS) should be applied
+            # @param args [Hash] OSS DLQ arguments
+            def dead_letter_queue(strategy: nil, **args)
+              return @dead_letter_queue if @dead_letter_queue
+
+              super(**args).tap do |config|
+                # If explicit strategy is not provided, use the default approach from OSS
+                config.strategy = strategy || lambda do |_errors_tracker, attempt|
+                  attempt > config.max_retries ? :dispatch : :retry
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end