karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/default.rb

@@ -0,0 +1,285 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          # Namespace for all the strategies starting with DLQ
+          module Dlq
+            # Only dead letter queue enabled
+            module Default
+              include Strategies::Default
+
+              # Features for this strategy
+              FEATURES = %i[
+                dead_letter_queue
+              ].freeze
+
+              # Override of the standard `#mark_as_consumed` in order to handle the pause tracker
+              # reset in case DLQ is marked as fully independent. When DLQ is marked independent,
+              # any offset marking causes the pause count tracker to reset. This is useful when
+              # the error is not due to the collective batch operations state but due to intermediate
+              # "crawling" errors that move with it
+              #
+              # @see `Strategies::Default#mark_as_consumed` for more details
+              # @param message [Messages::Message]
+              # @param offset_metadata [String, nil]
+              def mark_as_consumed(message, offset_metadata = @_current_offset_metadata)
+                return super unless retrying?
+                return super unless topic.dead_letter_queue.independent?
+                return false unless super
+
+                coordinator.pause_tracker.reset
+
+                true
+              ensure
+                @_current_offset_metadata = nil
+              end
+
+              # Override of the standard `#mark_as_consumed!`. Resets the pause tracker count in case
+              # DLQ was configured with the `independent` flag.
+              #
+              # @see `Strategies::Default#mark_as_consumed!` for more details
+              # @param message [Messages::Message]
+              # @param offset_metadata [String, nil]
+              def mark_as_consumed!(message, offset_metadata = @_current_offset_metadata)
+                return super unless retrying?
+                return super unless topic.dead_letter_queue.independent?
+                return false unless super
+
+                coordinator.pause_tracker.reset
+
+                true
+              ensure
+                @_current_offset_metadata = nil
+              end
+
+              # When we encounter non-recoverable message, we skip it and go on with our lives
+              def handle_after_consume
+                coordinator.on_finished do |last_group_message|
+                  return if revoked?
+
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    return if coordinator.manual_pause?
+
+                    mark_as_consumed(last_group_message)
+                  else
+                    apply_dlq_flow do
+                      dispatch_if_needed_and_mark_as_consumed
+                    end
+                  end
+                end
+              end
+
+              # Finds the message may want to skip (all, starting from first)
+              # @private
+              # @return [Array<Karafka::Messages::Message, Boolean>] message we may want to skip and
+              #   information if this message was from marked offset or figured out via mom flow
+              def find_skippable_message
+                skippable_message = messages.raw.find do |msg|
+                  coordinator.marked? && msg.offset == seek_offset
+                end
+
+                # If we don't have the message matching the last comitted offset, it means that
+                # user operates with manual offsets and we're beyond the batch in which things
+                # broke for the first time. Then we skip the first (as no markings) and we
+                # move on one by one.
+                skippable_message ? [skippable_message, true] : [messages.first, false]
+              end
+
+              # Moves the broken message into a separate queue defined via the settings
+              #
+              # @param skippable_message [Array<Karafka::Messages::Message>] message we want to
+              #   dispatch to DLQ
+              def dispatch_to_dlq(skippable_message)
+                # DLQ should never try to dispatch a message that was cleaned. It message was
+                # cleaned, we will not have all the needed data. If you see this error, it means
+                # that your processing flow is not as expected and you have cleaned message that
+                # should not be cleaned as it should go to the DLQ
+                raise(Cleaner::Errors::MessageCleanedError) if skippable_message.cleaned?
+
+                producer.public_send(
+                  topic.dead_letter_queue.dispatch_method,
+                  build_dlq_message(
+                    skippable_message
+                  )
+                )
+
+                # Notify about dispatch on the events bus
+                monitor.instrument(
+                  "dead_letter_queue.dispatched",
+                  caller: self,
+                  message: skippable_message
+                )
+              end
+
+              # Dispatches the message to the DLQ (when needed and when applicable based on settings)
+              #   and marks this message as consumed for non MOM flows.
+              #
+              # If producer is transactional and config allows, uses transaction to do that
+              def dispatch_if_needed_and_mark_as_consumed
+                skippable_message, = find_skippable_message
+
+                dispatch = lambda do
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+
+                  if mark_after_dispatch?
+                    mark_dispatched_to_dlq(skippable_message)
+                  else
+                    self.seek_offset = skippable_message.offset + 1
+                  end
+                end
+
+                if dispatch_in_a_transaction?
+                  transaction { dispatch.call }
+                else
+                  dispatch.call
+                end
+              end
+
+              # @param skippable_message [Array<Karafka::Messages::Message>]
+              # @return [Hash] dispatch DLQ message
+              def build_dlq_message(skippable_message)
+                source_partition = skippable_message.partition.to_s
+
+                dlq_message = {
+                  topic: @_dispatch_to_dlq_topic || topic.dead_letter_queue.topic,
+                  key: skippable_message.raw_key,
+                  partition_key: source_partition,
+                  payload: skippable_message.raw_payload,
+                  headers: skippable_message.raw_headers.merge(
+                    "source_topic" => topic.name,
+                    "source_partition" => source_partition,
+                    "source_offset" => skippable_message.offset.to_s,
+                    "source_consumer_group" => topic.group.id,
+                    "source_attempts" => attempt.to_s,
+                    "source_trace_id" => errors_tracker.trace_id
+                  )
+                }
+
+                # Optional method user can define in consumer to enhance the dlq message hash with
+                # some extra details if needed or to replace payload, etc
+                if respond_to?(:enhance_dlq_message, true)
+                  enhance_dlq_message(
+                    dlq_message,
+                    skippable_message
+                  )
+                end
+
+                dlq_message
+              end
+
+              # @return [Boolean] should we dispatch the message to DLQ or not. When the dispatch
+              #   topic is set to false, we will skip the dispatch, effectively ignoring the broken
+              #   message without taking any action.
+              def dispatch_to_dlq?
+                return false unless topic.dead_letter_queue.topic
+                return false unless @_dispatch_to_dlq
+
+                true
+              end
+
+              # @return [Boolean] should we use a transaction to move the data to the DLQ.
+              #   We can do it only when producer is transactional and configuration for DLQ
+              #   transactional dispatches is not set to false.
+              def dispatch_in_a_transaction?
+                producer.transactional? && topic.dead_letter_queue.transactional?
+              end
+
+              # @return [Boolean] should we mark given message as consumed after dispatch.
+              #  For default non MOM strategies if user did not explicitly tell us not to, we mark
+              #  it. Default is `nil`, which means `true` in this case. If user provided alternative
+              #  value, we go with it.
+              def mark_after_dispatch?
+                return true if topic.dead_letter_queue.mark_after_dispatch.nil?
+
+                topic.dead_letter_queue.mark_after_dispatch
+              end
+
+              # Runs the DLQ strategy and based on it it performs certain operations
+              #
+              # In case of `:skip` and `:dispatch` will run the exact flow provided in a block
+              # In case of `:retry` always `#retry_after_pause` is applied
+              def apply_dlq_flow
+                flow, target_topic = topic.dead_letter_queue.strategy.call(errors_tracker, attempt)
+
+                case flow
+                when :retry
+                  retry_after_pause
+
+                  return
+                when :skip
+                  @_dispatch_to_dlq = false
+                when :dispatch
+                  @_dispatch_to_dlq = true
+                  # Use custom topic if it was returned from the strategy
+                  @_dispatch_to_dlq_topic = target_topic || topic.dead_letter_queue.topic
+                else
+                  raise Karafka::UnsupportedCaseError, flow
+                end
+
+                yield
+
+                # We reset the pause to indicate we will now consider it as "ok".
+                coordinator.pause_tracker.reset
+
+                # Always backoff after DLQ dispatch even on skip to prevent overloads on errors
+                pause(seek_offset, nil, false)
+              ensure
+                @_dispatch_to_dlq_topic = nil
+              end
+
+              # Marks message that went to DLQ (if applicable) based on the requested method
+              # @param skippable_message [Karafka::Messages::Message]
+              def mark_dispatched_to_dlq(skippable_message)
+                case topic.dead_letter_queue.marking_method
+                when :mark_as_consumed
+                  mark_as_consumed(skippable_message)
+                when :mark_as_consumed!
+                  mark_as_consumed!(skippable_message)
+                else
+                  # This should never happen. Bug if encountered. Please report
+                  raise Karafka::Errors::UnsupportedCaseError
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/{strategies/dlq/lrj.rb → consumer_groups/strategies/dlq/ftr.rb}

@@ -31,41 +31,42 @@
 module Karafka
   module Pro
     module Processing
-      module Strategies
-        module Dlq
-          # DLQ enabled
-          # Long-Running Job enabled
-          module Lrj
-            # Order here matters, lrj needs to be second
-            include Strategies::Dlq::Default
-            include Strategies::Lrj::Default
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Dlq
+            # - DLQ
+            # - Ftr
+            module Ftr
+              include Strategies::Ftr::Default
+              include Strategies::Dlq::Default
 
-            # Features for this strategy
-            FEATURES = %i[
-              dead_letter_queue
-              long_running_job
-            ].freeze
+              # Features for this strategy
+              FEATURES = %i[
+                dead_letter_queue
+                filtering
+              ].freeze
 
-            # LRJ standard flow after consumption with DLQ dispatch
-            def handle_after_consume
-              coordinator.on_finished do |last_group_message|
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
+              # DLQ flow is standard here, what is not, is the success component where we need to
+              # take into consideration the filtering
+              def handle_after_consume
+                coordinator.on_finished do |last_group_message|
+                  return if revoked?
 
-                  return if coordinator.manual_pause?
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
 
-                  mark_as_consumed(last_group_message) unless revoked?
-                  # We should not overwrite user manual seel request with our seek
-                  unless revoked? || coordinator.manual_seek?
-                    seek(seek_offset, false, reset_offset: false)
-                  end
+                    return if coordinator.manual_pause?
 
-                  resume
-                else
-                  apply_dlq_flow do
-                    return resume if revoked?
+                    mark_as_consumed(last_group_message)
 
-                    dispatch_if_needed_and_mark_as_consumed
+                    handle_post_filtering
+                  else
+                    apply_dlq_flow do
+                      dispatch_if_needed_and_mark_as_consumed
+                    end
                   end
                 end
               end

data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Dlq
+            # Dead-Letter Queue enabled
+            # Filtering enabled
+            # Long-Running Job enabled
+            module FtrLrj
+              include Strategies::Dlq::Lrj
+              include Strategies::Lrj::Ftr
+
+              # Features for this strategy
+              FEATURES = %i[
+                dead_letter_queue
+                filtering
+                long_running_job
+              ].freeze
+
+              # This is one of more complex cases.
+              # We need to ensure, that we always resume (inline or via paused backoff) and we need
+              # to make sure we dispatch to DLQ when needed. Because revocation on LRJ can happen
+              # any time, we need to make sure we do not dispatch to DLQ when error happens but we
+              # no longer own the assignment. Throttling is another factor that has to be taken
+              # into consideration on the successful path
+              def handle_after_consume
+                coordinator.on_finished do |last_group_message|
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    return if coordinator.manual_pause?
+
+                    mark_as_consumed(last_group_message) unless revoked?
+
+                    if coordinator.filtered? && !revoked?
+                      handle_post_filtering
+
+                      # :seek and :pause are fully handled by handle_post_filtering
+                      # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
+                      return unless coordinator.filter.action == :skip
+                    elsif !revoked? && !coordinator.manual_seek?
+                      seek(seek_offset, false, reset_offset: false)
+                    end
+
+                    resume
+                  else
+                    apply_dlq_flow do
+                      return resume if revoked?
+
+                      dispatch_if_needed_and_mark_as_consumed
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_mom.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Dlq
+            # DLQ enabled
+            # Ftr enabled
+            # LRJ enabled
+            # MoM enabled
+            module FtrLrjMom
+              include Strategies::Ftr::Default
+              include Strategies::Dlq::LrjMom
+
+              # Features for this strategy
+              FEATURES = %i[
+                dead_letter_queue
+                filtering
+                long_running_job
+                manual_offset_management
+              ].freeze
+
+              # Post execution flow of this strategy
+              def handle_after_consume
+                coordinator.on_finished do |last_group_message|
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    return if coordinator.manual_pause?
+
+                    if coordinator.filtered? && !revoked?
+                      handle_post_filtering
+
+                      # :seek and :pause are fully handled by handle_post_filtering
+                      # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
+                      return unless coordinator.filter.action == :skip
+                    elsif !revoked? && !coordinator.manual_seek?
+                      seek(last_group_message.offset + 1, false, reset_offset: false)
+                    end
+
+                    resume
+                  else
+                    apply_dlq_flow do
+                      return resume if revoked?
+
+                      skippable_message, _marked = find_skippable_message
+                      dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+
+                      if mark_after_dispatch?
+                        mark_dispatched_to_dlq(skippable_message)
+                      else
+                        self.seek_offset = skippable_message.offset + 1
+                      end
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/{executor.rb → consumer_groups/strategies/dlq/ftr_lrj_mom_vp.rb}

@@ -31,22 +31,33 @@
 module Karafka
   module Pro
     module Processing
-      # Pro executor that supports periodic jobs
-      class Executor < Karafka::Processing::Executor
-        # Runs the code that should happen before periodic job is scheduled
-        #
-        # @note While jobs are called `Periodic`, from the consumer perspective it is "ticking".
-        #   This name was taken for a reason: we may want to introduce periodic ticking also not
-        #   only during polling but for example on wait and a name "poll" would not align well.
-        #   A name "periodic" is not a verb and our other consumer actions are verbs like:
-        #   consume or revoked. So for the sake of consistency we have ticking here.
-        def before_schedule_periodic
-          consumer.on_before_schedule_tick
-        end
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Dlq
+            # DLQ enabled
+            # Ftr enabled
+            # LRJ enabled
+            # MoM enabled
+            # VP enabled
+            module FtrLrjMomVp
+              include Strategies::Vp::Default
+              include Strategies::Dlq::Vp
+              # Same as non VP because of the coordinator post-execution lock
+              include Strategies::Dlq::FtrLrjMom
 
-        # Triggers consumer ticking
-        def periodic
-          consumer.on_tick
+              # Features for this strategy
+              FEATURES = %i[
+                dead_letter_queue
+                filtering
+                long_running_job
+                manual_offset_management
+                virtual_partitions
+              ].freeze
+            end
+          end
         end
       end
     end

data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_vp.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Dlq
+            # Dead-Letter Queue enabled
+            # Filtering enabled
+            # Long-Running Job enabled
+            # Virtual Partitions
+            module FtrLrjVp
+              include Strategies::Vp::Default
+              include Strategies::Dlq::Vp
+              # Same as non VP because of the coordinator post-execution lock
+              include Strategies::Dlq::FtrLrj
+
+              # Features for this strategy
+              FEATURES = %i[
+                dead_letter_queue
+                filtering
+                long_running_job
+                virtual_partitions
+              ].freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end