karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_mom.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Dlq
+            # - DLQ
+            # - Ftr
+            # - Mom
+            module FtrMom
+              include Strategies::Ftr::Default
+              include Strategies::Dlq::Default
+
+              # Features for this strategy
+              FEATURES = %i[
+                dead_letter_queue
+                filtering
+                manual_offset_management
+              ].freeze
+
+              # On mom we do not mark, throttling and seeking as in other strategies
+              def handle_after_consume
+                coordinator.on_finished do
+                  return if revoked?
+
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    return if coordinator.manual_pause?
+
+                    handle_post_filtering
+                  else
+                    apply_dlq_flow do
+                      skippable_message, _marked = find_skippable_message
+                      dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+
+                      if mark_after_dispatch?
+                        mark_dispatched_to_dlq(skippable_message)
+                      else
+                        self.seek_offset = skippable_message.offset + 1
+                      end
+                    end
+                  end
+                end
+              end
+
+              # @return [Boolean] should we mark given message as consumed after dispatch. For
+              #  MOM strategies if user did not explicitly tell us to mark, we do not mark. Default
+              #  is `nil`, which means `false` in this case. If user provided alternative value, we
+              #  go with it.
+              #
+              # @note Please note, this is the opposite behavior than in case of AOM strategies.
+              def mark_after_dispatch?
+                return false if topic.dead_letter_queue.mark_after_dispatch.nil?
+
+                topic.dead_letter_queue.mark_after_dispatch
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_mom_vp.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Dlq
+            # - DLQ
+            # - Ftr
+            # - Mom
+            # - VP
+            # We can use the `Strategies::FtrMom` because VPs collapse prior to DLQ
+            module FtrMomVp
+              include Strategies::Mom::Vp
+              include Strategies::Dlq::Vp
+              include Strategies::Dlq::FtrMom
+
+              # Features for this strategy
+              FEATURES = %i[
+                dead_letter_queue
+                filtering
+                manual_offset_management
+                virtual_partitions
+              ].freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/{routing/features/patterns/patterns.rb → processing/consumer_groups/strategies/dlq/ftr_vp.rb}

@@ -30,19 +30,29 @@
 
 module Karafka
   module Pro
-    module Routing
-      module Features
-        class Patterns < Base
-          # Representation of groups of topics
-          class Patterns < Karafka::Routing::Topics
-            # Finds first pattern matching given topic name
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Dlq
+            # - DLQ
+            # - Ftr
+            # - VPs
             #
-            # @param topic_name [String] topic name that may match a pattern
-            # @return [Karafka::Routing::Pattern, nil] pattern or nil if not found
-            # @note Please keep in mind, that there may be many patterns matching given topic name
-            #   and we always pick the first one (defined first)
-            def find(topic_name)
-              @accumulator.find { |pattern| pattern.regexp =~ topic_name }
+            # Behaves same as non-VP due to coordinator lock
+            module FtrVp
+              include Strategies::Vp::Default
+              include Strategies::Dlq::Vp
+              include Strategies::Dlq::Ftr
+
+              # Features for this strategy
+              FEATURES = %i[
+                dead_letter_queue
+                filtering
+                virtual_partitions
+              ].freeze
             end
           end
         end

data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Dlq
+            # DLQ enabled
+            # Long-Running Job enabled
+            module Lrj
+              # Order here matters, lrj needs to be second
+              include Strategies::Dlq::Default
+              include Strategies::Lrj::Default
+
+              # Features for this strategy
+              FEATURES = %i[
+                dead_letter_queue
+                long_running_job
+              ].freeze
+
+              # LRJ standard flow after consumption with DLQ dispatch
+              def handle_after_consume
+                coordinator.on_finished do |last_group_message|
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    return if coordinator.manual_pause?
+
+                    mark_as_consumed(last_group_message) unless revoked?
+                    # We should not overwrite user manual seel request with our seek
+                    unless revoked? || coordinator.manual_seek?
+                      seek(seek_offset, false, reset_offset: false)
+                    end
+
+                    resume
+                  else
+                    apply_dlq_flow do
+                      return resume if revoked?
+
+                      dispatch_if_needed_and_mark_as_consumed
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_mom.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Dlq
+            # Strategy for supporting DLQ with Mom and LRJ enabled
+            module LrjMom
+              # This strategy needs to pause and revoke same way as DlqLrj but without the offset
+              # management
+              include Strategies::Dlq::Lrj
+
+              # Features for this strategy
+              FEATURES = %i[
+                dead_letter_queue
+                long_running_job
+                manual_offset_management
+              ].freeze
+
+              # LRJ standard flow after consumption with DLQ dispatch and no offset management
+              def handle_after_consume
+                coordinator.on_finished do |last_group_message|
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    return if coordinator.manual_pause?
+
+                    unless revoked? || coordinator.manual_seek?
+                      seek(last_group_message.offset + 1, false, reset_offset: false)
+                    end
+
+                    resume
+                  else
+                    apply_dlq_flow do
+                      return resume if revoked?
+
+                      skippable_message, _marked = find_skippable_message
+                      dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+
+                      if mark_after_dispatch?
+                        mark_dispatched_to_dlq(skippable_message)
+                      else
+                        self.seek_offset = skippable_message.offset + 1
+                      end
+                    end
+                  end
+                end
+              end
+
+              # @return [Boolean] should we mark given message as consumed after dispatch. For
+              #  MOM strategies if user did not explicitly tell us to mark, we do not mark. Default
+              #  is `nil`, which means `false` in this case. If user provided alternative value, we
+              #  go with it.
+              #
+              # @note Please note, this is the opposite behavior than in case of AOM strategies.
+              def mark_after_dispatch?
+                return false if topic.dead_letter_queue.mark_after_dispatch.nil?
+
+                topic.dead_letter_queue.mark_after_dispatch
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/{strategies → consumer_groups/strategies}/dlq/lrj_mom_vp.rb

@@ -31,20 +31,25 @@
 module Karafka
   module Pro
     module Processing
-      module Strategies
-        module Dlq
-          # Strategy for supporting DLQ with Mom, LRJ and VP enabled
-          module LrjMomVp
-            # Feature set for this strategy
-            FEATURES = %i[
-              dead_letter_queue
-              long_running_job
-              manual_offset_management
-              virtual_partitions
-            ].freeze
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Dlq
+            # Strategy for supporting DLQ with Mom, LRJ and VP enabled
+            module LrjMomVp
+              # Feature set for this strategy
+              FEATURES = %i[
+                dead_letter_queue
+                long_running_job
+                manual_offset_management
+                virtual_partitions
+              ].freeze
 
-            include Strategies::Dlq::Vp
-            include Strategies::Dlq::LrjMom
+              include Strategies::Dlq::Vp
+              include Strategies::Dlq::LrjMom
+            end
           end
         end
       end

data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_vp.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Dlq
+            # Dead-Letter Queue enabled
+            # Long-Running Job enabled
+            # Virtual Partitions enabled
+            module LrjVp
+              # Same flow as the Dlq Lrj because VP collapses on errors, so DlqLrj can kick in
+              include Strategies::Vp::Default
+              include Strategies::Dlq::Vp
+              include Strategies::Dlq::Lrj
+
+              # Features for this strategy
+              FEATURES = %i[
+                dead_letter_queue
+                long_running_job
+                virtual_partitions
+              ].freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/mom.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Dlq
+            # Strategy for supporting DLQ with Mom enabled
+            module Mom
+              # The broken message lookup is the same in this scenario
+              include Strategies::Dlq::Default
+
+              # Features for this strategy
+              FEATURES = %i[
+                dead_letter_queue
+                manual_offset_management
+              ].freeze
+
+              # When manual offset management is on, we do not mark anything as consumed
+              # automatically and we rely on the user to figure things out
+              def handle_after_consume
+                coordinator.on_finished do
+                  return if revoked?
+
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+                  else
+                    apply_dlq_flow do
+                      skippable_message, = find_skippable_message
+                      dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+
+                      if mark_after_dispatch?
+                        mark_dispatched_to_dlq(skippable_message)
+                      else
+                        # Save the next offset we want to go with after moving given message to DLQ
+                        # Without this, we would not be able to move forward and we would end up
+                        # in an infinite loop trying to un-pause from the message we've already
+                        # processed. Of course, since it's a MoM a rebalance or kill, will move it
+                        # back as no offsets are being committed
+                        self.seek_offset = skippable_message.offset + 1
+                      end
+                    end
+                  end
+                end
+              end
+
+              # @return [Boolean] should we mark given message as consumed after dispatch. For
+              #  MOM strategies if user did not explicitly tell us to mark, we do not mark. Default
+              #  is `nil`, which means `false` in this case. If user provided alternative value, we
+              #  go with it.
+              #
+              # @note Please note, this is the opposite behavior than in case of AOM strategies.
+              def mark_after_dispatch?
+                return false if topic.dead_letter_queue.mark_after_dispatch.nil?
+
+                topic.dead_letter_queue.mark_after_dispatch
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/{strategies/lrj → consumer_groups/strategies/dlq}/mom_vp.rb

@@ -31,22 +31,26 @@
 module Karafka
   module Pro
     module Processing
-      module Strategies
-        # Namespace for all the LRJ starting strategies
-        module Lrj
-          # Long-Running Job enabled
-          # Manual offset management enabled
-          # Virtual Partitions enabled
-          module MomVp
-            include Strategies::Mom::Vp
-            include Strategies::Lrj::Mom
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Dlq
+            # Dead-Letter Queue enabled
+            # MoM enabled
+            # Virtual Partitions enabled
+            module MomVp
+              include Strategies::Dlq::Vp
+              include Strategies::Dlq::Mom
 
-            # Features for this strategy
-            FEATURES = %i[
-              long_running_job
-              manual_offset_management
-              virtual_partitions
-            ].freeze
+              # Features for this strategy
+              FEATURES = %i[
+                dead_letter_queue
+                manual_offset_management
+                virtual_partitions
+              ].freeze
+            end
           end
         end
       end