karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_mom.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Aj
+            # - Aj
+            # - Dlq
+            # - Ftr
+            # - Mom
+            module DlqFtrMom
+              include Strategies::Dlq::FtrMom
+
+              # Features for this strategy
+              FEATURES = %i[
+                active_job
+                dead_letter_queue
+                filtering
+                manual_offset_management
+              ].freeze
+
+              # We write our own custom handler for after consume here, because we mark as consumed
+              # per each job in the AJ consumer itself (for this strategy). This means, that for DLQ
+              # dispatch, we can mark this message as consumed as well.
+              def handle_after_consume
+                coordinator.on_finished do
+                  return if revoked?
+
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    return if coordinator.manual_pause?
+
+                    handle_post_filtering
+                  # If we've reached number of retries that we could, we need to skip the first
+                  # message that was not marked as consumed, pause and continue, while also moving
+                  # this message to the dead topic.
+                  #
+                  # For a Mom setup, this means, that user has to manage the checkpointing by
+                  # himself. If no checkpointing is ever done, we end up with an endless loop.
+                  else
+                    apply_dlq_flow do
+                      skippable_message, = find_skippable_message
+                      dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                      # We can commit the offset here because we know that we skip it "forever" and
+                      # since AJ consumer commits the offset after each job, we also know that the
+                      # previous job was successful
+                      mark_dispatched_to_dlq(skippable_message)
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_mom_vp.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Aj
+            # - Aj
+            # - Dlq
+            # - Ftr
+            # - Mom
+            # - VP
+            module DlqFtrMomVp
+              include Strategies::Aj::DlqMomVp
+              include Strategies::Aj::DlqFtrMom
+
+              # Features for this strategy
+              FEATURES = %i[
+                active_job
+                dead_letter_queue
+                filtering
+                manual_offset_management
+                virtual_partitions
+              ].freeze
+
+              # AJ VP does not early stop on shutdown, hence here we can mark as consumed at the
+              # end of all VPs
+              def handle_after_consume
+                coordinator.on_finished do |last_group_message|
+                  return if revoked?
+
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    return if coordinator.manual_pause?
+
+                    mark_as_consumed(last_group_message)
+
+                    handle_post_filtering
+                  else
+                    apply_dlq_flow do
+                      skippable_message, = find_skippable_message
+                      dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                      # We can commit the offset here because we know that we skip it "forever" and
+                      # since AJ consumer commits the offset after each job, we also know that the
+                      # previous job was successful
+                      mark_dispatched_to_dlq(skippable_message)
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/{strategies/aj/dlq_ftr_lrj_mom.rb → consumer_groups/strategies/aj/dlq_lrj_mom.rb}

@@ -31,51 +31,49 @@
 module Karafka
   module Pro
     module Processing
-      module Strategies
-        module Aj
-          # ActiveJob enabled
-          # DLQ enabled
-          # Filtering enabled
-          # Long-Running Job enabled
-          # Manual offset management enabled
-          module DlqFtrLrjMom
-            include Strategies::Aj::FtrMom
-            include Strategies::Aj::DlqMom
-            include Strategies::Aj::LrjMom
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Aj
+            # ActiveJob enabled
+            # DLQ enabled
+            # Long-Running Job enabled
+            # Manual offset management enabled
+            #
+            # This case is a bit of special. Please see the `AjDlqMom` for explanation on how the
+            # offset management works in this case.
+            module DlqLrjMom
+              include Strategies::Default
+              include Strategies::Dlq::Default
+              include Strategies::Aj::LrjMom
 
-            # Features for this strategy
-            FEATURES = %i[
-              active_job
-              dead_letter_queue
-              filtering
-              long_running_job
-              manual_offset_management
-            ].freeze
+              # Features for this strategy
+              FEATURES = %i[
+                active_job
+                dead_letter_queue
+                long_running_job
+                manual_offset_management
+              ].freeze
 
-            # This strategy assumes we do not early break on shutdown as it has VP
-            def handle_after_consume
-              coordinator.on_finished do
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
+              # We cannot use a VP version of this, because non-VP can early stop on shutdown
+              def handle_after_consume
+                coordinator.on_finished do
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
 
-                  if coordinator.filtered? && !revoked?
-                    handle_post_filtering
-
-                    # :seek and :pause are fully handled by handle_post_filtering
-                    # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
-                    return unless coordinator.filter.action == :skip
-                  elsif !revoked?
                     # no need to check for manual seek because AJ consumer is internal and
                     # fully controlled by us
-                    seek(seek_offset, false, reset_offset: false)
-                  end
+                    seek(seek_offset, false, reset_offset: false) unless revoked?
 
-                  resume
-                else
-                  apply_dlq_flow do
-                    skippable_message, = find_skippable_message
-                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                    mark_dispatched_to_dlq(skippable_message)
+                    resume
+                  else
+                    apply_dlq_flow do
+                      skippable_message, = find_skippable_message
+                      dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                      mark_dispatched_to_dlq(skippable_message)
+                    end
                   end
                 end
               end

data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_lrj_mom_vp.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Aj
+            # ActiveJob enabled
+            # DLQ enabled
+            # Long-Running Job enabled
+            # Manual offset management enabled
+            # Virtual Partitions enabled
+            #
+            # This case is a bit of special. Please see the `Aj::DlqMom` for explanation on how the
+            # offset management works in this case.
+            module DlqLrjMomVp
+              include Strategies::Aj::DlqMomVp
+              include Strategies::Aj::LrjMom
+
+              # Features for this strategy
+              FEATURES = %i[
+                active_job
+                dead_letter_queue
+                long_running_job
+                manual_offset_management
+                virtual_partitions
+              ].freeze
+
+              # This strategy assumes we do not early break on shutdown as it has VP
+              def handle_after_consume
+                coordinator.on_finished do |last_group_message|
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    # Since we have VP here we do not commit intermediate offsets and need to commit
+                    # them here. We do commit in collapsed mode but this is generalized.
+                    mark_as_consumed(last_group_message) unless revoked?
+                    # no need to check for manual seek because AJ consumer is internal and
+                    # fully controlled by us
+                    seek(seek_offset, false, reset_offset: false) unless revoked?
+
+                    resume
+                  else
+                    apply_dlq_flow do
+                      skippable_message, = find_skippable_message
+                      dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                      mark_dispatched_to_dlq(skippable_message)
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_mom.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Aj
+            # ActiveJob enabled
+            # DLQ enabled
+            # Manual offset management enabled
+            #
+            # AJ has manual offset management on by default and the offset management is delegated to
+            # the AJ consumer. This means, we cannot mark as consumed always. We can only mark as
+            # consumed when we skip given job upon errors. In all the other scenarios marking as
+            # consumed needs to happen in the AJ consumer on a per job basis.
+            module DlqMom
+              include Strategies::Dlq::Mom
+
+              # Features for this strategy
+              FEATURES = %i[
+                active_job
+                dead_letter_queue
+                manual_offset_management
+              ].freeze
+
+              # How should we post-finalize consumption.
+              def handle_after_consume
+                coordinator.on_finished do
+                  return if revoked?
+
+                  if coordinator.success?
+                    # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
+                    coordinator.pause_tracker.reset
+                  else
+                    apply_dlq_flow do
+                      skippable_message, = find_skippable_message
+                      dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                      # We can commit the offset here because we know that we skip it "forever" and
+                      # since AJ consumer commits the offset after each job, we also know that the
+                      # previous job was successful
+                      mark_dispatched_to_dlq(skippable_message)
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_mom_vp.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Aj
+            # ActiveJob enabled
+            # Manual offset management enabled
+            # Virtual Partitions enabled
+            module DlqMomVp
+              include Strategies::Default
+              include Strategies::Dlq::Vp
+              include Strategies::Vp::Default
+
+              # Features for this strategy
+              FEATURES = %i[
+                active_job
+                dead_letter_queue
+                manual_offset_management
+                virtual_partitions
+              ].freeze
+
+              # Flow including moving to DLQ in the collapsed mode
+              def handle_after_consume
+                coordinator.on_finished do |last_group_message|
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    # When this is an ActiveJob running via Pro with virtual partitions, we cannot
+                    # mark intermediate jobs as processed not to mess up with the ordering.
+                    # Only when all the jobs are processed and we did not loose the partition
+                    # assignment and we are not stopping (Pro ActiveJob has an early break) we can
+                    # commit offsets .
+                    # For a non virtual partitions case, the flow is regular and state is marked
+                    # after each successfully processed job
+                    return if revoked?
+
+                    mark_as_consumed(last_group_message)
+                  else
+                    apply_dlq_flow do
+                      # Here we are in a collapsed state, hence we can apply the same logic as
+                      # Aj::DlqMom
+                      skippable_message, = find_skippable_message
+                      dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                      mark_dispatched_to_dlq(skippable_message)
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/{strategies → consumer_groups/strategies}/aj/ftr_lrj_mom.rb

@@ -31,22 +31,27 @@
 module Karafka
   module Pro
     module Processing
-      module Strategies
-        module Aj
-          # ActiveJob enabled
-          # Filtering enabled
-          # Long-Running Job enabled
-          # Manual offset management enabled
-          module FtrLrjMom
-            include Strategies::Lrj::FtrMom
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Aj
+            # ActiveJob enabled
+            # Filtering enabled
+            # Long-Running Job enabled
+            # Manual offset management enabled
+            module FtrLrjMom
+              include Strategies::Lrj::FtrMom
 
-            # Features for this strategy
-            FEATURES = %i[
-              active_job
-              filtering
-              long_running_job
-              manual_offset_management
-            ].freeze
+              # Features for this strategy
+              FEATURES = %i[
+                active_job
+                filtering
+                long_running_job
+                manual_offset_management
+              ].freeze
+            end
           end
         end
       end

data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_lrj_mom_vp.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Aj
+            # ActiveJob enabled
+            # Filtering enabled
+            # Long-Running Job enabled
+            # Manual offset management enabled
+            # Virtual Partitions enabled
+            module FtrLrjMomVp
+              include Strategies::Vp::Default
+              include Strategies::Lrj::FtrMom
+
+              # Features for this strategy
+              FEATURES = %i[
+                active_job
+                filtering
+                long_running_job
+                manual_offset_management
+                virtual_partitions
+              ].freeze
+
+              # AJ MOM VP does not do intermediate marking, hence we need to make sure we mark as
+              # consumed here.
+              def handle_after_consume
+                coordinator.on_finished do |last_group_message|
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    mark_as_consumed(last_group_message) unless revoked?
+
+                    if coordinator.filtered? && !revoked?
+                      handle_post_filtering
+
+                      # :seek and :pause are fully handled by handle_post_filtering
+                      # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
+                      return unless coordinator.filter.action == :skip
+                    elsif !revoked?
+                      # no need to check for manual seek because AJ consumer is internal and
+                      # fully controlled by us
+                      seek(seek_offset, false, reset_offset: false)
+                    end
+
+                    resume
+                  else
+                    retry_after_pause
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end