karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/processing/{strategies → consumer_groups/strategies}/aj/ftr_mom.rb

@@ -31,22 +31,27 @@
 module Karafka
   module Pro
     module Processing
-      module Strategies
-        # Namespace for ActiveJob related strategies
-        module Aj
-          # ActiveJob enabled
-          # Filtering enabled
-          # Manual Offset management enabled
-          module FtrMom
-            # Same as standard Mom::Ftr
-            include Strategies::Mom::Ftr
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          # Namespace for ActiveJob related strategies
+          module Aj
+            # ActiveJob enabled
+            # Filtering enabled
+            # Manual Offset management enabled
+            module FtrMom
+              # Same as standard Mom::Ftr
+              include Strategies::Mom::Ftr
 
-            # Features for this strategy
-            FEATURES = %i[
-              active_job
-              filtering
-              manual_offset_management
-            ].freeze
+              # Features for this strategy
+              FEATURES = %i[
+                active_job
+                filtering
+                manual_offset_management
+              ].freeze
+            end
           end
         end
       end

data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_mom_vp.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          # Namespace for ActiveJob related strategies
+          module Aj
+            # ActiveJob enabled
+            # Filtering enabled
+            # Manual Offset management enabled
+            # Virtual partitions enabled
+            module FtrMomVp
+              include Strategies::Aj::FtrMom
+              include Strategies::Aj::MomVp
+
+              # Features for this strategy
+              FEATURES = %i[
+                active_job
+                filtering
+                manual_offset_management
+                virtual_partitions
+              ].freeze
+
+              # AJ with VPs always has intermediate marking disabled, hence we need to do it post
+              # execution always.
+              def handle_after_consume
+                coordinator.on_finished do |last_group_message|
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    return if revoked?
+
+                    mark_as_consumed(last_group_message)
+
+                    handle_post_filtering
+                  else
+                    retry_after_pause
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/{strategies/mom/default.rb → consumer_groups/strategies/aj/lrj_mom.rb}

@@ -31,29 +31,25 @@
 module Karafka
   module Pro
     module Processing
-      module Strategies
-        # Namespace for Mom starting strategies
-        module Mom
-          # Manual offset management enabled
-          module Default
-            include Strategies::Default
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Aj
+            # ActiveJob enabled
+            # Long-Running Job enabled
+            # Manual offset management enabled
+            module LrjMom
+              # Same behaviour as Lrj::Mom
+              include Strategies::Lrj::Mom
 
-            # Features for this strategy
-            FEATURES = %i[
-              manual_offset_management
-            ].freeze
-
-            # When mom is enabled, we do not mark messages as consumed after processing
-            def handle_after_consume
-              coordinator.on_finished do
-                return if revoked?
-
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
-                else
-                  retry_after_pause
-                end
-              end
+              # Features for this strategy
+              FEATURES = %i[
+                active_job
+                long_running_job
+                manual_offset_management
+              ].freeze
             end
           end
         end

data/lib/karafka/pro/processing/consumer_groups/strategies/aj/lrj_mom_vp.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Aj
+            # ActiveJob enabled
+            # Long-Running Job enabled
+            # Manual offset management enabled
+            # Virtual Partitions enabled
+            module LrjMomVp
+              include Strategies::Default
+              include Strategies::Vp::Default
+
+              # Features for this strategy
+              FEATURES = %i[
+                active_job
+                long_running_job
+                manual_offset_management
+                virtual_partitions
+              ].freeze
+
+              # No actions needed for the standard flow here
+              def handle_before_schedule_consume
+                super
+
+                coordinator.on_enqueued do
+                  pause(:consecutive, Strategies::Lrj::Default::MAX_PAUSE_TIME, false)
+                end
+              end
+
+              # Standard flow without any features
+              def handle_after_consume
+                coordinator.on_finished do |last_group_message|
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    mark_as_consumed(last_group_message) unless revoked?
+
+                    # no need to check for manual seek because AJ consumer is internal and
+                    # fully controlled by us
+                    seek(seek_offset, false, reset_offset: false) unless revoked?
+
+                    resume
+                  else
+                    # If processing failed, we need to pause
+                    # For long running job this will overwrite the default never-ending pause and
+                    # will cause the processing to keep going after the error backoff
+                    retry_after_pause
+                  end
+                end
+              end
+
+              # LRJ cannot resume here. Only in handling the after consumption
+              def handle_revoked
+                coordinator.on_revoked do
+                  coordinator.revoke
+                end
+
+                monitor.instrument("consumer.revoke", caller: self)
+                monitor.instrument("consumer.revoked", caller: self) do
+                  revoked
+                end
+              ensure
+                coordinator.decrement(:revoked)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/strategies/aj/mom.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          # Namespace for ActiveJob related strategies
+          module Aj
+            # ActiveJob enabled
+            # Manual Offset management enabled
+            module Mom
+              # Standard ActiveJob strategy is the same one we use for Mom
+              include Strategies::Mom::Default
+
+              # Features for this strategy
+              FEATURES = %i[
+                active_job
+                manual_offset_management
+              ].freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/strategies/aj/mom_vp.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Aj
+            # ActiveJob enabled
+            # Manual offset management enabled
+            # Virtual Partitions enabled
+            module MomVp
+              include Strategies::Default
+              include Strategies::Vp::Default
+
+              # Features for this strategy
+              FEATURES = %i[
+                active_job
+                manual_offset_management
+                virtual_partitions
+              ].freeze
+
+              # Standard flow without any features
+              def handle_after_consume
+                coordinator.on_finished do |last_group_message|
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    return if revoked?
+
+                    mark_as_consumed(last_group_message)
+                  else
+                    retry_after_pause
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/{strategies/lrj/vp.rb → consumer_groups/strategies/base.rb}

@@ -31,20 +31,15 @@
 module Karafka
   module Pro
     module Processing
-      module Strategies
-        module Lrj
-          # Long-Running Job enabled
-          # Virtual Partitions enabled
-          module Vp
-            # Same flow as the standard Lrj
-            include Strategies::Vp::Default
-            include Strategies::Lrj::Default
-
-            # Features for this strategy
-            FEATURES = %i[
-              long_running_job
-              virtual_partitions
-            ].freeze
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        # Pro processing strategies namespace
+        module Strategies
+          # Base strategy for Pro
+          module Base
+            include Karafka::Processing::ConsumerGroups::Strategies::Base
           end
         end
       end