karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/vp.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Dlq
+            # Dead Letter Queue enabled
+            # Virtual Partitions enabled
+            #
+            # In general because we collapse processing in virtual partitions to one on errors, there
+            # is no special action that needs to be taken because we warranty that even with VPs
+            # on errors a retry collapses into a single state and from this single state we can
+            # mark as consumed the message that we are moving to the DLQ.
+            module Vp
+              # Features for this strategy
+              FEATURES = %i[
+                dead_letter_queue
+                virtual_partitions
+              ].freeze
+
+              include Strategies::Dlq::Default
+              include Strategies::Vp::Default
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/strategies/ftr/default.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          # Filtering related init strategies
+          module Ftr
+            # Only filtering enabled
+            module Default
+              include Strategies::Default
+
+              # Just filtering enabled
+              FEATURES = %i[
+                filtering
+              ].freeze
+
+              # Empty run when running on idle means we need to filter
+              def handle_idle
+                handle_post_filtering
+              end
+
+              # Standard flow without any features
+              def handle_after_consume
+                coordinator.on_finished do |last_group_message|
+                  return if revoked?
+
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    # Do not mark last message if pause happened. This prevents a scenario where
+                    # pause is overridden upon rebalance by marking
+                    return if coordinator.manual_pause?
+
+                    mark_as_consumed(last_group_message)
+
+                    handle_post_filtering
+                  else
+                    retry_after_pause
+                  end
+                end
+              end
+
+              # Throttles by pausing for an expected time period if throttling is needed or seeks
+              # in case the throttle expired. Throttling may expire because we throttle before
+              # processing starts and we need to compensate for processing time. It may turn out
+              # that we don't have to pause but we need to move the offset because we skipped some
+              # messages due to throttling filtering.
+              # @return [Boolean] was any form of throttling operations (pause or seek) needed
+              def handle_post_filtering
+                filter = coordinator.filter
+
+                # We pick the timeout before the action because every action takes time. This time
+                # may then mean we end up having throttle time equal to zero when pause is needed
+                # and this should not happen
+                throttle_timeout = filter.timeout
+
+                # If user requested marking when applying filter, we mark. We may be in the user
+                # flow but even then this is not a problem. Older offsets will be ignored since
+                # we do not force the offset update (expected) and newer are on the user to control.
+                # This can be primarily used when filtering large quantities of data to mark on the
+                # idle runs, so lag reporting is aware that those messages were not consumed but also
+                # are no longer relevant
+                if filter.mark_as_consumed?
+                  send(
+                    filter.marking_method,
+                    filter.marking_cursor
+                  )
+                end
+
+                case filter.action
+                when :skip
+                  nil
+                when :seek
+                  # User direct actions take priority over automatic operations
+                  # If we've already seeked we can just resume operations, nothing extra needed
+                  return resume if coordinator.manual_seek?
+
+                  throttle_message = filter.cursor
+
+                  monitor.instrument(
+                    "filtering.seek",
+                    caller: self,
+                    message: throttle_message
+                  ) do
+                    seek(throttle_message.offset, false)
+                  end
+
+                  resume
+                when :pause
+                  # User direct actions take priority over automatic operations
+                  return nil if coordinator.manual_pause?
+
+                  throttle_message = filter.cursor
+
+                  monitor.instrument(
+                    "filtering.throttled",
+                    caller: self,
+                    message: throttle_message,
+                    timeout: throttle_timeout
+                  ) do
+                    pause(throttle_message.offset, throttle_timeout, false)
+                  end
+                else
+                  raise Karafka::Errors::UnsupportedCaseError filter.action
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/{strategies/mom/ftr.rb → consumer_groups/strategies/ftr/vp.rb}

@@ -31,36 +31,28 @@
 module Karafka
   module Pro
     module Processing
-      module Strategies
-        module Mom
-          # Filtering support for MoM
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          # Filtering related init strategies
           module Ftr
-            include Strategies::Ftr::Default
-            include Strategies::Mom::Default
+            # Filtering enabled
+            # VPs enabled
+            #
+            # VPs should operate without any problems with filtering because virtual partitioning
+            # happens on the limited set of messages and collective filtering applies the same
+            # way as for default cases
+            module Vp
+              # Filtering + VPs
+              FEATURES = %i[
+                filtering
+                virtual_partitions
+              ].freeze
 
-            # MoM + Ftr
-            FEATURES = %i[
-              filtering
-              manual_offset_management
-            ].freeze
-
-            # When mom is enabled, we do not mark messages as consumed after processing
-            # but we also need to keep in mind throttling here
-            def handle_after_consume
-              coordinator.on_finished do
-                return if revoked?
-
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
-
-                  # Do not throttle if paused
-                  return if coordinator.manual_pause?
-
-                  handle_post_filtering
-                else
-                  retry_after_pause
-                end
-              end
+              include Strategies::Vp::Default
+              include Strategies::Ftr::Default
             end
           end
         end

data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/default.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Lrj
+            # Long-Running Job enabled
+            module Default
+              include Strategies::Default
+
+              # Pause for tops 31 years
+              MAX_PAUSE_TIME = 1_000_000_000_000
+
+              # Features for this strategy
+              FEATURES = %i[
+                long_running_job
+              ].freeze
+
+              # We always need to pause prior to doing any jobs for LRJ
+              def handle_before_schedule_consume
+                super
+
+                # This ensures that when running LRJ with VP, things operate as expected run only
+                # once for all the virtual partitions collectively
+                coordinator.on_enqueued do
+                  # Pause and continue with another batch in case of a regular resume.
+                  # In case of an error, the `#retry_after_pause` will move the offset to the first
+                  # message out of this batch.
+                  pause(:consecutive, MAX_PAUSE_TIME, false)
+                end
+              end
+
+              # LRJ standard flow after consumption
+              def handle_after_consume
+                coordinator.on_finished do |last_group_message|
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    return if coordinator.manual_pause?
+
+                    mark_as_consumed(last_group_message) unless revoked?
+
+                    unless revoked? || coordinator.manual_seek?
+                      seek(seek_offset, false, reset_offset: false)
+                    end
+
+                    resume
+                  else
+                    # If processing failed, we need to pause
+                    # For long running job this will overwrite the default never-ending pause and
+                    # will cause the processing to keep going after the error backoff
+                    retry_after_pause
+                  end
+                end
+              end
+
+              # We do not un-pause on revokations for LRJ
+              def handle_revoked
+                coordinator.on_revoked do
+                  # We do not want to resume on revocation in case of a LRJ.
+                  # For LRJ we resume after the successful processing or do a backoff pause in case
+                  # of a failure. Double non-blocking resume could cause problems in coordination.
+                  coordinator.revoke
+                end
+
+                monitor.instrument("consumer.revoke", caller: self)
+                monitor.instrument("consumer.revoked", caller: self) do
+                  revoked
+                end
+              ensure
+                coordinator.decrement(:revoked)
+              end
+
+              # Allows for LRJ to synchronize its work. It may be needed because LRJ can run
+              # lifecycle events like revocation while the LRJ work is running and there may be a
+              # need for a critical section.
+              def synchronize(&)
+                coordinator.shared_mutex.synchronize(&)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Lrj
+            # Filtering enabled
+            # Long-Running Job enabled
+            #
+            # In general aside from throttling this one will behave the same way as the Lrj
+            module Ftr
+              include Strategies::Ftr::Default
+              include Strategies::Lrj::Default
+
+              # Features for this strategy
+              FEATURES = %i[
+                filtering
+                long_running_job
+              ].freeze
+
+              # LRJ standard flow after consumption with potential throttling on success
+              def handle_after_consume
+                coordinator.on_finished do |last_group_message|
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    # Manual pausing has the highest priority
+                    return if coordinator.manual_pause?
+
+                    # It's not a MoM, so for successful we need to mark as consumed
+                    mark_as_consumed(last_group_message) unless revoked?
+
+                    # If still not revoked and was throttled, we need to apply throttling logic
+                    if coordinator.filtered? && !revoked?
+                      handle_post_filtering
+
+                      # :seek and :pause are fully handled by handle_post_filtering
+                      # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
+                      return unless coordinator.filter.action == :skip
+                    elsif !revoked? && !coordinator.manual_seek?
+                      # If not revoked and not throttled, we move to where we were suppose to and
+                      # resume
+                      seek(seek_offset, false, reset_offset: false)
+                    end
+
+                    resume
+                  else
+                    # If processing failed, we need to pause
+                    # For long running job this will overwrite the default never-ending pause and
+                    # will cause the processing to keep going after the error backoff
+                    retry_after_pause
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_mom.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Lrj
+            # Long-Running Job enabled
+            # Filtering enabled
+            # Manual offset management enabled
+            #
+            # It is really similar to the Lrj::Ftr but we do not mark anything as consumed
+            module FtrMom
+              include Strategies::Lrj::Ftr
+
+              # Features for this strategy
+              FEATURES = %i[
+                filtering
+                long_running_job
+                manual_offset_management
+              ].freeze
+
+              # LRJ standard flow after consumption with potential filtering on success
+              def handle_after_consume
+                coordinator.on_finished do |last_group_message|
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    # Manual pausing has the highest priority
+                    return if coordinator.manual_pause?
+
+                    # If still not revoked and was throttled, we need to apply filtering logic
+                    if coordinator.filtered? && !revoked?
+                      handle_post_filtering
+
+                      # :seek and :pause are fully handled by handle_post_filtering
+                      # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
+                      return unless coordinator.filter.action == :skip
+                    elsif !revoked? && !coordinator.manual_seek?
+                      # If not revoked and not throttled, we move to where we were suppose to and
+                      # resume
+                      seek(last_group_message.offset + 1, false, reset_offset: false)
+                    end
+
+                    resume
+                  else
+                    # If processing failed, we need to pause
+                    # For long running job this will overwrite the default never-ending pause and
+                    # will cause the processing to keep going after the error backoff
+                    retry_after_pause
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_mom_vp.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          # Filtering related init strategies
+          module Lrj
+            # Filtering enabled
+            # LRJ enabled
+            # MoM enabled
+            # VPs enabled
+            module FtrMomVp
+              # Filtering + LRJ + Mom + VPs
+              FEATURES = %i[
+                filtering
+                long_running_job
+                manual_offset_management
+                virtual_partitions
+              ].freeze
+
+              include Strategies::Lrj::MomVp
+              include Strategies::Lrj::FtrMom
+            end
+          end
+        end
+      end
+    end
+  end
+end