karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_vp.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Lrj
+            # Long-Running Job enabled
+            # Filtering enabled
+            # Virtual Partitions enabled
+            #
+            # Behaves same as non-VP because of the aggregated flow in the coordinator.
+            module FtrVp
+              include Strategies::Vp::Default
+              include Strategies::Lrj::Ftr
+
+              # Features for this strategy
+              FEATURES = %i[
+                filtering
+                long_running_job
+                virtual_partitions
+              ].freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/mom.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          # Namespace for all the LRJ starting strategies
+          module Lrj
+            # Long-Running Job enabled
+            # Manual offset management enabled
+            module Mom
+              include Strategies::Default
+
+              # Features for this strategy
+              FEATURES = %i[
+                long_running_job
+                manual_offset_management
+              ].freeze
+
+              # We always need to pause prior to doing any jobs for LRJ
+              def handle_before_schedule_consume
+                super
+
+                # This ensures that when running LRJ with VP, things operate as expected run only
+                # once for all the virtual partitions collectively
+                coordinator.on_enqueued do
+                  pause(:consecutive, Strategies::Lrj::Default::MAX_PAUSE_TIME, false)
+                end
+              end
+
+              # No offset management, aside from that typical LRJ
+              def handle_after_consume
+                coordinator.on_finished do |last_group_message|
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    return if coordinator.manual_pause?
+
+                    unless revoked? || coordinator.manual_seek?
+                      seek(last_group_message.offset + 1, false, reset_offset: false)
+                    end
+
+                    resume
+                  else
+                    retry_after_pause
+                  end
+                end
+              end
+
+              # We do not un-pause on revokations for LRJ
+              def handle_revoked
+                coordinator.on_revoked do
+                  coordinator.revoke
+                end
+
+                monitor.instrument("consumer.revoke", caller: self)
+                monitor.instrument("consumer.revoked", caller: self) do
+                  revoked
+                end
+              ensure
+                coordinator.decrement(:revoked)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/mom_vp.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          # Namespace for all the LRJ starting strategies
+          module Lrj
+            # Long-Running Job enabled
+            # Manual offset management enabled
+            # Virtual Partitions enabled
+            module MomVp
+              include Strategies::Mom::Vp
+              include Strategies::Lrj::Mom
+
+              # Features for this strategy
+              FEATURES = %i[
+                long_running_job
+                manual_offset_management
+                virtual_partitions
+              ].freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/{strategies/mom → consumer_groups/strategies/lrj}/vp.rb

@@ -31,19 +31,25 @@
 module Karafka
   module Pro
     module Processing
-      module Strategies
-        module Mom
-          # - Mom enabled
-          # - Vp enabled
-          module Vp
-            include Strategies::Vp::Default
-            include Strategies::Mom::Default
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Lrj
+            # Long-Running Job enabled
+            # Virtual Partitions enabled
+            module Vp
+              # Same flow as the standard Lrj
+              include Strategies::Vp::Default
+              include Strategies::Lrj::Default
 
-            # Features of this strategy
-            FEATURES = %i[
-              manual_offset_management
-              virtual_partitions
-            ].freeze
+              # Features for this strategy
+              FEATURES = %i[
+                long_running_job
+                virtual_partitions
+              ].freeze
+            end
           end
         end
       end

data/lib/karafka/pro/processing/{strategies/aj/mom_vp.rb → consumer_groups/strategies/mom/default.rb}

@@ -31,33 +31,32 @@
 module Karafka
   module Pro
     module Processing
-      module Strategies
-        module Aj
-          # ActiveJob enabled
-          # Manual offset management enabled
-          # Virtual Partitions enabled
-          module MomVp
-            include Strategies::Default
-            include Strategies::Vp::Default
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          # Namespace for Mom starting strategies
+          module Mom
+            # Manual offset management enabled
+            module Default
+              include Strategies::Default
 
-            # Features for this strategy
-            FEATURES = %i[
-              active_job
-              manual_offset_management
-              virtual_partitions
-            ].freeze
-
-            # Standard flow without any features
-            def handle_after_consume
-              coordinator.on_finished do |last_group_message|
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
+              # Features for this strategy
+              FEATURES = %i[
+                manual_offset_management
+              ].freeze
 
+              # When mom is enabled, we do not mark messages as consumed after processing
+              def handle_after_consume
+                coordinator.on_finished do
                   return if revoked?
 
-                  mark_as_consumed(last_group_message)
-                else
-                  retry_after_pause
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+                  else
+                    retry_after_pause
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/{strategies/aj/ftr_mom_vp.rb → consumer_groups/strategies/mom/ftr.rb}

@@ -31,39 +31,39 @@
 module Karafka
   module Pro
     module Processing
-      module Strategies
-        # Namespace for ActiveJob related strategies
-        module Aj
-          # ActiveJob enabled
-          # Filtering enabled
-          # Manual Offset management enabled
-          # Virtual partitions enabled
-          module FtrMomVp
-            include Strategies::Aj::FtrMom
-            include Strategies::Aj::MomVp
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Mom
+            # Filtering support for MoM
+            module Ftr
+              include Strategies::Ftr::Default
+              include Strategies::Mom::Default
 
-            # Features for this strategy
-            FEATURES = %i[
-              active_job
-              filtering
-              manual_offset_management
-              virtual_partitions
-            ].freeze
-
-            # AJ with VPs always has intermediate marking disabled, hence we need to do it post
-            # execution always.
-            def handle_after_consume
-              coordinator.on_finished do |last_group_message|
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
+              # MoM + Ftr
+              FEATURES = %i[
+                filtering
+                manual_offset_management
+              ].freeze
 
+              # When mom is enabled, we do not mark messages as consumed after processing
+              # but we also need to keep in mind throttling here
+              def handle_after_consume
+                coordinator.on_finished do
                   return if revoked?
 
-                  mark_as_consumed(last_group_message)
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    # Do not throttle if paused
+                    return if coordinator.manual_pause?
 
-                  handle_post_filtering
-                else
-                  retry_after_pause
+                    handle_post_filtering
+                  else
+                    retry_after_pause
+                  end
                 end
               end
             end

data/lib/karafka/pro/processing/{virtual_partitions/distributors/base.rb → consumer_groups/strategies/mom/ftr_vp.rb}

@@ -31,21 +31,26 @@
 module Karafka
   module Pro
     module Processing
-      # Processing components for virtual partitions
-      module VirtualPartitions
-        # Distributors for virtual partitions
-        module Distributors
-          # Base class for all virtual partition distributors
-          class Base
-            # @param config [Karafka::Pro::Routing::Features::VirtualPartitions::Config]
-            def initialize(config)
-              @config = config
-            end
-
-            private
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Mom
+            # - Mom enabled
+            # - Ftr enabled
+            # - Vp enabled
+            module FtrVp
+              include Strategies::Mom::Vp
+              include Strategies::Mom::Ftr
 
-            # @return [Karafka::Pro::Routing::Features::VirtualPartitions::Config]
-            attr_reader :config
+              # Features of this strategy
+              FEATURES = %i[
+                filtering
+                manual_offset_management
+                virtual_partitions
+              ].freeze
+            end
           end
         end
       end

data/lib/karafka/pro/processing/{strategies/aj/mom.rb → consumer_groups/strategies/mom/vp.rb}

@@ -31,20 +31,24 @@
 module Karafka
   module Pro
     module Processing
-      module Strategies
-        # Namespace for ActiveJob related strategies
-        module Aj
-          # ActiveJob enabled
-          # Manual Offset management enabled
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
           module Mom
-            # Standard ActiveJob strategy is the same one we use for Mom
-            include Strategies::Mom::Default
+            # - Mom enabled
+            # - Vp enabled
+            module Vp
+              include Strategies::Vp::Default
+              include Strategies::Mom::Default
 
-            # Features for this strategy
-            FEATURES = %i[
-              active_job
-              manual_offset_management
-            ].freeze
+              # Features of this strategy
+              FEATURES = %i[
+                manual_offset_management
+                virtual_partitions
+              ].freeze
+            end
           end
         end
       end

data/lib/karafka/pro/processing/consumer_groups/strategies/vp/default.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          # VP starting strategies
+          module Vp
+            # Just Virtual Partitions enabled
+            module Default
+              # This flow is exactly the same as the default one because the default one is wrapper
+              # with `coordinator#on_finished`
+              include Strategies::Default
+
+              # Features for this strategy
+              FEATURES = %i[
+                virtual_partitions
+              ].freeze
+
+              # @param message [Karafka::Messages::Message] marks message as consumed
+              # @param offset_metadata [String, nil]
+              # @note This virtual offset management uses a regular default marking API underneath.
+              #   We do not alter the "real" marking API, as VPs are just one of many cases we want
+              #   to support and we do not want to impact them with collective offsets management
+              def mark_as_consumed(message, offset_metadata = @_current_offset_metadata)
+                if @_in_transaction && !collapsed?
+                  mark_in_transaction(message, offset_metadata, true)
+                elsif collapsed?
+                  super
+                else
+                  manager = coordinator.virtual_offset_manager
+
+                  coordinator.synchronize do
+                    manager.mark(message, offset_metadata)
+                    # If this is last marking on a finished flow, we can use the original
+                    # last message and in order to do so, we need to mark all previous messages as
+                    # consumed as otherwise the computed offset could be different
+                    # We mark until our offset just in case of a DLQ flow or similar, where we do not
+                    # want to mark all but until the expected location
+                    manager.mark_until(message, offset_metadata) if coordinator.finished?
+
+                    return revoked? unless manager.markable?
+
+                    manager.markable? ? super(*manager.markable) : revoked?
+                  end
+                end
+              ensure
+                @_current_offset_metadata = nil
+              end
+
+              # @param message [Karafka::Messages::Message] blocking marks message as consumed
+              # @param offset_metadata [String, nil]
+              def mark_as_consumed!(message, offset_metadata = @_current_offset_metadata)
+                if @_in_transaction && !collapsed?
+                  mark_in_transaction(message, offset_metadata, false)
+                elsif collapsed?
+                  super
+                else
+                  manager = coordinator.virtual_offset_manager
+
+                  coordinator.synchronize do
+                    manager.mark(message, offset_metadata)
+                    manager.mark_until(message, offset_metadata) if coordinator.finished?
+                    manager.markable? ? super(*manager.markable) : revoked?
+                  end
+                end
+              ensure
+                @_current_offset_metadata = nil
+              end
+
+              # Stores the next offset for processing inside of the transaction when collapsed and
+              # accumulates marking as consumed in the local buffer.
+              #
+              # Due to nature of VPs we cannot provide full EOS support but we can simulate it,
+              # making sure that no offset are stored unless transaction is finished. We do it by
+              # accumulating the post-transaction marking requests and after it is successfully done
+              # we mark each as consumed. This effectively on errors "rollbacks" the state and
+              # prevents offset storage.
+              #
+              # Since the EOS here is "weak", we do not have to worry about the race-conditions and
+              # we do not have to have any mutexes.
+              #
+              # @param message [Messages::Message] message we want to commit inside of a transaction
+              # @param offset_metadata [String, nil] offset metadata or nil if none
+              # @param async [Boolean] should we mark in async or sync way (applicable only to post
+              #   transaction state synchronization usage as within transaction it is always sync)
+              def mark_in_transaction(message, offset_metadata, async)
+                raise Errors::TransactionRequiredError unless @_in_transaction
+                # Prevent from attempts of offset storage when we no longer own the assignment
+                raise Errors::AssignmentLostError if revoked?
+
+                return super if collapsed?
+                # If this is user post-execution transaction (one initiated by the system) we should
+                # delegate to the original implementation that will store the offset via the producer
+                return super if @_transaction_internal
+
+                @_transaction_marked << [message, offset_metadata, async]
+              end
+
+              # @return [Boolean] is the virtual processing collapsed in the context of given
+              #   consumer.
+              def collapsed?
+                coordinator.collapsed?
+              end
+
+              # @param offset [Integer] first offset from which we should not operate in a collapsed
+              #   mode.
+              # @note Keep in mind, that if a batch contains this but also messages earlier messages
+              #   that should be collapsed, all will continue to operate in a collapsed mode until
+              #   first full batch with only messages that should not be collapsed.
+              def collapse_until!(offset)
+                coordinator.collapse_until!(offset)
+              end
+
+              # @return [Boolean] true if any of virtual partition we're operating in the entangled
+              #   mode has already failed and we know we are failing collectively.
+              #   Useful for early stop to minimize number of things processed twice.
+              #
+              # @note We've named it `#failing?` instead of `#failure?`  because it aims to be used
+              #   from within virtual partitions where we want to have notion of collective failing
+              #   not just "local" to our processing. We "are" failing with other virtual partitions
+              #   raising an error, but locally we are still processing.
+              def failing?
+                coordinator.failure?
+              end
+
+              # Allows for cross-virtual-partition consumers locks
+              #
+              # This is not needed in the non-VP flows except LRJ because there is always only one
+              # consumer per partition at the same time, so no coordination is needed directly for
+              # the end users. With LRJ it is needed and provided in the `LRJ::Default` strategy,
+              # because lifecycle events on revocation can run in parallel to the LRJ job as it is
+              # non-blocking.
+              def synchronize(&)
+                coordinator.shared_mutex.synchronize(&)
+              end
+
+              private
+
+              # Prior to adding work to the queue, registers all the messages offsets into the
+              # virtual offset group.
+              #
+              # @note This can be done without the mutex, because it happens from the same thread
+              #   for all the work (listener thread)
+              def handle_before_schedule_consume
+                super
+
+                # We should not register offsets in virtual manager when in collapse as virtual
+                # manager is not used then for offsets materialization.
+                #
+                # If we would do so, it would cause increased storage in cases of endless errors
+                # that are being retried in collapse without a DLQ.
+                return if collapsed?
+
+                coordinator.virtual_offset_manager.register(
+                  messages.raw.map(&:offset)
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end