karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/loader.rb

@@ -37,9 +37,10 @@ module Karafka
       # We require those not to deal with this and then all works as expected
       FORCE_LOADED = %w[
         active_job/dispatcher
-        processing/jobs/consume_non_blocking
-        processing/strategies/base
+        processing/consumer_groups/jobs/consume_non_blocking
+        processing/consumer_groups/strategies/base
         routing/features/base
+        routing/features/consumer_groups
         encryption
         encryption/cipher
         encryption/setup/config
@@ -48,8 +49,7 @@ module Karafka
         cli/topics
       ].freeze
 
-      # Zeitwerk pro loader
-      # We need to have one per process, that's why it's set as a constant
+      # Zeitwerk pro loader We need to have one per process, that's why it's set as a constant
       PRO_LOADER = Zeitwerk::Loader.new
 
       private_constant :PRO_LOADER
@@ -83,7 +83,7 @@ module Karafka
           features.each { |feature| feature.post_setup(config) }
 
           # We initialize it here so we don't initialize it during multi-threading work
-          Processing::SubscriptionGroupsCoordinator.instance
+          Processing::ConsumerGroups::SubscriptionGroupsCoordinator.instance
         end
 
         # Runs operations needed after fork in swarm for features that need it
@@ -113,20 +113,23 @@ module Karafka
         def reconfigure(config)
           icfg = config.internal
 
+          Karafka::Setup::DefaultsInjector.singleton_class.prepend(Setup::DefaultsInjector)
+
           icfg.cli.contract = Cli::Contracts::Server.new
 
           # Use manager that supports multiplexing
           icfg.connection.manager = Connection::Manager.new
 
-          icfg.processing.coordinator_class = Processing::Coordinator
-          icfg.processing.errors_tracker_class = Processing::Coordinators::ErrorsTracker
-          icfg.processing.partitioner_class = Processing::Partitioner
           icfg.processing.scheduler_class = Processing::Schedulers::Default
           icfg.processing.jobs_queue_class = Processing::JobsQueue
-          icfg.processing.executor_class = Processing::Executor
-          icfg.processing.jobs_builder = Processing::JobsBuilder.new
-          icfg.processing.strategy_selector = Processing::StrategySelector.new
-          icfg.processing.expansions_selector = Processing::ExpansionsSelector.new
+
+          icfg.processing.consumer_groups.coordinator_class = Processing::ConsumerGroups::Coordinator
+          icfg.processing.consumer_groups.errors_tracker_class = Processing::ConsumerGroups::Coordinators::ErrorsTracker
+          icfg.processing.consumer_groups.partitioner_class = Processing::ConsumerGroups::Partitioner
+          icfg.processing.consumer_groups.executor_class = Processing::ConsumerGroups::Executor
+          icfg.processing.consumer_groups.jobs_builder = Processing::ConsumerGroups::JobsBuilder.new
+          icfg.processing.consumer_groups.strategy_selector = Processing::ConsumerGroups::StrategySelector.new
+          icfg.processing.consumer_groups.expansions_selector = Processing::ConsumerGroups::ExpansionsSelector.new
 
           icfg.active_job.consumer_class = ActiveJob::Consumer
           icfg.active_job.dispatcher = ActiveJob::Dispatcher.new

data/lib/karafka/pro/processing/consumer_groups/adaptive_iterator/consumer.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        # Namespace for adaptive iterator consumer components
+        module AdaptiveIterator
+          # Consumer enhancements needed to wrap the batch iterator for adaptive iterating
+          # It automatically marks as consumed, ensures that we do not reach `max.poll.interval.ms`
+          # and does other stuff to simplify user per-message processing
+          module Consumer
+            # @param args [Array] anything accepted by `Karafka::Messages::Messages#each`
+            def each(*args)
+              adi_config = topic.adaptive_iterator
+
+              tracker = Tracker.new(
+                adi_config.safety_margin,
+                coordinator.last_polled_at,
+                topic.subscription_group.kafka.fetch(:"max.poll.interval.ms")
+              )
+
+              messages.each(*args) do |message|
+                # Always stop if we've lost the assignment
+                return if revoked?
+                # No automatic marking risk when mom is enabled so we can fast stop
+                return if Karafka::App.done? && topic.manual_offset_management?
+
+                # Seek request on done will allow us to stop without marking the offset when user had
+                # the automatic offset marking. This should not be a big network traffic issue for
+                # the end user as we're stopping anyhow but should improve shutdown time
+                if tracker.enough? || Karafka::App.done?
+                  # Enough means we no longer have time to process more data without polling as we
+                  # risk reaching max poll interval. Instead we seek and we will poll again soon.
+                  seek(message.offset, reset_offset: true)
+
+                  return
+                end
+
+                tracker.track { yield(message) }
+
+                # Clean if this is what user configured
+                message.clean! if adi_config.clean_after_yielding?
+
+                public_send(adi_config.marking_method, message)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/adaptive_iterator/tracker.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module AdaptiveIterator
+          # Tracker is responsible for monitoring the processing of messages within the poll
+          # interval limitation.
+          # It ensures that the consumer does not exceed the maximum poll interval by tracking the
+          # processing cost and determining when to halt further processing (if needed).
+          class Tracker
+            include Karafka::Core::Helpers::Time
+
+            # Initializes a new Tracker instance.
+            #
+            # @param safety_margin [Float] The safety margin percentage (0-100) to leave as a buffer.
+            # @param last_polled_at [Float] The timestamp of the last polling in milliseconds.
+            # @param max_poll_interval_ms [Integer] The maximum poll interval time in milliseconds.
+            def initialize(
+              safety_margin,
+              last_polled_at,
+              max_poll_interval_ms
+            )
+              @safety_margin = safety_margin / 100.0 # Convert percentage to decimal
+              @last_polled_at = last_polled_at
+              @max_processing_cost = 0
+              @max_poll_interval_ms = max_poll_interval_ms
+            end
+
+            # Tracks the processing time of a block and updates the maximum processing cost.
+            #
+            # @yield Executes the block, measuring the time taken for processing.
+            def track
+              before = monotonic_now
+
+              yield
+
+              time_taken = monotonic_now - before
+
+              return unless time_taken > @max_processing_cost
+
+              @max_processing_cost = time_taken
+            end
+
+            # Determines if there is enough time left to process more messages without exceeding the
+            # maximum poll interval, considering both the safety margin and adaptive margin.
+            #
+            # @return [Boolean] Returns true if it is time to stop processing. False otherwise.
+            def enough?
+              elapsed_time_ms = monotonic_now - @last_polled_at
+              remaining_time_ms = @max_poll_interval_ms - elapsed_time_ms
+
+              safety_margin_ms = @max_poll_interval_ms * @safety_margin
+
+              return true if remaining_time_ms <= safety_margin_ms
+              return true if remaining_time_ms - @max_processing_cost <= safety_margin_ms
+
+              false
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/collapser.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        # Manages the collapse of virtual partitions
+        # Since any non-virtual partition is actually a virtual partition of size one, we can use
+        # it in a generic manner without having to distinguish between those cases.
+        #
+        # We need to have notion of the offset until we want to collapse because upon pause and retry
+        # rdkafka may purge the buffer. This means, that we may end up with smaller or bigger
+        # (different) dataset and without tracking the end of collapse, there would be a chance for
+        # things to flicker. Tracking allows us to ensure, that collapse is happening until all the
+        # messages from the corrupted batch are processed.
+        class Collapser
+          # When initialized, nothing is collapsed
+          def initialize
+            @collapsed = false
+            @until_offset = -1
+            @mutex = Mutex.new
+          end
+
+          # @return [Boolean] Should we collapse into a single consumer
+          def collapsed?
+            @collapsed
+          end
+
+          # Collapse until given offset. Until given offset is encountered or offset bigger than that
+          # we keep collapsing.
+          # @param offset [Integer] offset until which we keep the collapse
+          def collapse_until!(offset)
+            @mutex.synchronize do
+              # We check it here in case after a pause and re-fetch we would get less messages and
+              # one of them would cause an error. We do not want to overwrite the offset here unless
+              # it is bigger.
+              @until_offset = offset if offset > @until_offset
+            end
+          end
+
+          # Sets the collapse state based on the first collective offset that we are going to process
+          # and makes the decision whether or not we need to still keep the collapse.
+          # @param first_offset [Integer] first offset from a collective batch
+          def refresh!(first_offset)
+            @mutex.synchronize do
+              @collapsed = first_offset < @until_offset
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/coordinator.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        # Pro coordinator that provides extra orchestration methods useful for parallel processing
+        # within the same partition
+        class Coordinator < Karafka::Processing::ConsumerGroups::Coordinator
+          extend Forwardable
+          include Helpers::ConfigImporter.new(
+            errors_tracker_class: %i[internal processing consumer_groups errors_tracker_class]
+          )
+
+          def_delegators :@collapser, :collapsed?, :collapse_until!
+
+          attr_reader :filter, :virtual_offset_manager, :shared_mutex, :errors_tracker
+
+          # @param args [Object] anything the base coordinator accepts
+          def initialize(*args)
+            super
+
+            @executed = []
+            @errors_tracker = errors_tracker_class.new(topic, partition)
+            @flow_mutex = Mutex.new
+            # Lock for user code synchronization
+            # We do not want to mix coordinator lock with the user lock not to create cases where
+            # user imposed lock would lock the internal operations of Karafka
+            # This shared lock can be used by the end user as it is not used internally by the
+            # framework and can be used for user-facing locking
+            @shared_mutex = Mutex.new
+            @collapser = Collapser.new
+            @filter = Coordinators::FiltersApplier.new(self)
+
+            return unless topic.virtual_partitions?
+
+            @virtual_offset_manager = Coordinators::VirtualOffsetManager.new(
+              topic.name,
+              partition,
+              topic.virtual_partitions.offset_metadata_strategy
+            )
+
+            # We register our own "internal" filter to support filtering of messages that were marked
+            # as consumed virtually
+            @filter.filters << Filters::VirtualLimiter.new(
+              @virtual_offset_manager,
+              @collapser
+            )
+          end
+
+          # Starts the coordination process
+          # @param messages [Array<Karafka::Messages::Message>] messages for which processing we are
+          #   going to coordinate.
+          def start(messages)
+            super
+
+            @collapser.refresh!(messages.first.offset)
+
+            @filter.apply!(messages)
+
+            # Do not clear coordinator errors storage when we are retrying, so we can reference the
+            # errors that have happened during recovery. This can be useful for implementing custom
+            # flows. There can be more errors than one when running with virtual partitions so we
+            # need to make sure we collect them all. Under collapse when we reference a given
+            # consumer we should be able to get all the errors and not just first/last.
+            #
+            # @note We use zero as the attempt mark because we are not "yet" in the attempt 1
+            @errors_tracker.clear if attempt.zero?
+            @executed.clear
+
+            # We keep the old processed offsets until the collapsing is done and regular processing
+            # with virtualization is restored
+            @virtual_offset_manager.clear if topic.virtual_partitions? && !collapsed?
+
+            @last_message = messages.last
+          end
+
+          # Sets the consumer failure status and additionally starts the collapse until
+          #
+          # @param consumer [Karafka::BaseConsumer] consumer that failed
+          # @param error [StandardError] error from the failure
+          def failure!(consumer, error)
+            super
+            @errors_tracker << error
+            collapse_until!(@last_message.offset + 1)
+          end
+
+          # @return [Boolean] did any of the filters apply any logic that would cause use to run
+          #   the filtering flow
+          def filtered?
+            @filter.applied?
+          end
+
+          # @return [Boolean] is the coordinated work finished or not
+          # @note Used only in the consume operation context
+          def finished?
+            @running_jobs[:consume].zero?
+          end
+
+          # Runs synchronized code once for a collective of virtual partitions prior to work being
+          # enqueued
+          def on_enqueued
+            @flow_mutex.synchronize do
+              return unless executable?(:on_enqueued)
+
+              yield(@last_message)
+            end
+          end
+
+          # Runs given code only once per all the coordinated jobs upon starting first of them
+          def on_started
+            @flow_mutex.synchronize do
+              return unless executable?(:on_started)
+
+              yield(@last_message)
+            end
+          end
+
+          # Runs given code once when all the work that is suppose to be coordinated is finished
+          # It runs once per all the coordinated jobs and should be used to run any type of post
+          # jobs coordination processing execution
+          def on_finished
+            @flow_mutex.synchronize do
+              return unless finished?
+              return unless executable?(:on_finished)
+
+              yield(@last_message)
+            end
+          end
+
+          # Runs once after a partition is revoked
+          def on_revoked
+            @flow_mutex.synchronize do
+              return unless executable?(:on_revoked)
+
+              yield(@last_message)
+            end
+          end
+
+          # @param interval [Integer] milliseconds of activity
+          # @return [Boolean] was this partition in activity within last `interval` milliseconds
+          # @note Will return true also if currently active
+          def active_within?(interval)
+            # its always active if there's any job related to this coordinator that is still
+            # enqueued or running
+            return true if @running_jobs.values.any?(:positive?)
+
+            # Otherwise we check last time any job of this coordinator was active
+            @changed_at + interval > monotonic_now
+          end
+
+          private
+
+          # Checks if given action is executable once. If it is and true is returned, this method
+          # will return false next time it is used.
+          #
+          # @param action [Symbol] what action we want to perform
+          # @return [Boolean] true if we can
+          # @note This method needs to run behind a mutex.
+          def executable?(action)
+            return false if @executed.include?(action)
+
+            @executed << action
+
+            true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/coordinators/errors_tracker.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        # Namespace for Pro coordinator related sub-components
+        module Coordinators
+          # Object used to track errors in between executions to be able to build error-type based
+          # recovery flows.
+          class ErrorsTracker
+            include Enumerable
+
+            # @return [Karafka::Routing::Topic] topic of this error tracker
+            attr_reader :topic
+
+            # @return [Integer] partition of this error tracker
+            attr_reader :partition
+
+            # @return [Hash]
+            attr_reader :counts
+
+            # @return [String]
+            attr_reader :trace_id
+
+            # Max errors we keep in memory.
+            # We do not want to keep more because for DLQ-less this would cause memory-leaks.
+            # We do however count per class for granular error counting
+            STORAGE_LIMIT = 100
+
+            private_constant :STORAGE_LIMIT
+
+            # @param topic [Karafka::Routing::Topic]
+            # @param partition [Integer]
+            # @param limit [Integer] max number of errors we want to keep for reference when
+            #   implementing custom error handling.
+            # @note `limit` does not apply to the counts. They will work beyond the number of errors
+            #   occurring
+            def initialize(topic, partition, limit: STORAGE_LIMIT)
+              @errors = []
+              @counts = Hash.new { |hash, key| hash[key] = 0 }
+              @topic = topic
+              @partition = partition
+              @limit = limit
+              @trace_id = SecureRandom.uuid
+            end
+
+            # Clears all the errors
+            def clear
+              @errors.clear
+              @counts.clear
+            end
+
+            # @param error [StandardError] adds the error to the tracker
+            def <<(error)
+              @errors.shift if @errors.size >= @limit
+              @errors << error
+              @counts[error.class] += 1
+              @trace_id = SecureRandom.uuid
+            end
+
+            # @return [Boolean] is the error tracker empty
+            def empty?
+              @errors.empty?
+            end
+
+            # @return [Integer] number of elements
+            def size
+              # We use counts reference of all errors and not the `@errors` array because it allows
+              # us to go beyond the whole errors storage limit
+              @counts.values.sum
+            end
+
+            # @return [StandardError, nil] last error that occurred or nil if no errors
+            def last
+              @errors.last
+            end
+
+            # Iterates over errors
+            def each(&)
+              @errors.each(&)
+            end
+
+            # @return [Array<StandardError>] array with all the errors that occurred
+            def all
+              @errors
+            end
+          end
+        end
+      end
+    end
+  end
+end