karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/routing/features/consumer_groups/virtual_partitions/topic.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class VirtualPartitions < Base
+            # Topic extensions to be able to manage virtual partitions feature
+            module Topic
+              # This method sets up the extra instance variable to nil before calling
+              # the parent class initializer. The explicit initialization
+              # to nil is included as an optimization for Ruby's object shapes system,
+              # which improves memory layout and access performance.
+              def initialize(...)
+                @virtual_partitions = nil
+                super
+              end
+
+              # @param max_partitions [Integer] max number of virtual partitions that can come out of
+              #   the single distribution flow. When set to more than the Karafka threading, will
+              #   create more work than workers. When less, can ensure we have spare resources to
+              #   process other things in parallel.
+              # @param partitioner [nil, #call] nil or callable partitioner
+              # @param offset_metadata_strategy [Symbol] how we should match the metadata for the
+              #   offset. `:exact` will match the offset matching metadata and `:current` will select
+              #   the most recently reported metadata
+              # @param reducer [nil, #call] reducer for VPs key. It allows for using a custom
+              #   reducer to achieve enhanced parallelization when the default reducer is not enough.
+              # @param distribution [Symbol] the strategy to use for virtual partitioning. Can be
+              #   either `:consistent` or `:balanced`. The `:balanced` strategy ensures balanced
+              #   distribution of work across available workers while maintaining message order
+              #   within groups.
+              # @return [VirtualPartitions] method that allows to set the virtual partitions details
+              #   during the routing configuration and then allows to retrieve it
+              def virtual_partitions(
+                max_partitions: Karafka::App.config.concurrency,
+                partitioner: nil,
+                offset_metadata_strategy: :current,
+                reducer: nil,
+                distribution: :consistent
+              )
+                @virtual_partitions ||= Config.new(
+                  active: !partitioner.nil?,
+                  max_partitions: max_partitions,
+                  partitioner: partitioner,
+                  offset_metadata_strategy: offset_metadata_strategy,
+                  # If no reducer provided, we use this one. It just runs a modulo on the sum of
+                  # a stringified version, providing fairly good distribution.
+                  reducer: reducer || ->(virtual_key) { virtual_key.to_s.sum % max_partitions },
+                  distribution: distribution
+                )
+              end
+
+              # @return [Boolean] are virtual partitions enabled for given topic
+              def virtual_partitions?
+                virtual_partitions.active?
+              end
+
+              # @return [Hash] topic with all its native configuration options plus manual offset
+              #   management namespace settings
+              def to_h
+                super.merge(
+                  virtual_partitions: virtual_partitions.to_h
+                ).freeze
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/virtual_partitions.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          # Virtual Partitions feature config and DSL namespace.
+          #
+          # Virtual Partitions allow you to parallelize the processing of data from a single
+          # partition. This can drastically increase throughput when IO operations are involved.
+          class VirtualPartitions < Base
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{scheduled_messages.rb → consumer_groups.rb}

@@ -32,8 +32,9 @@ module Karafka
   module Pro
     module Routing
       module Features
-        # Scheduled messages functionality routing namespace
-        class ScheduledMessages < Base
+        # Consumer-group-specific Pro routing features. A parallel `ShareGroups` namespace will
+        # hold share-group-specific feature implementations once KIP-932 lands.
+        module ConsumerGroups
         end
       end
     end

data/lib/karafka/pro/routing/features/expiring/topic.rb

@@ -35,13 +35,13 @@ module Karafka
         class Expiring < Base
           # Topic expiring API extensions
           module Topic
-            # This method calls the parent class initializer and then sets up the
-            # extra instance variable to nil. The explicit initialization
+            # This method sets up the extra instance variable to nil before calling
+            # the parent class initializer. The explicit initialization
             # to nil is included as an optimization for Ruby's object shapes system,
             # which improves memory layout and access performance.
             def initialize(...)
-              super
               @expiring = nil
+              super
             end
 
             # @param ttl [Integer, nil] maximum time in ms a message is considered alive
@@ -51,7 +51,7 @@ module Karafka
                 config = Config.new(active: !ttl.nil?, ttl: ttl)
 
                 if config.active?
-                  factory = ->(*) { Pro::Processing::Filters::Expirer.new(ttl) }
+                  factory = ->(*) { Pro::Processing::ConsumerGroups::Filters::Expirer.new(ttl) }
                   filter(factory)
                 end
 

data/lib/karafka/pro/routing/features/filtering/topic.rb

@@ -35,13 +35,13 @@ module Karafka
         class Filtering < Base
           # Filtering feature topic extensions
           module Topic
-            # This method calls the parent class initializer and then sets up the
-            # extra instance variable to nil. The explicit initialization
+            # This method sets up the extra instance variable to nil before calling
+            # the parent class initializer. The explicit initialization
             # to nil is included as an optimization for Ruby's object shapes system,
             # which improves memory layout and access performance.
             def initialize(...)
-              super
               @filtering = nil
+              super
             end
 
             # @param factory [#call, nil] Callable that can produce new filters instances per

data/lib/karafka/pro/routing/features/pausing/topic.rb

@@ -35,13 +35,13 @@ module Karafka
         class Pausing < Base
           # Expansion allowing for a per topic pause strategy definitions
           module Topic
-            # This method calls the parent class initializer and then sets up the
-            # extra instance variable to nil. The explicit initialization
+            # This method sets up the extra instance variable to nil before calling
+            # the parent class initializer. The explicit initialization
             # to nil is included as an optimization for Ruby's object shapes system,
             # which improves memory layout and access performance.
             def initialize(...)
-              super
               @pausing = nil
+              super
             end
 
             # Allows for per-topic pausing strategy setting

data/lib/karafka/pro/routing/features/throttling/topic.rb

@@ -35,13 +35,13 @@ module Karafka
         class Throttling < Base
           # Topic throttling API extensions
           module Topic
-            # This method calls the parent class initializer and then sets up the
-            # extra instance variable to nil. The explicit initialization
+            # This method sets up the extra instance variable to nil before calling
+            # the parent class initializer. The explicit initialization
             # to nil is included as an optimization for Ruby's object shapes system,
             # which improves memory layout and access performance.
             def initialize(...)
-              super
               @throttling = nil
+              super
             end
 
             # @param limit [Integer] max messages to process in an time interval
@@ -61,7 +61,7 @@ module Karafka
                 # If someone defined throttling setup, we need to create appropriate filter for it
                 # and inject it via filtering feature
                 if config.active?
-                  factory = ->(*) { Pro::Processing::Filters::Throttler.new(limit, interval) }
+                  factory = ->(*) { Pro::Processing::ConsumerGroups::Filters::Throttler.new(limit, interval) }
                   filter(factory)
                 end
 

data/lib/karafka/pro/setup/defaults_injector.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    # Namespace for Pro setup components
+    module Setup
+      # Pro defaults injector that extends the OSS defaults with Pro-specific settings
+      module DefaultsInjector
+        # Pro-specific consumer kafka defaults
+        # These defaults are carefully tuned to work with Pro's internal statistics aggregation,
+        # the Web UI dashboard, and the performance tracker. They depend on Pro's extended
+        # instrumentation pipeline and should not be applied outside of Pro as they may cause
+        # incomplete or inconsistent metrics collection and other unexpected behaviours.
+        CONSUMER_KAFKA_DEFAULTS = {
+          "statistics.unassigned.include": false
+        }.freeze
+
+        private_constant :CONSUMER_KAFKA_DEFAULTS
+
+        # Pro actively manages these keys via its own DefaultsInjector so users are allowed
+        # to set them if needed.
+        #
+        # @return [Set<Symbol>] empty set since Pro handles these keys
+        def managed_keys
+          @managed_keys ||= Set.new
+        end
+
+        # Enriches consumer kafka config with Pro-specific defaults
+        # @param kafka_config [Hash] kafka scoped config
+        def consumer(kafka_config)
+          super
+
+          CONSUMER_KAFKA_DEFAULTS.each do |key, value|
+            next if kafka_config.key?(key)
+
+            kafka_config[key] = value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/swarm/liveness_listener.rb

@@ -73,6 +73,13 @@ module Karafka
           super()
         end
 
+        # Report status before the first fetch loop iteration so the supervisor gets an initial
+        # healthy report even if the first consumption takes longer than the report timeout.
+        # @param _event [Karafka::Core::Monitoring::Event]
+        def on_connection_listener_before_fetch_loop(_event)
+          report_status
+        end
+
         # Tick on each fetch and report liveness so it works even when statistics are disabled
         #
         # @param _event [Karafka::Core::Monitoring::Event]
@@ -143,42 +150,47 @@ module Karafka
           periodically do
             return unless node
 
+            Kernel.exit!(orphaned_exit_code) if node.orphaned?
+
             current_status = status
 
             current_status.positive? ? node.unhealthy(current_status) : node.healthy
           end
         end
 
-        # @return [Integer] object id of the current thread
-        def thread_id
-          Thread.current.object_id
+        # @return [Integer] object id of the current fiber
+        # @note We use fiber object id instead of thread object id to ensure fiber-safety.
+        #   Multiple fibers can run on the same thread, and using thread id would cause them
+        #   to overwrite each other's timestamps.
+        def fiber_id
+          Fiber.current.object_id
         end
 
-        # Update the polling tick time for current thread
+        # Update the polling tick time for current fiber
         def mark_polling_tick
           synchronize do
-            @pollings[thread_id] = monotonic_now
+            @pollings[fiber_id] = monotonic_now
           end
         end
 
-        # Clear current thread polling time tracker
+        # Clear current fiber polling time tracker
         def clear_polling_tick
           synchronize do
-            @pollings.delete(thread_id)
+            @pollings.delete(fiber_id)
           end
         end
 
         # Update the processing tick time
         def mark_consumption_tick
           synchronize do
-            @consumptions[thread_id] = monotonic_now
+            @consumptions[fiber_id] = monotonic_now
           end
         end
 
-        # Clear current thread consumption time tracker
+        # Clear current fiber consumption time tracker
         def clear_consumption_tick
           synchronize do
-            @consumptions.delete(thread_id)
+            @consumptions.delete(fiber_id)
           end
         end
 

data/lib/karafka/processing/consumer_groups/coordinator.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
+    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      # Basic coordinator that allows us to provide coordination objects into consumers.
+      #
+      # This is a wrapping layer to simplify management of work to be handled around consumption.
+      #
+      # @note This coordinator needs to be thread safe. Some operations are performed only in the
+      #   listener thread, but we go with thread-safe by default for all not to worry about potential
+      #   future mistakes.
+      class Coordinator
+        extend Forwardable
+        include Core::Helpers::Time
+
+        attr_reader :pause_tracker, :seek_offset, :topic, :partition
+
+        # This can be set directly on the listener because it can be triggered on first run without
+        # any messages
+        attr_accessor :eofed
+
+        # Last polled at time set based on the incoming last poll time
+        attr_accessor :last_polled_at
+
+        def_delegators :@pause_tracker, :attempt, :paused?
+
+        # @param topic [Karafka::Routing::Topic]
+        # @param partition [Integer]
+        # @param pause_tracker [Karafka::TimeTrackers::Pause] pause tracker for given topic partition
+        def initialize(topic, partition, pause_tracker)
+          @topic = topic
+          @partition = partition
+          @pause_tracker = pause_tracker
+          @revoked = false
+          @consumptions = {}
+          @running_jobs = Hash.new { |h, k| h[k] = 0 }
+          @manual_pause = false
+          @manual_seek = false
+          @mutex = Mutex.new
+          @marked = false
+          @failure = false
+          @eofed = false
+          @changed_at = monotonic_now
+          @last_polled_at = @changed_at
+        end
+
+        # Starts the coordinator for given consumption jobs
+        # @param messages [Array<Karafka::Messages::Message>] batch of message for which we are
+        #   going to coordinate work. Not used with regular coordinator.
+        def start(messages)
+          @failure = false
+          @running_jobs[:consume] = 0
+          # We need to clear the consumption results hash here, otherwise we could end up storing
+          # consumption results of consumer instances we no longer control
+          @consumptions.clear
+
+          # When starting to run, no pause is expected and no manual pause as well
+          @manual_pause = false
+
+          # No user invoked seeks on a new run
+          @manual_seek = false
+
+          # We set it on the first encounter and never again, because then the offset setting
+          # should be up to the consumers logic (our or the end user)
+          # Seek offset needs to be always initialized as for case where manual offset management
+          # is turned on, we need to have reference to the first offset even in case of running
+          # multiple batches without marking any messages as consumed. Rollback needs to happen to
+          # the last place we know of or the last message + 1 that was marked
+          #
+          # It is however worth keeping in mind, that this may need to be used with `#marked?` to
+          # make sure that the first offset is an offset that has been marked.
+          @seek_offset ||= messages.first.offset
+        end
+
+        # @param offset [Integer] message offset
+        def seek_offset=(offset)
+          synchronize do
+            @marked = true
+            @seek_offset = offset
+          end
+        end
+
+        # Increases number of jobs that we handle with this coordinator
+        # @param job_type [Symbol] type of job that we want to increment
+        def increment(job_type)
+          synchronize do
+            @running_jobs[job_type] += 1
+            @changed_at = monotonic_now
+          end
+        end
+
+        # Decrements number of jobs we handle at the moment
+        # @param job_type [Symbol] type of job that we want to decrement
+        def decrement(job_type)
+          synchronize do
+            @running_jobs[job_type] -= 1
+            @changed_at = monotonic_now
+
+            return @running_jobs[job_type] unless @running_jobs[job_type].negative?
+
+            # This should never happen. If it does, something is heavily out of sync. Please reach
+            # out to us if you encounter this
+            raise Karafka::Errors::InvalidCoordinatorStateError, "Was zero before decrementation"
+          end
+        end
+
+        # Is all the consumption done and finished successfully for this coordinator
+        # We do not say we're successful until all work is done, because running work may still
+        # crash.
+        # @note This is only used for consume synchronization
+        def success?
+          synchronize do
+            @running_jobs[:consume].zero? && @consumptions.values.all?(&:success?)
+          end
+        end
+
+        # Mark given consumption on consumer as successful
+        # @param consumer [Karafka::BaseConsumer] consumer that finished successfully
+        def success!(consumer)
+          synchronize do
+            consumption(consumer).success!
+          end
+        end
+
+        # Mark given consumption on consumer as failed
+        # @param consumer [Karafka::BaseConsumer] consumer that failed
+        # @param error [StandardError] error that occurred
+        def failure!(consumer, error)
+          synchronize do
+            @failure = true
+            consumption(consumer).failure!(error)
+          end
+        end
+
+        # @return [Boolean] true if any of work we were running failed
+        def failure?
+          @failure
+        end
+
+        # Marks given coordinator for processing group as revoked
+        #
+        # This is invoked in two places:
+        #   - from the main listener loop when we detect revoked partitions
+        #   - from the consumer in case checkpointing fails
+        #
+        # This means, we can end up having consumer being aware that it was revoked prior to the
+        # listener loop dispatching the revocation job. It is ok, as effectively nothing will be
+        # processed until revocation jobs are done.
+        def revoke
+          synchronize { @revoked = true }
+        end
+
+        # @return [Boolean] is the partition we are processing revoked or not
+        def revoked?
+          @revoked
+        end
+
+        # @return [Boolean] did we reach end of partition when polling data
+        def eofed?
+          @eofed
+        end
+
+        # @return [Boolean] was the new seek offset assigned at least once. This is needed because
+        #   by default we assign seek offset of a first message ever, however this is insufficient
+        #   for DLQ in a scenario where the first message would be broken. We would never move
+        #   out of it and would end up in an endless loop.
+        def marked?
+          @marked
+        end
+
+        # Store in the coordinator info, that this pause was done manually by the end user and not
+        # by the system itself
+        def manual_pause
+          @manual_pause = true
+        end
+
+        # @return [Boolean] are we in a pause that was initiated by the user
+        def manual_pause?
+          paused? && @manual_pause
+        end
+
+        # Marks seek as manual for coordination purposes
+        def manual_seek
+          @manual_seek = true
+        end
+
+        # @return [Boolean] did a user invoke seek in the current operations scope
+        def manual_seek?
+          @manual_seek
+        end
+
+        # @param consumer [Object] karafka consumer (normal or pro)
+        # @return [Karafka::Processing::Result] result object which we can use to indicate
+        #   consumption processing state.
+        def consumption(consumer)
+          @consumptions[consumer] ||= Processing::Result.new
+        end
+
+        # Allows to run synchronized (locked) code that can operate only from a given thread
+        #
+        # @note We check if mutex is not owned already by the current thread so we won't end up with
+        #   a deadlock in case user runs coordinated code from inside of his own lock
+        #
+        # @note This is internal and should **not** be used to synchronize user-facing code.
+        #   Otherwise user indirectly could cause deadlocks or prolonged locks by running his logic.
+        #   This can and should however be used for multi-thread strategy applications and other
+        #   internal operations locks.
+        def synchronize(&)
+          if @mutex.owned?
+            yield
+          else
+            @mutex.synchronize(&)
+          end
+        end
+      end
+    end
+  end
+end