karafka 2.5.2 → 2.5.4.rc1

data/lib/karafka/routing/subscription_groups_builder.rb

@@ -24,22 +24,38 @@ module Karafka
 
       private_constant :DISTRIBUTION_KEYS
 
+      # Initializes the subscription groups builder
       def initialize
         @position = -1
       end
 
       # @param topics [Karafka::Routing::Topics] all the topics based on which we want to build
       #   subscription groups
+      # @param base_position [Integer, nil] optional starting position for subscription groups.
+      #   When provided, positions will start from this value instead of continuing from the
+      #   global counter. This is used when rebuilding subscription groups to maintain stable
+      #   positions for static group membership.
       # @return [Array<SubscriptionGroup>] all subscription groups we need in separate threads
-      def call(topics)
+      def call(topics, base_position: nil)
+        # If base_position is provided, use it for stable rebuilding (consumer group reopening).
+        # Otherwise continue from global counter for new subscription groups.
+        # We subtract 1 from base_position because position is incremented in the map block below.
+        # This ensures the first subscription group gets the correct base_position value.
+        use_base = !base_position.nil?
+        position = use_base ? base_position - 1 : @position
+
         topics
           .map { |topic| [checksum(topic), topic] }
           .group_by(&:first)
           .values
           .map { |value| value.map(&:last) }
           .flat_map { |value| expand(value) }
-          .map { |grouped_topics| SubscriptionGroup.new(@position += 1, grouped_topics) }
+          .map { |grouped_topics| SubscriptionGroup.new(position += 1, grouped_topics) }
           .tap do |subscription_groups|
+            # Always ensure global counter is at least as high as the highest position used.
+            # This prevents position collisions when new consumer groups are created after
+            # existing ones are rebuilt with base_position.
+            @position = position if position > @position
             subscription_groups.each do |subscription_group|
               subscription_group.topics.each do |topic|
                 topic.subscription_group = subscription_group

data/lib/karafka/routing/topic.rb

@@ -29,7 +29,7 @@ module Karafka
 
       private_constant :INHERITABLE_ATTRIBUTES
 
-      # @param [String, Symbol] name of a topic on which we want to listen
+      # @param name [String, Symbol] name of a topic on which we want to listen
       # @param consumer_group [Karafka::Routing::ConsumerGroup] owning consumer group of this topic
       def initialize(name, consumer_group)
         @name = name.to_s
@@ -92,7 +92,7 @@ module Karafka
           # consumer class is defined with a name. It won't support code reload for anonymous
           # consumer classes, but this is an edge case
           begin
-            ::Object.const_get(@consumer.to_s)
+            Object.const_get(@consumer.to_s)
           rescue NameError
             # It will only fail if the in case of anonymous classes
             @consumer
@@ -139,7 +139,7 @@ module Karafka
           [attribute, public_send(attribute)]
         end
 
-        Hash[map].merge!(
+        map.to_h.merge!(
           id: id,
           name: name,
           active: active?,

data/lib/karafka/server.rb

@@ -174,7 +174,7 @@ module Karafka
           # This ensures that if users have configured the default pool, it is closed correctly
           #
           # Custom pools need to be closed by users themselves
-          ::WaterDrop::ConnectionPool.close
+          WaterDrop::ConnectionPool.close
 
           Karafka::App.terminate!
         end

data/lib/karafka/setup/attributes_map.rb

@@ -122,6 +122,7 @@ module Karafka
         sasl.oauthbearer.config
         sasl.oauthbearer.extensions
         sasl.oauthbearer.grant.type
+        sasl.oauthbearer.metadata.authentication.type
         sasl.oauthbearer.method
         sasl.oauthbearer.scope
         sasl.oauthbearer.token.endpoint.url
@@ -279,6 +280,7 @@ module Karafka
         sasl.oauthbearer.config
         sasl.oauthbearer.extensions
         sasl.oauthbearer.grant.type
+        sasl.oauthbearer.metadata.authentication.type
         sasl.oauthbearer.method
         sasl.oauthbearer.scope
         sasl.oauthbearer.token.endpoint.url
@@ -357,7 +359,7 @@ module Karafka
         end
 
         # @private
-        # @return [Hash<Symbol, Array<Symbol>>] hash with consumer and producer attributes list
+        # @return [Hash{Symbol => Array<Symbol>}] hash with consumer and producer attributes list
         #   that is sorted.
         # @note This method should not be used directly. It is only used to generate appropriate
         #   options list in case it would change
@@ -367,7 +369,7 @@ module Karafka
 
           attributes = { consumer: Set.new, producer: Set.new }
 
-          ::URI.parse(SOURCE).open.readlines.each do |line|
+          URI.parse(SOURCE).open.readlines.each do |line|
             next unless line.include?('|')
 
             attribute, attribute_type = line.split('|').map(&:strip)

data/lib/karafka/setup/config.rb

@@ -12,7 +12,7 @@ module Karafka
     #   enough and will still keep the code simple
     # @see Karafka::Setup::Configurators::Base for more details about configurators api
     class Config
-      extend ::Karafka::Core::Configurable
+      extend Karafka::Core::Configurable
 
       # Available settings
 
@@ -33,9 +33,9 @@ module Karafka
       #   Used only for logging.
       setting :client_id, default: 'karafka'
       # option logger [Instance] logger that we want to use
-      setting :logger, default: ::Karafka::Instrumentation::Logger.new
+      setting :logger, default: Karafka::Instrumentation::Logger.new
       # option monitor [Instance] monitor that we will to use (defaults to Karafka::Monitor)
-      setting :monitor, default: ::Karafka::Instrumentation::Monitor.new
+      setting :monitor, default: Karafka::Instrumentation::Monitor.new
       # option [Boolean] should we reload consumers with each incoming batch thus effectively
       # supporting code reload (if someone reloads code) or should we keep the persistence
       setting :consumer_persistence, default: true
@@ -211,7 +211,7 @@ module Karafka
         setting :cli do
           # option contract [Object] cli setup validation contract (in the context of options and
           # topics)
-          setting :contract, default: ::Karafka::Cli::Contracts::Server.new
+          setting :contract, default: Karafka::Cli::Contracts::Server.new
         end
 
         setting :routing do
@@ -345,7 +345,7 @@ module Karafka
           #   (incoming). Can be replaced with a custom implementation for formats like Avro,
           #   Protobuf, etc. This is a global setting because Rails serializes jobs before
           #   Karafka receives them, so we need a consistent approach across all ActiveJob topics.
-          setting :deserializer, default: ::Karafka::ActiveJob::Deserializer.new
+          setting :deserializer, default: Karafka::ActiveJob::Deserializer.new
         end
       end
 
@@ -415,17 +415,21 @@ module Karafka
           # of the pro defaults with custom components
           Pro::Loader.pre_setup_all(config) if Karafka.pro?
 
-          configure(&)
+          # Wrap config in a proxy that intercepts producer block configuration
+          proxy = ConfigProxy.new(config)
+          # We need to check for the block presence here because user can just run setup without
+          # any block given
+          configure { yield(proxy) if block_given? }
 
           Contracts::Config.new.validate!(
             config.to_h,
             scope: %w[config]
           )
 
-          configure_components
+          configure_components(proxy)
 
           # Refreshes the references that are cached that might have been changed by the config
-          ::Karafka.refresh!
+          Karafka.refresh!
 
           # Post-setup configure all routing features that would need this
           Routing::Features::Base.post_setup_all(config)
@@ -443,14 +447,16 @@ module Karafka
         private
 
         # Sets up all the components that are based on the user configuration
+        # @param config_proxy [ConfigProxy] the configuration proxy containing deferred setup
+        #   blocks
         # @note At the moment it is only WaterDrop
-        def configure_components
+        def configure_components(config_proxy)
           oauth_listener = config.oauth.token_provider_listener
           # We need to subscribe the oauth listener here because we want it to be ready before
           # any consumer/admin runs
           Karafka::App.monitor.subscribe(oauth_listener) if oauth_listener
 
-          config.producer ||= ::WaterDrop::Producer.new do |producer_config|
+          config.producer ||= WaterDrop::Producer.new do |producer_config|
             # In some cases WaterDrop updates the config and we don't want our consumer config to
             # be polluted by those updates, that's why we copy
             producer_kafka = AttributesMap.producer(config.kafka.dup)
@@ -463,6 +469,11 @@ module Karafka
             producer_config.oauth.token_provider_listener = oauth_listener
             producer_config.logger = config.logger
           end
+
+          # Execute user's producer configuration block
+          # This happens after the default producer setup, allowing users to customize settings
+          # If no block was provided during setup, this will be an empty lambda that does nothing
+          config_proxy.producer_initialization_block.call(config.producer.config)
         end
       end
     end

data/lib/karafka/setup/config_proxy.rb

@@ -0,0 +1,209 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Setup
+    # Configuration proxy that wraps the actual config object during the setup phase.
+    #
+    # ## Purpose
+    #
+    # This proxy exists to intercept specific configuration methods during the setup block
+    # execution, allowing for deferred initialization and special handling of certain
+    # configuration aspects without permanently modifying the config object's API.
+    #
+    # The key design principle is: **the proxy only exists during setup and doesn't pollute
+    # the permanent config API**. Once setup is complete, all access goes directly to the
+    # real config object.
+    #
+    # ## Why Use a Proxy?
+    #
+    # During Karafka setup, there's a specific order of operations:
+    #
+    # 1. User configures basic settings (kafka, client_id, etc.)
+    # 2. Config validation runs
+    # 3. Components are initialized based on finalized config
+    # 4. Post-setup hooks execute
+    #
+    # Some configuration needs to happen **after** user settings but **during** component
+    # initialization. The proxy intercepts these special cases during step 1, stores the
+    # instructions, and applies them during step 3.
+    #
+    # ## Current Use Case: Producer Configuration
+    #
+    # The proxy currently intercepts `producer` calls with blocks:
+    #
+    # ```ruby
+    # Karafka::App.setup do |config|
+    #   config.kafka = { 'bootstrap.servers': 'localhost:9092' }
+    #
+    #   # This is intercepted by the proxy
+    #   config.producer do |producer_config|
+    #     producer_config.kafka['compression.type'] = 'snappy'
+    #   end
+    # end
+    # ```
+    #
+    # Without the proxy, we'd have two problems:
+    #
+    # 1. **Permanent API pollution**: Adding a `producer` method to config that accepts blocks
+    #    would change its permanent API, even though this functionality is only needed during
+    #    setup.
+    #
+    # 2. **Timing issues**: The producer doesn't exist yet when the user's setup block runs.
+    #    The producer is created in `configure_components` after all user configuration is
+    #    complete. We need to store the user's producer configuration block and apply it
+    #    later at the right time.
+    #
+    # The proxy solves both:
+    # - It only exists during the setup call
+    # - It stores the producer configuration block in an instance variable
+    # - After setup, the block is passed to `configure_components` for execution
+    # - The proxy is then discarded
+    #
+    # ## Future Use Cases
+    #
+    # This pattern can be extended for other deferred configuration needs:
+    #
+    # - **Monitor configuration**: Intercept monitor setup to configure it after all components
+    #   are initialized
+    # - **Custom component initialization**: Allow users to configure internal components that
+    #   need access to the fully-configured environment
+    # - **Feature toggles**: Enable/disable features based on the complete configuration state
+    #
+    # ## Implementation Details
+    #
+    # The proxy uses Ruby's SimpleDelegator pattern:
+    # 1. Inherits from SimpleDelegator for automatic delegation
+    # 2. Implements specific interceptor methods (currently just `producer`)
+    # 3. Delegates everything else to the wrapped config automatically
+    # 4. Stores deferred configuration in instance variables
+    #
+    # ## Lifecycle
+    #
+    # ```
+    # Setup.setup(&block)
+    #   ↓
+    # proxy = ConfigProxy.new(config)  # Create proxy
+    #   ↓
+    # configure { yield(proxy) }       # User block receives proxy
+    #   ↓
+    # [User calls config methods]      # Most delegate to real config
+    #   ↓
+    # [User calls config.producer {}]  # Intercepted by proxy
+    #   ↓
+    # configure_components(proxy.producer_initialization_block)  # Block retrieved
+    #   ↓
+    # [Block executed with real producer config]
+    #   ↓
+    # proxy = nil                      # Proxy discarded
+    # ```
+    #
+    # ## Example Usage
+    #
+    # ```ruby
+    # class KarafkaApp < Karafka::App
+    #   setup do |config|
+    #     # Standard config access - delegated to real config
+    #     config.kafka = { 'bootstrap.servers': 'localhost:9092' }
+    #     config.client_id = 'my_app'
+    #
+    #     # Special intercepted method - handled by proxy
+    #     config.producer do |producer_config|
+    #       producer_config.kafka['compression.type'] = 'snappy'
+    #       producer_config.kafka['linger.ms'] = 10
+    #       producer_config.max_wait_timeout = 60_000
+    #     end
+    #   end
+    # end
+    # ```
+    #
+    # @see Karafka::Setup::Config.setup
+    # @see Karafka::Setup::Config.configure_components
+    class ConfigProxy < SimpleDelegator
+      # @return [Proc] the stored producer initialization block (defaults to empty lambda)
+      attr_reader :producer_initialization_block
+
+      # Creates a new configuration proxy wrapping the actual config object.
+      #
+      # Uses SimpleDelegator to automatically delegate all method calls to the wrapped config
+      # except for specifically intercepted methods like {#producer}.
+      #
+      # The producer initialization block defaults to an empty lambda, eliminating the need for
+      # nil checks when executing the block in {#configure_components}.
+      #
+      # @param config [Karafka::Setup::Config::Node] the actual config object to wrap
+      #
+      # @example
+      #   proxy = ConfigProxy.new(Karafka::App.config)
+      def initialize(config)
+        super
+        @producer_initialization_block = ->(_) {}
+      end
+
+      # Captures a block for producer configuration or delegates producer assignment to config.
+      #
+      # This method has dual behavior:
+      #
+      # 1. **With a block**: Stores the block for later execution after the producer is created.
+      #    This allows users to customize producer settings without manually creating a producer
+      #    instance.
+      #
+      # 2. **With an instance**: Delegates to `config.producer=` for direct producer assignment.
+      #    This preserves the existing API for users who want to provide their own producer.
+      #
+      # The block is stored in `@producer_initialization_block` and later passed to
+      # `configure_components` where it's executed with the producer's config object.
+      #
+      # ## Why This Exists
+      #
+      # The producer is created in `configure_components` after all user configuration is
+      # complete. This ensures the producer inherits the correct kafka settings. However,
+      # users may want to customize the producer further (add middleware, change timeouts,
+      # etc.) without creating their own producer instance.
+      #
+      # This method bridges the gap: it lets users configure the producer **as if it exists**
+      # during setup, but actually defers the configuration until after it's created.
+      #
+      # ## Block Execution Timing
+      #
+      # ```
+      # setup do |config|
+      #   config.kafka = { ... }        # Runs immediately
+      #
+      #   config.producer do |pc|       # Block STORED (not executed yet)
+      #     pc.kafka['...'] = '...'
+      #   end
+      # end
+      # # User block complete
+      # # Config validation runs
+      # # configure_components creates producer
+      # # NOW the stored block executes: block.call(producer.config)
+      # ```
+      #
+      # @param instance [WaterDrop::Producer, nil] optional producer instance for direct
+      #   assignment
+      # @param block [Proc] optional block for producer configuration. Will be called with
+      #   the producer's config object after the producer is created.
+      # @return [void]
+      #
+      # @example Configuring producer with a block
+      #   config.producer do |producer_config|
+      #     producer_config.kafka['compression.type'] = 'snappy'
+      #     producer_config.max_wait_timeout = 30_000
+      #     producer_config.middleware.append(MyMiddleware.new)
+      #   end
+      #
+      # @example Direct producer assignment
+      #   custom_producer = WaterDrop::Producer.new { |c| c.kafka = { 'bootstrap.servers' => 'localhost:9092' } }
+      #   config.producer = custom_producer
+      def producer(instance = nil, &block)
+        if block
+          # Store the configuration block for later execution
+          @producer_initialization_block = block
+        else
+          # Direct assignment - delegate to real config via __getobj__
+          __getobj__.producer = instance
+        end
+      end
+    end
+  end
+end

data/lib/karafka/setup/contracts/config.rb

@@ -18,7 +18,7 @@ module Karafka
         end
 
         # Topics regexp constant reference for easier usage
-        TOPIC_REGEXP = ::Karafka::Contracts::TOPIC_REGEXP
+        TOPIC_REGEXP = Karafka::Contracts::TOPIC_REGEXP
 
         private_constant :TOPIC_REGEXP
 

data/lib/karafka/swarm/liveness_listener.rb

@@ -13,6 +13,7 @@ module Karafka
         orphaned_exit_code: %i[internal swarm orphaned_exit_code]
       )
 
+      # Initializes the liveness listener
       def initialize
         @last_checked_at = 0
         @mutex = Mutex.new

data/lib/karafka/swarm/manager.rb

@@ -29,6 +29,7 @@ module Karafka
       # @return [Array<Node>] All nodes that manager manages
       attr_reader :nodes
 
+      # Initializes the swarm manager with empty nodes
       def initialize
         @nodes = []
         @statuses = Hash.new { |h, k| h[k] = {} }
@@ -97,7 +98,7 @@ module Karafka
       # If we've issued a stop to this process and it does not want to stop in the period, kills it
       #
       # @param statuses [Hash] hash with statuses transitions with times
-      # @param [Swarm::Node] node we're checking
+      # @param node [Swarm::Node] node we're checking
       # @return [Boolean] should it be the last action taken on this node in this run
       def terminate_if_hanging(statuses, node)
         return false unless statuses.key?(:stop)
@@ -118,7 +119,7 @@ module Karafka
       # reported it is not healthy.
       #
       # @param statuses [Hash] hash with statuses transitions with times
-      # @param [Swarm::Node] node we're checking
+      # @param node [Swarm::Node] node we're checking
       # @return [Boolean] should it be the last action taken on this node in this run
       def stop_if_not_healthy(statuses, node)
         status = node.status
@@ -146,7 +147,7 @@ module Karafka
       # If node stopped responding, starts the stopping procedure.
       #
       # @param statuses [Hash] hash with statuses transitions with times
-      # @param [Swarm::Node] node we're checking
+      # @param node [Swarm::Node] node we're checking
       # @return [Boolean] should it be the last action taken on this node in this run
       def stop_if_not_responding(statuses, node)
         # Do nothing if already stopping
@@ -171,7 +172,7 @@ module Karafka
       # Cleans up a dead process and remembers time of death for restart after a period.
       #
       # @param statuses [Hash] hash with statuses transitions with times
-      # @param [Swarm::Node] node we're checking
+      # @param node [Swarm::Node] node we're checking
       # @return [Boolean] should it be the last action taken on this node in this run
       def cleanup_one(statuses, node)
         return false if statuses.key?(:dead_since)
@@ -188,7 +189,7 @@ module Karafka
       # killed for some external reason.
       #
       # @param statuses [Hash] hash with statuses transitions with times
-      # @param [Swarm::Node] node we're checking
+      # @param node [Swarm::Node] node we're checking
       # @return [Boolean] should it be the last action taken on this node in this run
       def restart_after_timeout(statuses, node)
         return false unless over?(statuses[:dead_since], node_restart_timeout)
@@ -200,7 +201,7 @@ module Karafka
 
       # Starts a new node (or restarts dead)
       #
-      # @param [Swarm::Node] node we're starting
+      # @param node [Swarm::Node] node we're starting
       def start_one(node)
         instr_args = { caller: self, node: node }
 

data/lib/karafka/swarm/node.rb

@@ -73,7 +73,7 @@ module Karafka
           old_producer_config = old_producer.config
 
           # Supervisor producer is closed, hence we need a new one here
-          config.producer = ::WaterDrop::Producer.new do |p_config|
+          config.producer = WaterDrop::Producer.new do |p_config|
             p_config.kafka = Setup::AttributesMap.producer(kafka.dup)
             p_config.logger = config.logger
 

data/lib/karafka/swarm/supervisor.rb

@@ -34,6 +34,7 @@ module Karafka
 
       private_constant :SHUTDOWN_GRACE_PERIOD
 
+      # Initializes the swarm supervisor
       def initialize
         @mutex = Mutex.new
         @queue = Queue.new

data/lib/karafka/time_trackers/base.rb

@@ -8,7 +8,7 @@ module Karafka
   module TimeTrackers
     # Base class for all the time-trackers.
     class Base
-      include ::Karafka::Core::Helpers::Time
+      include Karafka::Core::Helpers::Time
     end
   end
 end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.5.2'
+  VERSION = '2.5.4.rc1'
 end

data/lib/karafka.rb

@@ -11,7 +11,6 @@ require 'fileutils'
 require 'openssl'
 require 'optparse'
 require 'socket'
-require 'base64'
 require 'date'
 require 'singleton'
 require 'digest'
@@ -34,7 +33,7 @@ module Karafka
       env.replace(environment.to_s)
     end
 
-    # @return [Logger] logger that we want to use. Will use ::Karafka::Logger by default
+    # @return [Logger] logger that we want to use. Will use Karafka::Logger by default
     def logger
       @logger ||= App.config.logger
     end
@@ -148,7 +147,7 @@ module Karafka
     #
     # This method refreshes the things that might have been altered by the configuration
     def refresh!
-      config = ::Karafka::App.config
+      config = Karafka::App.config
 
       @logger = config.logger
       @producer = config.producer