karafka 2.5.1 → 2.5.3

data/lib/karafka/routing/contracts/topic.rb

@@ -6,10 +6,8 @@ module Karafka
       # Consumer group topic validation rules.
       class Topic < Karafka::Contracts::Base
         configure do |config|
-          config.error_messages = YAML.safe_load(
-            File.read(
-              File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
-            )
+          config.error_messages = YAML.safe_load_file(
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
           ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
         end
 
@@ -68,10 +66,13 @@ module Karafka
 
         virtual do |data, errors|
           next unless errors.empty?
-          next unless ::Karafka::App.config.strict_topics_namespacing
+          next unless Karafka::App.config.strict_topics_namespacing
 
           value = data.fetch(:name)
-          namespacing_chars_count = value.chars.find_all { |c| ['.', '_'].include?(c) }.uniq.size
+          namespace_chars = ['.', '_'].freeze
+          namespacing_chars_count = value.chars.find_all do |c|
+            namespace_chars.include?(c)
+          end.uniq.size
 
           next if namespacing_chars_count <= 1
 

data/lib/karafka/routing/features/active_job/contracts/topic.rb

@@ -10,10 +10,8 @@ module Karafka
           # in order to be able to use active job routing
           class Topic < Karafka::Contracts::Base
             configure do |config|
-              config.error_messages = YAML.safe_load(
-                File.read(
-                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
-                )
+              config.error_messages = YAML.safe_load_file(
+                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
               ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
             end
 

data/lib/karafka/routing/features/active_job/topic.rb

@@ -5,6 +5,12 @@ module Karafka
     module Features
       class ActiveJob < Base
         # Topic extensions to be able to check if given topic is ActiveJob topic
+        #
+        # @note ActiveJob topics do not have per-topic deserializer configuration. The deserializer
+        #   is configured globally via `config.internal.active_job.deserializer` because Rails
+        #   serializes jobs before dispatching them, requiring a consistent serialization format
+        #   across all ActiveJob topics. If you need custom serialization (e.g., Avro, Protobuf),
+        #   configure it once at the application level rather than per-topic.
         module Topic
           # This method calls the parent class initializer and then sets up the
           # extra instance variable to nil. The explicit initialization

data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb

@@ -9,10 +9,8 @@ module Karafka
           # Rules around dead letter queue settings
           class Topic < Karafka::Contracts::Base
             configure do |config|
-              config.error_messages = YAML.safe_load(
-                File.read(
-                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
-                )
+              config.error_messages = YAML.safe_load_file(
+                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
               ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
             end
 
@@ -41,7 +39,7 @@ module Karafka
               next unless dead_letter_queue[:active]
 
               topic = dead_letter_queue[:topic]
-              topic_regexp = ::Karafka::Contracts::TOPIC_REGEXP
+              topic_regexp = Karafka::Contracts::TOPIC_REGEXP
 
               # When topic is set to false, it means we just want to skip dispatch on DLQ
               next if topic == false

data/lib/karafka/routing/features/declaratives/contracts/topic.rb

@@ -9,10 +9,8 @@ module Karafka
           # Basic validation of the Kafka expected config details
           class Topic < Karafka::Contracts::Base
             configure do |config|
-              config.error_messages = YAML.safe_load(
-                File.read(
-                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
-                )
+              config.error_messages = YAML.safe_load_file(
+                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
               ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
             end
 
@@ -22,7 +20,7 @@ module Karafka
               required(:replication_factor) { |val| val.is_a?(Integer) && val.positive? }
               required(:details) do |val|
                 val.is_a?(Hash) &&
-                  val.keys.all? { |key| key.is_a?(Symbol) }
+                  val.keys.all?(Symbol)
               end
             end
           end

data/lib/karafka/routing/features/declaratives/topic.rb

@@ -16,9 +16,12 @@ module Karafka
           end
 
           # @param active [Boolean] is the topic structure management feature active
-          # @param partitions [Integer]
-          # @param replication_factor [Integer]
+          # @param partitions [Integer] number of partitions for the topic
+          # @param replication_factor [Integer] replication factor for the topic
           # @param details [Hash] extra configuration for the topic
+          # @option details [String] :retention.ms retention time in milliseconds
+          # @option details [String] :compression.type compression type
+          #   (none, gzip, snappy, lz4, zstd)
           # @return [Config] defined structure
           def config(active: true, partitions: 1, replication_factor: 1, **details)
             @declaratives ||= Config.new(

data/lib/karafka/routing/features/deserializers/contracts/topic.rb

@@ -9,10 +9,8 @@ module Karafka
           # Basic validation of the Kafka expected config details
           class Topic < Karafka::Contracts::Base
             configure do |config|
-              config.error_messages = YAML.safe_load(
-                File.read(
-                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
-                )
+              config.error_messages = YAML.safe_load_file(
+                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
               ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
             end
 

data/lib/karafka/routing/features/deserializers/topic.rb

@@ -22,9 +22,9 @@ module Karafka
           # @param key [Object] deserializer for the message key
           # @param headers [Object] deserializer for the message headers
           def deserializers(
-            payload: ::Karafka::Deserializers::Payload.new,
-            key: ::Karafka::Deserializers::Key.new,
-            headers: ::Karafka::Deserializers::Headers.new
+            payload: Karafka::Deserializers::Payload.new,
+            key: Karafka::Deserializers::Key.new,
+            headers: Karafka::Deserializers::Headers.new
           )
             @deserializers ||= Config.new(
               active: true,

data/lib/karafka/routing/features/eofed/contracts/topic.rb

@@ -9,10 +9,8 @@ module Karafka
           # Contract for eofed topic setup
           class Topic < Karafka::Contracts::Base
             configure do |config|
-              config.error_messages = YAML.safe_load(
-                File.read(
-                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
-                )
+              config.error_messages = YAML.safe_load_file(
+                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
               ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
             end
 

data/lib/karafka/routing/features/inline_insights/contracts/topic.rb

@@ -9,10 +9,8 @@ module Karafka
           # Contract for inline insights topic setup
           class Topic < Karafka::Contracts::Base
             configure do |config|
-              config.error_messages = YAML.safe_load(
-                File.read(
-                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
-                )
+              config.error_messages = YAML.safe_load_file(
+                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
               ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
             end
 

data/lib/karafka/routing/features/inline_insights.rb

@@ -12,11 +12,11 @@ module Karafka
           #
           # @param _config [Karafka::Core::Configurable::Node] app config
           def post_setup(_config)
-            ::Karafka::App.monitor.subscribe('app.running') do
+            Karafka::App.monitor.subscribe('app.running') do
               # Do not activate tracking of statistics if none of our active topics uses it
               # This prevents us from tracking metrics when user just runs a subset of topics
               # in a given process and none of those actually utilizes this feature
-              next unless ::Karafka::App
+              next unless Karafka::App
                           .subscription_groups
                           .values
                           .flat_map(&:itself)
@@ -25,11 +25,11 @@ module Karafka
                           .any?(&:inline_insights?)
 
               # Initialize the tracker prior to becoming multi-threaded
-              ::Karafka::Processing::InlineInsights::Tracker.instance
+              Karafka::Processing::InlineInsights::Tracker.instance
 
               # Subscribe to the statistics reports and collect them
-              ::Karafka.monitor.subscribe(
-                ::Karafka::Processing::InlineInsights::Listener.new
+              Karafka.monitor.subscribe(
+                Karafka::Processing::InlineInsights::Listener.new
               )
             end
           end

data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb

@@ -9,10 +9,8 @@ module Karafka
           # Rules around manual offset management settings
           class Topic < Karafka::Contracts::Base
             configure do |config|
-              config.error_messages = YAML.safe_load(
-                File.read(
-                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
-                )
+              config.error_messages = YAML.safe_load_file(
+                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
               ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
             end
 

data/lib/karafka/routing/router.rb

@@ -8,7 +8,7 @@ module Karafka
     #  structure so all the routes are being stored in a single level array
     module Router
       # Finds first reference of a given topic based on provided lookup attribute
-      # @param lookup [Hash<Symbol, String>] hash with attribute - value key pairs
+      # @param lookup [Hash{Symbol => String}] hash with attribute - value key pairs
       # @return [Karafka::Routing::Topic, nil] proper route details or nil if not found
       def find_by(lookup)
         App.consumer_groups.each do |consumer_group|

data/lib/karafka/routing/subscription_group.rb

@@ -30,7 +30,7 @@ module Karafka
             @group_counter ||= 0
             @group_counter += 1
 
-            ::Digest::SHA256.hexdigest(
+            Digest::SHA256.hexdigest(
               @group_counter.to_s
             )[0..11]
           end

data/lib/karafka/routing/subscription_groups_builder.rb

@@ -24,6 +24,7 @@ module Karafka
 
       private_constant :DISTRIBUTION_KEYS
 
+      # Initializes the subscription groups builder
       def initialize
         @position = -1
       end

data/lib/karafka/routing/topic.rb

@@ -29,7 +29,7 @@ module Karafka
 
       private_constant :INHERITABLE_ATTRIBUTES
 
-      # @param [String, Symbol] name of a topic on which we want to listen
+      # @param name [String, Symbol] name of a topic on which we want to listen
       # @param consumer_group [Karafka::Routing::ConsumerGroup] owning consumer group of this topic
       def initialize(name, consumer_group)
         @name = name.to_s
@@ -92,7 +92,7 @@ module Karafka
           # consumer class is defined with a name. It won't support code reload for anonymous
           # consumer classes, but this is an edge case
           begin
-            ::Object.const_get(@consumer.to_s)
+            Object.const_get(@consumer.to_s)
           rescue NameError
             # It will only fail if the in case of anonymous classes
             @consumer
@@ -139,7 +139,7 @@ module Karafka
           [attribute, public_send(attribute)]
         end
 
-        Hash[map].merge!(
+        map.to_h.merge!(
           id: id,
           name: name,
           active: active?,

data/lib/karafka/routing/topics.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-# frozen_string_literal: true
-
 module Karafka
   module Routing
     # Abstraction layer on top of groups of topics
@@ -17,18 +15,15 @@ module Karafka
       end
 
       # Yields each topic
-      #
-      # @param [Proc] block we want to yield with on each topic
-      def each(&block)
-        @accumulator.each(&block)
+      def each(&)
+        @accumulator.each(&)
       end
 
       # Allows us to remove elements from the topics
       #
       # Block to decide what to delete
-      # @param block [Proc]
-      def delete_if(&block)
-        @accumulator.delete_if(&block)
+      def delete_if(&)
+        @accumulator.delete_if(&)
       end
 
       # Finds topic by its name

data/lib/karafka/server.rb

@@ -64,7 +64,7 @@ module Karafka
         process.on_sigterm { stop }
         process.on_sigtstp { quiet }
         # Needed for instrumentation
-        process.on_sigttin {}
+        process.on_sigttin { nil }
         process.supervise
 
         # This will only run when not in a swarm mode. In swarm mode the server runs post-fork, so
@@ -174,7 +174,7 @@ module Karafka
           # This ensures that if users have configured the default pool, it is closed correctly
           #
           # Custom pools need to be closed by users themselves
-          ::WaterDrop::ConnectionPool.close
+          WaterDrop::ConnectionPool.close
 
           Karafka::App.terminate!
         end

data/lib/karafka/setup/attributes_map.rb

@@ -122,6 +122,7 @@ module Karafka
         sasl.oauthbearer.config
         sasl.oauthbearer.extensions
         sasl.oauthbearer.grant.type
+        sasl.oauthbearer.metadata.authentication.type
         sasl.oauthbearer.method
         sasl.oauthbearer.scope
         sasl.oauthbearer.token.endpoint.url
@@ -279,6 +280,7 @@ module Karafka
         sasl.oauthbearer.config
         sasl.oauthbearer.extensions
         sasl.oauthbearer.grant.type
+        sasl.oauthbearer.metadata.authentication.type
         sasl.oauthbearer.method
         sasl.oauthbearer.scope
         sasl.oauthbearer.token.endpoint.url
@@ -357,7 +359,7 @@ module Karafka
         end
 
         # @private
-        # @return [Hash<Symbol, Array<Symbol>>] hash with consumer and producer attributes list
+        # @return [Hash{Symbol => Array<Symbol>}] hash with consumer and producer attributes list
         #   that is sorted.
         # @note This method should not be used directly. It is only used to generate appropriate
         #   options list in case it would change
@@ -367,7 +369,7 @@ module Karafka
 
           attributes = { consumer: Set.new, producer: Set.new }
 
-          ::URI.parse(SOURCE).open.readlines.each do |line|
+          URI.parse(SOURCE).open.readlines.each do |line|
             next unless line.include?('|')
 
             attribute, attribute_type = line.split('|').map(&:strip)

data/lib/karafka/setup/config.rb

@@ -12,7 +12,7 @@ module Karafka
     #   enough and will still keep the code simple
     # @see Karafka::Setup::Configurators::Base for more details about configurators api
     class Config
-      extend ::Karafka::Core::Configurable
+      extend Karafka::Core::Configurable
 
       # Available settings
 
@@ -33,9 +33,9 @@ module Karafka
       #   Used only for logging.
       setting :client_id, default: 'karafka'
       # option logger [Instance] logger that we want to use
-      setting :logger, default: ::Karafka::Instrumentation::Logger.new
+      setting :logger, default: Karafka::Instrumentation::Logger.new
       # option monitor [Instance] monitor that we will to use (defaults to Karafka::Monitor)
-      setting :monitor, default: ::Karafka::Instrumentation::Monitor.new
+      setting :monitor, default: Karafka::Instrumentation::Monitor.new
       # option [Boolean] should we reload consumers with each incoming batch thus effectively
       # supporting code reload (if someone reloads code) or should we keep the persistence
       setting :consumer_persistence, default: true
@@ -52,12 +52,16 @@ module Karafka
       setting :shutdown_timeout, default: 60_000
       # option [Integer] number of threads in which we want to do parallel processing
       setting :concurrency, default: 5
-      # option [Integer] how long should we wait upon processing error (milliseconds)
-      setting :pause_timeout, default: 1_000
-      # option [Integer] what is the max timeout in case of an exponential backoff (milliseconds)
-      setting :pause_max_timeout, default: 30_000
-      # option [Boolean] should we use exponential backoff
-      setting :pause_with_exponential_backoff, default: true
+      # Namespace for pause-related settings
+      setting :pause do
+        # option [Integer] how long should we wait upon processing error (milliseconds)
+        setting :timeout, default: 1_000
+        # option [Integer] what is the max timeout in case of an exponential backoff (milliseconds)
+        setting :max_timeout, default: 30_000
+        # option [Boolean] should we use exponential backoff
+        setting :with_exponential_backoff, default: true
+      end
+
       # option [::WaterDrop::Producer, nil]
       # Unless configured, will be created once Karafka is configured based on user Karafka setup
       setting :producer, default: nil
@@ -207,7 +211,7 @@ module Karafka
         setting :cli do
           # option contract [Object] cli setup validation contract (in the context of options and
           # topics)
-          setting :contract, default: ::Karafka::Cli::Contracts::Server.new
+          setting :contract, default: Karafka::Cli::Contracts::Server.new
         end
 
         setting :routing do
@@ -336,6 +340,12 @@ module Karafka
           setting :job_options_contract, default: ActiveJob::JobOptionsContract.new
           # option consumer [Class] consumer class that should be used to consume ActiveJob data
           setting :consumer_class, default: ActiveJob::Consumer
+          # option deserializer [Karafka::ActiveJob::Deserializer] deserializer for ActiveJob jobs
+          #   Despite the name, handles both serialization (outgoing) and deserialization
+          #   (incoming). Can be replaced with a custom implementation for formats like Avro,
+          #   Protobuf, etc. This is a global setting because Rails serializes jobs before
+          #   Karafka receives them, so we need a consistent approach across all ActiveJob topics.
+          setting :deserializer, default: Karafka::ActiveJob::Deserializer.new
         end
       end
 
@@ -343,10 +353,57 @@ module Karafka
       # Thanks to that we have an initial state out of the box.
       configure
 
+      # Backwards compatibility: Add old flat API methods to the config instance
+      # These delegate to the new nested pause config
+      # @deprecated Will be removed in Karafka 2.6
+      #
+      # Prior to the introduction of nested pause configuration, pause-related settings were
+      # accessed directly on the config object (e.g., `config.pause_timeout`). With the nested
+      # structure introduced, these settings moved to `config.pause.timeout`, etc.
+      #
+      # This instance_eval block adds delegation methods to maintain backwards compatibility,
+      # allowing existing code using the old flat API to continue working without modification.
+      config.instance_eval do
+        # @return [Integer] delegated timeout value from pause.timeout
+        # @deprecated Use config.pause.timeout instead
+        def pause_timeout
+          pause.timeout
+        end
+
+        # @param value [Integer] timeout value to set
+        # @deprecated Use config.pause.timeout= instead
+        def pause_timeout=(value)
+          pause.timeout = value
+        end
+
+        # @return [Integer] delegated max_timeout value from pause.max_timeout
+        # @deprecated Use config.pause.max_timeout instead
+        def pause_max_timeout
+          pause.max_timeout
+        end
+
+        # @param value [Integer] max timeout value to set
+        # @deprecated Use config.pause.max_timeout= instead
+        def pause_max_timeout=(value)
+          pause.max_timeout = value
+        end
+
+        # @return [Boolean] delegated exponential backoff flag from pause.with_exponential_backoff
+        # @deprecated Use config.pause.with_exponential_backoff instead
+        def pause_with_exponential_backoff
+          pause.with_exponential_backoff
+        end
+
+        # @param value [Boolean] exponential backoff flag to set
+        # @deprecated Use config.pause.with_exponential_backoff= instead
+        def pause_with_exponential_backoff=(value)
+          pause.with_exponential_backoff = value
+        end
+      end
+
       class << self
         # Configuring method
-        # @param block [Proc] block we want to execute with the config instance
-        def setup(&block)
+        def setup(&)
           # Will prepare and verify license if present
           Licenser.prepare_and_verify(config.license)
 
@@ -358,17 +415,21 @@ module Karafka
           # of the pro defaults with custom components
           Pro::Loader.pre_setup_all(config) if Karafka.pro?
 
-          configure(&block)
+          # Wrap config in a proxy that intercepts producer block configuration
+          proxy = ConfigProxy.new(config)
+          # We need to check for the block presence here because user can just run setup without
+          # any block given
+          configure { yield(proxy) if block_given? }
 
           Contracts::Config.new.validate!(
             config.to_h,
             scope: %w[config]
           )
 
-          configure_components
+          configure_components(proxy)
 
           # Refreshes the references that are cached that might have been changed by the config
-          ::Karafka.refresh!
+          Karafka.refresh!
 
           # Post-setup configure all routing features that would need this
           Routing::Features::Base.post_setup_all(config)
@@ -386,14 +447,16 @@ module Karafka
         private
 
         # Sets up all the components that are based on the user configuration
+        # @param config_proxy [ConfigProxy] the configuration proxy containing deferred setup
+        #   blocks
         # @note At the moment it is only WaterDrop
-        def configure_components
+        def configure_components(config_proxy)
           oauth_listener = config.oauth.token_provider_listener
           # We need to subscribe the oauth listener here because we want it to be ready before
           # any consumer/admin runs
           Karafka::App.monitor.subscribe(oauth_listener) if oauth_listener
 
-          config.producer ||= ::WaterDrop::Producer.new do |producer_config|
+          config.producer ||= WaterDrop::Producer.new do |producer_config|
             # In some cases WaterDrop updates the config and we don't want our consumer config to
             # be polluted by those updates, that's why we copy
             producer_kafka = AttributesMap.producer(config.kafka.dup)
@@ -406,6 +469,11 @@ module Karafka
             producer_config.oauth.token_provider_listener = oauth_listener
             producer_config.logger = config.logger
           end
+
+          # Execute user's producer configuration block
+          # This happens after the default producer setup, allowing users to customize settings
+          # If no block was provided during setup, this will be an empty lambda that does nothing
+          config_proxy.producer_initialization_block.call(config.producer.config)
         end
       end
     end