karafka 2.6.0.rc1 → 2.6.0.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc797708a61eca1cc0309dce91d05ab1dd623f3c7f6abfd74ac7af69c98f5254
-  data.tar.gz: 9e0e3bace7cc9f5393dfe788035ddc6e89cd64c8afd7ed5a2400d841cce95637
+  metadata.gz: d69bd2578f4c712b61a5d1a8eab6448337c0c5e88324fa4e3b87c1593f444b9f
+  data.tar.gz: c8652e3bc72ea6e5e7d14b89cbdcc5c33ff42519c10c515dc5a4426bd2770342
 SHA512:
-  metadata.gz: 512bed7912b22916225f0428cf1ef949e534765eaca459606155e451566e847c3c0993a83978f230f72ac202e2d6e797415df4086ed00b37bfdbcdb94f5f4e06
-  data.tar.gz: f0152dec64b5ba30ebae473db2236848b6a3298473be0dec7956330e44e973593311a70e616eb68cc8705fdbcdf55a53eff1d54a230658a0307d7ba6b00cb413
+  metadata.gz: d59922e671375a20a74f46b8b8efe74e6f779367977c94e2c7c3ce42fdcb4dfe2ef1cd1d3a46689bf7acd00cfc6498c9f62ea39d78df4b1fc6b2c2dff9dfd073
+  data.tar.gz: b484c30e72d838db5d2d4fcc8183d80980cbcbdcd85b3a8214c9165e990577e837ea8824c0e7833a9a6dcf567d6c05df5a7339cc94b0ede1004395f95c701ff7

data/CHANGELOG.md

@@ -1,6 +1,8 @@
 # Karafka Framework Changelog
 
 ## 2.6.0 (Unreleased)
+- **[Breaking]** Remove the flat global pause configuration accessors (`config.pause_timeout`, `config.pause_max_timeout`, `config.pause_with_exponential_backoff`) deprecated in 2.5.2. Use the nested `config.pause.*` namespace (`config.pause.timeout`, `config.pause.max_timeout`, `config.pause.with_exponential_backoff`) instead.
+- **[Breaking]** Nest the per-topic pause configuration under `topic.pause`. The flat `topic.pause_timeout`, `topic.pause_max_timeout`, and `topic.pause_with_exponential_backoff` readers are removed in favor of `topic.pause.timeout`, `topic.pause.max_timeout`, and `topic.pause.with_exponential_backoff`; `topic.to_h` now emits a nested `pause:` hash instead of the flat keys. The Pro Granular Backoffs override DSL is unchanged (`pause(timeout:, max_timeout:, with_exponential_backoff:)`), and `topic.pausing`/`topic.pausing?` are now `topic.pause`/`topic.pause?`. In OSS `topic.pause` reflects the global `config.pause.*` settings; per-topic overriding remains a Pro feature.
 - **[Feature]** Add `Karafka::Admin.read_partition_offsets` (and `Admin::Topics#read_partition_offsets`) exposing the `rd_kafka_ListOffsets` admin API to query partition offsets by spec (`:earliest`, `:latest`, `:max_timestamp`, or a timestamp in ms) without a consumer group. Accepts an optional `isolation_level:` keyword (pass `READ_COMMITTED` for the Last Stable Offset instead of the high-watermark, giving accurate lag on transactional topics).
 - **[Feature]** Add `Karafka::Connection::Client#read_partition_offsets` (and `Connection::Proxy#read_partition_offsets`) exposing the batched `ListOffsets` query on a consumer's own connection, forwarding its `isolation.level`. Note that `:latest` resolves to the high watermark regardless of isolation level, so it includes uncommitted messages on topics with in-flight transactions.
 - **[Feature]** Allow `Karafka::Admin` (and subsystems like `Admin::ConsumerGroups`, `Admin::Topics`) to operate on an external client via `Karafka::Admin.new(external_client: client)`. Reuses the caller's client (no per-call bootstrap); its lifecycle stays with its owner.

data/config/locales/pro_errors.yml

@@ -32,11 +32,11 @@ en:
         delaying.delay_format: 'needs to be equal or more than 0 and an integer'
         delaying.active_format: 'needs to be boolean'
 
-        pausing.active_format: 'needs to be boolean'
-        pausing.timeout_format: needs to be an integer bigger than 0
-        pausing.max_timeout_format: needs to be an integer bigger than 0
-        pausing.with_exponential_backoff_format: needs to be either true or false
-        pausing.timeout_max_timeout_vs_pause_max_timeout: timeout must be less or equal to max_timeout
+        pause.active_format: 'needs to be boolean'
+        pause.timeout_format: needs to be an integer bigger than 0
+        pause.max_timeout_format: needs to be an integer bigger than 0
+        pause.with_exponential_backoff_format: needs to be either true or false
+        pause.timeout_max_timeout_vs_pause_max_timeout: timeout must be less or equal to max_timeout
 
         patterns.active_format: 'needs to be boolean'
         patterns.type_format: 'needs to be :matcher, :discovered or :regular'

data/lib/karafka/connection/pauses_manager.rb

@@ -19,9 +19,9 @@ module Karafka
       # @return [Karafka::TimeTrackers::Pause] pause tracker instance
       def fetch(topic, partition)
         @pauses[topic][partition] ||= TimeTrackers::Pause.new(
-          timeout: topic.pause_timeout,
-          max_timeout: topic.pause_max_timeout,
-          exponential_backoff: topic.pause_with_exponential_backoff
+          timeout: topic.pause.timeout,
+          max_timeout: topic.pause.max_timeout,
+          exponential_backoff: topic.pause.with_exponential_backoff
         )
       end
 

data/lib/karafka/pro/routing/features/consumer_groups/recurring_tasks/builder.rb

@@ -117,8 +117,7 @@ module Karafka
                     # Since the execution of particular tasks is isolated and guarded, it should not
                     # leak. This means, that this is to handle errors like schedule version
                     # incompatibility and other errors that will not go away without a redeployment
-                    pause_timeout(60 * 1_000)
-                    pause_max_timeout(60 * 1_000)
+                    pause(timeout: 60 * 1_000, max_timeout: 60 * 1_000)
 
                     # This is the core of execution. Since we're producers of states, we need a way
                     # to tick without having new data

data/lib/karafka/pro/routing/features/pausing/config.rb

@@ -33,17 +33,13 @@ module Karafka
     module Routing
       module Features
         class Pausing < Base
-          # Config for pausing feature
-          Config = Struct.new(
-            :active,
-            :timeout,
-            :max_timeout,
-            :with_exponential_backoff,
-            keyword_init: true
-          ) do
-            alias_method :active?, :active
-            alias_method :with_exponential_backoff?, :with_exponential_backoff
-          end
+          # Config for pausing feature.
+          #
+          # By convention each routing feature exposes a `Config`. The pause value object is defined
+          # in OSS (`Karafka::Routing::Features::Pausing::Config`) because the per-topic `#pause`
+          # reader defaults to the global config there, so the feature points its `Config` at that
+          # shared class rather than redefining it.
+          Config = Karafka::Routing::Features::Pausing::Config
         end
       end
     end

data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb

@@ -42,10 +42,8 @@ module Karafka
                   File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
                 ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
 
-                # Validate the nested pausing configuration
-                # Both old setters and new pause() method update the pausing config,
-                # so we only need to validate this one format
-                nested :pausing do
+                # Validate the nested pause configuration
+                nested :pause do
                   required(:active) { |val| [true, false].include?(val) }
                   required(:timeout) { |val| val.is_a?(Integer) && val.positive? }
                   required(:max_timeout) { |val| val.is_a?(Integer) && val.positive? }
@@ -56,13 +54,13 @@ module Karafka
                 virtual do |data, errors|
                   next unless errors.empty?
 
-                  pausing = data.fetch(:pausing)
-                  timeout = pausing.fetch(:timeout)
-                  max_timeout = pausing.fetch(:max_timeout)
+                  pause = data.fetch(:pause)
+                  timeout = pause.fetch(:timeout)
+                  max_timeout = pause.fetch(:max_timeout)
 
                   next if timeout <= max_timeout
 
-                  [[%i[pausing timeout], :max_timeout_vs_pause_max_timeout]]
+                  [[%i[pause timeout], :max_timeout_vs_pause_max_timeout]]
                 end
               end
             end

data/lib/karafka/pro/routing/features/pausing/topic.rb

@@ -35,84 +35,39 @@ module Karafka
         class Pausing < Base
           # Expansion allowing for a per topic pause strategy definitions
           module Topic
-            # This method sets up the extra instance variable to nil before calling
-            # the parent class initializer. The explicit initialization
-            # to nil is included as an optimization for Ruby's object shapes system,
-            # which improves memory layout and access performance.
-            def initialize(...)
-              @pausing = nil
-              super
-            end
-
-            # Allows for per-topic pausing strategy setting
+            # Allows for per-topic pausing strategy setting.
+            #
+            # Overrides the OSS `#pause` reader (this module is prepended onto `Routing::Topic`).
+            # With no arguments it returns the current configuration, defaulting to the global
+            # `config.pause.*` settings via `super`. With arguments it overrides the pausing
+            # strategy for this topic and marks it as active.
             #
             # @param timeout [Integer] how long should we wait upon processing error (milliseconds)
             # @param max_timeout [Integer] what is the max timeout in case of an exponential
             #   backoff (milliseconds)
             # @param with_exponential_backoff [Boolean] should we use exponential backoff
-            # @return [Config] pausing config object
+            # @return [Karafka::Routing::Features::Pausing::Config] pausing config object
             def pause(timeout: nil, max_timeout: nil, with_exponential_backoff: nil)
-              # If no arguments provided, just return or initialize the config
-              return pausing if timeout.nil? && max_timeout.nil? && with_exponential_backoff.nil?
+              config = super()
 
-              # Update instance variables for backwards compatibility
-              # This ensures code reading @pause_timeout directly or via the inherited getter
-              # will get the correct values
-              @pause_timeout = timeout if timeout
-              @pause_max_timeout = max_timeout if max_timeout
-
-              unless with_exponential_backoff.nil?
-                @pause_with_exponential_backoff = with_exponential_backoff
-              end
+              # If no arguments provided, just return the current (default or overridden) config
+              return config if timeout.nil? && max_timeout.nil? && with_exponential_backoff.nil?
 
-              # Create or update the config
-              @pausing ||= Config.new(
-                active: false,
-                timeout: @pause_timeout || Karafka::App.config.pause.timeout,
-                max_timeout: @pause_max_timeout || Karafka::App.config.pause.max_timeout,
-                with_exponential_backoff: if @pause_with_exponential_backoff.nil?
-                                            Karafka::App.config.pause.with_exponential_backoff
-                                          else
-                                            @pause_with_exponential_backoff
-                                          end
-              )
-
-              @pausing.timeout = timeout if timeout
-              @pausing.max_timeout = max_timeout if max_timeout
+              config.timeout = timeout if timeout
+              config.max_timeout = max_timeout if max_timeout
 
               unless with_exponential_backoff.nil?
-                @pausing.with_exponential_backoff = with_exponential_backoff
+                config.with_exponential_backoff = with_exponential_backoff
               end
 
-              @pausing.active = true
-
-              @pausing
-            end
-
-            # @return [Config] pausing configuration object
-            def pausing
-              @pausing ||= Config.new(
-                active: false,
-                timeout: @pause_timeout || Karafka::App.config.pause.timeout,
-                max_timeout: @pause_max_timeout || Karafka::App.config.pause.max_timeout,
-                with_exponential_backoff: if @pause_with_exponential_backoff.nil?
-                                            Karafka::App.config.pause.with_exponential_backoff
-                                          else
-                                            @pause_with_exponential_backoff
-                                          end
-              )
-            end
+              config.active = true
 
-            # @return [Boolean] is pausing explicitly configured
-            def pausing?
-              pausing.active?
+              config
             end
 
-            # @return [Hash] topic with all its native configuration options plus pausing settings
-            def to_h
-              super.merge(
-                pausing: pausing.to_h
-              ).freeze
+            # @return [Boolean] is pausing explicitly configured on a per-topic basis
+            def pause?
+              pause.active?
             end
           end
         end

data/lib/karafka/routing/features/pausing/config.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module Pausing
+        # Per-topic pause (backoff) configuration value object.
+        #
+        # Defaults to the global `config.pause.*` settings unless overridden on a per-topic basis.
+        #
+        # Unlike most feature configs, `active` here does not switch the behavior on or off - the
+        # pause/backoff always applies. It only marks whether these settings were explicitly set
+        # for this topic (`true`) or inherited from the global defaults (`false`).
+        Config = Struct.new(
+          :active,
+          :timeout,
+          :max_timeout,
+          :with_exponential_backoff,
+          keyword_init: true
+        ) do
+          alias_method :active?, :active
+          alias_method :with_exponential_backoff?, :with_exponential_backoff
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/pausing.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # Namespace holding the pause (backoff) routing configuration. There is no OSS pausing
+      # feature to activate here - the backoff behavior is part of the topic itself and its
+      # settings default to the global `config.pause.*`.
+      module Pausing
+      end
+    end
+  end
+end

data/lib/karafka/routing/topic.rb

@@ -32,9 +32,6 @@ module Karafka
         max_wait_time
         initial_offset
         consumer_persistence
-        pause_timeout
-        pause_max_timeout
-        pause_with_exponential_backoff
       ].freeze
 
       private_constant :INHERITABLE_ATTRIBUTES
@@ -58,6 +55,10 @@ module Karafka
         INHERITABLE_ATTRIBUTES.each do |attribute|
           instance_variable_set("@#{attribute}", nil)
         end
+
+        # Explicit nil initialization for Ruby's object shapes optimization. The per-topic pause
+        # config is built lazily on first read, defaulting to the global `config.pause.*` settings.
+        @pause = nil
       end
 
       INHERITABLE_ATTRIBUTES.each do |attribute|
@@ -73,6 +74,17 @@ module Karafka
         RUBY
       end
 
+      # @return [Karafka::Routing::Features::Pausing::Config] per-topic pause configuration,
+      #   reflecting the root `config.pause.*` settings.
+      def pause
+        @pause ||= Features::Pausing::Config.new(
+          active: false,
+          timeout: Karafka::App.config.pause.timeout,
+          max_timeout: Karafka::App.config.pause.max_timeout,
+          with_exponential_backoff: Karafka::App.config.pause.with_exponential_backoff
+        )
+      end
+
       # Often users want to have the same basic cluster setup with small setting alterations
       # This method allows us to do so by setting `inherit` to `true`. Whe inherit is enabled,
       # settings will be merged with defaults.
@@ -155,6 +167,7 @@ module Karafka
           name: name,
           active: active?,
           consumer: consumer,
+          pause: pause.to_h,
           group_id: group.id,
           # Kept as a reference alongside `group_id` for backwards compatibility. Will be removed
           # in Karafka 3.0.

data/lib/karafka/setup/config.rb

@@ -405,53 +405,6 @@ module Karafka
       # Thanks to that we have an initial state out of the box.
       configure
 
-      # Backwards compatibility: Add old flat API methods to the config instance. These delegate
-      # to the new nested pause config. Deprecated: Will be removed in Karafka 2.6
-      #
-      # Prior to the introduction of nested pause configuration, pause-related settings were
-      # accessed directly on the config object (e.g., `config.pause_timeout`). With the nested
-      # structure introduced, these settings moved to `config.pause.timeout`, etc.
-      #
-      # This instance_eval block adds delegation methods to maintain backwards compatibility,
-      # allowing existing code using the old flat API to continue working without modification.
-      config.instance_eval do
-        # @return [Integer] delegated timeout value from pause.timeout
-        # @deprecated Use config.pause.timeout instead
-        def pause_timeout
-          pause.timeout
-        end
-
-        # @param value [Integer] timeout value to set
-        # @deprecated Use config.pause.timeout= instead
-        def pause_timeout=(value)
-          pause.timeout = value
-        end
-
-        # @return [Integer] delegated max_timeout value from pause.max_timeout
-        # @deprecated Use config.pause.max_timeout instead
-        def pause_max_timeout
-          pause.max_timeout
-        end
-
-        # @param value [Integer] max timeout value to set
-        # @deprecated Use config.pause.max_timeout= instead
-        def pause_max_timeout=(value)
-          pause.max_timeout = value
-        end
-
-        # @return [Boolean] delegated exponential backoff flag from pause.with_exponential_backoff
-        # @deprecated Use config.pause.with_exponential_backoff instead
-        def pause_with_exponential_backoff
-          pause.with_exponential_backoff
-        end
-
-        # @param value [Boolean] exponential backoff flag to set
-        # @deprecated Use config.pause.with_exponential_backoff= instead
-        def pause_with_exponential_backoff=(value)
-          pause.with_exponential_backoff = value
-        end
-      end
-
       class << self
         # Configuring method
         def setup(&)

data/lib/karafka/version.rb

@@ -2,5 +2,5 @@
 
 module Karafka
   # Current Karafka version
-  VERSION = "2.6.0.rc1"
+  VERSION = "2.6.0.rc2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.6.0.rc1
+  version: 2.6.0.rc2
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -547,6 +547,8 @@ files:
 - lib/karafka/routing/features/deserializers/config.rb
 - lib/karafka/routing/features/deserializers/contracts/topic.rb
 - lib/karafka/routing/features/deserializers/topic.rb
+- lib/karafka/routing/features/pausing.rb
+- lib/karafka/routing/features/pausing/config.rb
 - lib/karafka/routing/proxy.rb
 - lib/karafka/routing/router.rb
 - lib/karafka/routing/subscription_group.rb