karafka 2.6.0.beta1 → 2.6.0.rc2

data/lib/karafka/processing/consumer_groups/strategies/dlq.rb

@@ -2,8 +2,6 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
       module Strategies
         # When using dead letter queue, processing won't stop after defined number of retries
@@ -64,6 +62,11 @@ module Karafka
               return if coordinator.manual_pause?
 
               mark_as_consumed(messages.last)
+            # Process-critical errors are never dispatched to the DLQ regardless of the retries
+            # state: the retry pause protects the partition during the critical shutdown and
+            # the failed batch is redelivered after the restart
+            elsif critical_error?(coordinator.consumption(self).cause)
+              retry_after_pause
             elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
               retry_after_pause
             # If we've reached number of retries that we could, we need to skip the first message

data/lib/karafka/processing/consumer_groups/strategies/dlq_mom.rb

@@ -2,8 +2,6 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
       module Strategies
         # Same as pure dead letter queue but we do not marked failed message as consumed
@@ -23,6 +21,11 @@ module Karafka
 
             if coordinator.success?
               coordinator.pause_tracker.reset
+            # Process-critical errors are never dispatched to the DLQ regardless of the retries
+            # state: the retry pause protects the partition during the critical shutdown and
+            # the failed batch is redelivered after the restart
+            elsif critical_error?(coordinator.consumption(self).cause)
+              retry_after_pause
             elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
               retry_after_pause
             # If we've reached number of retries that we could, we need to skip the first message

data/lib/karafka/processing/consumer_groups/strategies/mom.rb

@@ -2,11 +2,10 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
       module Strategies
-        # When using manual offset management, we do not mark as consumed after successful processing
+        # When using manual offset management, we do not mark as consumed after successful
+        # processing
         module Mom
           include Default
 

data/lib/karafka/processing/consumer_groups/strategy_selector.rb

@@ -2,8 +2,6 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
       # Selector of appropriate processing strategy matching topic combinations
       class StrategySelector

data/lib/karafka/processing/coordinator.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-# Backwards-compatible alias kept for karafka-web gem that references the old, un-namespaced
-# constant. Will be removed in Karafka 3.0.
 module Karafka
   module Processing
     # @see ConsumerGroups::Coordinator

data/lib/karafka/processing/coordinators_buffer.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-# Backwards-compatible alias kept for karafka-testing gem that references the old, un-namespaced
-# constant. Will be removed in Karafka 3.0.
 module Karafka
   module Processing
     # @see ConsumerGroups::CoordinatorsBuffer

data/lib/karafka/processing/partitioner.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-# Backwards-compatible alias kept for external code that references the old, un-namespaced
-# constant. Will be removed in Karafka 3.0.
 module Karafka
   module Processing
     # @see ConsumerGroups::Partitioner

data/lib/karafka/processing/strategy_selector.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-# Backwards-compatible alias kept for external code that references the old, un-namespaced
-# constant. Will be removed in Karafka 3.0.
 module Karafka
   module Processing
     # @see ConsumerGroups::StrategySelector

data/lib/karafka/processing/worker.rb

@@ -45,10 +45,16 @@ module Karafka
 
       # Fetches a single job, processes it and marks as completed.
       #
-      # @note We do not have error handling here, as no errors should propagate this far. If they
-      #   do, it is a critical error and should bubble up.
+      # @note No user or framework errors should propagate this far. If they do, it is a critical
+      #   error that we instrument while keeping the worker alive.
       #
       # @note Upon closing the jobs queue, worker will close it's thread
+      #
+      # @return [Boolean] should the worker keep running. False only when the jobs queue is
+      #   closed (full shutdown) or a downscale sentinel was received. This decision is
+      #   deliberately decoupled from the processing flow result: the flow passes through the
+      #   user-redefinable `#wrap`, whose return value (or an error raised in user code or in an
+      #   instrumentation subscriber) must never be able to silently kill the worker thread.
       def process
         job = @jobs_queue.pop
 
@@ -76,10 +82,10 @@ module Karafka
               end
 
               job.after_call
-
-              true
             end
           end
+
+          true
         else
           # nil means either queue closed (full shutdown) or pool downscaling.
           # Either way, deregister from pool so it reflects the actual worker count.
@@ -87,9 +93,7 @@ module Karafka
           false
         end
       # We signal critical exceptions, notify and do not allow worker to fail
-      # rubocop:disable Lint/RescueException
       rescue Exception => e
-        # rubocop:enable Lint/RescueException
         monitor.instrument(
           "error.occurred",
           caller: self,
@@ -98,6 +102,10 @@ module Karafka
           error: e,
           type: "worker.process.error"
         )
+
+        # Keep the worker alive. Without an explicit true this arm would return the result of
+        # the instrumentation call (nil) and the worker thread would silently exit
+        true
       ensure
         # job can be nil when the queue is being closed or during pool downscaling
         if job

data/lib/karafka/processing/workers_pool.rb

@@ -39,6 +39,13 @@ module Karafka
         # after a worker exits, so thread names remain unique across the lifetime of the process
         # and make it easy to correlate log entries with specific worker generations.
         @next_index = 0
+        # Number of downscale sentinels enqueued but not yet picked up by a worker. Workers
+        # deregister asynchronously, so the live thread count alone overstates the pool between
+        # a scale-down request and its completion. All sizing decisions use the committed size
+        # (live workers minus pending shrinks) so repeated or overlapping scale-down requests do
+        # not enqueue duplicate sentinels and drain the pool below the requested target (or
+        # below the minimum of one worker).
+        @pending_shrinks = 0
       end
 
       # Scale pool towards `target` workers (minimum 1).
@@ -63,7 +70,9 @@ module Karafka
         event = nil
 
         @mutex.synchronize do
-          current = @workers.size
+          # Compute against the committed size, not the live thread count: sentinels already in
+          # flight are removals this pool is committed to, just not yet executed by the workers
+          current = @workers.size - @pending_shrinks
           delta = target - current
 
           if delta.positive?
@@ -106,6 +115,11 @@ module Karafka
         @mutex.synchronize do
           @workers.delete(worker)
           @size = @workers.size
+          # A worker exits either because it picked up a downscale sentinel or because the jobs
+          # queue was closed for shutdown. The worker cannot tell those apart, so pending
+          # removals are settled here, with a floor because shutdown exits may have no pending
+          # shrink to consume
+          @pending_shrinks -= 1 if @pending_shrinks.positive?
         end
       end
 
@@ -142,14 +156,17 @@ module Karafka
       #
       # @param count [Integer] number of workers to remove
       # @return [Array, nil] instrumentation event args or nil if no-op
-      # @note Never shrinks below 1 worker.
+      # @note Never shrinks below 1 worker, including removals already in flight - the cap is
+      #   computed against the committed size, not the live thread count.
       def shrink(count)
-        effective = [count, @workers.size - 1].min
+        committed = @workers.size - @pending_shrinks
+        effective = [count, committed - 1].min
         return if effective <= 0
 
         from = @workers.size
+        @pending_shrinks += effective
         effective.times { @jobs_queue << nil }
-        to = from - effective
+        to = committed - effective
 
         ["worker.scaling.down", { workers_pool: self, from: from, to: to }]
       end

data/lib/karafka/railtie.rb

@@ -3,7 +3,6 @@
 # This file contains Railtie for auto-configuration
 
 unless Karafka.rails?
-  # Without defining this in any way, Zeitwerk ain't happy so we do it that way
   module Karafka
     class Railtie
     end

data/lib/karafka/routing/contracts/routing.rb

@@ -12,8 +12,8 @@ module Karafka
         end
 
         # Ensures, that when declarative topics strict requirement is on, all topics have
-        # declarative definition (including DLQ topics)
-        # @note It will ignore routing pattern topics because those topics are virtual
+        # declarative definition (including DLQ topics). It will ignore routing pattern topics
+        # because those topics are virtual
         virtual do |data, errors|
           next unless errors.empty?
           # Do not validate declaratives unless required and explicitly enabled

data/lib/karafka/routing/features/consumer_groups/active_job/topic.rb

@@ -7,11 +7,12 @@ module Karafka
         class ActiveJob < Base
           # Topic extensions to be able to check if given topic is ActiveJob topic
           #
-          # @note ActiveJob topics do not have per-topic deserializer configuration. The deserializer
-          #   is configured globally via `config.internal.active_job.deserializer` because Rails
-          #   serializes jobs before dispatching them, requiring a consistent serialization format
-          #   across all ActiveJob topics. If you need custom serialization (e.g., Avro, Protobuf),
-          #   configure it once at the application level rather than per-topic.
+          # @note ActiveJob topics do not have per-topic deserializer configuration. The
+          #   deserializer is configured globally via `config.internal.active_job.deserializer`
+          #   because Rails serializes jobs before dispatching them, requiring a consistent
+          #   serialization format across all ActiveJob topics. If you need custom serialization
+          #   (e.g., Avro, Protobuf), configure it once at the application level rather than
+          #   per-topic.
           module Topic
             # This method sets up the extra instance variable to nil before calling
             # the parent class initializer. The explicit initialization

data/lib/karafka/routing/features/consumer_groups/manual_offset_management/topic.rb

@@ -16,8 +16,8 @@ module Karafka
               super
             end
 
-            # @param active [Boolean] should we stop managing the offset in Karafka and make the user
-            #   responsible for marking messages as consumed.
+            # @param active [Boolean] should we stop managing the offset in Karafka and make the
+            #   user responsible for marking messages as consumed.
             # @return [Config] defined config
             #
             # @note Since this feature supports only one setting (active), we can use the old API

data/lib/karafka/routing/features/deserializers/topic.rb

@@ -3,7 +3,6 @@
 module Karafka
   module Routing
     module Features
-      # Deserializers for all the message details (payload, headers, key)
       class Deserializers < Base
         # Routing topic deserializers API. It allows to configure deserializers for various
         # components of each message.

data/lib/karafka/routing/features/pausing/config.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module Pausing
+        # Per-topic pause (backoff) configuration value object.
+        #
+        # Defaults to the global `config.pause.*` settings unless overridden on a per-topic basis.
+        #
+        # Unlike most feature configs, `active` here does not switch the behavior on or off - the
+        # pause/backoff always applies. It only marks whether these settings were explicitly set
+        # for this topic (`true`) or inherited from the global defaults (`false`).
+        Config = Struct.new(
+          :active,
+          :timeout,
+          :max_timeout,
+          :with_exponential_backoff,
+          keyword_init: true
+        ) do
+          alias_method :active?, :active
+          alias_method :with_exponential_backoff?, :with_exponential_backoff
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/pausing.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # Namespace holding the pause (backoff) routing configuration. There is no OSS pausing
+      # feature to activate here - the backoff behavior is part of the topic itself and its
+      # settings default to the global `config.pause.*`.
+      module Pausing
+      end
+    end
+  end
+end

data/lib/karafka/routing/subscription_group.rb

@@ -16,7 +16,7 @@ module Karafka
 
       attr_reader :id, :name, :topics, :kafka, :group, :position
 
-      # Backwards compatible alias for `#group`. Kept purely for compatibility — this is an
+      # Backwards compatible alias for `#group`. Kept purely for compatibility - this is an
       # unconditional alias and performs no type validation, so callers should prefer `#group`
       # once additional group types (e.g. KIP-932 share groups) land.
       alias_method :consumer_group, :group

data/lib/karafka/routing/topic.rb

@@ -14,7 +14,7 @@ module Karafka
       attr_reader :id, :name, :group
       attr_writer :consumer
 
-      # Backwards compatible alias for `#group`. Kept purely for compatibility — this is an
+      # Backwards compatible alias for `#group`. Kept purely for compatibility - this is an
       # unconditional alias and performs no type validation, so callers should prefer `#group`
       # once additional group types (e.g. KIP-932 share groups) land.
       alias_method :consumer_group, :group
@@ -32,9 +32,6 @@ module Karafka
         max_wait_time
         initial_offset
         consumer_persistence
-        pause_timeout
-        pause_max_timeout
-        pause_with_exponential_backoff
       ].freeze
 
       private_constant :INHERITABLE_ATTRIBUTES
@@ -47,9 +44,9 @@ module Karafka
         @group = group
         @attributes = {}
         @active = true
-        # @note We use identifier related to the group that owns a topic, because from Karafka 0.6
-        #   we can handle multiple Kafka instances with the same process and we can have same
-        #   topic name across multiple groups
+        # We use identifier related to the group that owns a topic, because from Karafka 0.6 we can
+        # handle multiple Kafka instances with the same process and we can have same topic name
+        # across multiple groups
         @id = "#{group.id}_#{@name}"
         @consumer = nil
         @active_assigned = false
@@ -58,6 +55,10 @@ module Karafka
         INHERITABLE_ATTRIBUTES.each do |attribute|
           instance_variable_set("@#{attribute}", nil)
         end
+
+        # Explicit nil initialization for Ruby's object shapes optimization. The per-topic pause
+        # config is built lazily on first read, defaulting to the global `config.pause.*` settings.
+        @pause = nil
       end
 
       INHERITABLE_ATTRIBUTES.each do |attribute|
@@ -73,6 +74,17 @@ module Karafka
         RUBY
       end
 
+      # @return [Karafka::Routing::Features::Pausing::Config] per-topic pause configuration,
+      #   reflecting the root `config.pause.*` settings.
+      def pause
+        @pause ||= Features::Pausing::Config.new(
+          active: false,
+          timeout: Karafka::App.config.pause.timeout,
+          max_timeout: Karafka::App.config.pause.max_timeout,
+          with_exponential_backoff: Karafka::App.config.pause.with_exponential_backoff
+        )
+      end
+
       # Often users want to have the same basic cluster setup with small setting alterations
       # This method allows us to do so by setting `inherit` to `true`. Whe inherit is enabled,
       # settings will be merged with defaults.
@@ -155,6 +167,7 @@ module Karafka
           name: name,
           active: active?,
           consumer: consumer,
+          pause: pause.to_h,
           group_id: group.id,
           # Kept as a reference alongside `group_id` for backwards compatibility. Will be removed
           # in Karafka 3.0.

data/lib/karafka/routing/topics.rb

@@ -7,13 +7,33 @@ module Karafka
       include Enumerable
       extend Forwardable
 
-      def_delegators :@accumulator, :[], :size, :empty?, :last, :<<, :map!, :sort_by!, :reverse!
+      def_delegators :@accumulator, :[], :size, :empty?, :last, :map!, :sort_by!, :reverse!
 
       # @param topics_array [Array<Karafka::Routing::Topic>] array with topics
       def initialize(topics_array)
         @accumulator = topics_array.dup
       end
 
+      # Adds a topic using copy-on-write: we publish a brand-new accumulator array instead of
+      # mutating the existing one in place.
+      #
+      # @param topic [Karafka::Routing::Topic] topic that should become part of this consumer
+      #   group's topics
+      # @return [self]
+      #
+      # @note Topics can be appended to this collection at runtime from one thread while other
+      #   threads iterate the same collection (for example during routing lookups). Swapping the
+      #   `@accumulator` reference atomically (rather than appending in place) guarantees a
+      #   concurrent `#each` always traverses a complete, immutable snapshot: it either sees the
+      #   newly added topic or it does not, but never a torn array. This holds on every Ruby
+      #   runtime, not only on MRI's GIL. The in-place mutators above (`map!`, `sort_by!`,
+      #   `reverse!`) only run while routes are built at boot, single-threaded, so they do not
+      #   need the same treatment.
+      def <<(topic)
+        @accumulator += [topic]
+        self
+      end
+
       # Yields each topic
       def each(&)
         @accumulator.each(&)

data/lib/karafka/server.rb

@@ -168,6 +168,15 @@ module Karafka
         # otherwise we would overwrite the shutdown process of the process that started Karafka
         return unless process.supervised?
 
+        # We intentionally do NOT flush or close the producer before the `Kernel.exit!` below.
+        # Forceful shutdown is a last resort - a job already exceeded `shutdown_timeout` - and it
+        # is expected that in-flight, async-buffered data (user `produce_async` calls and DLQ
+        # `produce_async` copies) may be lost. Flushing here would be risky precisely when it
+        # matters: the producer (or its connection pool) may itself be the resource that is blocked
+        # (e.g. an unreachable broker), so waiting on it could stall or hang the very forceful exit
+        # that must guarantee the process terminates. On a normal (graceful) shutdown the producer
+        # is still closed via the `ensure` block below, which `exit!` deliberately skips here.
+        #
         # exit! is not within the instrumentation as it would not trigger due to exit
         Kernel.exit!(forceful_exit_code)
       ensure

data/lib/karafka/setup/config.rb

@@ -22,8 +22,8 @@ module Karafka
         # option token [String, false] - license token issued when you acquire a Pro license
         # Leave false if using the LGPL version and all is going to work just fine :)
         #
-        # @note By using the commercial components, you accept the LICENSE-COMM commercial license
-        #   terms and conditions
+        # By using the commercial components, you accept the LICENSE-COMM commercial license terms
+        # and conditions
         setting :token, default: false
         # option entity [String] for whom we did issue the license
         setting :entity, default: ""
@@ -88,8 +88,8 @@ module Karafka
         setting :token_provider_listener, default: false
       end
 
-      # rdkafka default options
-      # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+      # rdkafka default options See
+      # https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
       setting :kafka, default: {}
 
       # Public configuration for swarm operations
@@ -163,9 +163,8 @@ module Karafka
       setting :internal do
         # option status [Karafka::Status] app status
         setting :status, default: Status.new
-        # option process [Karafka::Process] process status
-        # @note In the future, we need to have a single process representation for all the karafka
-        #   instances
+        # option process [Karafka::Process] process status. In the future, we need to have a single
+        # process representation for all the karafka instances
         setting :process, default: Process.new
         # Interval of "ticking". This is used to define the maximum time between consecutive
         # polling of the main rdkafka queue. It should match also the `statistics.interval.ms`
@@ -314,6 +313,17 @@ module Karafka
           # option worker_job_call_wrapper [Proc, false] callable object that will be used to wrap
           #   the worker execution of a job or false if no wrapper needed
           setting :worker_job_call_wrapper, default: false
+          # option critical_errors [Array<Class>] errors that indicate that the process integrity
+          #   is compromised or that termination was explicitly requested. They are not retried
+          #   in-process when raised from the consumption flow: SystemExit means someone called
+          #   `exit` and retrying would invert that intent (and loop forever on a deterministic
+          #   raise), while NoMemoryError means the VM cannot be trusted to execute a retry.
+          #   Their failure is still recorded (so nothing gets marked, they are never dispatched
+          #   to a DLQ and the retry pause protects the partition) and a graceful shutdown is
+          #   initiated instead - redelivery after restart preserves the processing guarantees
+          #   through the process death. Can be extended with user-specific fatal error classes
+          #   that should terminate the process rather than be retried.
+          setting :critical_errors, default: [SystemExit, SignalException, NoMemoryError].freeze
 
           # Consumer-group-specific processing defaults. When share groups land, a parallel
           # `share_groups` namespace will hold their equivalents.
@@ -343,6 +353,36 @@ module Karafka
           setting :parser, default: Messages::Parser.new
         end
 
+        # Things related to librdkafka statistics decoration prior to their emission
+        setting :statistics do
+          # Consumer groups related statistics settings
+          setting :consumer_groups do
+            # option decorator_class [Class] class used to decorate raw librdkafka statistics
+            #   (adds delta/freeze-duration values) before `statistics.emitted` is instrumented.
+            #   Exposed so it can be replaced with a custom decorator (e.g. one that also
+            #   enriches or corrects specific values) without touching the callback that uses
+            #   it.
+            setting :decorator_class, default: Instrumentation::Callbacks::ConsumerGroups::Decorator
+
+            # Active refreshing of watermarks and lags for long-paused partitions (Pro).
+            # librdkafka only updates watermark offsets and lags from fetch responses, so
+            # partitions paused for a long time report frozen values in statistics. When
+            # enabled, they are refreshed via the running consumer connection and overlaid onto
+            # the emitted statistics.
+            setting :lag_compensation do
+              # option interval [Integer] how often (ms) at most to refresh watermarks and lags
+              #   of long-paused partitions. 0 disables the feature entirely.
+              setting :interval, default: 0
+
+              # option pause_age [Integer] how long (ms) a partition needs to stay paused to
+              #   qualify for the compensation. Short pause/resume cycles reset the clock and
+              #   never qualify. Minimum accepted value is 5 seconds as statistics of partitions
+              #   paused shorter are fresh enough to make compensating them pointless.
+              setting :pause_age, default: 30_000
+            end
+          end
+        end
+
         # Karafka components for ActiveJob
         setting :active_job do
           # option dispatcher [Karafka::ActiveJob::Dispatcher] default dispatcher for ActiveJob
@@ -365,54 +405,6 @@ module Karafka
       # Thanks to that we have an initial state out of the box.
       configure
 
-      # Backwards compatibility: Add old flat API methods to the config instance
-      # These delegate to the new nested pause config
-      # @deprecated Will be removed in Karafka 2.6
-      #
-      # Prior to the introduction of nested pause configuration, pause-related settings were
-      # accessed directly on the config object (e.g., `config.pause_timeout`). With the nested
-      # structure introduced, these settings moved to `config.pause.timeout`, etc.
-      #
-      # This instance_eval block adds delegation methods to maintain backwards compatibility,
-      # allowing existing code using the old flat API to continue working without modification.
-      config.instance_eval do
-        # @return [Integer] delegated timeout value from pause.timeout
-        # @deprecated Use config.pause.timeout instead
-        def pause_timeout
-          pause.timeout
-        end
-
-        # @param value [Integer] timeout value to set
-        # @deprecated Use config.pause.timeout= instead
-        def pause_timeout=(value)
-          pause.timeout = value
-        end
-
-        # @return [Integer] delegated max_timeout value from pause.max_timeout
-        # @deprecated Use config.pause.max_timeout instead
-        def pause_max_timeout
-          pause.max_timeout
-        end
-
-        # @param value [Integer] max timeout value to set
-        # @deprecated Use config.pause.max_timeout= instead
-        def pause_max_timeout=(value)
-          pause.max_timeout = value
-        end
-
-        # @return [Boolean] delegated exponential backoff flag from pause.with_exponential_backoff
-        # @deprecated Use config.pause.with_exponential_backoff instead
-        def pause_with_exponential_backoff
-          pause.with_exponential_backoff
-        end
-
-        # @param value [Boolean] exponential backoff flag to set
-        # @deprecated Use config.pause.with_exponential_backoff= instead
-        def pause_with_exponential_backoff=(value)
-          pause.with_exponential_backoff = value
-        end
-      end
-
       class << self
         # Configuring method
         def setup(&)
@@ -451,6 +443,11 @@ module Karafka
           # Post-setup configure all routing features that would need this
           Routing::Features::Base.post_setup_all(config)
 
+          # Subscribe the critical errors listener so process-critical errors reported anywhere
+          # in the framework escalate to a graceful shutdown. Subscribed before any other
+          # internal listener (including the Pro post-setup subscriptions) so it reacts first
+          config.monitor.subscribe(Instrumentation::CriticalErrorsListener.instance)
+
           # Runs things that need to be executed after config is defined and all the components
           # are also configured
           Pro::Loader.post_setup_all(config) if Karafka.pro?

data/lib/karafka/setup/config_proxy.rb

@@ -197,7 +197,8 @@ module Karafka
       #   end
       #
       # @example Direct producer assignment
-      #   custom_producer = WaterDrop::Producer.new { |c| c.kafka = { 'bootstrap.servers' => 'localhost:9092' } }
+      #   custom_producer = WaterDrop::Producer.new { |c|
+      #     c.kafka = { 'bootstrap.servers' => 'localhost:9092' } }
       #   config.producer = custom_producer
       def producer(instance = READ, &block)
         if block

data/lib/karafka/setup/contracts/config.rb

@@ -124,6 +124,9 @@ module Karafka
             required(:jobs_queue_class) { |val| !val.nil? }
             required(:scheduler_class) { |val| !val.nil? }
             required(:worker_job_call_wrapper) { |val| val == false || val.respond_to?(:wrap) }
+            required(:critical_errors) do |val|
+              val.is_a?(Array) && val.all? { |klass| klass.is_a?(Class) && klass <= Exception }
+            end
 
             nested(:consumer_groups) do
               required(:jobs_builder) { |val| !val.nil? }
@@ -136,6 +139,17 @@ module Karafka
             end
           end
 
+          nested(:statistics) do
+            nested(:consumer_groups) do
+              required(:decorator_class) { |val| !val.nil? }
+
+              nested(:lag_compensation) do
+                required(:interval) { |val| val.is_a?(Integer) && val >= 0 }
+                required(:pause_age) { |val| val.is_a?(Integer) && val >= 5_000 }
+              end
+            end
+          end
+
           nested(:active_job) do
             required(:dispatcher) { |val| !val.nil? }
             required(:job_options_contract) { |val| !val.nil? }