karafka 2.6.0.beta1 → 2.6.0.rc1

data/lib/karafka/connection/proxy.rb

@@ -65,6 +65,51 @@ module Karafka
         end
       end
 
+      # Resolves offsets of many partitions with one batched `ListOffsets` request instead of one
+      # roundtrip per partition. This is the same broker query as `#query_watermark_offsets`,
+      # only batched, so it is sensitive to the same latencies and leader-election hiccups and we
+      # recover from them the same way and with the same settings.
+      #
+      # Named after `Admin::Topics#read_partition_offsets` (same specs in, same offsets out) and
+      # **not** after the underlying `#list_offsets`, so we do not shadow the rdkafka method of
+      # that name: it returns a handle to wait on, while we return the already resolved offsets.
+      #
+      # @param topic_partition_offsets [Hash{String => Array<Hash>}] topics with arrays of
+      #   partition offset specs, each with a `:partition` and an `:offset` (`:earliest`,
+      #   `:latest`, `:max_timestamp` or an integer timestamp in ms)
+      # @param isolation_level [Integer, nil] isolation level with which the offsets should be
+      #   resolved or nil for the librdkafka default (read_uncommitted)
+      # @return [Array<Hash>] resolved offsets, each with `:topic`, `:partition` and `:offset`
+      # @note The `ListOffsets` admin API resolves `:latest` to the high watermark even with
+      #   `isolation_level` set to read_committed, unlike `#query_watermark_offsets`, which returns
+      #   the last stable offset for a read_committed consumer. On a topic with an in-flight
+      #   transaction `:latest` therefore includes the uncommitted messages. Non-transactional
+      #   topics are unaffected.
+      # @note Specs given as a literal hash need explicit curly braces: a brace-less trailing
+      #   hash is parsed by Ruby as keyword arguments, which this method (having the
+      #   `isolation_level:` keyword) would reject as a missing positional argument
+      def read_partition_offsets(topic_partition_offsets, isolation_level: nil)
+        l_config = proxy_config.query_watermark_offsets
+
+        with_broker_errors_retry(
+          # required to be in seconds, not ms
+          wait_time: l_config.wait_time / 1_000.to_f,
+          max_attempts: l_config.max_attempts
+        ) do
+          @wrapped
+            .list_offsets(topic_partition_offsets, isolation_level: isolation_level)
+            .wait(max_wait_timeout_ms: l_config.timeout)
+            .offsets
+        rescue Rdkafka::AbstractHandle::WaitTimeoutError
+          # Align with the error contract of the per-partition watermark query, so a slow broker
+          # is retried here instead of escaping as a handle wait timeout
+          raise Rdkafka::RdkafkaError.new(
+            Rdkafka::Bindings::RD_KAFKA_RESP_ERR__TIMED_OUT,
+            "Error querying offsets of '#{topic_partition_offsets.keys.join(", ")}'"
+          )
+        end
+      end
+
       # Similar to `#query_watermark_offsets`, this method can be sensitive to latency. We handle
       # this the same way
       #

data/lib/karafka/connection/status.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 module Karafka
-  # Namespace for Kafka connection related logic
   module Connection
     # Listener connection status representation
     class Status

data/lib/karafka/instrumentation/assignments_tracker.rb

@@ -14,6 +14,16 @@ module Karafka
     class AssignmentsTracker
       include Singleton
 
+      # Upper bound on the number of distinct topics tracked in `@generations`. Generations are
+      # intentionally retained after revocation (so they can keep incrementing across
+      # reassignments), which means the map only ever grows. With static topologies that is
+      # bounded by the routing topic count, but pattern subscriptions inject a fresh, permanent
+      # `Routing::Topic` per discovered name, so without a bound the map (and the frozen deep copy
+      # built in `#generations`) would grow for the whole process lifetime. When the bound is
+      # exceeded we evict the least-recently-assigned topics. It is high enough never to be reached
+      # by realistic static or even large dynamic topologies.
+      GENERATIONS_TOPICS_LIMIT = 100_000
+
       class << self
         # @return [Hash{Karafka::Routing::Topic => Array<Integer>}]
         # @see #current
@@ -74,6 +84,10 @@ module Karafka
       #   reassignment. Revoked partitions remain in the hash with their last generation value.
       #
       # @note Returns a frozen deep copy to prevent external mutation
+      #
+      # @note The map is bounded to `GENERATIONS_TOPICS_LIMIT` topics; once exceeded, the
+      #   least-recently-assigned topics are evicted. This only ever matters for very large dynamic
+      #   (pattern-subscribed) topologies - static ones never reach the bound.
       def generations
         result = {}
 
@@ -188,11 +202,31 @@ module Karafka
               @generations[topic][partition_id] += 1
             end
 
+            # Move this topic to the end of the insertion-ordered map so it counts as
+            # most-recently-assigned for the bounded eviction below (keeps actively reassigned
+            # topics, drops genuinely stale ones first).
+            @generations[topic] = @generations.delete(topic)
+
             @assignments[topic] += partition_ids
             @assignments[topic].sort!
           end
+
+          evict_stale_generations
         end
       end
+
+      private
+
+      # Keeps `@generations` bounded by evicting the least-recently-assigned topics once the limit
+      # is exceeded. Without this, pattern subscriptions (which keep injecting new permanent topics)
+      # would grow the map without bound for the whole process lifetime.
+      def evict_stale_generations
+        excess = @generations.size - GENERATIONS_TOPICS_LIMIT
+
+        return unless excess.positive?
+
+        @generations.keys.first(excess).each { |topic| @generations.delete(topic) }
+      end
     end
   end
 end

data/lib/karafka/instrumentation/callbacks/consumer_groups/decorator.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    module Callbacks
+      module ConsumerGroups
+        # Default statistics decorator used for consumer group statistics decoration.
+        #
+        # This is a thin subclass (rather than referencing
+        # `Karafka::Core::Monitoring::StatisticsDecorator` directly) so it can be swapped out via
+        # `config.internal.statistics.consumer_groups.decorator_class` for a custom decorator (for
+        # example one that also enriches or corrects specific statistics values) without having to
+        # alter the callback that uses it.
+        class Decorator < Karafka::Core::Monitoring::StatisticsDecorator
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/callbacks/consumer_groups/error.rb

@@ -4,7 +4,6 @@ module Karafka
   module Instrumentation
     # Callbacks used to transport things from rdkafka
     module Callbacks
-      # Consumer-group-specific librdkafka callbacks
       module ConsumerGroups
         # Callback that kicks in when consumer error occurs and is published in a background thread
         class Error

data/lib/karafka/instrumentation/callbacks/consumer_groups/statistics.rb

@@ -9,7 +9,8 @@ module Karafka
         #   those statistics
         class Statistics
           include Helpers::ConfigImporter.new(
-            monitor: %i[monitor]
+            monitor: %i[monitor],
+            decorator_class: %i[internal statistics consumer_groups decorator_class]
           )
 
           # @param subscription_group_id [String]
@@ -19,7 +20,7 @@ module Karafka
             @subscription_group_id = subscription_group_id
             @group_id = group_id
             @client_name = client_name
-            @statistics_decorator = Karafka::Core::Monitoring::StatisticsDecorator.new
+            @statistics_decorator = decorator_class.new
           end
 
           # Emits decorated statistics to the monitor

data/lib/karafka/instrumentation/critical_errors_listener.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Listener that watches the `error.occurred` bus for process-critical errors and initiates a
+    # graceful shutdown when one appears, regardless of where in the framework it surfaced
+    # (consumption flow, after-consume flow, worker, etc.).
+    #
+    # Process-critical errors (see the `internal.processing.critical_errors` setting) are not
+    # something a retry can handle: SystemExit means termination was explicitly requested and
+    # NoMemoryError means the VM cannot be trusted to continue. The error containment in the
+    # processing flow keeps the affected partition paused and its offsets uncommitted, so after
+    # the restart the failed work is redelivered - the stop initiated here is what converts that
+    # containment into the actual process replacement.
+    #
+    # It is auto-subscribed as part of Karafka itself.
+    class CriticalErrorsListener
+      include Singleton
+      include Helpers::ConfigImporter.new(
+        critical_errors: %i[internal processing critical_errors]
+      )
+
+      # Sets up the one-shot initiation state
+      def initialize
+        @mutex = Mutex.new
+        @initiated = false
+      end
+
+      # Re-arms the listener when the app (re)starts so in-process restarts (embedded mode,
+      # integration flows) can escalate again
+      # @param _event [Karafka::Core::Monitoring::Event]
+      def on_app_running(_event)
+        @mutex.synchronize { @initiated = false }
+      end
+
+      # @param event [Karafka::Core::Monitoring::Event] error.occurred event. Its `:error` key
+      #   carries the error that was reported.
+      def on_error_occurred(event)
+        error = event[:error]
+
+        return unless critical_errors.any? { |type| error.is_a?(type) }
+
+        # The one-shot flag makes the check-and-spawn atomic: critical errors reported by many
+        # consumers at the same moment (or a duplicated bus subscription) spawn exactly one
+        # supervising stop thread
+        @mutex.synchronize do
+          return if @initiated
+          # No need to initiate the stop if one is already in motion (shutdown requested before
+          # the error surfaced). We deliberately do not use `App.done?` here: it includes the
+          # quieting/quiet states, in which the process is parked but alive - a critical error
+          # raised by in-flight work during quiet must still escalate to a full stop
+          return if Karafka::App.stopping?
+          return if Karafka::App.stopped?
+          return if Karafka::App.terminated?
+
+          @initiated = true
+
+          # `Karafka::Server.stop` supervises the whole shutdown (including the forceful timeout
+          # path), so it runs from a dedicated thread: the reporting thread must return so the
+          # processing flow can finish and engage the pause that protects the affected partition
+          # during the shutdown window
+          Thread.new do
+            Thread.current.name = "karafka.critical_shutdown"
+
+            Karafka::Server.stop
+          end
+        end
+      # The spawn itself can fail exactly under the conditions this listener handles (e.g.
+      # NoMemoryError, ThreadError at the thread limit). It must never escape: an error raised
+      # from an `error.occurred` subscriber would propagate into the reporting flow and could
+      # bypass the containment that protects the failed batch. We leave a best-effort fatal
+      # trace because in this degenerate state the process will NOT shut down despite the
+      # critical error - without it the why would be undiagnosable
+      rescue Exception
+        begin
+          Karafka.logger.fatal(
+            "Failed to initiate the critical error shutdown - process continues without it"
+          )
+        rescue Exception
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -467,6 +467,8 @@ module Karafka
         when "virtual_partitions.partitioner.error"
           error "Virtual partitions partitioner error occurred: #{details}"
           error backtrace
+        when "lag_compensation.refresher.error"
+          warn "Lag compensation refresher error occurred: #{details}"
         # This handles any custom errors coming from places like Web-UI, etc
         else
           error "#{type} error occurred: #{error.class} - #{details}"

data/lib/karafka/instrumentation/monitor.rb

@@ -2,9 +2,8 @@
 
 module Karafka
   module Instrumentation
-    # Karafka instrumentation monitor that we use to publish events
-    # By default uses our internal notifications bus but can be used with
-    # `ActiveSupport::Notifications` as well
+    # Karafka instrumentation monitor that we use to publish events. By default uses our internal
+    # notifications bus but can be used with `ActiveSupport::Notifications` as well
     class Monitor < Karafka::Core::Monitoring::Monitor
       attr_reader :notifications_bus
 

data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb

@@ -5,7 +5,6 @@ require_relative "base"
 module Karafka
   module Instrumentation
     module Vendors
-      # Namespace for Appsignal instrumentation
       module Appsignal
         # Listener for reporting errors from both consumers and producers
         # Since we have the same API for WaterDrop and Karafka, we can use one listener with

data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb

@@ -5,7 +5,6 @@ require_relative "base"
 module Karafka
   module Instrumentation
     module Vendors
-      # Namespace for Appsignal instrumentation
       module Appsignal
         # Listener that ships metrics to Appsignal
         class MetricsListener < Base

data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb

@@ -6,8 +6,7 @@ module Karafka
     module Vendors
       # Datadog specific instrumentation
       module Datadog
-        # A karafka's logger listener for Datadog
-        # It depends on the 'ddtrace' gem
+        # A karafka's logger listener for Datadog. It depends on the 'ddtrace' gem
         class LoggerListener
           include Karafka::Core::Configurable
           extend Forwardable
@@ -17,7 +16,7 @@ module Karafka
           # `Datadog::Tracing` client that we should use to trace stuff
           setting :client
 
-          # @see https://docs.datadoghq.com/tracing/trace_collection/dd_libraries/ruby
+          # See https://docs.datadoghq.com/tracing/trace_collection/dd_libraries/ruby
           setting :service_name, default: nil
 
           configure
@@ -145,6 +144,8 @@ module Karafka
               error "Parallel segments partitioner error occurred: #{error}"
             when "virtual_partitions.partitioner.error"
               error "Virtual partitions partitioner error occurred: #{error}"
+            when "lag_compensation.refresher.error"
+              warn "Lag compensation refresher error occurred: #{error}"
             else
               error "#{event[:type]} error occurred: #{error}"
             end

data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb

@@ -2,9 +2,7 @@
 
 module Karafka
   module Instrumentation
-    # Namespace for vendor specific instrumentation
     module Vendors
-      # Datadog specific instrumentation
       module Datadog
         # Listener that can be used to subscribe to Karafka to receive stats via StatsD
         # and/or Datadog

data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb

@@ -5,7 +5,6 @@ require "karafka/instrumentation/vendors/kubernetes/base_listener"
 module Karafka
   module Instrumentation
     module Vendors
-      # Namespace for instrumentation related with Kubernetes
       module Kubernetes
         # Kubernetes HTTP listener that does not only reply when process is not fully hanging, but
         # also allows to define max time of processing and looping.
@@ -26,6 +25,21 @@ module Karafka
         #
         # @note Please use `Kubernetes::SwarmLivenessListener` when operating in the swarm mode
         class LivenessListener < BaseListener
+          # Default time in ms after which we consider consumption hanging (5 minutes)
+          DEFAULT_CONSUMING_TTL = 5 * 60 * 1_000
+          # Default max time in ms between polls before the process is considered dead (5 minutes)
+          DEFAULT_POLLING_TTL = 5 * 60 * 1_000
+          # Multiplier applied to max.poll.interval.ms when deriving the default stability_ttl.
+          # 2x gives headroom above the slowest legitimate rebalance phase (cooperative
+          # `wait-unassign-to-complete` can approach max.poll.interval.ms).
+          STABILITY_TTL_MAX_POLL_MULTIPLIER = 2
+
+          private_constant(
+            :DEFAULT_CONSUMING_TTL,
+            :DEFAULT_POLLING_TTL,
+            :STABILITY_TTL_MAX_POLL_MULTIPLIER
+          )
+
           # @param hostname [String, nil] hostname or nil to bind on all
           # @param port [Integer] TCP port on which we want to run our HTTP status server
           # @param consuming_ttl [Integer] time in ms after which we consider consumption hanging.
@@ -33,12 +47,36 @@ module Karafka
           #   process as hanging
           # @param polling_ttl [Integer] max time in ms for polling. If polling (any) does not
           #   happen that often, process should be considered dead.
-          # @note The default TTL matches the default `max.poll.interval.ms`
+          # @param stability_ttl [Integer, nil] max time in ms a subscription group can remain in
+          #   the same tracked non-"steady" librdkafka `cgrp.join_state` (e.g. `wait-join`,
+          #   `wait-assn`, `wait-sync`) before the process is considered unhealthy. When nil
+          #   (default), it is derived lazily on first health evaluation as the maximum
+          #   `max.poll.interval.ms` across all active subscription groups, multiplied by 2,
+          #   which keeps the threshold safe relative to the user's actual poll intervals rather
+          #   than a fixed constant. Per subscription group overrides of `max.poll.interval.ms`
+          #   set via the routing DSL are respected - the largest value wins. When no subscription
+          #   groups are available (routes not drawn), the root `Karafka::App.config.kafka` value
+          #   is used instead (with librdkafka's default applied). `wait-metadata`
+          #   and `init` are
+          #   excluded from tracking and additionally clear any stale prior tracking for the
+          #   subscription group: `wait-metadata` appears in normal scenarios (auto-create waits,
+          #   unmatched pattern subscriptions, and as the group leader's post-JoinGroup state while
+          #   fetching cluster metadata to compute the partition assignment); `init` indicates a
+          #   client reset. Both states are covered by `polling_ttl` for genuine freezes.
+          #   The timer resets on every join_state transition between tracked states because any
+          #   change indicates the join protocol is still progressing; only a consumer frozen in
+          #   the same non-steady state continuously triggers the alarm. A consumer cycling
+          #   rapidly between non-steady states (e.g. wait-join -> wait-assn -> wait-join) will
+          #   not trip this check - it only catches consumers frozen in a single join state.
+          #   Requires `statistics.interval.ms` to be configured in the Kafka client settings;
+          #   without it `statistics.emitted` never fires, tracking hashes remain empty, and this
+          #   check has no effect (no misreporting).
           def initialize(
             hostname: nil,
             port: 3000,
-            consuming_ttl: 5 * 60 * 1_000,
-            polling_ttl: 5 * 60 * 1_000
+            consuming_ttl: DEFAULT_CONSUMING_TTL,
+            polling_ttl: DEFAULT_POLLING_TTL,
+            stability_ttl: nil
           )
             # If this is set to a symbol, it indicates unrecoverable error like fencing
             # While fencing can be partial (for one of the SGs), we still should consider this
@@ -47,9 +85,21 @@ module Karafka
             @unrecoverable = false
             @polling_ttl = polling_ttl
             @consuming_ttl = consuming_ttl
+            # When nil, resolved lazily via #stability_ttl on first health evaluation because
+            # the default derives from the routing subscription groups, which may not be drawn
+            # yet when the listener is constructed in karafka.rb
+            @stability_ttl = stability_ttl
             @mutex = Mutex.new
             @pollings = {}
             @consumptions = {}
+            # Maps subscription_group_id => last observed cgrp.join_state string.
+            # Used to detect state changes so the stuck timer resets on any transition.
+            # Both hashes are only populated by on_statistics_emitted, which requires
+            # statistics.interval.ms to be set. Without it they remain empty and the
+            # stability flag in evaluate_ttl_flags is always false - no misreporting.
+            @join_states = {}
+            # Maps subscription_group_id => monotonic_now when the current non-steady state began.
+            @instabilities = {}
             super(hostname: hostname, port: port)
           end
 
@@ -70,6 +120,43 @@ module Karafka
             mark_polling_tick
           end
 
+          # Track when each subscription group's consumer enters or leaves a non-steady join state.
+          # A prolonged non-steady state (e.g., wait-join, wait-assn caused by a broker stuck in
+          # CompletingRebalance) is reported as unhealthy.
+          #
+          # `steady`, `init`, and `wait-metadata` all clear any stale tracking for the subscription
+          # group rather than skip the event. Clearing on `wait-metadata` matters because the
+          # state legitimately appears mid-join (leader fetching cluster metadata to compute
+          # assignment), during auto-create waits, and on unmatched pattern subscriptions - if we
+          # only `return` from these the previous non-steady timer keeps running and eventually
+          # trips a false positive. Clearing on `init` covers client resets / subscription-group
+          # removal. Genuine freezes in these states are covered by `polling_ttl`.
+          #
+          # The stuck timer resets on every transition between tracked states because any change
+          # indicates the join protocol is still progressing. Only a consumer frozen in the same
+          # non-steady state continuously for stability_ttl is flagged. A consumer cycling
+          # rapidly between non-steady states (e.g. wait-join -> wait-assn -> wait-join) will not
+          # trip this alarm - only a frozen-in-one-state consumer is caught.
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_statistics_emitted(event)
+            cgrp = event[:statistics]&.dig("cgrp")
+            return unless cgrp
+
+            sg_id = event[:subscription_group_id]
+            join_state = cgrp["join_state"]
+            return unless join_state
+
+            synchronize do
+              if join_state == "steady" || join_state == "init" || join_state == "wait-metadata"
+                @instabilities.delete(sg_id)
+                @join_states.delete(sg_id)
+              elsif @join_states[sg_id] != join_state
+                @join_states[sg_id] = join_state
+                @instabilities[sg_id] = monotonic_now
+              end
+            end
+          end
+
           {
             consume: :consumed,
             revoke: :revoked,
@@ -127,30 +214,95 @@ module Karafka
             clear_polling_tick
           end
 
+          # Clear instability tracking for a subscription group whose listener finished its fetch
+          # loop (dynamic downscale or shutdown). This event fires after the listener's client has
+          # already been stopped, so no further statistics can move the join_state back to steady
+          # and no late statistics callback can re-create the entry. Without this cleanup a
+          # lingering non-steady entry (e.g. a downscale mid-rebalance) would guarantee a false
+          # positive once stability_ttl elapses. Genuine shutdown freezes remain covered by
+          # polling_ttl.
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_connection_listener_after_fetch_loop(event)
+            clear_instability_tracking(event[:subscription_group])
+          end
+
           # Did we exceed any of the ttls
-          # @return [String] 204 string if ok, 500 otherwise
+          # @return [Boolean] true if all TTLs are within bounds and no unrecoverable error
+          # @note When called from inside `status_body`, reuses the cached snapshot so the
+          #   per-request HTTP response stays internally consistent. When called standalone,
+          #   takes its own snapshot.
           def healthy?
             return false if @unrecoverable
-            return false if polling_ttl_exceeded?
-            return false if consuming_ttl_exceeded?
 
-            true
+            flags = @evaluation_snapshot || evaluate_ttl_flags
+            !flags[:polling] && !flags[:consuming] && !flags[:stability]
           end
 
           private
 
-          # @return [Boolean] true if the consumer exceeded the polling ttl
-          def polling_ttl_exceeded?
+          # Single-snapshot evaluation of all three TTL flags under one lock. Used by both
+          # `healthy?` and `status_body` so the HTTP status code (derived from healthy?) and the
+          # error details in the response body are always derived from the same atomic snapshot.
+          # @return [Hash{Symbol => Boolean}] flags for polling, consuming and stability
+          def evaluate_ttl_flags
             time = monotonic_now
+            synchronize do
+              {
+                polling: @pollings.values.any? { |tick| (time - tick) > @polling_ttl },
+                consuming: @consumptions.values.any? { |tick| (time - tick) > @consuming_ttl },
+                stability: @instabilities.values.any? { |start| (time - start) > stability_ttl }
+              }
+            end
+          end
 
-            @pollings.values.any? { |tick| (time - tick) > @polling_ttl }
+          # @return [Integer] effective stability ttl in ms. Resolved lazily (instead of in
+          #   `#initialize`) because the default derives from the routing subscription groups,
+          #   which may not be drawn yet when the listener is constructed in karafka.rb. By the
+          #   time health is evaluated, routing is final.
+          def stability_ttl
+            @stability_ttl ||= default_stability_ttl
           end
 
-          # @return [Boolean] true if the consumer exceeded the consuming ttl
-          def consuming_ttl_exceeded?
-            time = monotonic_now
+          # @return [Integer] default stability_ttl derived from the configured
+          #   max.poll.interval.ms, multiplied by 2 for headroom above the slowest legitimate
+          #   rebalance phase. Uses the maximum value across all active subscription groups
+          #   (their kafka configs already have librdkafka defaults injected), so per
+          #   subscription group overrides set via the routing DSL are respected. Falls back to
+          #   the root kafka config when no subscription groups are available.
+          def default_stability_ttl
+            max_poll_interval = Karafka::App
+              .subscription_groups
+              .values
+              .flatten
+              .map { |sg| sg.kafka.fetch(:"max.poll.interval.ms") }
+              .max
+
+            max_poll_interval ||= begin
+              # DefaultsInjector#consumer mutates a kafka config hash to ensure
+              # max.poll.interval.ms is set (using the same librdkafka default Karafka injects
+              # per subscription group). We dup so we don't mutate the global config.
+              kafka_config = Karafka::App.config.kafka.dup
+              Karafka::Setup::DefaultsInjector.consumer(kafka_config)
+              kafka_config.fetch(:"max.poll.interval.ms")
+            end
 
-            @consumptions.values.any? { |tick| (time - tick) > @consuming_ttl }
+            max_poll_interval * STABILITY_TTL_MAX_POLL_MULTIPLIER
+          end
+
+          # Removes join_state tracking for the given subscription group. Used when its listener
+          # stops, since no further statistics events will arrive to move the state back to
+          # steady.
+          # @param subscription_group [Karafka::Routing::SubscriptionGroup, nil] subscription
+          #   group from the event payload (nil-safe for events emitted without one)
+          def clear_instability_tracking(subscription_group)
+            return unless subscription_group
+
+            sg_id = subscription_group.id
+
+            synchronize do
+              @instabilities.delete(sg_id)
+              @join_states.delete(sg_id)
+            end
           end
 
           # Wraps the logic with a mutex
@@ -194,15 +346,26 @@ module Karafka
             end
           end
 
-          # @return [Hash] response body status
+          # @return [Hash] response body status. Takes a single locked snapshot of all three TTL
+          #   flags and caches it for the duration of this call so that `super` (which calls
+          #   `healthy?` to populate the status field) and the merged `errors` hash see the same
+          #   values. Avoids the previous pattern of taking the lock six times per response, where
+          #   state could change between acquisitions and leave the status field disagreeing with
+          #   the errors hash. The base envelope (timestamp, port, process_id, ...) still comes
+          #   from `super` so any future field added to `BaseListener#status_body` is preserved.
           def status_body
+            @evaluation_snapshot = evaluate_ttl_flags
+
             super.merge!(
               errors: {
-                polling_ttl_exceeded: polling_ttl_exceeded?,
-                consumption_ttl_exceeded: consuming_ttl_exceeded?,
+                polling_ttl_exceeded: @evaluation_snapshot[:polling],
+                consumption_ttl_exceeded: @evaluation_snapshot[:consuming],
+                stability_ttl_exceeded: @evaluation_snapshot[:stability],
                 unrecoverable: @unrecoverable
               }
             )
+          ensure
+            @evaluation_snapshot = nil
           end
         end
       end

data/lib/karafka/pro/admin/recovery/errors.rb

@@ -30,7 +30,6 @@
 
 module Karafka
   module Pro
-    # Pro Admin utilities
     module Admin
       class Recovery < Karafka::Admin
         # Recovery related errors

data/lib/karafka/pro/admin/recovery.rb

@@ -159,14 +159,16 @@ module Karafka
         #   )
         #
         # @example Read offsets from a specific point in time
-        #   Karafka::Admin::Recovery.read_committed_offsets('sync', last_committed_at: Time.new(2025, 3, 1))
+        #   Karafka::Admin::Recovery.read_committed_offsets(
+        #     'sync', last_committed_at: Time.new(2025, 3, 1)
+        #   )
         #
         # @example Migrate a stuck consumer group to a new name (two-step workflow)
         #   # Step 1: Read committed offsets from the broken group (bypasses coordinator)
         #   offsets = Karafka::Admin::Recovery.read_committed_offsets('sync')
         #   #=> { 'events' => { 0 => 1400, 1 => 1402 }, 'orders' => { 0 => 890 } }
         #
-        #   # Step 2: Inspect the recovered offsets — verify all expected topics and partitions
+        #   # Step 2: Inspect the recovered offsets - verify all expected topics and partitions
         #   # are present and the offset values look reasonable before committing them
         #
         #   # Step 3: Write the offsets to the target group using standard Admin APIs
@@ -193,11 +195,11 @@ module Karafka
             next unless parsed[:group] == group_id
 
             if parsed[:offset].nil?
-              # Tombstone — offset was deleted, remove from results
+              # Tombstone - offset was deleted, remove from results
               committed[parsed[:topic]].delete(parsed[:partition])
               committed.delete(parsed[:topic]) if committed[parsed[:topic]].empty?
             else
-              # Last write wins — scanning forward means we naturally end up with the most
+              # Last write wins - scanning forward means we naturally end up with the most
               # recent commit per partition
               committed[parsed[:topic]][parsed[:partition]] = parsed[:offset]
             end