karafka 2.6.0.beta1 → 2.6.0.rc1

data/lib/karafka/admin.rb

@@ -3,8 +3,10 @@
 module Karafka
   # Admin actions that we can perform via Karafka on our Kafka cluster
   #
-  # @note It always initializes a new admin instance as we want to ensure it is always closed
-  #   Since admin actions are not performed that often, that should be ok.
+  # @note Each operation initializes its own dedicated instance as we want to ensure it is
+  #   always closed. Since admin actions are not performed that often, that should be ok. When
+  #   an external client is provided, it is reused instead and its lifecycle stays fully with
+  #   its owner.
   #
   # @note By default it uses the primary defined cluster. For multi-cluster operations, create
   #   an Admin instance with custom kafka configuration:
@@ -22,34 +24,80 @@ module Karafka
       admin_kafka: %i[admin kafka]
     )
 
-    # Custom kafka configuration for this admin instance
-    # @return [Hash] custom kafka settings to merge with defaults
-    attr_reader :custom_kafka
-
     # Creates a new Admin instance
     #
     # @param kafka [Hash] custom kafka configuration to merge with app defaults.
     #   Useful for multi-cluster operations where you want to target a different cluster.
+    # @param external_client [Object, nil] active rdkafka client (raw, wrapped with
+    #   `Karafka::Connection::Proxy` or a `Karafka::Connection::Client` of a running consumer)
+    #   on which admin operations should run, instead of each operation creating its own
+    #   short-lived instance. Routing is capability based: rdkafka admin instances are used by
+    #   admin-based operations (`with_admin` and everything built on top of it, e.g.
+    #   `cluster_info`), any other external client is used by consumer-based operations
+    #   (`with_consumer` and everything built on top of it). The lifecycle of an external
+    #   client belongs fully to its owner: it is never configured, started or closed here and
+    #   all operations run within its identity, including its `group.id`. This is a low-level
+    #   internal API: the caller is responsible for providing a client capable of the invoked
+    #   operations and for invoking only operations that are safe to run on a live client.
+    #
+    # @note Raw and proxied rdkafka instances are resolved once at construction, so the admin
+    #   instance should not outlive them. `Karafka::Connection::Client` instances are resolved
+    #   on each use instead, so the admin instance follows such a client across the underlying
+    #   connection recovery resets.
     #
     # @example Create admin for a different cluster
     #   admin = Karafka::Admin.new(kafka: { 'bootstrap.servers': 'other-cluster:9092' })
     #   admin.cluster_info
-    def initialize(kafka: {})
+    #
+    # @example Read lags of a running consumer via its own client connection
+    #   admin = Karafka::Admin.new(external_client: client)
+    #   admin.read_lags_with_offsets({ 'my-group' => ['events'] })
+    def initialize(kafka: {}, external_client: nil)
       @custom_kafka = kafka
+      @external_client = nil
+      @external_admin = false
+      @external_consumer = false
+
+      return unless external_client
+
+      # A running consumer connection client is kept as-is and its current handle is resolved
+      # on each use: such clients swap their underlying rdkafka instance across recovery
+      # resets, so a handle resolved once at construction could go stale while the client
+      # itself remains fully operational
+      if external_client.is_a?(Karafka::Connection::Client)
+        @external_client = external_client
+        @external_consumer = true
+
+        return
+      end
+
+      # Raw and proxied rdkafka instances are wrapped upfront - the proxy itself prevents
+      # double wrapping
+      @external_client = Karafka::Connection::Proxy.new(external_client)
+
+      # Capability based routing resolved once upfront: rdkafka admin instances serve
+      # admin-based operations, any other external client is assumed consumer capable
+      if @external_client.wrapped.is_a?(Rdkafka::Admin)
+        @external_admin = true
+      else
+        @external_consumer = true
+      end
     end
 
     # No-op close to normalize the API surface.
     #
-    # Each admin operation currently opens and closes its own underlying rdkafka admin instance
-    # internally, so there is nothing to release at the `Karafka::Admin` level right now. This
-    # method exists so that callers who hold an instance and call `#close` on it (matching the
-    # pattern of other closeable resources) do not raise `NoMethodError`.
+    # Each admin operation opens and closes its own dedicated rdkafka instance internally,
+    # while external clients are owned and closed by their providers, so there is nothing to
+    # release at the `Karafka::Admin` level right now. This method exists so that callers who
+    # hold an instance and call `#close` on it (matching the pattern of other closeable
+    # resources) do not raise `NoMethodError`.
     #
-    # In the future, `Karafka::Admin` is planned to be refactored to reuse a single rdkafka admin
-    # instance across multiple operations rather than creating and tearing one down per call. When
-    # that happens, this method will need to release that shared instance. The no-op is here now
-    # so that all callers are already written against the correct API and require no changes when
-    # the real implementation lands.
+    # In the future, `Karafka::Admin` is planned to be refactored to reuse a single owned
+    # rdkafka admin instance across multiple operations rather than creating and tearing one
+    # down per call. When that happens, this method will need to release that shared owned
+    # instance - though never an external client. The no-op is here now so that all callers are
+    # already written against the correct API and require no changes when the real
+    # implementation lands.
     def close
     end
 
@@ -257,7 +305,7 @@ module Karafka
     # @param settings [Hash] kafka extra settings (optional)
     # @see Topics#read
     def read_topic(name, partition, count, start_offset = -1, settings = {})
-      Topics.new(kafka: @custom_kafka).read(name, partition, count, start_offset, settings)
+      topics_admin.read(name, partition, count, start_offset, settings)
     end
 
     # @param name [String] topic name
@@ -266,47 +314,47 @@ module Karafka
     # @param topic_config [Hash] topic config details
     # @see Topics#create
     def create_topic(name, partitions, replication_factor, topic_config = {})
-      Topics.new(kafka: @custom_kafka).create(name, partitions, replication_factor, topic_config)
+      topics_admin.create(name, partitions, replication_factor, topic_config)
     end
 
     # @param name [String] topic name
     # @see Topics#delete
     def delete_topic(name)
-      Topics.new(kafka: @custom_kafka).delete(name)
+      topics_admin.delete(name)
     end
 
     # @param name [String] topic name
     # @param partitions [Integer] total number of partitions we expect to end up with
     # @see Topics#create_partitions
     def create_partitions(name, partitions)
-      Topics.new(kafka: @custom_kafka).create_partitions(name, partitions)
+      topics_admin.create_partitions(name, partitions)
     end
 
     # @param name_or_hash [String, Symbol, Hash] topic name or hash with topics and partitions
     # @param partition [Integer, nil] partition (nil when using hash format)
     # @see Topics#read_watermark_offsets
     def read_watermark_offsets(name_or_hash, partition = nil)
-      Topics.new(kafka: @custom_kafka).read_watermark_offsets(name_or_hash, partition)
+      topics_admin.read_watermark_offsets(name_or_hash, partition)
     end
 
     # @param topic_partition_offsets [Hash{String => Array<Hash>}] topics with partition specs
     # @param isolation_level [Integer, nil] optional isolation level constant
     # @see Topics#read_partition_offsets
     def read_partition_offsets(topic_partition_offsets, isolation_level: nil)
-      Topics.new(kafka: @custom_kafka).read_partition_offsets(topic_partition_offsets, isolation_level: isolation_level)
+      topics_admin.read_partition_offsets(topic_partition_offsets, isolation_level: isolation_level)
     end
 
     # @param topic_name [String] name of the topic we're interested in
     # @see Topics#info
     def topic_info(topic_name)
-      Topics.new(kafka: @custom_kafka).info(topic_name)
+      topics_admin.info(topic_name)
     end
 
     # @param group_id [String] group for which we want to move offsets
     # @param topics_with_partitions_and_offsets [Hash] hash with topics and settings
     # @see ConsumerGroups#seek
     def seek_consumer_group(group_id, topics_with_partitions_and_offsets)
-      ConsumerGroups.new(kafka: @custom_kafka).seek(
+      consumer_groups_admin.seek(
         group_id,
         topics_with_partitions_and_offsets
       )
@@ -317,7 +365,7 @@ module Karafka
     # @param topics [Array<String>] topics for which we want to copy offsets
     # @see ConsumerGroups#copy
     def copy_consumer_group(previous_name, new_name, topics)
-      ConsumerGroups.new(kafka: @custom_kafka).copy(previous_name, new_name, topics)
+      consumer_groups_admin.copy(previous_name, new_name, topics)
     end
 
     # @param previous_name [String] old consumer group name
@@ -326,7 +374,7 @@ module Karafka
     # @param delete_previous [Boolean] should we delete previous consumer group after rename
     # @see ConsumerGroups#rename
     def rename_consumer_group(previous_name, new_name, topics, delete_previous: true)
-      ConsumerGroups.new(kafka: @custom_kafka).rename(
+      consumer_groups_admin.rename(
         previous_name,
         new_name,
         topics,
@@ -337,13 +385,13 @@ module Karafka
     # @param group_id [String] group name
     # @see ConsumerGroups#delete
     def delete_consumer_group(group_id)
-      ConsumerGroups.new(kafka: @custom_kafka).delete(group_id)
+      consumer_groups_admin.delete(group_id)
     end
 
     # @param group_id [String] group id to trigger rebalance for
     # @see ConsumerGroups#trigger_rebalance
     def trigger_rebalance(group_id)
-      ConsumerGroups.new(kafka: @custom_kafka).trigger_rebalance(group_id)
+      consumer_groups_admin.trigger_rebalance(group_id)
     end
 
     # @param groups_with_topics [Hash{String => Array<String>}] hash with group
@@ -351,7 +399,7 @@ module Karafka
     # @param active_topics_only [Boolean] if set to false, will select also inactive topics
     # @see ConsumerGroups#read_lags_with_offsets
     def read_lags_with_offsets(groups_with_topics = {}, active_topics_only: true)
-      ConsumerGroups.new(kafka: @custom_kafka).read_lags_with_offsets(
+      consumer_groups_admin.read_lags_with_offsets(
         groups_with_topics,
         active_topics_only: active_topics_only
       )
@@ -362,7 +410,7 @@ module Karafka
     # @param brokers [Hash, nil] optional manual broker assignments per partition
     # @see Replication#plan
     def plan_topic_replication(topic:, replication_factor:, brokers: nil)
-      Replication.new(kafka: @custom_kafka).plan(
+      Replication.new(kafka: @custom_kafka, external_client: @external_client).plan(
         topic: topic,
         to: replication_factor,
         brokers: brokers
@@ -381,53 +429,101 @@ module Karafka
     #
     # @note We always ship and yield a proxied consumer because admin API performance is not
     #   that relevant. That is, there are no high frequency calls that would have to be delegated
+    #
+    # @note When an external client is present, it is yielded wrapped with
+    #   `Karafka::Connection::Proxy` (which unwraps pre-proxied clients, so no double wrapping
+    #   occurs) and its lifecycle is not managed here in any way: no oauth binding, no start and
+    #   no closing. `settings` are
+    #   ignored as the external instance is already configured and operations run within its
+    #   identity, including its `group.id`. External rdkafka admin instances are not used here
+    #   as they are not capable of consumer operations - they are used by `#with_admin` instead.
     def with_consumer(settings = {})
-      bind_id = SecureRandom.uuid
+      return yield(external_client_proxy) if @external_consumer
 
-      consumer = config(:consumer, settings).consumer(native_kafka_auto_start: false)
-      bind_oauth(bind_id, consumer)
+      bind_id = SecureRandom.uuid
+      consumer = nil
 
-      consumer.start
-      proxy = Karafka::Connection::Proxy.new(consumer)
-      yield(proxy)
-    ensure
-      # Always unsubscribe consumer just to be sure, that no metadata requests are running
-      # when we close the consumer. This in theory should prevent from some race-conditions
-      # that originate from librdkafka
       begin
-        consumer&.unsubscribe
-      # Ignore any errors and continue to close consumer despite them
-      rescue Rdkafka::RdkafkaError
-        nil
+        consumer = config(:consumer, settings).consumer(native_kafka_auto_start: false)
+        bind_oauth(bind_id, consumer)
+
+        consumer.start
+        proxy = Karafka::Connection::Proxy.new(consumer)
+        yield(proxy)
+      ensure
+        # Always unsubscribe consumer just to be sure, that no metadata requests are running
+        # when we close the consumer. This in theory should prevent from some race-conditions
+        # that originate from librdkafka
+        begin
+          consumer&.unsubscribe
+        # Ignore any errors and continue to close consumer despite them
+        rescue Rdkafka::RdkafkaError
+          nil
+        end
+
+        consumer&.close
+
+        unbind_oauth(bind_id)
       end
-
-      consumer&.close
-
-      unbind_oauth(bind_id)
     end
 
     # Creates admin instance and yields it. After usage it closes the admin instance
+    #
+    # @note When the external client is an rdkafka admin instance, it is yielded wrapped with
+    #   `Karafka::Connection::Proxy` (which unwraps pre-proxied clients, so no double wrapping
+    #   occurs) and its lifecycle is not managed here in any way, same as with `#with_consumer`.
+    #   External
+    #   clients of other types (consumers, producers) are not capable of admin operations, thus
+    #   a dedicated admin instance is created for them as usual.
     def with_admin
+      return yield(external_client_proxy) if @external_admin
+
       bind_id = SecureRandom.uuid
+      admin = nil
 
-      admin = config(:producer, {}).admin(
-        native_kafka_auto_start: false,
-        native_kafka_poll_timeout_ms: self.class.poll_timeout
-      )
+      begin
+        admin = config(:producer, {}).admin(
+          native_kafka_auto_start: false,
+          native_kafka_poll_timeout_ms: self.class.poll_timeout
+        )
 
-      bind_oauth(bind_id, admin)
+        bind_oauth(bind_id, admin)
 
-      admin.start
-      proxy = Karafka::Connection::Proxy.new(admin)
-      yield(proxy)
-    ensure
-      admin&.close
+        admin.start
+        proxy = Karafka::Connection::Proxy.new(admin)
+        yield(proxy)
+      ensure
+        admin&.close
 
-      unbind_oauth(bind_id)
+        unbind_oauth(bind_id)
+      end
     end
 
     private
 
+    # @return [Karafka::Connection::Proxy] proxy wrapping the current external client handle.
+    #   For connection clients the handle is resolved on each use as they swap their underlying
+    #   rdkafka instance across recovery resets while remaining operational
+    def external_client_proxy
+      if @external_client.is_a?(Karafka::Connection::Client)
+        @external_client.wrapped_kafka
+      else
+        @external_client
+      end
+    end
+
+    # @return [Topics] topics admin operating within this instance context (custom kafka and
+    #   external client carried over)
+    def topics_admin
+      Topics.new(kafka: @custom_kafka, external_client: @external_client)
+    end
+
+    # @return [ConsumerGroups] consumer groups admin operating within this instance context
+    #   (custom kafka and external client carried over)
+    def consumer_groups_admin
+      ConsumerGroups.new(kafka: @custom_kafka, external_client: @external_client)
+    end
+
     # @return [Integer] max wait time in ms
     def max_wait_time_ms
       self.class.max_wait_time

data/lib/karafka/app.rb

@@ -57,7 +57,7 @@ module Karafka
 
       # Just a nicer name for the consumer groups
       alias_method :routes, :consumer_groups
-      # Generalized alias — routing entries are "groups" (consumer groups today, other kinds
+      # Generalized alias - routing entries are "groups" (consumer groups today, other kinds
       # of groups like KIP-932 share groups in the future).
       alias_method :groups, :consumer_groups
 

data/lib/karafka/base_consumer.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 
-# Karafka module namespace
 module Karafka
   # Base consumer from which all Karafka consumers should inherit
   class BaseConsumer
     # Allow for consumer instance tagging for instrumentation
     include Karafka::Core::Taggable
     include Helpers::ConfigImporter.new(
-      monitor: %i[monitor]
+      monitor: %i[monitor],
+      critical_errors: %i[internal processing critical_errors]
     )
 
     extend Forwardable
@@ -104,7 +104,17 @@ module Karafka
     #   message.
     def on_consume
       handle_consume
-    rescue => e
+    # Containment is intentionally broader than StandardError: any error escaping this method
+    # would bypass `#on_after_consume` in the worker, skipping the retry/pause flow entirely.
+    # The next successful batch would then auto-mark its offsets, durably committing past the
+    # failed batch - a silent at-least-once violation. Errors like SystemStackError or the
+    # ScriptError family (e.g. LoadError surfacing from a Rails autoload hiccup) are per-message
+    # failures and go through the regular retry flow like any other processing error.
+    # Process-critical errors additionally trigger a graceful shutdown via the auto-subscribed
+    # `Instrumentation::CriticalErrorsListener` watching this very instrumentation - the
+    # recorded failure plus the retry pause keep this partition protected during the shutdown
+    # window and the batch is redelivered after restart.
+    rescue Exception => e
       monitor.instrument(
         "error.occurred",
         error: e,
@@ -124,7 +134,11 @@ module Karafka
     #   flows do not interact with external systems and their errors are expected to bubble up
     def on_after_consume
       handle_after_consume
-    rescue => e
+    # Same containment rationale as in `#on_consume`: an error escaping this method would skip
+    # the retry below and the next successful batch would auto-mark its offsets, committing
+    # past the failed batch. The after-consume flow runs user-extensible code as well (DLQ
+    # strategies, dispatch enhancements), so it needs the same class-agnostic protection
+    rescue Exception => e
       monitor.instrument(
         "error.occurred",
         error: e,
@@ -380,7 +394,6 @@ module Karafka
     # @note Please note, that if you are seeking to a time offset, getting the offset is blocking
     def seek(offset, manual_seek = true, reset_offset: true)
       coordinator.manual_seek if manual_seek
-      self.seek_offset = nil if reset_offset
 
       message = Karafka::Messages::Seek.new(
         topic.name,
@@ -398,6 +411,12 @@ module Karafka
         reset_offset: reset_offset
       ) do
         client.seek(message)
+
+        # We reset the seek offset only after the seek actually succeeded. Resetting it before the
+        # seek would, on a raising seek (for example an unresolvable time-based offset), leave
+        # seek_offset nil so the failure-driven retry would pause without seeking back and skip the
+        # rest of the batch.
+        self.seek_offset = nil if reset_offset
       end
     end
 
@@ -446,5 +465,14 @@ module Karafka
         attempt: attempt
       )
     end
+
+    # @param error [Exception, nil] error to check or nil when none was recorded
+    # @return [Boolean] is the error one of the process-critical ones (configurable via the
+    #   `internal.processing.critical_errors` setting)
+    def critical_error?(error)
+      return false unless error
+
+      critical_errors.any? { |type| error.is_a?(type) }
+    end
   end
 end

data/lib/karafka/cli/console.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 module Karafka
-  # Karafka framework Cli
   class Cli
     # Console Karafka Cli action
     class Console < Base

data/lib/karafka/cli/help.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 module Karafka
-  # Karafka framework Cli
   class Cli
     # Prints info with list of commands available
     class Help < Base

data/lib/karafka/cli/info.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 module Karafka
-  # Karafka framework Cli
   class Cli
     # Info Karafka Cli action
     class Info < Base

data/lib/karafka/cli/install.rb

@@ -3,7 +3,6 @@
 require "erb"
 
 module Karafka
-  # Karafka framework Cli
   class Cli
     # Install Karafka Cli action
     class Install < Base

data/lib/karafka/cli/server.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 module Karafka
-  # Karafka framework Cli
   class Cli
     # Server Karafka Cli action
     class Server < Base

data/lib/karafka/cli/swarm.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 module Karafka
-  # Karafka framework Cli
   class Cli
     # Starts swarm of consumers forked from the supervisor
     class Swarm < Base

data/lib/karafka/cli.rb

@@ -50,7 +50,7 @@ end
 # so this method is only available when the Karafka console is running
 #
 # We skip this because this should exist and be only valid in the console
-# :nocov:
+# simplecov:disable
 if ENV["KARAFKA_CONSOLE"]
   # Reloads Karafka irb console session
   def reload!
@@ -58,4 +58,4 @@ if ENV["KARAFKA_CONSOLE"]
     Kernel.exec Karafka::Cli::Console.command
   end
 end
-# :nocov:
+# simplecov:enable

data/lib/karafka/connection/client.rb

@@ -28,6 +28,12 @@ module Karafka
       # @return [String] id of the client
       attr_reader :id
 
+      # @return [Karafka::Connection::Proxy, nil] proxy wrapping the underlying rdkafka consumer
+      #   instance or nil when the connection was not yet established. Used by the admin APIs
+      #   external client support to run admin operations on top of a running consumer
+      #   connection. Low-level internal API.
+      attr_reader :wrapped_kafka
+
       # How many times should we retry polling in case of a failure
       MAX_POLL_RETRIES = 20
 
@@ -320,8 +326,8 @@ module Karafka
         # won't get into this loop again. This can happen when supervision decides it should close
         # things faster
         #
-        # @see https://github.com/confluentinc/librdkafka/issues/4792
-        # @see https://github.com/confluentinc/librdkafka/issues/4527
+        # See https://github.com/confluentinc/librdkafka/issues/4792
+        # See https://github.com/confluentinc/librdkafka/issues/4527
         if unsubscribe?
           @unsubscribing = true
 
@@ -454,6 +460,26 @@ module Karafka
         @wrapped_kafka.query_watermark_offsets(topic, partition)
       end
 
+      # Resolves offsets of many partitions with one batched request instead of one broker
+      # roundtrip per partition.
+      #
+      # @param topic_partition_offsets [Hash{String => Array<Hash>}] topics with arrays of
+      #   partition offset specs, each with a `:partition` and an `:offset` (`:earliest`,
+      #   `:latest`, `:max_timestamp` or an integer timestamp in ms)
+      # @return [Array<Hash>] resolved offsets, each with `:topic`, `:partition` and `:offset`
+      # @note This consumer own isolation level is forwarded to the query. Be aware that the
+      #   underlying batched `ListOffsets` resolves `:latest` to the high watermark regardless of
+      #   the isolation level (unlike the consumer `query_watermark_offsets`, which returns the
+      #   last stable offset for a read_committed consumer). On a topic with an in-flight
+      #   transaction `:latest` therefore includes the uncommitted messages a read_committed
+      #   consumer will not see. Non-transactional topics are unaffected (LSO == HWM).
+      def read_partition_offsets(topic_partition_offsets)
+        @wrapped_kafka.read_partition_offsets(
+          topic_partition_offsets,
+          isolation_level: isolation_level
+        )
+      end
+
       # @return [String] safe inspection string that is causing circular dependencies and other
       #   issues
       def inspect
@@ -463,6 +489,17 @@ module Karafka
 
       private
 
+      # @return [Integer] isolation level of this consumer mapped to the constant expected by the
+      #   offsets queries. It cannot change after the client is created, hence memoized.
+      # @note When not configured, librdkafka reads committed, so we mirror that default here.
+      def isolation_level
+        @isolation_level ||=
+          case @subscription_group.kafka.fetch(:"isolation.level", "read_committed").to_s
+          when "read_uncommitted" then Admin::IsolationLevels::READ_UNCOMMITTED
+          else Admin::IsolationLevels::READ_COMMITTED
+          end
+      end
+
       # When we cannot store an offset, it means we no longer own the partition
       #
       # Non thread-safe offset storing method
@@ -560,8 +597,8 @@ module Karafka
         @kafka = nil
         @wrapped_kafka = nil
         @buffer.clear
-        # @note We do not clear rebalance manager here as we may still have revocation info
-        # here that we want to consider valid prior to running another reconnection
+        # We do not clear rebalance manager here as we may still have revocation info here that we
+        # want to consider valid prior to running another reconnection
       end
 
       # Unsubscribes from all the subscriptions
@@ -665,7 +702,7 @@ module Karafka
         case e.code
         when *EARLY_REPORT_ERRORS
           early_report = true
-        # @see
+        # See
         # https://github.com/confluentinc/confluent-kafka-dotnet/issues/1366#issuecomment-821842990
         # This will be raised each time poll detects a non-existing topic. When auto creation is
         # on, we can safely ignore it
@@ -700,8 +737,8 @@ module Karafka
         raise unless retryable
 
         # Most of the errors can be safely ignored as librdkafka will recover from them
-        # @see https://github.com/edenhill/librdkafka/issues/1987#issuecomment-422008750
-        # @see https://github.com/edenhill/librdkafka/wiki/Error-handling
+        # See https://github.com/edenhill/librdkafka/issues/1987#issuecomment-422008750
+        # See https://github.com/edenhill/librdkafka/wiki/Error-handling
 
         time_poll.checkpoint
         time_poll.backoff

data/lib/karafka/connection/listener.rb

@@ -26,9 +26,9 @@ module Karafka
       # @return [Karafka::Routing::SubscriptionGroup] subscription group that this listener handles
       attr_reader :subscription_group
 
-      # @return [Processing::ConsumerGroups::CoordinatorsBuffer] coordinator buffers that can be used directly in
-      #   advanced cases of changes to the polling flow (like triggered seek back without messages
-      #   ahead in the topic)
+      # @return [Processing::ConsumerGroups::CoordinatorsBuffer] coordinator buffers that can be
+      #   used directly in advanced cases of changes to the polling flow (like triggered seek back
+      #   without messages ahead in the topic)
       attr_reader :coordinators
 
       # How long to wait in the initial events poll. Increases chances of having the initial events

data/lib/karafka/connection/manager.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 module Karafka
-  # Namespace for Kafka connection related logic
   module Connection
     # Connections manager responsible for starting and managing listeners connections
     #

data/lib/karafka/connection/pauses_manager.rb

@@ -25,6 +25,23 @@ module Karafka
         )
       end
 
+      # Resets the attempt count of a given topic partition pause tracker.
+      #
+      # Used on revocation so that a later reclaim of the same partition starts counting retry
+      # attempts from zero instead of carrying the stale count across the rebalance. We reset
+      # rather than remove the tracker because the pause itself may still need to be resumed after
+      # the reclaim (the partition can be re-paused via the retained paused offsets on rebalance).
+      #
+      # A coordinator and its pause tracker are created together in
+      # `CoordinatorsBuffer#find_or_create`, and `CoordinatorsBuffer#revoke` only calls us once it
+      # has confirmed the coordinator exists - so the tracker is always present here.
+      #
+      # @param topic [::Karafka::Routing::Topic] topic
+      # @param partition [Integer] partition number
+      def revoke(topic, partition)
+        @pauses[topic][partition].reset
+      end
+
       # Resumes processing of partitions for which pause time has ended.
       #
       # @yieldparam [Karafka::Routing::Topic] topic