@platformatic/kafka 2.0.1 → 2.2.0

package/dist/clients/consumer/consumer.js

@@ -2,12 +2,12 @@ import { randomUUID } from 'node:crypto';
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
 import { FetchIsolationLevels, FindCoordinatorKeyTypes } from "../../apis/enumerations.js";
 import { consumerCommitsChannel, consumerConsumesChannel, consumerFetchesChannel, consumerGroupChannel, consumerHeartbeatChannel, consumerLagChannel, consumerOffsetsChannel, createDiagnosticContext } from "../../diagnostic.js";
-import { protocolErrors, UserError } from "../../errors.js";
+import { findErrorBy, NetworkError, protocolErrors, UserError } from "../../errors.js";
 import { INT32_SIZE } from "../../protocol/definitions.js";
 import { Reader } from "../../protocol/reader.js";
 import { IS_CONTROL } from "../../protocol/records.js";
 import { Writer } from "../../protocol/writer.js";
-import { kAutocommit, kRefreshOffsetsAndFetch } from "../../symbols.js";
+import { kAutocommit, kGetFetchNode, kRefreshOffsetsAndFetch } from "../../symbols.js";
 import { emitExperimentalApiWarning } from "../../utils.js";
 import { Base, kAfterCreate, kCheckNotClosed, kClosed, kConnections, kCreateConnectionPool, kFormatValidationErrors, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
 import { ensureMetric } from "../metrics.js";
@@ -15,6 +15,7 @@ import { MessagesStream } from "./messages-stream.js";
 import { commitOptionsValidator, consumeOptionsValidator, consumerOptionsValidator, defaultConsumerOptions, fetchOptionsValidator, getLagOptionsValidator, groupIdAndOptionsValidator, groupOptionsValidator, listCommitsOptionsValidator, listOffsetsOptionsValidator } from "./options.js";
 import { roundRobinAssigner } from "./partitions-assigners.js";
 import { TopicsMap } from "./topics-map.js";
+import { nextFetchEpoch, partitionKey } from "./utils.js";
 export class Consumer extends Base {
     groupId;
     groupInstanceId;
@@ -31,9 +32,16 @@ export class Consumer extends Base {
     #heartbeatInterval;
     #lastHeartbeatIntervalMs;
     #lastHeartbeat;
+    #heartbeatStallTimer;
+    #heartbeatStallStart;
+    #heartbeatStalled;
+    #heartbeatStallError;
     #useConsumerGroupProtocol;
     #memberEpoch;
     #groupRemoteAssignor;
+    #clientRack;
+    #fetchSessions;
+    #preferredReadReplicas;
     #streams;
     #lagMonitoring;
     #streamContext;
@@ -73,11 +81,18 @@ export class Consumer extends Base {
         this.#heartbeatInterval = null;
         this.#lastHeartbeatIntervalMs = 0;
         this.#lastHeartbeat = null;
+        this.#heartbeatStallTimer = null;
+        this.#heartbeatStallStart = null;
+        this.#heartbeatStalled = false;
+        this.#heartbeatStallError = undefined;
         this.#streams = new Set();
         this.#lagMonitoring = null;
         this.#memberEpoch = 0;
         this.#useConsumerGroupProtocol = this[kOptions].groupProtocol === 'consumer';
         this.#groupRemoteAssignor = this[kOptions].groupRemoteAssignor ?? null;
+        this.#clientRack = this[kOptions].clientRack ?? '';
+        this.#fetchSessions = new Map();
+        this.#preferredReadReplicas = new Map();
         this.#streamContext = options.streamContext ?? options.context;
         this.#validateGroupOptions(this[kOptions], groupIdAndOptionsValidator);
         if (this[kPrometheus]) {
@@ -114,6 +129,7 @@ export class Consumer extends Base {
         }
         this[kClosed] = true;
         clearTimeout(this.#lagMonitoring);
+        this.#resetHeartbeatStall();
         let closer;
         if (this.#useConsumerGroupProtocol) {
             closer = this.#leaveGroupConsumerProtocol.bind(this);
@@ -139,6 +155,7 @@ export class Consumer extends Base {
                     return;
                 }
                 this.topics.clear();
+                this.#fetchSessions.clear();
                 if (this[kPrometheus]) {
                     ensureMetric(this[kPrometheus], 'Gauge', 'kafka_consumers', 'Number of active Kafka consumers').dec();
                 }
@@ -170,6 +187,10 @@ export class Consumer extends Base {
             callback(validationError);
             return callback[kCallbackPromise];
         }
+        if (options.onCorruptedMessage && options.onDeserializationError) {
+            callback(new UserError('Cannot specify both onCorruptedMessage and onDeserializationError.'));
+            return callback[kCallbackPromise];
+        }
         options.autocommit ??= this[kOptions].autocommit ?? true;
         options.maxBytes ??= this[kOptions].maxBytes;
         options.maxBytesPerPartition ??= this[kOptions].maxBytesPerPartition ?? options.maxBytes;
@@ -354,7 +375,7 @@ export class Consumer extends Base {
         if (this[kCheckNotClosed](callback)) {
             return callback[kCallbackPromise];
         }
-        if (this.#coordinatorId) {
+        if (this.#coordinatorId !== null) {
             callback(null, this.#coordinatorId);
             return callback[kCallbackPromise];
         }
@@ -387,6 +408,10 @@ export class Consumer extends Base {
         options.rebalanceTimeout ??= this[kOptions].rebalanceTimeout;
         options.heartbeatInterval ??= this[kOptions].heartbeatInterval;
         options.protocols ??= this[kOptions].protocols;
+        const protocolsMetadata = this[kOptions].protocolsMetadata;
+        if (!options.protocolsMetadata && protocolsMetadata) {
+            options.protocolsMetadata = protocolsMetadata;
+        }
         this.#validateGroupOptions(options);
         this.#membershipActive = true;
         this.#joinGroup(options, callback);
@@ -408,6 +433,7 @@ export class Consumer extends Base {
             return callback[kCallbackPromise];
         }
         this.#membershipActive = false;
+        this.#resetHeartbeatStall();
         this.#leaveGroupClassicProtocol(force, error => {
             if (error) {
                 this.#membershipActive = true;
@@ -422,22 +448,76 @@ export class Consumer extends Base {
     #consume(options, callback) {
         consumerConsumesChannel.traceCallback(this.#performConsume, 2, createDiagnosticContext({ client: this, operation: 'consume', options }), this, options, true, callback);
     }
+    [kGetFetchNode](metadata, topic, partition, now) {
+        const partitionMetadata = metadata.topics.get(topic).partitions[partition];
+        const key = partitionKey(topic, partition);
+        const preferredReadReplica = this.#preferredReadReplicas.get(key);
+        if (preferredReadReplica === undefined) {
+            return partitionMetadata.leader;
+        }
+        if (now > preferredReadReplica.expiresAt) {
+            this.#preferredReadReplicas.delete(key);
+            return partitionMetadata.leader;
+        }
+        if (metadata.brokers.has(preferredReadReplica.node) &&
+            partitionMetadata.replicas.includes(preferredReadReplica.node) &&
+            !partitionMetadata.offlineReplicas.includes(preferredReadReplica.node)) {
+            return preferredReadReplica.node;
+        }
+        this.#preferredReadReplicas.delete(key);
+        this.clearMetadata();
+        return partitionMetadata.leader;
+    }
+    #topicIdsById(metadata) {
+        const topicIds = new Map();
+        for (const [topic, { id }] of metadata.topics) {
+            topicIds.set(id, topic);
+        }
+        return topicIds;
+    }
+    #fetchNodeForRequest(metadata, fallbackNode, topics, topicIds, now) {
+        let requestNode;
+        for (const topicRequest of topics) {
+            const topic = topicIds.get(topicRequest.topicId);
+            if (!topic) {
+                return fallbackNode;
+            }
+            for (const { partition } of topicRequest.partitions) {
+                const node = this[kGetFetchNode](metadata, topic, partition, now);
+                if (requestNode === undefined) {
+                    requestNode = node;
+                }
+                else if (requestNode !== node) {
+                    return fallbackNode;
+                }
+            }
+        }
+        return requestNode ?? fallbackNode;
+    }
     #fetch(options, callback) {
         const isolationLevel = options.isolationLevel ?? this[kOptions].isolationLevel;
+        let refreshMetadata = false;
         this[kPerformWithRetry]('fetch', retryCallback => {
-            this[kMetadata]({ topics: this.topics.current }, (error, metadata) => {
+            this[kMetadata]({ topics: this.topics.current, forceUpdate: refreshMetadata }, (error, metadata) => {
                 if (error) {
+                    refreshMetadata = NetworkError.isRetryable(error);
                     retryCallback(error);
                     return;
                 }
-                const broker = metadata.brokers.get(options.node);
+                refreshMetadata = false;
+                const topicIds = this.#topicIdsById(metadata);
+                const node = this.#fetchNodeForRequest(metadata, options.node, options.topics, topicIds, Date.now());
+                const broker = metadata.brokers.get(node);
                 if (!broker) {
-                    retryCallback(new UserError(`Cannot find broker with node id ${options.node}`));
+                    retryCallback(new UserError(`Cannot find broker with node id ${node}`));
                     return;
                 }
                 const pool = options.connectionPool ?? this[kConnections];
                 pool.get(broker, (error, connection) => {
                     if (error) {
+                        refreshMetadata = NetworkError.isRetryable(error);
+                        this.#clearPreferredReadReplicas(options.topics, topicIds);
+                        this.#fetchSessions.delete(node);
                         // When a connection was not available (either interrupted or not available) we
                         // reset the leader epoch in the options so that when connection is re-established again we can continue
                         for (const topic of options.topics) {
@@ -451,13 +531,107 @@ export class Consumer extends Base {
                     }
                     this[kGetApi]('Fetch', (error, api) => {
                         if (error) {
+                            refreshMetadata = NetworkError.isRetryable(error);
                             retryCallback(error);
                             return;
                         }
-                        api(connection, options.maxWaitTime ?? this[kOptions].maxWaitTime, options.minBytes ?? this[kOptions].minBytes, options.maxBytes ?? this[kOptions].maxBytes, isolationLevel, 0, 0, options.topics, [], '', (error, result) => {
+                        let session = this.#fetchSessions.get(node);
+                        if (session && session.pool !== pool) {
+                            session = undefined;
+                        }
+                        const sessionId = session?.id ?? 0;
+                        const sessionEpoch = session?.epoch ?? 0;
+                        // The desired state of the session: options.topics always lists all the wanted partitions
+                        const desired = new Map();
+                        for (const topic of options.topics) {
+                            let partitions = desired.get(topic.topicId);
+                            if (!partitions) {
+                                partitions = new Map();
+                                desired.set(topic.topicId, partitions);
+                            }
+                            for (const partition of topic.partitions) {
+                                // Copy the partition since the data is mutated in the error paths
+                                partitions.set(partition.partition, { ...partition });
+                            }
+                        }
+                        let requestTopics = options.topics;
+                        const forgottenTopicsData = [];
+                        if (session) {
+                            // Only send the partitions which are new or whose state changed: the broker
+                            // keeps serving the unchanged ones from the session (KIP-227)
+                            requestTopics = [];
+                            for (const [topicId, partitions] of desired) {
+                                const cached = session.partitions.get(topicId);
+                                const changed = [];
+                                for (const partition of partitions.values()) {
+                                    const previous = cached?.get(partition.partition);
+                                    if (!previous ||
+                                        previous.fetchOffset !== partition.fetchOffset ||
+                                        previous.currentLeaderEpoch !== partition.currentLeaderEpoch ||
+                                        previous.lastFetchedEpoch !== partition.lastFetchedEpoch ||
+                                        previous.partitionMaxBytes !== partition.partitionMaxBytes) {
+                                        changed.push(partition);
+                                    }
+                                }
+                                if (changed.length > 0) {
+                                    requestTopics.push({ topicId, partitions: changed });
+                                }
+                            }
+                            // Ask the broker to remove from the session the partitions which are no
+                            // longer wanted, otherwise it keeps serving them forever
+                            for (const [topicId, partitions] of session.partitions) {
+                                const stillDesired = desired.get(topicId);
+                                const forgotten = [];
+                                for (const partition of partitions.keys()) {
+                                    if (!stillDesired?.has(partition)) {
+                                        forgotten.push(partition);
+                                    }
+                                }
+                                if (forgotten.length > 0) {
+                                    forgottenTopicsData.push({ topic: topicId, partitions: forgotten });
+                                }
+                            }
+                        }
+                        // Fetch v12 identifies topics by name, while later versions use UUIDs.
+                        const useTopicNames = api.version <= 12;
+                        let topicIdsByName;
+                        if (useTopicNames) {
+                            // Built here rather than in the response callback so that the fetch loop allocates
+                            // it once per request instead of once per request and once per response.
+                            topicIdsByName = new Map();
+                            for (const [id, name] of topicIds) {
+                                topicIdsByName.set(name, id);
+                            }
+                            requestTopics = requestTopics.map(topic => ({
+                                ...topic,
+                                topicId: topicIds.get(topic.topicId) ?? topic.topicId
+                            }));
+                            for (const forgotten of forgottenTopicsData) {
+                                const topicId = forgotten.topicId ?? forgotten.topic;
+                                forgotten.topicId = topicIds.get(topicId) ?? topicId;
+                                delete forgotten.topic;
+                            }
+                        }
+                        api(connection, options.maxWaitTime ?? this[kOptions].maxWaitTime, options.minBytes ?? this[kOptions].minBytes, options.maxBytes ?? this[kOptions].maxBytes, isolationLevel, sessionId, sessionEpoch, requestTopics, forgottenTopicsData, this.#clientRack, (error, result) => {
+                            const response = result ?? error?.response;
+                            if (response && !response.nodeEndpoints) {
+                                response.nodeEndpoints = [];
+                            }
+                            if (useTopicNames && response) {
+                                for (const topic of response.responses) {
+                                    topic.topicId = topicIdsByName.get(topic.topicId) ?? topic.topicId;
+                                }
+                            }
                             if (error) {
-                                const genericError = error;
-                                if (genericError.findBy?.('apiId', 'FENCED_LEADER_EPOCH')) {
+                                refreshMetadata = NetworkError.isRetryable(error);
+                                this.#clearPreferredReadReplicas(options.topics, topicIds);
+                                if (findErrorBy(error, 'apiId', 'FETCH_SESSION_ID_NOT_FOUND') ||
+                                    findErrorBy(error, 'apiId', 'INVALID_FETCH_SESSION_EPOCH') ||
+                                    findErrorBy(error, 'apiId', 'FETCH_SESSION_TOPIC_ID_ERROR')) {
+                                    this.#fetchSessions.delete(node);
+                                }
+                                if (findErrorBy(error, 'apiId', 'FENCED_LEADER_EPOCH')) {
+                                    this.#fetchSessions.delete(node);
                                     this.clearMetadata();
                                     for (const topic of options.topics) {
                                         for (const partition of topic.partitions) {
@@ -467,6 +641,10 @@ export class Consumer extends Base {
                                     }
                                 }
                             }
+                            else {
+                                this.#updatePreferredReadReplicas(metadata, topicIds, result);
+                                this.#updateFetchSession(node, pool, sessionEpoch, result.sessionId, desired);
+                            }
                             retryCallback(error, result);
                         });
                     });
@@ -484,6 +662,20 @@ export class Consumer extends Base {
             callback(error, response);
         }, 0);
     }
+    #updateFetchSession(node, pool, sentEpoch, responseSessionId, desired) {
+        if (!responseSessionId) {
+            this.#fetchSessions.delete(node);
+            return;
+        }
+        // After a successful fetch the broker session contains exactly the desired partitions:
+        // unchanged ones already matched it, changed ones were sent and removed ones were forgotten
+        this.#fetchSessions.set(node, {
+            id: responseSessionId,
+            epoch: nextFetchEpoch(sentEpoch),
+            pool,
+            partitions: desired
+        });
+    }
     #commit(options, callback, rejoinAttempts = 0) {
         this.#performGroupOperation('commit', (connection, groupCallback) => {
             const topics = new Map();
@@ -521,12 +713,12 @@ export class Consumer extends Base {
             callback(error);
         });
     }
-    #listOffsets(withTimestamps, options, callback) {
+    #listOffsets(withTimestamps, options, callback, forceUpdateMetadata = false) {
         let topics = options.topics;
         if (!topics || topics.length === 0) {
             topics = this.topics.current;
         }
-        this[kMetadata]({ topics }, (error, metadata) => {
+        this[kMetadata]({ topics, forceUpdate: forceUpdateMetadata }, (error, metadata) => {
             if (error) {
                 callback(error);
                 return;
@@ -581,7 +773,12 @@ export class Consumer extends Base {
                 }, concurrentCallback, 0);
             }, (error, responses) => {
                 if (error) {
-                    callback(this.#handleMetadataError(error));
+                    if (!forceUpdateMetadata && NetworkError.isRetryable(error)) {
+                        this.clearMetadata();
+                        this.#listOffsets(withTimestamps, options, callback, true);
+                        return;
+                    }
+                    callback(this.#handleError(error));
                     return;
                 }
                 let offsets = new Map();
@@ -637,23 +834,33 @@ export class Consumer extends Base {
                         memberEpoch: this.#useConsumerGroupProtocol ? this.#memberEpoch : -1,
                         topics
                     }
-                ], false, groupCallback);
+                ], false, (error, response) => {
+                    if (error || api.version >= 8) {
+                        groupCallback(error, response);
+                        return;
+                    }
+                    const legacyResponse = response;
+                    groupCallback(null, {
+                        throttleTimeMs: legacyResponse.throttleTimeMs,
+                        groups: [{ groupId: this.groupId, topics: legacyResponse.topics, errorCode: legacyResponse.errorCode }]
+                    });
+                });
             });
         }, (error, response) => {
             if (error) {
                 if (this.#useConsumerGroupProtocol &&
                     staleMemberEpochRetries < this[kOptions].retries &&
-                    error.findBy?.('apiId', 'STALE_MEMBER_EPOCH')) {
+                    findErrorBy(error, 'apiId', 'STALE_MEMBER_EPOCH')) {
                     this.#consumerGroupHeartbeat(this[kOptions], heartbeatError => {
                         if (heartbeatError) {
-                            callback(this.#handleMetadataError(heartbeatError));
+                            callback(this.#handleError(heartbeatError));
                             return;
                         }
                         this.#listCommittedOffsets(options, callback, staleMemberEpochRetries + 1);
                     });
                     return;
                 }
-                callback(this.#handleMetadataError(error));
+                callback(this.#handleError(error));
                 return;
             }
             const committed = new Map();
@@ -671,7 +878,7 @@ export class Consumer extends Base {
         });
     }
     #findGroupCoordinator(callback) {
-        if (this.#coordinatorId) {
+        if (this.#coordinatorId !== null) {
             callback(null, this.#coordinatorId);
             return;
         }
@@ -695,6 +902,7 @@ export class Consumer extends Base {
                 return;
             }
             this.emitWithDebug('consumer:heartbeat', 'start', eventPayload);
+            this.#trackHeartbeatStall(options);
             this[kGetApi]('Heartbeat', (error, api) => {
                 if (error) {
                     groupCallback(error);
@@ -711,21 +919,43 @@ export class Consumer extends Base {
             if (error) {
                 this.#cancelHeartbeat();
                 if (this.#getRejoinError(error)) {
+                    // A rejoin can legitimately block up to rebalanceTimeout:
+                    // - extend the stall window by that bound rather than pausing it
+                    // - a rejoin that never completes still surfaces as a stall
+                    this.#resetHeartbeatStall();
+                    this.#trackHeartbeatStall(options, error, options.rebalanceTimeout + this.#heartbeatStallTimeoutFor(options));
                     this[kPerformWithRetry]('rejoinGroup', retryCallback => {
                         this.#joinGroup(options, retryCallback);
                     }, error => {
                         if (error) {
                             this.emitWithDebug(null, 'error', error);
+                            if (!this.#heartbeatStalled) {
+                                // No more heartbeats will be scheduled: restart the stall window without
+                                // the rejoin grace, keeping the triggering heartbeat error as lastError.
+                                const heartbeatError = this.#heartbeatStallError;
+                                this.#resetHeartbeatStall();
+                                this.#trackHeartbeatStall(options, heartbeatError);
+                            }
+                        }
+                        else {
+                            // Rejoin succeeded: drop the extended window; the next heartbeat opens the normal one.
+                            this.#resetHeartbeatStall();
                         }
                         this.emitWithDebug('consumer', 'rejoin');
                     }, 0);
                     return;
                 }
+                // The error was not group related, so we schedule another heartbeat.
+                // Do this before notifying listeners so they can still cancel it.
+                this.#heartbeatInterval = setTimeout(() => {
+                    this.#heartbeat(options);
+                }, options.heartbeatInterval);
                 this.emitWithDebug('consumer:heartbeat', 'error', { ...eventPayload, error });
-                // Note that here we purposely do not return, since it was not a group related problem we schedule another heartbeat
+                this.#trackHeartbeatStall(options, error);
             }
             else {
                 this.#lastHeartbeat = new Date();
+                this.#resetHeartbeatStall();
                 this.emitWithDebug('consumer:heartbeat', 'end', eventPayload);
             }
             this.#heartbeatInterval?.refresh();
@@ -735,6 +965,47 @@ export class Consumer extends Base {
         clearTimeout(this.#heartbeatInterval);
         this.#heartbeatInterval = null;
     }
+    #heartbeatStallTimeoutFor(options) {
+        return (options.heartbeatStallTimeout ??
+            this[kOptions].heartbeatStallTimeout ??
+            options.rebalanceTimeout ??
+            this[kOptions].rebalanceTimeout ??
+            defaultConsumerOptions.rebalanceTimeout);
+    }
+    // Opens the stall window on the first heartbeat attempt after the last success.
+    // The window survives retries and only #resetHeartbeatStall closes it.
+    #trackHeartbeatStall(options, error, timeout) {
+        if (error) {
+            this.#heartbeatStallError = error;
+        }
+        if (this.#heartbeatStalled || this.#heartbeatStallTimer) {
+            return;
+        }
+        timeout ??= this.#heartbeatStallTimeoutFor(options);
+        this.#heartbeatStallStart = Date.now();
+        this.#heartbeatStallTimer = setTimeout(() => {
+            this.#heartbeatStallTimer = null;
+            // Emit once per stall episode. The next successful heartbeat re-arms the tracking.
+            this.#heartbeatStalled = true;
+            if (this[kClosed] || !this.#membershipActive) {
+                return;
+            }
+            this.emitWithDebug('consumer:heartbeat', 'stalled', {
+                lastError: this.#heartbeatStallError,
+                lastHeartbeat: this.#lastHeartbeat,
+                stalledFor: Date.now() - this.#heartbeatStallStart
+            });
+        }, timeout);
+    }
+    #resetHeartbeatStall() {
+        if (this.#heartbeatStallTimer) {
+            clearTimeout(this.#heartbeatStallTimer);
+            this.#heartbeatStallTimer = null;
+        }
+        this.#heartbeatStallStart = null;
+        this.#heartbeatStalled = false;
+        this.#heartbeatStallError = undefined;
+    }
     #consumerGroupHeartbeat(options, callback) {
         options.rebalanceTimeout ??= this[kOptions].rebalanceTimeout;
         consumerHeartbeatChannel.traceCallback(this.#performConsumerGroupHeartbeat, 1, createDiagnosticContext({ client: this, operation: 'consumerGroupHeartbeat' }), this, options, callback);
@@ -742,17 +1013,18 @@ export class Consumer extends Base {
     #performConsumerGroupHeartbeat(options, callback) {
         this.#performGroupOperation('consumerGroupHeartbeat', (connection, groupCallback) => {
             this.emitWithDebug('consumer:heartbeat', 'start');
+            this.#trackHeartbeatStall(options);
             this[kGetApi]('ConsumerGroupHeartbeat', (error, api) => {
                 if (error) {
                     groupCallback(error);
                     return;
                 }
                 const memberId = this.#getConsumerGroupHeartbeatMemberId(api.version);
-                api(connection, this.groupId, memberId, this.#memberEpoch, this.groupInstanceId, null, // rackId
-                options.rebalanceTimeout, this.topics.current, null, this.#groupRemoteAssignor, this.#assignments, groupCallback);
+                api(connection, this.groupId, memberId, this.#memberEpoch, this.groupInstanceId, this.#clientRack || null, options.rebalanceTimeout, this.topics.current, null, this.#groupRemoteAssignor, this.#assignments, groupCallback);
             });
         }, (error, response) => {
             if (this[kClosed]) {
+                this.#resetHeartbeatStall();
                 this.emitWithDebug('consumer:heartbeat', 'end');
                 callback(null);
                 return;
@@ -760,10 +1032,12 @@ export class Consumer extends Base {
             if (error) {
                 this.#cancelHeartbeat();
                 this.emitWithDebug('consumer:heartbeat', 'error', { error });
+                this.#trackHeartbeatStall(options, error);
                 const fenced = error.response?.errorCode === protocolErrors.FENCED_MEMBER_EPOCH.code;
                 if (fenced) {
                     this.#assignments = [];
                     this.assignments = [];
+                    this.#syncPreferredReadReplicas();
                     this.#memberEpoch = 0;
                     this.#consumerGroupHeartbeat(options, () => { });
                     callback(error);
@@ -776,6 +1050,7 @@ export class Consumer extends Base {
                 return;
             }
             this.#lastHeartbeat = new Date();
+            this.#resetHeartbeatStall();
             this.#memberEpoch = response.memberEpoch;
             if (response.memberId) {
                 const changed = this.memberId !== response.memberId;
@@ -825,6 +1100,7 @@ export class Consumer extends Base {
         if (toRevoke.length === 0) {
             return;
         }
+        this.emitWithDebug('consumer', 'group:rebalance', { groupId: this.groupId });
         for (const stream of this.#streams) {
             stream.pause();
             stream[kAutocommit]();
@@ -878,6 +1154,7 @@ export class Consumer extends Base {
             }));
             this.#assignments = newAssignments;
             this.assignments = assignments;
+            this.#syncPreferredReadReplicas();
             callback(null);
         });
     }
@@ -892,7 +1169,35 @@ export class Consumer extends Base {
             callback(err);
         });
     }
-    #leaveGroupConsumerProtocol(_, callback) {
+    #leaveGroupConsumerProtocol(force, callback) {
+        // Remove streams that might have been exited in the meanwhile
+        for (const stream of this.#streams) {
+            if (stream.closed || stream.destroyed) {
+                this.#streams.delete(stream);
+            }
+        }
+        // Active streams must be closed before leaving the group, exactly like in the classic protocol.
+        // Their fetch loop keeps running until they are closed and, since Consumer.close marks the client
+        // as closed before leaving, any fetch scheduled in the meanwhile would fail with "Client is closed."
+        // and destroy the stream with an error the user never asked for.
+        if (this.#streams.size) {
+            if (!force) {
+                callback(new UserError('Cannot leave group while consuming messages.'));
+                return;
+            }
+            runConcurrentCallbacks('Closing streams failed.', this.#streams, (stream, concurrentCallback) => {
+                stream.close(concurrentCallback);
+            }, error => {
+                /* c8 ignore next 4 - Hard to test */
+                if (error) {
+                    callback(error);
+                    return;
+                }
+                // All streams are closed, try the operation again without force
+                this.#leaveGroupConsumerProtocol(false, callback);
+            });
+            return;
+        }
         // Leave by sending a heartbeat with memberEpoch = -1
         this.#cancelHeartbeat();
         this.#performDeduplicateGroupOperaton('leaveGroupConsumerProtocol', (connection, groupCallback) => {
@@ -903,8 +1208,7 @@ export class Consumer extends Base {
                 }
                 const memberId = this.#getConsumerGroupHeartbeatMemberId(api.version);
                 api(connection, this.groupId, memberId, -1, // memberEpoch = -1 signals leave
-                this.groupInstanceId, null, // rackId
-                0, // rebalanceTimeout
+                this.groupInstanceId, this.#clientRack || null, 0, // rebalanceTimeout
                 [], // subscribedTopicNames
                 null, // subscribedTopicRegex
                 this.#groupRemoteAssignor, [], // topicPartitions
@@ -916,6 +1220,7 @@ export class Consumer extends Base {
             this.#memberEpoch = -1;
             this.#assignments = [];
             this.assignments = [];
+            this.#syncPreferredReadReplicas();
             callback(null);
         });
     }
@@ -985,7 +1290,12 @@ export class Consumer extends Base {
                             retryCallback(error);
                             return;
                         }
-                        api(connection, FindCoordinatorKeyTypes.GROUP, [this.groupId], retryCallback);
+                        api(connection, FindCoordinatorKeyTypes.GROUP, [this.groupId], (error, response) => {
+                            if (response && api.version < 4) {
+                                response.coordinators[0].key = this.groupId;
+                            }
+                            retryCallback(error, response);
+                        });
                     });
                 });
             }, (error, response) => {
@@ -1005,45 +1315,20 @@ export class Consumer extends Base {
             return;
         }
         this.#cancelHeartbeat();
-        const protocols = [];
-        for (const protocol of options.protocols) {
-            protocols.push({
-                name: protocol.name,
-                metadata: this.#encodeProtocolSubscriptionMetadata(protocol, this.topics.current)
-            });
-        }
-        this.#performDeduplicateGroupOperaton('joinGroup', (connection, groupCallback) => {
-            this[kGetApi]('JoinGroup', (error, api) => {
-                if (error) {
-                    groupCallback(error);
-                    return;
-                }
-                api(connection, this.groupId, options.sessionTimeout, options.rebalanceTimeout, this.memberId ?? '', this.groupInstanceId, 'consumer', protocols, '', groupCallback);
-            });
-        }, (error, response) => {
-            if (!this.#membershipActive) {
-                callback(null);
-                return;
-            }
+        this.#createJoinGroupProtocols(options, (error, protocols) => {
             if (error) {
-                if (this.#getRejoinError(error)) {
-                    this.#performJoinGroup(options, callback);
-                    return;
-                }
                 callback(error);
                 return;
             }
-            // This is for Azure Event Hubs compatibility, which does not respond with an error on the first join
-            this.memberId = response.memberId;
-            this.generationId = response.generationId;
-            this.#isLeader = response.leader === this.memberId;
-            this.#protocol = response.protocolName;
-            this.#members = new Map();
-            for (const member of response.members) {
-                this.#members.set(member.memberId, this.#decodeProtocolSubscriptionMetadata(member.memberId, member.metadata));
-            }
-            // Send a syncGroup request
-            this.#syncGroup(options.partitionAssigner, (error, response) => {
+            this.#performDeduplicateGroupOperaton('joinGroup', (connection, groupCallback) => {
+                this[kGetApi]('JoinGroup', (error, api) => {
+                    if (error) {
+                        groupCallback(error);
+                        return;
+                    }
+                    api(connection, this.groupId, options.sessionTimeout, options.rebalanceTimeout, this.memberId ?? '', this.groupInstanceId, 'consumer', protocols, '', groupCallback);
+                });
+            }, (error, response) => {
                 if (!this.#membershipActive) {
                     callback(null);
                     return;
@@ -1056,19 +1341,44 @@ export class Consumer extends Base {
                     callback(error);
                     return;
                 }
-                this.assignments = response;
-                this.#cancelHeartbeat();
-                this.#heartbeatInterval = setTimeout(() => {
-                    this.#heartbeat(options);
-                }, options.heartbeatInterval);
-                this.emitWithDebug('consumer', 'group:join', {
-                    groupId: this.groupId,
-                    memberId: this.memberId,
-                    generationId: this.generationId,
-                    isLeader: this.#isLeader,
-                    assignments: this.assignments
+                // This is for Azure Event Hubs compatibility, which does not respond with an error on the first join
+                this.memberId = response.memberId;
+                this.generationId = response.generationId;
+                this.#isLeader = response.leader === this.memberId;
+                this.#protocol = response.protocolName;
+                this.#members = new Map();
+                for (const member of response.members) {
+                    this.#members.set(member.memberId, this.#decodeProtocolSubscriptionMetadata(member.memberId, member.metadata));
+                }
+                // Send a syncGroup request
+                this.#syncGroup(options.partitionAssigner, (error, response) => {
+                    if (!this.#membershipActive) {
+                        callback(null);
+                        return;
+                    }
+                    if (error) {
+                        if (this.#getRejoinError(error)) {
+                            this.#performJoinGroup(options, callback);
+                            return;
+                        }
+                        callback(error);
+                        return;
+                    }
+                    this.assignments = response;
+                    this.#syncPreferredReadReplicas();
+                    this.#cancelHeartbeat();
+                    this.#heartbeatInterval = setTimeout(() => {
+                        this.#heartbeat(options);
+                    }, options.heartbeatInterval);
+                    this.emitWithDebug('consumer', 'group:join', {
+                        groupId: this.groupId,
+                        memberId: this.memberId,
+                        generationId: this.generationId,
+                        isLeader: this.#isLeader,
+                        assignments: this.assignments
+                    });
+                    callback(null, this.memberId);
                 });
-                callback(null, this.memberId);
             });
         });
     }
@@ -1111,7 +1421,7 @@ export class Consumer extends Base {
             });
         }, error => {
             if (error) {
-                const unknownMemberError = error.findBy?.('unknownMemberId', true);
+                const unknownMemberError = findErrorBy(error, 'unknownMemberId', true);
                 // This is to avoid throwing an error if a group join was cancelled.
                 if (!unknownMemberError) {
                     callback(error);
@@ -1126,6 +1436,7 @@ export class Consumer extends Base {
             this.memberId = null;
             this.generationId = 0;
             this.assignments = null;
+            this.#syncPreferredReadReplicas();
             callback(null);
         });
     }
@@ -1148,9 +1459,15 @@ export class Consumer extends Base {
                         topicSubscriptions.push(subscription);
                     }
                 }
-                this[kMetadata]({ topics: Array.from(topicsSubscriptions.keys()) }, (error, metadata) => {
+                const subscribedTopics = Array.from(topicsSubscriptions.keys());
+                this[kMetadata]({ topics: subscribedTopics }, (error, metadata) => {
                     if (error) {
-                        callback(this.#handleMetadataError(error));
+                        callback(this.#handleError(error));
+                        return;
+                    }
+                    const missingTopic = subscribedTopics.find(topic => !metadata.topics.has(topic));
+                    if (missingTopic) {
+                        callback(this.#handleError(new NetworkError(`Cannot compute group assignments: metadata for subscribed topic "${missingTopic}" is not available in the current cluster metadata.`, { canRetry: true, hasStaleMetadata: true })));
                         return;
                     }
                     this.#performSyncGroup(partitionsAssigner, this.#createAssignments(partitionsAssigner, metadata), callback);
@@ -1188,26 +1505,30 @@ export class Consumer extends Base {
         }, callback);
     }
     #performGroupOperation(operationId, operation, callback) {
-        this.#findGroupCoordinator((error, coordinatorId) => {
-            if (error) {
-                callback(error);
-                return;
-            }
-            this[kMetadata]({ topics: this.topics.current }, (error, metadata) => {
+        this[kPerformWithRetry](operationId, retryCallback => {
+            this.#findGroupCoordinator((error, coordinatorId) => {
                 if (error) {
-                    callback(this.#handleMetadataError(error));
+                    retryCallback(error);
                     return;
                 }
-                this[kPerformWithRetry](operationId, retryCallback => {
+                this[kMetadata]({ topics: this.topics.current }, (error, metadata) => {
+                    if (error) {
+                        retryCallback(this.#handleError(error));
+                        return;
+                    }
                     this[kGetConnection](metadata.brokers.get(coordinatorId), (error, connection) => {
                         if (error) {
-                            retryCallback(error);
+                            retryCallback(this.#handleError(error));
                             return;
                         }
-                        operation(connection, retryCallback);
+                        operation(connection, (error, result) => {
+                            retryCallback(this.#handleError(error), result);
+                        });
                     });
-                }, callback);
+                });
             });
+        }, (error, result) => {
+            callback(this.#handleError(error), result);
         });
     }
     #validateGroupOptions(options, validator) {
@@ -1217,6 +1538,61 @@ export class Consumer extends Base {
             throw new UserError(this[kFormatValidationErrors](validator, '/options'));
         }
     }
+    #createJoinGroupProtocols(options, callback) {
+        if (typeof options.protocolsMetadata !== 'function') {
+            callback(null, options.protocols.map(protocol => ({
+                name: protocol.name,
+                metadata: this.#encodeProtocolSubscriptionMetadata(protocol, this.topics.current)
+            })));
+            return;
+        }
+        this[kMetadata]({ topics: this.topics.current, forceUpdate: true }, (error, metadata) => {
+            if (error) {
+                callback(this.#handleError(error));
+                return;
+            }
+            this.#resolveProtocolsMetadata(options.protocolsMetadata, options.protocols, this.topics.current, metadata, (error, userData) => {
+                if (error) {
+                    callback(error);
+                    return;
+                }
+                else if (!Buffer.isBuffer(userData)) {
+                    callback(new UserError('protocolsMetadata must resolve to a Buffer.'));
+                    return;
+                }
+                callback(null, options.protocols.map(protocol => ({
+                    name: protocol.name,
+                    metadata: this.#encodeProtocolSubscriptionMetadata({ ...protocol, metadata: userData }, this.topics.current)
+                })));
+            });
+        });
+    }
+    #resolveProtocolsMetadata(protocolsMetadata, protocols, topics, metadata, callback) {
+        let settled = false;
+        function resolveCallback(error, userData) {
+            if (settled) {
+                return;
+            }
+            settled = true;
+            callback(error, userData);
+        }
+        let result;
+        try {
+            result = protocolsMetadata(protocols, topics, metadata, resolveCallback);
+            if (typeof result !== 'undefined') {
+                if (typeof result.then === 'function') {
+                    ;
+                    result.then(userData => process.nextTick(resolveCallback, null, userData), error => process.nextTick(resolveCallback, error));
+                }
+                else {
+                    resolveCallback(null, result);
+                }
+            }
+        }
+        catch (error) {
+            resolveCallback(error);
+        }
+    }
     /*
       The following two methods follow:
       https://github.com/apache/kafka/blob/trunk/clients/src/main/resources/common/message/ConsumerProtocolSubscription.json
@@ -1300,12 +1676,13 @@ export class Consumer extends Base {
                 if (!partition.records) {
                     continue;
                 }
-                if (partition.abortedTransactions.length) {
+                const abortedTransactions = partition.abortedTransactions ?? [];
+                if (abortedTransactions.length) {
                     const abortedRanges = new Map();
                     // Find first offsets
                     // For last offsets we set -1n as special value. It allows us to detect open ranges, whichs means
                     // that Kafka has not sent the control marker. In that case we ignore the range for safety.
-                    for (const aborted of partition.abortedTransactions) {
+                    for (const aborted of abortedTransactions) {
                         abortedRanges.set(aborted.producerId, [aborted.firstOffset, -1n]);
                     }
                     // Now find last offsets
@@ -1326,7 +1703,7 @@ export class Consumer extends Base {
         }
     }
     #getRejoinError(error) {
-        const protocolError = error.findBy?.('needsRejoin', true);
+        const protocolError = findErrorBy(error, 'needsRejoin', true);
         if (!protocolError) {
             return null;
         }
@@ -1345,10 +1722,82 @@ export class Consumer extends Base {
         }
         return protocolError;
     }
-    #handleMetadataError(error) {
-        if (error && error?.findBy('hasStaleMetadata', true)) {
+    #updatePreferredReadReplicas(metadata, topicIds, response) {
+        for (const topicResponse of response.responses) {
+            const topic = topicIds.get(topicResponse.topicId);
+            if (!topic) {
+                continue;
+            }
+            for (const { partitionIndex: partition, preferredReadReplica } of topicResponse.partitions) {
+                if (preferredReadReplica < 0) {
+                    continue;
+                }
+                const partitionMetadata = metadata.topics.get(topic).partitions[partition];
+                const key = partitionKey(topic, partition);
+                if (metadata.brokers.has(preferredReadReplica) &&
+                    partitionMetadata.replicas.includes(preferredReadReplica) &&
+                    !partitionMetadata.offlineReplicas.includes(preferredReadReplica)) {
+                    const cachedPreferredReadReplica = this.#preferredReadReplicas.get(key);
+                    if (cachedPreferredReadReplica?.node !== preferredReadReplica) {
+                        this.#preferredReadReplicas.set(key, {
+                            node: preferredReadReplica,
+                            expiresAt: Date.now() + this[kOptions].metadataMaxAge
+                        });
+                    }
+                }
+                else {
+                    this.#preferredReadReplicas.delete(key);
+                    this.clearMetadata();
+                }
+            }
+        }
+    }
+    #clearPreferredReadReplicas(topics, topicIds) {
+        let cleared = false;
+        for (const topicRequest of topics) {
+            const topic = topicIds.get(topicRequest.topicId);
+            if (!topic) {
+                continue;
+            }
+            for (const { partition } of topicRequest.partitions) {
+                cleared = this.#preferredReadReplicas.delete(partitionKey(topic, partition)) || cleared;
+            }
+        }
+        return cleared;
+    }
+    // Drops cached preferred replicas for partitions that are no longer assigned.
+    // Called whenever `this.assignments` changes so the cache does not retain
+    // entries for partitions the consumer has lost during a rebalance.
+    #syncPreferredReadReplicas() {
+        if (this.#preferredReadReplicas.size === 0) {
+            return;
+        }
+        if (!this.assignments?.length) {
+            this.#preferredReadReplicas.clear();
+            return;
+        }
+        const assignedKeys = new Set();
+        for (const { topic, partitions } of this.assignments) {
+            for (const partition of partitions) {
+                assignedKeys.add(partitionKey(topic, partition));
+            }
+        }
+        for (const key of this.#preferredReadReplicas.keys()) {
+            if (!assignedKeys.has(key)) {
+                this.#preferredReadReplicas.delete(key);
+            }
+        }
+    }
+    #handleError(error) {
+        if (findErrorBy(error, 'hasStaleMetadata', true)) {
             this.clearMetadata();
         }
+        else if (findErrorBy(error, 'code', 'PLT_KFK_NETWORK') ||
+            findErrorBy(error, 'apiId', 'COORDINATOR_LOAD_IN_PROGRESS') ||
+            findErrorBy(error, 'apiId', 'COORDINATOR_NOT_AVAILABLE') ||
+            findErrorBy(error, 'apiId', 'NOT_COORDINATOR')) {
+            this.#coordinatorId = null;
+        }
         return error;
     }
 }