@platformatic/kafka 2.10.0 → 2.12.0

package/dist/clients/consumer/consumer.js

@@ -2,18 +2,17 @@ import { randomUUID } from 'node:crypto';
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
 import { FetchIsolationLevels, FindCoordinatorKeyTypes } from "../../apis/enumerations.js";
 import { consumerCommitsChannel, consumerConsumesChannel, consumerFetchesChannel, consumerGroupChannel, consumerHeartbeatChannel, consumerLagChannel, consumerOffsetsChannel, createDiagnosticContext } from "../../diagnostic.js";
-import { findErrorBy, NetworkError, protocolErrors, UserError } from "../../errors.js";
-import { INT32_SIZE } from "../../protocol/definitions.js";
-import { Reader } from "../../protocol/reader.js";
+import { findErrorBy, MultipleErrors, NetworkError, protocolErrors, UserError } from "../../errors.js";
 import { IS_CONTROL } from "../../protocol/records.js";
 import { Writer } from "../../protocol/writer.js";
-import { kAutocommit, kGetFetchNode, kRefreshOffsetsAndFetch } from "../../symbols.js";
+import { kAutocommit, kGetFetchNode, kRefreshOffsetsAndFetch, kUpdateCommittedOffset } from "../../symbols.js";
 import { emitExperimentalApiWarning } from "../../utils.js";
 import { Base, kAfterCreate, kCheckNotClosed, kClosed, kConnections, kCreateConnectionPool, kFormatValidationErrors, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
 import { ensureMetric } from "../metrics.js";
 import { MessagesStream } from "./messages-stream.js";
+import { CONSUMER_PROTOCOL_HIGHEST_VERSION, decodeCooperativeStickyGeneration, decodeConsumerProtocolAssignment, decodeConsumerProtocolSubscription, encodeConsumerProtocolAssignment, encodeConsumerProtocolSubscription } from "./consumer-protocol.js";
 import { commitOptionsValidator, consumeOptionsValidator, consumerOptionsValidator, defaultConsumerOptions, fetchOptionsValidator, getLagOptionsValidator, groupIdAndOptionsValidator, groupOptionsValidator, listCommitsOptionsValidator, listOffsetsOptionsValidator } from "./options.js";
-import { roundRobinAssigner } from "./partitions-assigners.js";
+import { COOPERATIVE_STICKY_ASSIGNOR, cooperativeStickyAssigner, roundRobinAssigner } from "./partitions-assigners.js";
 import { TopicsMap } from "./topics-map.js";
 import { nextFetchEpoch, partitionKey } from "./utils.js";
 export class Consumer extends Base {
@@ -170,7 +169,7 @@ export class Consumer extends Base {
             return false;
         }
         if (this.#useConsumerGroupProtocol) {
-            return !!this.memberId && this.#memberEpoch >= 0;
+            return !!this.memberId && this.#memberEpoch > 0;
         }
         // We consider the group ready if we have a groupId, a memberId and heartbeat interval
         return this.#membershipActive && Boolean(this.groupId) && Boolean(this.memberId) && this.#heartbeatInterval !== null;
@@ -362,6 +361,13 @@ export class Consumer extends Base {
             throw validationError;
         }
         this.#lagMonitoring = setTimeout(() => {
+            // Skip this tick while mid-(re)join: listOffsets adds load to a coordinator/cluster that
+            // may already be struggling, and any lag figure computed against stale/absent assignments
+            // would be meaningless anyway. Just reschedule and try again next tick.
+            if (!this.isActive()) {
+                this.#lagMonitoring.refresh();
+                return;
+            }
             this.getLag(options, () => this.#lagMonitoring.refresh());
         }, interval);
     }
@@ -592,7 +598,36 @@ export class Consumer extends Base {
                                 }
                             }
                         }
+                        // Fetch v12 identifies topics by name, while later versions use UUIDs.
+                        const useTopicNames = api.version <= 12;
+                        let topicIdsByName;
+                        if (useTopicNames) {
+                            // Built here rather than in the response callback so that the fetch loop allocates
+                            // it once per request instead of once per request and once per response.
+                            topicIdsByName = new Map();
+                            for (const [id, name] of topicIds) {
+                                topicIdsByName.set(name, id);
+                            }
+                            requestTopics = requestTopics.map(topic => ({
+                                ...topic,
+                                topicId: topicIds.get(topic.topicId) ?? topic.topicId
+                            }));
+                            for (const forgotten of forgottenTopicsData) {
+                                const topicId = forgotten.topicId ?? forgotten.topic;
+                                forgotten.topicId = topicIds.get(topicId) ?? topicId;
+                                delete forgotten.topic;
+                            }
+                        }
                         api(connection, options.maxWaitTime ?? this[kOptions].maxWaitTime, options.minBytes ?? this[kOptions].minBytes, options.maxBytes ?? this[kOptions].maxBytes, isolationLevel, sessionId, sessionEpoch, requestTopics, forgottenTopicsData, this.#clientRack, (error, result) => {
+                            const response = result ?? error?.response;
+                            if (response && !response.nodeEndpoints) {
+                                response.nodeEndpoints = [];
+                            }
+                            if (useTopicNames && response) {
+                                for (const topic of response.responses) {
+                                    topic.topicId = topicIdsByName.get(topic.topicId) ?? topic.topicId;
+                                }
+                            }
                             if (error) {
                                 refreshMetadata = NetworkError.isRetryable(error);
                                 this.#clearPreferredReadReplicas(options.topics, topicIds);
@@ -601,7 +636,14 @@ export class Consumer extends Base {
                                     findErrorBy(error, 'apiId', 'FETCH_SESSION_TOPIC_ID_ERROR')) {
                                     this.#fetchSessions.delete(node);
                                 }
-                                if (findErrorBy(error, 'apiId', 'FENCED_LEADER_EPOCH')) {
+                                // NOT_LEADER_OR_FOLLOWER/LEADER_NOT_AVAILABLE mean leadership just moved (broker
+                                // rebalance/leader election); without a refresh here, kPerformWithRetry repeats
+                                // the same request against the stale cached leader until retries exhaust.
+                                // UNKNOWN_TOPIC_OR_PARTITION also carries hasStaleMetadata but is deliberately
+                                // excluded: broadening this to it breaks deleted-topic handling (see git history).
+                                if (findErrorBy(error, 'apiId', 'FENCED_LEADER_EPOCH') ||
+                                    findErrorBy(error, 'apiId', 'NOT_LEADER_OR_FOLLOWER') ||
+                                    findErrorBy(error, 'apiId', 'LEADER_NOT_AVAILABLE')) {
                                     this.#fetchSessions.delete(node);
                                     this.clearMetadata();
                                     for (const topic of options.topics) {
@@ -681,6 +723,16 @@ export class Consumer extends Base {
                 });
                 return;
             }
+            if (!error) {
+                for (const { topic, partition, offset } of options.offsets) {
+                    const key = partitionKey(topic, partition);
+                    for (const stream of this.#streams) {
+                        if (stream.offsetsCommitted.has(key)) {
+                            stream[kUpdateCommittedOffset](topic, partition, offset);
+                        }
+                    }
+                }
+            }
             callback(error);
         });
     }
@@ -744,7 +796,16 @@ export class Consumer extends Base {
                 }, concurrentCallback, 0);
             }, (error, responses) => {
                 if (error) {
-                    if (!forceUpdateMetadata && NetworkError.isRetryable(error)) {
+                    // Mirrors the fetch() handling: right after a topic is created or its leadership moves,
+                    // the cached leader is briefly stale and the broker answers with one of these three
+                    // errors. Retrying once against fresh metadata clears the race instead of surfacing it.
+                    // UNKNOWN_TOPIC_OR_PARTITION also carries hasStaleMetadata but is deliberately excluded,
+                    // same as in fetch(), to avoid masking deleted-topic handling.
+                    if (!forceUpdateMetadata &&
+                        (NetworkError.isRetryable(error) ||
+                            findErrorBy(error, 'apiId', 'FENCED_LEADER_EPOCH') ||
+                            findErrorBy(error, 'apiId', 'NOT_LEADER_OR_FOLLOWER') ||
+                            findErrorBy(error, 'apiId', 'LEADER_NOT_AVAILABLE'))) {
                         this.clearMetadata();
                         this.#listOffsets(withTimestamps, options, callback, true);
                         return;
@@ -805,7 +866,17 @@ export class Consumer extends Base {
                         memberEpoch: this.#useConsumerGroupProtocol ? this.#memberEpoch : -1,
                         topics
                     }
-                ], false, groupCallback);
+                ], false, (error, response) => {
+                    if (error || api.version >= 8) {
+                        groupCallback(error, response);
+                        return;
+                    }
+                    const legacyResponse = response;
+                    groupCallback(null, {
+                        throttleTimeMs: legacyResponse.throttleTimeMs,
+                        groups: [{ groupId: this.groupId, topics: legacyResponse.topics, errorCode: legacyResponse.errorCode }]
+                    });
+                });
             });
         }, (error, response) => {
             if (error) {
@@ -885,9 +956,7 @@ export class Consumer extends Base {
                     // - a rejoin that never completes still surfaces as a stall
                     this.#resetHeartbeatStall();
                     this.#trackHeartbeatStall(options, error, options.rebalanceTimeout + this.#heartbeatStallTimeoutFor(options));
-                    this[kPerformWithRetry]('rejoinGroup', retryCallback => {
-                        this.#joinGroup(options, retryCallback);
-                    }, error => {
+                    this.#joinGroup(options, error => {
                         if (error) {
                             this.emitWithDebug(null, 'error', error);
                             if (!this.#heartbeatStalled) {
@@ -903,7 +972,7 @@ export class Consumer extends Base {
                             this.#resetHeartbeatStall();
                         }
                         this.emitWithDebug('consumer', 'rejoin');
-                    }, 0);
+                    });
                     return;
                 }
                 // The error was not group related, so we schedule another heartbeat.
@@ -1000,7 +1069,12 @@ export class Consumer extends Base {
                     this.assignments = [];
                     this.#syncPreferredReadReplicas();
                     this.#memberEpoch = 0;
-                    this.#consumerGroupHeartbeat(options, () => { });
+                    // Retry after a delay, not immediately: if many members get fenced around the same
+                    // rebalance, resending ConsumerGroupHeartbeat with no delay at all can flood the
+                    // coordinator faster than the group can converge.
+                    this.#heartbeatInterval = setTimeout(() => {
+                        this.#consumerGroupHeartbeat(options, () => { });
+                    }, this.#lastHeartbeatIntervalMs || 1000);
                     callback(error);
                     return;
                 }
@@ -1056,6 +1130,22 @@ export class Consumer extends Base {
         }
         return result;
     }
+    #diffGroupAssignments(A, B) {
+        const result = [];
+        for (const a of A) {
+            const b = B.find(assignment => assignment.topic === a.topic);
+            if (!b) {
+                result.push({ topic: a.topic, partitions: [...a.partitions] });
+            }
+            else {
+                const diff = a.partitions.filter(partition => !b.partitions.includes(partition));
+                if (diff.length > 0) {
+                    result.push({ topic: a.topic, partitions: diff });
+                }
+            }
+        }
+        return result;
+    }
     #revokePartitions(newAssignment) {
         const toRevoke = this.#diffAssignments(this.#assignments, newAssignment);
         if (toRevoke.length === 0) {
@@ -1081,6 +1171,14 @@ export class Consumer extends Base {
     #assignPartitions(newAssignment) {
         const toAssign = this.#diffAssignments(newAssignment, this.#assignments);
         if (toAssign.length === 0) {
+            // An explicit empty initial assignment confirms an idle member, even though
+            // there are no partitions to add. An omitted assignment must remain unknown.
+            if (this.assignments === null && newAssignment.length === 0) {
+                this.assignments = [];
+                for (const stream of this.#streams) {
+                    stream[kRefreshOffsetsAndFetch]();
+                }
+            }
             return;
         }
         this.#updateAssignments(newAssignment, error => {
@@ -1159,9 +1257,18 @@ export class Consumer extends Base {
             });
             return;
         }
+        const finish = () => {
+            this.emitWithDebug('consumer', 'group:leave', { groupId: this.groupId, memberId: this.memberId });
+            this.memberId = null;
+            this.#memberEpoch = -1;
+            this.#assignments = [];
+            this.assignments = [];
+            this.#syncPreferredReadReplicas();
+            callback(null);
+        };
         // Leave by sending a heartbeat with memberEpoch = -1
         this.#cancelHeartbeat();
-        this.#performDeduplicateGroupOperaton('leaveGroupConsumerProtocol', (connection, groupCallback) => {
+        const operation = (connection, groupCallback) => {
             this[kGetApi]('ConsumerGroupHeartbeat', (error, api) => {
                 if (error) {
                     groupCallback(error);
@@ -1175,15 +1282,18 @@ export class Consumer extends Base {
                 this.#groupRemoteAssignor, [], // topicPartitions
                 groupCallback);
             });
-        }, _error => {
-            this.emitWithDebug('consumer', 'group:leave', { groupId: this.groupId, memberId: this.memberId });
-            this.memberId = null;
-            this.#memberEpoch = -1;
-            this.#assignments = [];
-            this.assignments = [];
-            this.#syncPreferredReadReplicas();
-            callback(null);
-        });
+        };
+        if (this[kClosed]) {
+            const connection = this.#getGroupCoordinatorConnection();
+            if (connection) {
+                operation(connection, finish);
+            }
+            else {
+                finish();
+            }
+            return;
+        }
+        this.#performDeduplicateGroupOperaton('leaveGroupConsumerProtocol', operation, finish);
     }
     #getConsumerGroupHeartbeatMemberId(apiVersion) {
         if (this.memberId) {
@@ -1251,7 +1361,12 @@ export class Consumer extends Base {
                             retryCallback(error);
                             return;
                         }
-                        api(connection, FindCoordinatorKeyTypes.GROUP, [this.groupId], retryCallback);
+                        api(connection, FindCoordinatorKeyTypes.GROUP, [this.groupId], (error, response) => {
+                            if (response && api.version < 4) {
+                                response.coordinators[0].key = this.groupId;
+                            }
+                            retryCallback(error, response);
+                        });
                     });
                 });
             }, (error, response) => {
@@ -1266,6 +1381,13 @@ export class Consumer extends Base {
         }, callback);
     }
     #performJoinGroup(options, callback) {
+        if (!this.#membershipActive) {
+            callback(null);
+            return;
+        }
+        this.#performJoinGroupAttempt(options, callback, 0, []);
+    }
+    #performJoinGroupAttempt(options, callback, attempt, attemptErrors) {
         if (!this.#membershipActive) {
             callback(null);
             return;
@@ -1273,7 +1395,7 @@ export class Consumer extends Base {
         this.#cancelHeartbeat();
         this.#createJoinGroupProtocols(options, (error, protocols) => {
             if (error) {
-                callback(error);
+                this.#retryJoinGroup(options, callback, attempt, attemptErrors, error);
                 return;
             }
             this.#performDeduplicateGroupOperaton('joinGroup', (connection, groupCallback) => {
@@ -1290,11 +1412,7 @@ export class Consumer extends Base {
                     return;
                 }
                 if (error) {
-                    if (this.#getRejoinError(error)) {
-                        this.#performJoinGroup(options, callback);
-                        return;
-                    }
-                    callback(error);
+                    this.#retryJoinGroup(options, callback, attempt, attemptErrors, error);
                     return;
                 }
                 // This is for Azure Event Hubs compatibility, which does not respond with an error on the first join
@@ -1313,31 +1431,120 @@ export class Consumer extends Base {
                         return;
                     }
                     if (error) {
-                        if (this.#getRejoinError(error)) {
-                            this.#performJoinGroup(options, callback);
-                            return;
-                        }
-                        callback(error);
+                        this.#retryJoinGroup(options, callback, attempt, attemptErrors, error);
                         return;
                     }
-                    this.assignments = response;
-                    this.#syncPreferredReadReplicas();
-                    this.#cancelHeartbeat();
-                    this.#heartbeatInterval = setTimeout(() => {
-                        this.#heartbeat(options);
-                    }, options.heartbeatInterval);
-                    this.emitWithDebug('consumer', 'group:join', {
-                        groupId: this.groupId,
-                        memberId: this.memberId,
-                        generationId: this.generationId,
-                        isLeader: this.#isLeader,
-                        assignments: this.assignments
-                    });
-                    callback(null, this.memberId);
+                    this.#completeClassicSyncGroup(options, response, callback);
                 });
             });
         });
     }
+    #retryJoinGroup(options, callback, attempt, attemptErrors, error) {
+        const retries = this[kOptions].retries;
+        const memberId = this.memberId;
+        const rejoinError = this.#getRejoinError(error);
+        // MEMBER_ID_REQUIRED is the normal first-join handshake. The broker assigns a member ID and
+        // expects the JoinGroup request to be sent again; this progress must not consume a retry.
+        if (rejoinError?.apiId === 'MEMBER_ID_REQUIRED' && this.memberId !== memberId) {
+            this.#performJoinGroupAttempt(options, callback, attempt, attemptErrors);
+            return;
+        }
+        attemptErrors.push(error);
+        if (!rejoinError) {
+            if (error instanceof MultipleErrors && error.errors.length > 0) {
+                const nestedMessages = error.errors.map(error => error.message).join(' ');
+                callback(new MultipleErrors(`${error.message} ${nestedMessages}`, error.errors));
+            }
+            else {
+                callback(error);
+            }
+            return;
+        }
+        if (attempt >= retries) {
+            callback(attempt > 0 ? new MultipleErrors(`joinGroup failed ${attempt + 1} times.`, attemptErrors) : error);
+            return;
+        }
+        let delay = this[kOptions].retryDelay;
+        if (typeof delay === 'function') {
+            delay = delay(this, 'joinGroup', attempt + 1, retries, error);
+        }
+        function onClose() {
+            clearTimeout(timeout);
+            attemptErrors.push(new UserError('Client closed while retrying joinGroup.'));
+            callback(new MultipleErrors(`joinGroup failed ${attempt + 1} times.`, attemptErrors));
+        }
+        const timeout = setTimeout(() => {
+            this.removeListener('client:close', onClose);
+            this.#performJoinGroupAttempt(options, callback, attempt + 1, attemptErrors);
+        }, delay);
+        this.once('client:close', onClose);
+    }
+    #completeClassicSyncGroup(options, assignments, callback) {
+        if (this.#isCooperativeProtocol()) {
+            const previousAssignments = this.assignments ?? [];
+            const revoked = this.#diffGroupAssignments(previousAssignments, assignments);
+            const addedOnRevokeRound = this.#diffGroupAssignments(assignments, previousAssignments);
+            if (revoked.length > 0) {
+                for (const stream of this.#streams) {
+                    stream.pause();
+                }
+                // Snapshot the set: a failing autocommit destroys its stream, which removes it from
+                // #streams while the concurrent callbacks are still iterating.
+                runConcurrentCallbacks('Autocommit before cooperative rebalance failed.', [...this.#streams], (stream, concurrentCallback) => {
+                    // A destroyed stream never settles an autocommit, so do not wait on one.
+                    if (stream.destroyed || stream.closed) {
+                        concurrentCallback(null);
+                        return;
+                    }
+                    stream[kAutocommit](concurrentCallback);
+                }, autocommitError => {
+                    // Mirror Kafka's reference client: capture autocommit errors but
+                    // continue the rebalance so the group does not get stuck mid-cycle.
+                    // Per-stream autocommit errors are already surfaced through the
+                    // `autocommit` event on each MessagesStream.
+                    if (autocommitError) {
+                        this.emitWithDebug('consumer', 'group:autocommit:error', {
+                            groupId: this.groupId,
+                            error: autocommitError
+                        });
+                    }
+                    this.assignments = assignments;
+                    this.#syncPreferredReadReplicas();
+                    for (const stream of this.#streams) {
+                        stream.resume();
+                    }
+                    if (addedOnRevokeRound.length > 0) {
+                        for (const stream of this.#streams) {
+                            stream[kRefreshOffsetsAndFetch]();
+                        }
+                    }
+                    this.emitWithDebug('consumer', 'group:rebalance', { groupId: this.groupId });
+                    this.#performJoinGroup(options, callback);
+                });
+                return;
+            }
+        }
+        const added = this.#diffGroupAssignments(assignments, this.assignments ?? []);
+        this.assignments = assignments;
+        this.#syncPreferredReadReplicas();
+        this.#cancelHeartbeat();
+        this.#heartbeatInterval = setTimeout(() => {
+            this.#heartbeat(options);
+        }, options.heartbeatInterval);
+        this.emitWithDebug('consumer', 'group:join', {
+            groupId: this.groupId,
+            memberId: this.memberId,
+            generationId: this.generationId,
+            isLeader: this.#isLeader,
+            assignments: this.assignments
+        });
+        if (this.#isCooperativeProtocol() && added.length > 0) {
+            for (const stream of this.#streams) {
+                stream[kRefreshOffsetsAndFetch]();
+            }
+        }
+        callback(null, this.memberId);
+    }
     #performLeaveGroup(force, callback) {
         if (!this.memberId) {
             callback(null);
@@ -1366,8 +1573,20 @@ export class Consumer extends Base {
             });
             return;
         }
+        const finish = () => {
+            this.emitWithDebug('consumer', 'group:leave', {
+                groupId: this.groupId,
+                memberId: this.memberId,
+                generationId: this.generationId
+            });
+            this.memberId = null;
+            this.generationId = 0;
+            this.assignments = null;
+            this.#syncPreferredReadReplicas();
+            callback(null);
+        };
         this.#cancelHeartbeat();
-        this.#performDeduplicateGroupOperaton('leaveGroup', (connection, groupCallback) => {
+        const operation = (connection, groupCallback) => {
             this[kGetApi]('LeaveGroup', (error, api) => {
                 if (error) {
                     groupCallback(error);
@@ -1375,26 +1594,30 @@ export class Consumer extends Base {
                 }
                 api(connection, this.groupId, [{ memberId: this.memberId }], groupCallback);
             });
-        }, error => {
+        };
+        const onLeave = (error) => {
             if (error) {
                 const unknownMemberError = findErrorBy(error, 'unknownMemberId', true);
+                const unavailableBrokerWhileClosing = this[kClosed] && NetworkError.isRetryable(error);
                 // This is to avoid throwing an error if a group join was cancelled.
-                if (!unknownMemberError) {
+                if (!unknownMemberError && !unavailableBrokerWhileClosing) {
                     callback(error);
                     return;
                 }
             }
-            this.emitWithDebug('consumer', 'group:leave', {
-                groupId: this.groupId,
-                memberId: this.memberId,
-                generationId: this.generationId
-            });
-            this.memberId = null;
-            this.generationId = 0;
-            this.assignments = null;
-            this.#syncPreferredReadReplicas();
-            callback(null);
-        });
+            finish();
+        };
+        if (this[kClosed]) {
+            const connection = this.#getGroupCoordinatorConnection();
+            if (connection) {
+                operation(connection, onLeave);
+            }
+            else {
+                finish();
+            }
+            return;
+        }
+        this.#performDeduplicateGroupOperaton('leaveGroup', operation, onLeave);
     }
     #performSyncGroup(partitionsAssigner, assignments, callback) {
         if (!this.#membershipActive) {
@@ -1485,7 +1708,7 @@ export class Consumer extends Base {
             });
         }, (error, result) => {
             callback(this.#handleError(error), result);
-        });
+        }, 0, [], () => this[kClosed]);
     }
     #validateGroupOptions(options, validator) {
         validator ??= groupOptionsValidator;
@@ -1554,19 +1777,37 @@ export class Consumer extends Base {
       https://github.com/apache/kafka/blob/trunk/clients/src/main/resources/common/message/ConsumerProtocolSubscription.json
     */
     #encodeProtocolSubscriptionMetadata(metadata, topics) {
-        return Writer.create()
-            .appendInt16(metadata.version)
-            .appendArray(topics, (w, t) => w.appendString(t, false), false, false)
-            .appendBytes(typeof metadata.metadata === 'string' ? Buffer.from(metadata.metadata) : metadata.metadata, false)
-            .buffer;
+        const cooperative = metadata.name === COOPERATIVE_STICKY_ASSIGNOR;
+        const ownedPartitions = cooperative ? this.#ownedPartitionsForSubscription(topics) : [];
+        let userData = typeof metadata.metadata === 'string' ? Buffer.from(metadata.metadata) : metadata.metadata;
+        if (cooperative && !userData && metadata.version < 2) {
+            userData = Writer.create()
+                .appendInt32(this.memberId && ownedPartitions.length > 0 ? this.generationId : -1)
+                .buffer;
+        }
+        return encodeConsumerProtocolSubscription({
+            version: metadata.version,
+            topics,
+            userData,
+            ownedPartitions,
+            generationId: cooperative && this.memberId && ownedPartitions.length > 0 ? this.generationId : -1,
+            rackId: cooperative ? this.#clientRack || null : null
+        });
     }
     #decodeProtocolSubscriptionMetadata(memberId, buffer) {
-        const reader = Reader.from(buffer);
+        const subscription = decodeConsumerProtocolSubscription(buffer);
+        const cooperative = this.#protocol === COOPERATIVE_STICKY_ASSIGNOR;
+        const generationId = cooperative && subscription.version < 2
+            ? decodeCooperativeStickyGeneration(subscription.userData)
+            : subscription.generationId;
         return {
             memberId,
-            version: reader.readInt16(),
-            topics: reader.readArray(r => r.readString(false), false, false),
-            metadata: reader.readBytes(false)
+            version: subscription.version,
+            topics: subscription.topics,
+            metadata: subscription.userData,
+            ownedPartitions: subscription.ownedPartitions,
+            generationId,
+            rackId: subscription.rackId
         };
     }
     /*
@@ -1574,27 +1815,14 @@ export class Consumer extends Base {
       https://github.com/apache/kafka/blob/trunk/clients/src/main/resources/common/message/ConsumerProtocolAssignment.json
     */
     #encodeProtocolAssignment(assignments) {
-        const userData = this[kOptions].assignmentUserData;
-        const writer = Writer.create()
-            .appendInt16(0) // Version information
-            .appendArray(assignments, (w, { topic, partitions }) => {
-            w.appendString(topic, false).appendArray(partitions, (w, a) => w.appendInt32(a), false, false);
-        }, false, false);
-        writer.appendBytes(userData ?? null, false);
-        return writer.buffer;
+        return encodeConsumerProtocolAssignment({
+            version: this.#isCooperativeProtocol() ? CONSUMER_PROTOCOL_HIGHEST_VERSION : 0,
+            assignedPartitions: assignments,
+            userData: this[kOptions].assignmentUserData ?? null
+        });
     }
     #decodeProtocolAssignment(buffer) {
-        const reader = Reader.from(buffer);
-        reader.skip(2); // Ignore Version information
-        if (reader.remaining < INT32_SIZE) {
-            return [];
-        }
-        return reader.readArray(r => {
-            return {
-                topic: r.readString(false),
-                partitions: r.readArray(r => r.readInt32(), false, false)
-            };
-        }, false, false);
+        return decodeConsumerProtocolAssignment(buffer).assignedPartitions;
     }
     #createAssignments(partitionsAssigner, metadata) {
         const partitionTracker = new Map();
@@ -1617,7 +1845,7 @@ export class Consumer extends Base {
             return [{ memberId: this.memberId, assignment: this.#encodeProtocolAssignment(assignments) }];
         }
         const encodedAssignments = [];
-        partitionsAssigner ??= this[kOptions].partitionAssigner ?? roundRobinAssigner;
+        partitionsAssigner ??= this.#defaultPartitionsAssigner();
         for (const member of partitionsAssigner(this.memberId, this.#members, new Set(this.topics.current), metadata)) {
             encodedAssignments.push({
                 memberId: member.memberId,
@@ -1626,24 +1854,50 @@ export class Consumer extends Base {
         }
         return encodedAssignments;
     }
+    #defaultPartitionsAssigner() {
+        const configuredAssigner = this[kOptions].partitionAssigner;
+        if (configuredAssigner) {
+            return configuredAssigner;
+        }
+        if (this.#isCooperativeProtocol()) {
+            return cooperativeStickyAssigner;
+        }
+        return roundRobinAssigner;
+    }
+    #isCooperativeProtocol() {
+        return this.#protocol === COOPERATIVE_STICKY_ASSIGNOR;
+    }
+    #ownedPartitionsForSubscription(topics) {
+        if (!this.assignments) {
+            return [];
+        }
+        const currentTopics = new Set(topics);
+        return this.assignments
+            .filter(assignment => currentTopics.has(assignment.topic))
+            .map(assignment => ({ topic: assignment.topic, partitions: [...assignment.partitions] }));
+    }
     #filterUncommittedMessages(response) {
         for (const topic of response.responses) {
             for (const partition of topic.partitions) {
                 if (!partition.records) {
                     continue;
                 }
-                if (partition.abortedTransactions.length) {
+                const abortedTransactions = partition.abortedTransactions ?? [];
+                if (abortedTransactions.length) {
                     const abortedRanges = new Map();
                     // Find first offsets
                     // For last offsets we set -1n as special value. It allows us to detect open ranges, whichs means
                     // that Kafka has not sent the control marker. In that case we ignore the range for safety.
-                    for (const aborted of partition.abortedTransactions) {
+                    for (const aborted of abortedTransactions) {
                         abortedRanges.set(aborted.producerId, [aborted.firstOffset, -1n]);
                     }
                     // Now find last offsets
                     for (const batch of partition.records) {
                         if (batch.attributes & IS_CONTROL && batch.records[0].key?.readInt16BE(2) === 0) {
-                            abortedRanges.get(batch.producerId)[1] = batch.firstOffset;
+                            const abortedRange = abortedRanges.get(batch.producerId);
+                            if (abortedRange) {
+                                abortedRange[1] = batch.firstOffset;
+                            }
                         }
                     }
                     // Filter all records in the aborted ranges
@@ -1665,7 +1919,14 @@ export class Consumer extends Base {
         if (protocolError.rebalanceInProgress) {
             this.emitWithDebug('consumer', 'group:rebalance', { groupId: this.groupId });
         }
-        if (protocolError.unknownMemberId) {
+        // A cooperative member which lost its membership or generation no longer owns anything:
+        // clear the assignments so the next JoinGroup advertises an empty ownedPartitions set
+        // instead of re-claiming partitions the group has already reassigned elsewhere.
+        if (this.#isCooperativeProtocol() && ['UNKNOWN_MEMBER_ID', 'ILLEGAL_GENERATION'].includes(protocolError.apiId)) {
+            this.assignments = [];
+            this.#syncPreferredReadReplicas();
+        }
+        if (protocolError.unknownMemberId || protocolError.fencedInstanceId) {
             this.memberId = null;
         }
         else if (protocolError.memberId && !this.memberId) {
@@ -1755,4 +2016,12 @@ export class Consumer extends Base {
         }
         return error;
     }
+    #getGroupCoordinatorConnection() {
+        if (this.#coordinatorId === null) {
+            return null;
+        }
+        const broker = this.currentMetadata?.brokers.get(this.#coordinatorId);
+        const connection = broker ? this[kConnections].getEstablishedConnection(broker) : undefined;
+        return connection?.isConnected() === true ? connection : null;
+    }
 }