@platformatic/kafka 1.17.0 → 1.18.0

package/dist/clients/admin/types.d.ts

@@ -1,3 +1,6 @@
+import { type AlterClientQuotasRequestEntry } from '../../apis/admin/alter-client-quotas-v1.ts';
+import { type DescribeClientQuotasRequestComponent } from '../../apis/admin/describe-client-quotas-v0.ts';
+import { type DescribeLogDirsResponse, type DescribeLogDirsResponseResult, type DescribeLogDirsRequestTopic } from '../../apis/admin/describe-log-dirs-v4.ts';
 import { type CreateTopicsRequestTopicConfig } from '../../apis/admin/create-topics-v7.ts';
 import { type ConsumerGroupState } from '../../apis/enumerations.ts';
 import { type NullableString } from '../../protocol/definitions.ts';
@@ -59,3 +62,19 @@ export interface DescribeGroupsOptions {
 export interface DeleteGroupsOptions {
     groups: string[];
 }
+export interface DescribeClientQuotasOptions {
+    components: DescribeClientQuotasRequestComponent[];
+    strict?: boolean;
+}
+export interface AlterClientQuotasOptions {
+    entries: AlterClientQuotasRequestEntry[];
+    validateOnly?: boolean;
+}
+export interface DescribeLogDirsOptions {
+    topics: DescribeLogDirsRequestTopic[];
+}
+export interface BrokerLogDirDescription {
+    broker: number;
+    throttleTimeMs: DescribeLogDirsResponse['throttleTimeMs'];
+    results: Omit<DescribeLogDirsResponseResult, 'errorCode'>[];
+}

package/dist/clients/base/base.js

@@ -232,7 +232,9 @@ export class Base extends EventEmitter {
             return;
         }
         const autocreateTopics = options.autocreateTopics ?? this[kOptions].autocreateTopics;
-        this[kPerformDeduplicated]('metadata', deduplicateCallback => {
+        this[kPerformDeduplicated](
+        // Unique key to avoid mixing callbacks
+        `metadata-${options.topics.sort().join(',')}-${autocreateTopics}-${options.forceUpdate}`, deduplicateCallback => {
             this[kPerformWithRetry]('metadata', retryCallback => {
                 this[kGetBootstrapConnection]((error, connection) => {
                     if (error) {

package/dist/clients/consumer/consumer.d.ts

@@ -4,7 +4,7 @@ import { type ConnectionPool } from '../../network/connection-pool.ts';
 import { Base, kFetchConnections } from '../base/base.ts';
 import { MessagesStream } from './messages-stream.ts';
 import { TopicsMap } from './topics-map.ts';
-import { type CommitOptions, type ConsumeOptions, type ConsumerOptions, type FetchOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from './types.ts';
+import { type CommitOptions, type ConsumeOptions, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from './types.ts';
 export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends Base<ConsumerOptions<Key, Value, HeaderKey, HeaderValue>> {
     #private;
     groupId: string;
@@ -31,6 +31,10 @@ export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
     listOffsetsWithTimestamps(options: ListOffsetsOptions): Promise<OffsetsWithTimestamps>;
     listCommittedOffsets(options: ListCommitsOptions, callback: CallbackWithPromise<Offsets>): void;
     listCommittedOffsets(options: ListCommitsOptions): Promise<Offsets>;
+    getLag(options: GetLagOptions, callback: CallbackWithPromise<Offsets>): void;
+    getLag(options: GetLagOptions): Promise<Offsets>;
+    startLagMonitoring(options: GetLagOptions, interval: number): void;
+    stopLagMonitoring(): void;
     findGroupCoordinator(callback: CallbackWithPromise<number>): void;
     findGroupCoordinator(): Promise<number>;
     joinGroup(options: GroupOptions, callback: CallbackWithPromise<string>): void;

package/dist/clients/consumer/consumer.js

@@ -1,14 +1,16 @@
-import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
+import { createPromisifiedCallback, createTimeoutCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
 import { FetchIsolationLevels, FindCoordinatorKeyTypes } from "../../apis/enumerations.js";
-import { consumerCommitsChannel, consumerConsumesChannel, consumerFetchesChannel, consumerGroupChannel, consumerHeartbeatChannel, consumerOffsetsChannel, createDiagnosticContext } from "../../diagnostic.js";
-import { UserError } from "../../errors.js";
+import { consumerCommitsChannel, consumerConsumesChannel, consumerFetchesChannel, consumerGroupChannel, consumerHeartbeatChannel, consumerLagChannel, consumerOffsetsChannel, createDiagnosticContext } from "../../diagnostic.js";
+import { protocolErrors, UserError } from "../../errors.js";
+import { INT32_SIZE } from "../../protocol/definitions.js";
 import { Reader } from "../../protocol/reader.js";
 import { Writer } from "../../protocol/writer.js";
+import { kAutocommit, kRefreshOffsetsAndFetch } from "../../symbols.js";
 import { Base, kAfterCreate, kCheckNotClosed, kClearMetadata, kClosed, kCreateConnectionPool, kFetchConnections, kFormatValidationErrors, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
 import { defaultBaseOptions } from "../base/options.js";
 import { ensureMetric } from "../metrics.js";
 import { MessagesStream } from "./messages-stream.js";
-import { commitOptionsValidator, consumeOptionsValidator, consumerOptionsValidator, defaultConsumerOptions, fetchOptionsValidator, groupIdAndOptionsValidator, groupOptionsValidator, listCommitsOptionsValidator, listOffsetsOptionsValidator } from "./options.js";
+import { commitOptionsValidator, consumeOptionsValidator, consumerOptionsValidator, defaultConsumerOptions, fetchOptionsValidator, getLagOptionsValidator, groupIdAndOptionsValidator, groupOptionsValidator, listCommitsOptionsValidator, listOffsetsOptionsValidator } from "./options.js";
 import { roundRobinAssigner } from "./partitions-assigners.js";
 import { TopicsMap } from "./topics-map.js";
 export class Consumer extends Base {
@@ -17,15 +19,20 @@ export class Consumer extends Base {
     memberId;
     topics;
     assignments;
+    #assignments;
     #members;
     #membershipActive;
     #isLeader;
     #protocol;
     #coordinatorId;
     #heartbeatInterval;
+    #lastHeartbeatIntervalMs;
     #lastHeartbeat;
+    #useConsumerGroupProtocol;
+    #memberEpoch;
+    #groupRemoteAssignor;
     #streams;
-    #partitionsAssigner;
+    #lagMonitoring;
     /*
       The following requests are blocking in Kafka:
   
@@ -43,6 +50,7 @@ export class Consumer extends Base {
     [kFetchConnections];
     // Metrics
     #metricActiveStreams;
+    #metricLags;
     constructor(options) {
         super(options);
         this[kOptions] = Object.assign({}, defaultBaseOptions, defaultConsumerOptions, options);
@@ -52,15 +60,20 @@ export class Consumer extends Base {
         this.memberId = null;
         this.topics = new TopicsMap();
         this.assignments = null;
+        this.#assignments = [];
         this.#members = new Map();
         this.#membershipActive = false;
         this.#isLeader = false;
         this.#protocol = null;
         this.#coordinatorId = null;
         this.#heartbeatInterval = null;
+        this.#lastHeartbeatIntervalMs = 0;
         this.#lastHeartbeat = null;
         this.#streams = new Set();
-        this.#partitionsAssigner = this[kOptions].partitionAssigner ?? roundRobinAssigner;
+        this.#lagMonitoring = null;
+        this.#memberEpoch = 0;
+        this.#useConsumerGroupProtocol = this[kOptions].groupProtocol === 'consumer';
+        this.#groupRemoteAssignor = this[kOptions].groupRemoteAssignor ?? null;
         this.#validateGroupOptions(this[kOptions], groupIdAndOptionsValidator);
         // Initialize connection pool
         this[kFetchConnections] = this[kCreateConnectionPool]();
@@ -68,6 +81,7 @@ export class Consumer extends Base {
             ensureMetric(this[kPrometheus], 'Gauge', 'kafka_consumers', 'Number of active Kafka consumers').inc();
             this.#metricActiveStreams = ensureMetric(this[kPrometheus], 'Gauge', 'kafka_consumers_streams', 'Number of active Kafka consumers streams');
             this.topics.setMetric(ensureMetric(this[kPrometheus], 'Gauge', 'kafka_consumers_topics', 'Number of topics being consumed'));
+            this.#metricLags = ensureMetric(this[kPrometheus], 'Histogram', 'kafka_consumers_lags', 'Lag of active Kafka consumers');
         }
         this[kAfterCreate]('consumer');
     }
@@ -90,11 +104,19 @@ export class Consumer extends Base {
             return callback[kCallbackPromise];
         }
         this[kClosed] = true;
-        const closer = this.#membershipActive
-            ? this.#leaveGroup.bind(this)
-            : function noopCloser(_, callback) {
+        clearTimeout(this.#lagMonitoring);
+        let closer;
+        if (this.#useConsumerGroupProtocol) {
+            closer = this.#leaveGroupConsumerProtocol.bind(this);
+        }
+        else if (this.#membershipActive) {
+            closer = this.#leaveGroupClassicProtocol.bind(this);
+        }
+        else {
+            closer = function noopCloser(_, callback) {
                 callback(null);
             };
+        }
         closer(force, error => {
             if (error) {
                 this[kClosed] = false;
@@ -128,6 +150,9 @@ export class Consumer extends Base {
         if (!baseReady) {
             return false;
         }
+        if (this.#useConsumerGroupProtocol) {
+            return !!this.memberId && this.#memberEpoch >= 0;
+        }
         // We consider the group ready if we have a groupId, a memberId and heartbeat interval
         return this.#membershipActive && Boolean(this.groupId) && Boolean(this.memberId) && this.#heartbeatInterval !== null;
     }
@@ -225,6 +250,79 @@ export class Consumer extends Base {
         consumerOffsetsChannel.traceCallback(this.#listCommittedOffsets, 1, createDiagnosticContext({ client: this, operation: 'listCommittedOffsets', options }), this, options, callback);
         return callback[kCallbackPromise];
     }
+    getLag(options, callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        if (this[kCheckNotClosed](callback)) {
+            return callback[kCallbackPromise];
+        }
+        const validationError = this[kValidateOptions](options, getLagOptionsValidator, '/options', false);
+        if (validationError) {
+            callback(validationError, undefined);
+            return callback[kCallbackPromise];
+        }
+        this.listOffsets(options, (error, offsets) => {
+            if (error) {
+                this.emit('consumer:lag:error', error);
+                callback(error, undefined);
+                return;
+            }
+            // Now gather the last committed offsets from each stream
+            const committeds = new Map();
+            for (const stream of this.#streams) {
+                for (const [topic, offset] of stream.committedOffsets) {
+                    committeds.set(topic, offset);
+                }
+            }
+            // Build the lag map back. A -1n denotes that the consumer is not assigned to a certain partition
+            const lag = new Map();
+            for (const [topic, partitions] of offsets) {
+                const toInclude = new Set(options.partitions?.[topic] ?? []);
+                const hasPartitionsFilter = toInclude.size > 0;
+                const partitionLags = [];
+                for (let i = 0; i < partitions.length; i++) {
+                    if (hasPartitionsFilter && !toInclude.has(i)) {
+                        partitionLags.push(-2n);
+                        continue;
+                    }
+                    const latest = partitions[i];
+                    const committed = committeds.get(`${topic}:${i}`);
+                    // If the consumer is not assigned to this partition, we return -1n.
+                    // Otherwise we compute the lag as latest - committed - 1. The -1 is because latest is the offset of the next message to be produced.
+                    partitionLags.push(typeof committed === 'undefined' ? -1n : latest - committed - 1n);
+                }
+                lag.set(topic, partitionLags);
+            }
+            // Publish to the diagnostic channel
+            consumerLagChannel.publish({ client: this, lag });
+            // Publish to the metric if available
+            if (this.#metricLags) {
+                for (const partitions of lag.values()) {
+                    for (const l of partitions) {
+                        if (l >= 0n) {
+                            this.#metricLags.observe(Number(l));
+                        }
+                    }
+                }
+            }
+            this.emit('consumer:lag', lag);
+            callback(null, lag);
+        });
+        return callback[kCallbackPromise];
+    }
+    startLagMonitoring(options, interval) {
+        const validationError = this[kValidateOptions](options, getLagOptionsValidator, '/options', false);
+        if (validationError) {
+            throw validationError;
+        }
+        this.#lagMonitoring = setTimeout(() => {
+            this.getLag(options, () => this.#lagMonitoring.refresh());
+        }, interval);
+    }
+    stopLagMonitoring() {
+        clearTimeout(this.#lagMonitoring);
+    }
     findGroupCoordinator(callback) {
         if (!callback) {
             callback = createPromisifiedCallback();
@@ -246,6 +344,10 @@ export class Consumer extends Base {
         if (this[kCheckNotClosed](callback)) {
             return callback[kCallbackPromise];
         }
+        if (this.#useConsumerGroupProtocol) {
+            callback(null, '');
+            return callback[kCallbackPromise];
+        }
         const validationError = this[kValidateOptions](options, groupOptionsValidator, '/options', false);
         if (validationError) {
             callback(validationError, undefined);
@@ -271,8 +373,12 @@ export class Consumer extends Base {
         if (this[kCheckNotClosed](callback)) {
             return callback[kCallbackPromise];
         }
+        if (this.#useConsumerGroupProtocol) {
+            callback(null);
+            return callback[kCallbackPromise];
+        }
         this.#membershipActive = false;
-        this.#leaveGroup(force, error => {
+        this.#leaveGroupClassicProtocol(force, error => {
             if (error) {
                 this.#membershipActive = true;
                 callback(error);
@@ -335,7 +441,7 @@ export class Consumer extends Base {
                     groupCallback(error, undefined);
                     return;
                 }
-                api(connection, this.groupId, this.generationId, this.memberId, null, Array.from(topics.values()), groupCallback);
+                api(connection, this.groupId, this.#useConsumerGroupProtocol ? this.#memberEpoch : this.generationId, this.memberId, null, Array.from(topics.values()), groupCallback);
             });
         }, error => {
             callback(error);
@@ -446,9 +552,14 @@ export class Consumer extends Base {
                     groupCallback(error, undefined);
                     return;
                 }
-                api(connection, 
-                // Note: once we start implementing KIP-848, the memberEpoch must be obtained
-                [{ groupId: this.groupId, memberId: this.memberId, memberEpoch: -1, topics }], false, groupCallback);
+                api(connection, [
+                    {
+                        groupId: this.groupId,
+                        memberId: this.memberId,
+                        memberEpoch: this.#useConsumerGroupProtocol ? this.#memberEpoch : -1,
+                        topics
+                    }
+                ], false, groupCallback);
             });
         }, (error, response) => {
             if (error) {
@@ -479,11 +590,11 @@ export class Consumer extends Base {
     #joinGroup(options, callback) {
         consumerGroupChannel.traceCallback(this.#performJoinGroup, 1, createDiagnosticContext({ client: this, operation: 'joinGroup', options }), this, options, callback);
     }
-    #leaveGroup(force, callback) {
+    #leaveGroupClassicProtocol(force, callback) {
         consumerGroupChannel.traceCallback(this.#performLeaveGroup, 1, createDiagnosticContext({ client: this, operation: 'leaveGroup', force }), this, force, callback);
     }
-    #syncGroup(callback) {
-        consumerGroupChannel.traceCallback(this.#performSyncGroup, 1, createDiagnosticContext({ client: this, operation: 'syncGroup' }), this, null, callback);
+    #syncGroup(partitionsAssigner, callback) {
+        consumerGroupChannel.traceCallback(this.#performSyncGroup, 2, createDiagnosticContext({ client: this, operation: 'syncGroup' }), this, partitionsAssigner, null, callback);
     }
     #heartbeat(options) {
         const eventPayload = { groupId: this.groupId, memberId: this.memberId, generationId: this.generationId };
@@ -534,6 +645,186 @@ export class Consumer extends Base {
         clearTimeout(this.#heartbeatInterval);
         this.#heartbeatInterval = null;
     }
+    #consumerGroupHeartbeat(options, callback) {
+        options.rebalanceTimeout ??= this[kOptions].rebalanceTimeout;
+        consumerHeartbeatChannel.traceCallback(this.#performConsumerGroupHeartbeat, 1, createDiagnosticContext({ client: this, operation: 'consumerGroupHeartbeat' }), this, options, callback);
+    }
+    #performConsumerGroupHeartbeat(options, callback) {
+        this.#performGroupOperation('consumerGroupHeartbeat', (connection, groupCallback) => {
+            this.emitWithDebug('consumer:heartbeat', 'start');
+            this[kGetApi]('ConsumerGroupHeartbeat', (error, api) => {
+                if (error) {
+                    groupCallback(error, undefined);
+                    return;
+                }
+                const timeoutCallback = createTimeoutCallback(groupCallback, this[kOptions].timeout, 'Heartbeat timeout.');
+                api(connection, this.groupId, this.memberId || '', this.#memberEpoch, null, // instanceId
+                null, // rackId
+                options.rebalanceTimeout, this.topics.current, this.#groupRemoteAssignor, this.#assignments, timeoutCallback);
+            });
+        }, (error, response) => {
+            if (this[kClosed]) {
+                this.emitWithDebug('consumer:heartbeat', 'end');
+                callback(null);
+                return;
+            }
+            if (error) {
+                this.#cancelHeartbeat();
+                this.emitWithDebug('consumer:heartbeat', 'error', { error });
+                const fenced = error.response?.errorCode === protocolErrors.FENCED_MEMBER_EPOCH.code;
+                if (fenced) {
+                    this.#assignments = [];
+                    this.assignments = [];
+                    this.#memberEpoch = 0;
+                    this.#consumerGroupHeartbeat(options, () => { });
+                    callback(error);
+                    return;
+                }
+                this.#heartbeatInterval = setTimeout(() => {
+                    this.#consumerGroupHeartbeat(options, () => { });
+                }, this.#lastHeartbeatIntervalMs || 1000);
+                callback(error);
+                return;
+            }
+            this.#lastHeartbeat = new Date();
+            this.#memberEpoch = response.memberEpoch;
+            if (response.memberId) {
+                const changed = this.memberId !== response.memberId;
+                this.memberId = response.memberId;
+                if (changed) {
+                    this.memberId = response.memberId;
+                    this.#consumerGroupHeartbeat(options, () => { });
+                    this.emitWithDebug('consumer:heartbeat', 'end');
+                    callback(null);
+                    return;
+                }
+            }
+            if (response.heartbeatIntervalMs > 0) {
+                this.#cancelHeartbeat();
+                this.#lastHeartbeatIntervalMs = response.heartbeatIntervalMs;
+                this.#heartbeatInterval = setTimeout(() => {
+                    this.#consumerGroupHeartbeat(options, () => { });
+                }, response.heartbeatIntervalMs);
+            }
+            const newAssignments = response.assignment?.topicPartitions;
+            if (newAssignments) {
+                this.#revokePartitions(newAssignments);
+                this.#assignPartitions(newAssignments);
+            }
+            this.emitWithDebug('consumer:heartbeat', 'end');
+            callback(null);
+        });
+    }
+    #diffAssignments(A, B) {
+        const result = [];
+        for (const a of A) {
+            const b = B.find(tp => tp.topicId === a.topicId);
+            if (!b) {
+                result.push(a);
+            }
+            else {
+                const diff = a.partitions.filter(partition => !b.partitions.includes(partition));
+                if (diff.length > 0) {
+                    result.push({ topicId: a.topicId, partitions: diff });
+                }
+            }
+        }
+        return result;
+    }
+    #revokePartitions(newAssignment) {
+        const toRevoke = this.#diffAssignments(this.#assignments, newAssignment);
+        if (toRevoke.length === 0) {
+            return;
+        }
+        for (const stream of this.#streams) {
+            stream.pause();
+            stream[kAutocommit]();
+        }
+        this.#updateAssignments(newAssignment, error => {
+            for (const stream of this.#streams) {
+                stream.resume();
+            }
+            /* c8 ignore next 3 - Hard to test */
+            if (error) {
+                return;
+            }
+            this.#cancelHeartbeat();
+            this.#consumerGroupHeartbeat(this[kOptions], () => { });
+        });
+    }
+    #assignPartitions(newAssignment) {
+        const toAssign = this.#diffAssignments(newAssignment, this.#assignments);
+        if (toAssign.length === 0) {
+            return;
+        }
+        this.#updateAssignments(newAssignment, error => {
+            if (error) {
+                return;
+            }
+            this.#cancelHeartbeat();
+            this.#consumerGroupHeartbeat(this[kOptions], () => { });
+            for (const stream of this.#streams) {
+                // 3. Refresh partition offsets
+                stream[kRefreshOffsetsAndFetch]();
+            }
+        });
+    }
+    #updateAssignments(newAssignments, callback) {
+        this[kMetadata]({ topics: this.topics.current }, (error, metadata) => {
+            if (error) {
+                callback(error);
+                return;
+            }
+            const topicIdToTopic = new Map();
+            for (const [topic, topicMetadata] of metadata.topics) {
+                topicIdToTopic.set(topicMetadata.id, topic);
+            }
+            const assignments = newAssignments.map(tp => ({
+                topic: topicIdToTopic.get(tp.topicId),
+                partitions: tp.partitions
+            }));
+            this.#assignments = newAssignments;
+            this.assignments = assignments;
+            callback(null);
+        });
+    }
+    #joinGroupConsumerProtocol(options, callback) {
+        this.#memberEpoch = 0;
+        this.#assignments = [];
+        this.#membershipActive = true;
+        this.#consumerGroupHeartbeat(options, err => {
+            if (this.memberId) {
+                this.emitWithDebug('consumer', 'group:join', { groupId: this.groupId, memberId: this.memberId });
+            }
+            callback(err);
+        });
+    }
+    #leaveGroupConsumerProtocol(_, callback) {
+        // Leave by sending a heartbeat with memberEpoch = -1
+        this.#cancelHeartbeat();
+        this.#performDeduplicateGroupOperaton('leaveGroupConsumerProtocol', (connection, groupCallback) => {
+            this[kGetApi]('ConsumerGroupHeartbeat', (error, api) => {
+                if (error) {
+                    groupCallback(error, undefined);
+                    return;
+                }
+                api(connection, this.groupId, this.memberId, -1, // memberEpoch = -1 signals leave
+                null, // instanceId
+                null, // rackId
+                0, // rebalanceTimeout
+                [], // subscribedTopicNames
+                this.#groupRemoteAssignor, [], // topicPartitions
+                groupCallback);
+            });
+        }, _error => {
+            this.emitWithDebug('consumer', 'group:leave', { groupId: this.groupId, memberId: this.memberId });
+            this.memberId = null;
+            this.#memberEpoch = -1;
+            this.#assignments = [];
+            this.assignments = [];
+            callback(null);
+        });
+    }
     #performConsume(options, trackTopics, callback) {
         // Subscribe all topics
         let joinNeeded = this.memberId === null;
@@ -546,6 +837,17 @@ export class Consumer extends Base {
         }
         // If we need to (re)join the group, do that first and then try again
         if (joinNeeded) {
+            if (this.#useConsumerGroupProtocol) {
+                this.#joinGroupConsumerProtocol(options, error => {
+                    if (error) {
+                        callback(error, undefined);
+                        return;
+                    }
+                    this.#performConsume(options, false, callback);
+                });
+                return;
+            }
+            // Classic consumer protocol join
             this.joinGroup(options, error => {
                 if (error) {
                     callback(error, undefined);
@@ -635,7 +937,7 @@ export class Consumer extends Base {
                 this.#members.set(member.memberId, this.#decodeProtocolSubscriptionMetadata(member.memberId, member.metadata));
             }
             // Send a syncGroup request
-            this.#syncGroup((error, response) => {
+            this.#syncGroup(options.partitionAssigner, (error, response) => {
                 if (!this.#membershipActive) {
                     callback(null, undefined);
                     return;
@@ -721,7 +1023,7 @@ export class Consumer extends Base {
             callback(null);
         });
     }
-    #performSyncGroup(assignments, callback) {
+    #performSyncGroup(partitionsAssigner, assignments, callback) {
         if (!this.#membershipActive) {
             callback(null, []);
             return;
@@ -745,7 +1047,7 @@ export class Consumer extends Base {
                         callback(this.#handleMetadataError(error), undefined);
                         return;
                     }
-                    this.#performSyncGroup(this.#createAssignments(metadata), callback);
+                    this.#performSyncGroup(partitionsAssigner, this.#createAssignments(partitionsAssigner, metadata), callback);
                 });
                 return;
             }
@@ -844,6 +1146,9 @@ export class Consumer extends Base {
     #decodeProtocolAssignment(buffer) {
         const reader = Reader.from(buffer);
         reader.skip(2); // Ignore Version information
+        if (reader.remaining < INT32_SIZE) {
+            return [];
+        }
         return reader.readArray(r => {
             return {
                 topic: r.readString(false),
@@ -851,7 +1156,7 @@ export class Consumer extends Base {
             };
         }, false, false);
     }
-    #createAssignments(metadata) {
+    #createAssignments(partitionsAssigner, metadata) {
         const partitionTracker = new Map();
         // First of all, layout topics-partitions in a list
         for (const [topic, partitions] of metadata.topics) {
@@ -872,7 +1177,8 @@ export class Consumer extends Base {
             return [{ memberId: this.memberId, assignment: this.#encodeProtocolAssignment(assignments) }];
         }
         const encodedAssignments = [];
-        for (const member of this.#partitionsAssigner(this.memberId, this.#members, new Set(this.topics.current), metadata)) {
+        partitionsAssigner ??= this[kOptions].partitionAssigner ?? roundRobinAssigner;
+        for (const member of partitionsAssigner(this.memberId, this.#members, new Set(this.topics.current), metadata)) {
             encodedAssignments.push({
                 memberId: member.memberId,
                 assignment: this.#encodeProtocolAssignment(Array.from(member.assignments.values()))

package/dist/clients/consumer/messages-stream.d.ts

@@ -1,6 +1,7 @@
 import { Readable } from 'node:stream';
 import { type CallbackWithPromise } from '../../apis/callbacks.ts';
 import { type Message } from '../../protocol/records.ts';
+import { kAutocommit, kInstance, kRefreshOffsetsAndFetch } from '../../symbols.ts';
 import { kInspect } from '../base/base.ts';
 import { type Consumer } from './consumer.ts';
 import { type CommitOptionsPartition, type ConsumeOptions } from './types.ts';
@@ -8,12 +9,17 @@ export declare function noopDeserializer(data?: Buffer): Buffer | undefined;
 export declare function defaultCorruptedMessageHandler(): boolean;
 export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends Readable {
     #private;
+    [kInstance]: number;
     constructor(consumer: Consumer<Key, Value, HeaderKey, HeaderValue>, options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>);
+    get committedOffsets(): Map<string, bigint>;
     close(callback: CallbackWithPromise<void>): void;
     close(): Promise<void>;
     isActive(): boolean;
     isConnected(): boolean;
+    resume(): this;
+    pause(): this;
     addListener(event: 'autocommit', listener: (err: Error, offsets: CommitOptionsPartition[]) => void): this;
+    addListener(event: 'fetch', listener: () => void): this;
     addListener(event: 'data', listener: (message: Message<Key, Value, HeaderKey, HeaderValue>) => void): this;
     addListener(event: 'close', listener: () => void): this;
     addListener(event: 'end', listener: () => void): this;
@@ -22,6 +28,7 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     addListener(event: 'readable', listener: () => void): this;
     addListener(event: 'resume', listener: () => void): this;
     on(event: 'autocommit', listener: (err: Error, offsets: CommitOptionsPartition[]) => void): this;
+    on(event: 'fetch', listener: () => void): this;
     on(event: 'data', listener: (message: Message<Key, Value, HeaderKey, HeaderValue>) => void): this;
     on(event: 'close', listener: () => void): this;
     on(event: 'end', listener: () => void): this;
@@ -30,6 +37,7 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     on(event: 'readable', listener: () => void): this;
     on(event: 'resume', listener: () => void): this;
     once(event: 'autocommit', listener: (err: Error, offsets: CommitOptionsPartition[]) => void): this;
+    once(event: 'fetch', listener: () => void): this;
     once(event: 'data', listener: (message: Message<Key, Value, HeaderKey, HeaderValue>) => void): this;
     once(event: 'close', listener: () => void): this;
     once(event: 'end', listener: () => void): this;
@@ -37,6 +45,8 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     once(event: 'pause', listener: () => void): this;
     once(event: 'readable', listener: () => void): this;
     once(event: 'resume', listener: () => void): this;
+    prependListener(event: 'autocommit', listener: (err: Error, offsets: CommitOptionsPartition[]) => void): this;
+    prependListener(event: 'fetch', listener: () => void): this;
     prependListener(event: 'data', listener: (message: Message<Key, Value, HeaderKey, HeaderValue>) => void): this;
     prependListener(event: 'close', listener: () => void): this;
     prependListener(event: 'end', listener: () => void): this;
@@ -44,6 +54,8 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     prependListener(event: 'pause', listener: () => void): this;
     prependListener(event: 'readable', listener: () => void): this;
     prependListener(event: 'resume', listener: () => void): this;
+    prependOnceListener(event: 'autocommit', listener: (err: Error, offsets: CommitOptionsPartition[]) => void): this;
+    prependOnceListener(event: 'fetch', listener: () => void): this;
     prependOnceListener(event: 'data', listener: (message: Message<Key, Value, HeaderKey, HeaderValue>) => void): this;
     prependOnceListener(event: 'close', listener: () => void): this;
     prependOnceListener(event: 'end', listener: () => void): this;
@@ -55,5 +67,7 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     _construct(callback: (error?: Error) => void): void;
     _destroy(error: Error | null, callback: (error?: Error | null) => void): void;
     _read(): void;
+    [kAutocommit](): void;
+    [kRefreshOffsetsAndFetch](): void;
     [kInspect](...args: unknown[]): void;
 }