@platformatic/kafka 2.12.0 → 2.13.0

package/dist/clients/admin/admin.js

@@ -648,29 +648,34 @@ export class Admin extends Base {
                                 members: new Map(),
                                 authorizedOperations: raw.authorizedOperations
                             };
-                            for (const member of raw.members) {
-                                let memberMetadata;
-                                let memberAssignments;
-                                if (member.memberMetadata.length > 0) {
-                                    const subscription = decodeConsumerProtocolSubscription(member.memberMetadata);
-                                    memberMetadata = {
-                                        version: subscription.version,
-                                        topics: subscription.topics,
-                                        metadata: subscription.userData,
-                                        ownedPartitions: subscription.ownedPartitions,
-                                        generationId: subscription.generationId,
-                                        rackId: subscription.rackId
-                                    };
-                                    memberAssignments = new Map(decodeConsumerProtocolAssignment(member.memberAssignment).assignedPartitions.map(assignment => [assignment.topic, assignment]));
+                            try {
+                                for (const member of raw.members) {
+                                    let memberMetadata;
+                                    let memberAssignments;
+                                    if (raw.protocolType === 'consumer' && member.memberMetadata.length > 0) {
+                                        const subscription = decodeConsumerProtocolSubscription(member.memberMetadata);
+                                        memberMetadata = {
+                                            version: subscription.version,
+                                            topics: subscription.topics,
+                                            metadata: subscription.userData,
+                                            ownedPartitions: subscription.ownedPartitions,
+                                            generationId: subscription.generationId,
+                                            rackId: subscription.rackId
+                                        };
+                                        memberAssignments = new Map(decodeConsumerProtocolAssignment(member.memberAssignment).assignedPartitions.map(assignment => [assignment.topic, assignment]));
+                                    }
+                                    group.members.set(member.memberId, {
+                                        id: member.memberId,
+                                        groupInstanceId: member.groupInstanceId,
+                                        clientId: member.clientId,
+                                        clientHost: member.clientHost,
+                                        metadata: memberMetadata,
+                                        assignments: memberAssignments
+                                    });
                                 }
-                                group.members.set(member.memberId, {
-                                    id: member.memberId,
-                                    groupInstanceId: member.groupInstanceId,
-                                    clientId: member.clientId,
-                                    clientHost: member.clientHost,
-                                    metadata: memberMetadata,
-                                    assignments: memberAssignments
-                                });
+                            }
+                            catch (e) {
+                                group.error = e;
                             }
                             groups.set(group.id, group);
                         }

package/dist/clients/admin/types.d.ts

@@ -40,6 +40,7 @@ export interface Group extends Omit<GroupBase, 'groupType'> {
     protocol: string;
     members: Map<string, GroupMember>;
     authorizedOperations: number;
+    error?: Error;
 }
 export interface AdminOptions extends BaseOptions {
 }

package/dist/clients/base/options.d.ts

@@ -108,6 +108,19 @@ export declare const baseOptionsSchema: {
                     type: string;
                     enum: ("PLAIN" | "SCRAM-SHA-256" | "SCRAM-SHA-512" | "OAUTHBEARER" | "GSSAPI")[];
                 };
+                reauthFraction: {
+                    type: string;
+                    exclusiveMinimum: number;
+                    maximum: number;
+                    default: number;
+                };
+                reauthLeadTime: {
+                    type: string;
+                    minimum: number;
+                };
+                lazyReauthentication: {
+                    type: string;
+                };
                 username: {
                     oneOf: ({
                         type: string;

package/dist/clients/base/options.js

@@ -40,6 +40,9 @@ export const baseOptionsSchema = {
             type: 'object',
             properties: {
                 mechanism: { type: 'string', enum: allowedSASLMechanisms },
+                reauthFraction: { type: 'number', exclusiveMinimum: 0, maximum: 1, default: 0.8 },
+                reauthLeadTime: { type: 'number', minimum: 0 },
+                lazyReauthentication: { type: 'boolean' },
                 username: { oneOf: [{ type: 'string' }, { function: true }] },
                 password: { oneOf: [{ type: 'string' }, { function: true }] },
                 token: { oneOf: [{ type: 'string' }, { function: true }] },

package/dist/clients/consumer/consumer-protocol.js

@@ -1,3 +1,4 @@
+import { UnsupportedFormatError } from "../../errors.js";
 import { Reader } from "../../protocol/reader.js";
 import { Writer } from "../../protocol/writer.js";
 export const CONSUMER_PROTOCOL_LOWEST_VERSION = 0;
@@ -48,26 +49,33 @@ export function encodeConsumerProtocolSubscription(data) {
 }
 export function decodeConsumerProtocolSubscription(buffer) {
     const reader = Reader.from(buffer);
-    const encodedVersion = reader.readInt16();
-    const version = supportedVersion(encodedVersion);
-    const subscription = {
-        version: encodedVersion,
-        topics: reader.readArray(r => r.readString(false), false, false),
-        userData: reader.readBytes(false),
-        ownedPartitions: [],
-        generationId: -1,
-        rackId: null
-    };
-    if (version >= 1) {
-        subscription.ownedPartitions = readTopicPartitions(reader);
+    try {
+        const encodedVersion = reader.readInt16();
+        const version = supportedVersion(encodedVersion);
+        const subscription = {
+            version: encodedVersion,
+            topics: reader.readArray(r => r.readString(false), false, false),
+            userData: reader.readBytes(false),
+            ownedPartitions: [],
+            generationId: -1,
+            rackId: null
+        };
+        // KafkaJS and Platformatic <= 2.11 may advertise newer versions with a v0 body.
+        // Default absent trailing fields, but still reject partially encoded fields.
+        if (version >= 1 && reader.remaining > 0) {
+            subscription.ownedPartitions = readTopicPartitions(reader);
+        }
+        if (version >= 2 && reader.remaining > 0) {
+            subscription.generationId = reader.readInt32();
+        }
+        if (version >= 3 && reader.remaining > 0) {
+            subscription.rackId = reader.readNullableString(false);
+        }
+        return subscription;
     }
-    if (version >= 2) {
-        subscription.generationId = reader.readInt32();
+    catch (e) {
+        throw new UnsupportedFormatError('Malformed consumer protocol subscription', { cause: e });
     }
-    if (version >= 3) {
-        subscription.rackId = reader.readNullableString(false);
-    }
-    return subscription;
 }
 export function decodeCooperativeStickyGeneration(userData) {
     if (userData.length < 4) {
@@ -96,19 +104,24 @@ export function decodeConsumerProtocolAssignment(buffer) {
         };
     }
     const reader = Reader.from(buffer);
-    const encodedVersion = reader.readInt16();
-    if (reader.remaining === 0) {
+    try {
+        const encodedVersion = reader.readInt16();
+        if (reader.remaining === 0) {
+            return {
+                version: encodedVersion,
+                assignedPartitions: [],
+                userData: Buffer.alloc(0)
+            };
+        }
+        const assignedPartitions = readTopicPartitions(reader);
+        const userData = reader.remaining > 0 ? reader.readBytes(false) : Buffer.alloc(0);
         return {
             version: encodedVersion,
-            assignedPartitions: [],
-            userData: Buffer.alloc(0)
+            assignedPartitions,
+            userData
         };
     }
-    const assignedPartitions = readTopicPartitions(reader);
-    const userData = reader.remaining > 0 ? reader.readBytes(false) : Buffer.alloc(0);
-    return {
-        version: encodedVersion,
-        assignedPartitions,
-        userData
-    };
+    catch (e) {
+        throw new UnsupportedFormatError('Malformed consumer protocol assignment', { cause: e });
+    }
 }

package/dist/clients/consumer/consumer.d.ts

@@ -6,7 +6,7 @@ import { Base, type BaseEvents, kCreateConnectionPool } from '../base/base.ts';
 import { type ClusterMetadata } from '../base/types.ts';
 import { MessagesStream } from './messages-stream.ts';
 import { TopicsMap } from './topics-map.ts';
-import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupAutocommitErrorPayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerHeartbeatStalledPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from './types.ts';
+import { type CommitOptions, type ConsumeOptions, type ConsumerGroupAutocommitErrorPayload, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerHeartbeatStalledPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from './types.ts';
 export interface ConsumerEvents extends BaseEvents {
     'consumer:group:join': (payload: ConsumerGroupJoinPayload) => void;
     'consumer:group:leave': (payload: ConsumerGroupLeavePayload) => void;

package/dist/clients/consumer/consumer.js

@@ -9,8 +9,8 @@ import { kAutocommit, kGetFetchNode, kRefreshOffsetsAndFetch, kUpdateCommittedOf
 import { emitExperimentalApiWarning } from "../../utils.js";
 import { Base, kAfterCreate, kCheckNotClosed, kClosed, kConnections, kCreateConnectionPool, kFormatValidationErrors, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
 import { ensureMetric } from "../metrics.js";
+import { CONSUMER_PROTOCOL_HIGHEST_VERSION, decodeConsumerProtocolAssignment, decodeConsumerProtocolSubscription, decodeCooperativeStickyGeneration, encodeConsumerProtocolAssignment, encodeConsumerProtocolSubscription } from "./consumer-protocol.js";
 import { MessagesStream } from "./messages-stream.js";
-import { CONSUMER_PROTOCOL_HIGHEST_VERSION, decodeCooperativeStickyGeneration, decodeConsumerProtocolAssignment, decodeConsumerProtocolSubscription, encodeConsumerProtocolAssignment, encodeConsumerProtocolSubscription } from "./consumer-protocol.js";
 import { commitOptionsValidator, consumeOptionsValidator, consumerOptionsValidator, defaultConsumerOptions, fetchOptionsValidator, getLagOptionsValidator, groupIdAndOptionsValidator, groupOptionsValidator, listCommitsOptionsValidator, listOffsetsOptionsValidator } from "./options.js";
 import { COOPERATIVE_STICKY_ASSIGNOR, cooperativeStickyAssigner, roundRobinAssigner } from "./partitions-assigners.js";
 import { TopicsMap } from "./topics-map.js";
@@ -418,6 +418,12 @@ export class Consumer extends Base {
         if (!options.protocolsMetadata && protocolsMetadata) {
             options.protocolsMetadata = protocolsMetadata;
         }
+        if (!options.partitionAssigner && this[kOptions].partitionAssigner) {
+            options.partitionAssigner = this[kOptions].partitionAssigner;
+        }
+        if (!options.partitionAssignerTopicsSelector && this[kOptions].partitionAssignerTopicsSelector) {
+            options.partitionAssignerTopicsSelector = this[kOptions].partitionAssignerTopicsSelector;
+        }
         this.#validateGroupOptions(options);
         this.#membershipActive = true;
         this.#joinGroup(options, callback);
@@ -922,8 +928,8 @@ export class Consumer extends Base {
     #leaveGroupClassicProtocol(force, callback) {
         consumerGroupChannel.traceCallback(this.#performLeaveGroup, 1, createDiagnosticContext({ client: this, operation: 'leaveGroup', force }), this, force, callback);
     }
-    #syncGroup(partitionsAssigner, callback) {
-        consumerGroupChannel.traceCallback(this.#performSyncGroup, 2, createDiagnosticContext({ client: this, operation: 'syncGroup' }), this, partitionsAssigner, null, callback);
+    #syncGroup(partitionsAssigner, partitionAssignerTopicsSelector, callback) {
+        consumerGroupChannel.traceCallback(this.#performSyncGroup, 3, createDiagnosticContext({ client: this, operation: 'syncGroup' }), this, partitionsAssigner, partitionAssignerTopicsSelector, null, callback);
     }
     #heartbeat(options) {
         const eventPayload = { groupId: this.groupId, memberId: this.memberId, generationId: this.generationId };
@@ -1425,7 +1431,7 @@ export class Consumer extends Base {
                     this.#members.set(member.memberId, this.#decodeProtocolSubscriptionMetadata(member.memberId, member.metadata));
                 }
                 // Send a syncGroup request
-                this.#syncGroup(options.partitionAssigner, (error, response) => {
+                this.#syncGroup(options.partitionAssigner, options.partitionAssignerTopicsSelector, (error, response) => {
                     if (!this.#membershipActive) {
                         callback(null);
                         return;
@@ -1619,26 +1625,43 @@ export class Consumer extends Base {
         }
         this.#performDeduplicateGroupOperaton('leaveGroup', operation, onLeave);
     }
-    #performSyncGroup(partitionsAssigner, assignments, callback) {
+    #performSyncGroup(partitionsAssigner, partitionAssignerTopicsSelector, assignments, callback) {
         if (!this.#membershipActive) {
             callback(null, []);
             return;
         }
         if (!Array.isArray(assignments)) {
             if (this.#isLeader) {
-                // Get all the metadata for  the topics the consumer are listening to, then compute the assignments
-                const topicsSubscriptions = new Map();
-                for (const subscription of this.#members.values()) {
-                    for (const topic of subscription.topics) {
-                        let topicSubscriptions = topicsSubscriptions.get(topic);
-                        if (!topicSubscriptions) {
-                            topicSubscriptions = [];
-                            topicsSubscriptions.set(topic, topicSubscriptions);
+                const customTopics = !!(partitionsAssigner && partitionAssignerTopicsSelector);
+                let subscribedTopics;
+                if (customTopics) {
+                    try {
+                        subscribedTopics = partitionAssignerTopicsSelector(this.#members.get(this.memberId), this.#members);
+                    }
+                    catch (error) {
+                        callback(this.#handleError(new UserError('partitionAssignerTopicsSelector failed.', { cause: error })));
+                        return;
+                    }
+                    if (!Array.isArray(subscribedTopics) || subscribedTopics.some(topic => typeof topic !== 'string')) {
+                        callback(this.#handleError(new UserError('partitionAssignerTopicsSelector must return an array of topic names.')));
+                        return;
+                    }
+                }
+                else {
+                    // Get all the metadata for the topics the consumers are listening to, then compute the assignments.
+                    const topicsSubscriptions = new Map();
+                    for (const subscription of this.#members.values()) {
+                        for (const topic of subscription.topics) {
+                            let topicSubscriptions = topicsSubscriptions.get(topic);
+                            if (!topicSubscriptions) {
+                                topicSubscriptions = [];
+                                topicsSubscriptions.set(topic, topicSubscriptions);
+                            }
+                            topicSubscriptions.push(subscription);
                         }
-                        topicSubscriptions.push(subscription);
                     }
+                    subscribedTopics = Array.from(topicsSubscriptions.keys());
                 }
-                const subscribedTopics = Array.from(topicsSubscriptions.keys());
                 this[kMetadata]({ topics: subscribedTopics }, (error, metadata) => {
                     if (error) {
                         callback(this.#handleError(error));
@@ -1649,7 +1672,7 @@ export class Consumer extends Base {
                         callback(this.#handleError(new NetworkError(`Cannot compute group assignments: metadata for subscribed topic "${missingTopic}" is not available in the current cluster metadata.`, { canRetry: true, hasStaleMetadata: true })));
                         return;
                     }
-                    this.#performSyncGroup(partitionsAssigner, this.#createAssignments(partitionsAssigner, metadata), callback);
+                    this.#performSyncGroup(partitionsAssigner, partitionAssignerTopicsSelector, this.#createAssignments(partitionsAssigner, metadata), callback);
                 });
                 return;
             }
@@ -1781,9 +1804,7 @@ export class Consumer extends Base {
         const ownedPartitions = cooperative ? this.#ownedPartitionsForSubscription(topics) : [];
         let userData = typeof metadata.metadata === 'string' ? Buffer.from(metadata.metadata) : metadata.metadata;
         if (cooperative && !userData && metadata.version < 2) {
-            userData = Writer.create()
-                .appendInt32(this.memberId && ownedPartitions.length > 0 ? this.generationId : -1)
-                .buffer;
+            userData = Writer.create().appendInt32(this.memberId && ownedPartitions.length > 0 ? this.generationId : -1).buffer;
         }
         return encodeConsumerProtocolSubscription({
             version: metadata.version,
@@ -1832,7 +1853,7 @@ export class Consumer extends Base {
         }
         // We are the only member of the group, assign all partitions to us
         const membersSize = this.#members.size;
-        if (membersSize === 1) {
+        if (membersSize === 1 && !partitionsAssigner) {
             const assignments = [];
             for (const topic of this.topics.current) {
                 const partitionsCount = metadata.topics.get(topic).partitionsCount;

package/dist/clients/consumer/messages-stream.js

@@ -42,6 +42,7 @@ export class MessagesStream extends Readable {
     #refreshOffsetsInflight;
     #refreshOffsetsPending;
     #refreshOffsetsDestroyOnError;
+    #offsetsEpoch;
     #fetches;
     #maxFetches;
     #options;
@@ -88,7 +89,7 @@ export class MessagesStream extends Readable {
     constructor(consumer, options) {
         const { autocommit, mode, fallbackMode, maxFetches, offsets, context, deserializers, onCorruptedMessage, onDeserializationError, registry, beforeDeserialization, 
         // The options below are only destructured to avoid being part of structuredClone below
-        partitionAssigner: _partitionAssigner, protocolsMetadata: _protocolsMetadata, ...otherOptions } = options;
+        partitionAssigner: _partitionAssigner, partitionAssignerTopicsSelector: _partitionAssignerTopicsSelector, protocolsMetadata: _protocolsMetadata, ...otherOptions } = options;
         if (onCorruptedMessage && onDeserializationError) {
             throw new UserError('Cannot specify both onCorruptedMessage and onDeserializationError.');
         }
@@ -116,6 +117,7 @@ export class MessagesStream extends Readable {
         this.#refreshOffsetsInflight = false;
         this.#refreshOffsetsPending = false;
         this.#refreshOffsetsDestroyOnError = false;
+        this.#offsetsEpoch = 0;
         this.#fetches = 0;
         this.#maxFetches = maxFetches ?? 0;
         this.#topics = structuredClone(options.topics);
@@ -352,7 +354,11 @@ export class MessagesStream extends Readable {
         if (this.readableFlowing === null || this.#paused || this.#refreshOffsetsInflight || this.#refreshOffsetsPending) {
             return;
         }
+        const offsetsEpoch = this.#offsetsEpoch;
         this.#consumer.metadata({ topics: this.#consumer.topics.current }, (error, metadata) => {
+            if (offsetsEpoch !== this.#offsetsEpoch) {
+                return;
+            }
             if (error) {
                 this.emit('fetch');
                 // The stream has been closed, ignore any error
@@ -373,8 +379,8 @@ export class MessagesStream extends Readable {
             // Remove stale inflight entries that have been pending for too long.
             // This prevents permanent partition starvation if a fetch callback is never invoked.
             const now = Date.now();
-            for (const [node, timestamp] of this.#inflightNodes) {
-                if (now - timestamp > 120_000) {
+            for (const [node, inflight] of this.#inflightNodes) {
+                if (now - inflight.startedAt > 120_000) {
                     this.#inflightNodes.delete(node);
                 }
             }
@@ -423,10 +429,29 @@ export class MessagesStream extends Readable {
                 return;
             }
             for (const [node, nodeRequests] of requests) {
-                this.#inflightNodes.set(node, Date.now());
+                const inflight = { startedAt: Date.now() };
+                this.#inflightNodes.set(node, inflight);
                 this.#consumer.fetch({ ...this.#options, node, topics: nodeRequests, connectionPool: this[kConnections] }, (error, response) => {
-                    this.#inflightNodes.delete(node);
+                    const ownsNode = this.#inflightNodes.get(node) === inflight;
+                    if (ownsNode) {
+                        this.#inflightNodes.delete(node);
+                    }
                     this.emit('fetch');
+                    // A refresh (including a rejoin) invalidates requests issued before it.
+                    // Never let an old response or error undo the restored offsets.
+                    if (offsetsEpoch !== this.#offsetsEpoch || !ownsNode) {
+                        if (this.#closed || this.closed || this.destroyed) {
+                            if (this.#inflightNodes.size === 0) {
+                                this.push(null);
+                            }
+                        }
+                        else if (ownsNode) {
+                            // The refresh may have finished while this node was still inflight.
+                            // Now that it is free, resume fetching from the restored position.
+                            process.nextTick(() => this.#fetch());
+                        }
+                        return;
+                    }
                     if (error) {
                         // The stream has been closed, ignore the error
                         /* c8 ignore next 4 - Hard to test */
@@ -464,7 +489,7 @@ export class MessagesStream extends Readable {
                         }
                         return;
                     }
-                    this.#pushRecordsOperation(metadata, topicIds, response, requestedOffsets);
+                    this.#pushRecordsOperation(metadata, topicIds, response, requestedOffsets, offsetsEpoch);
                 });
             }
         });
@@ -553,7 +578,10 @@ export class MessagesStream extends Readable {
         }
         return recoveredOffsets.length > 0;
     }
-    #pushRecords(metadata, topicIds, response, requestedOffsets) {
+    #pushRecords(metadata, topicIds, response, requestedOffsets, offsetsEpoch) {
+        if (offsetsEpoch !== this.#offsetsEpoch) {
+            return;
+        }
         const autocommit = this.#autocommitEnabled;
         const keyDeserializer = this.#keyDeserializer;
         const valueDeserializer = this.#valueDeserializer;
@@ -995,6 +1023,8 @@ export class MessagesStream extends Readable {
         });
     }
     #scheduleRefreshOffsetsAndFetch(destroyOnError = true) {
+        // Invalidate outstanding fetches immediately, not only after restoration completes.
+        this.#offsetsEpoch++;
         this.#refreshOffsetsDestroyOnError ||= destroyOnError;
         /* c8 ignore next 4 - Hard to test */
         if (this.#refreshOffsetsInflight) {
@@ -1078,7 +1108,7 @@ export class MessagesStream extends Readable {
     [kInspect](...args) {
         this.#consumer[kInspect](...args);
     }
-    #beforeDeserialization(hook, metadata, topicIds, response, requestedOffsets) {
+    #beforeDeserialization(hook, metadata, topicIds, response, requestedOffsets, offsetsEpoch) {
         const requests = [];
         // Create the pre-deserialization requests
         for (const topicResponse of response.responses) {
@@ -1113,7 +1143,7 @@ export class MessagesStream extends Readable {
             }
         }
         if (requests.length === 0) {
-            this.#pushRecords(metadata, topicIds, response, requestedOffsets);
+            this.#pushRecords(metadata, topicIds, response, requestedOffsets, offsetsEpoch);
             return;
         }
         runAsyncSeries((request, cb) => {
@@ -1137,11 +1167,14 @@ export class MessagesStream extends Readable {
                 result.then(() => onHookSettled(null), onHookSettled);
             }
         }, requests, 0, error => {
+            if (offsetsEpoch !== this.#offsetsEpoch) {
+                return;
+            }
             if (error) {
                 this.destroy(error);
                 return;
             }
-            this.#pushRecords(metadata, topicIds, response, requestedOffsets);
+            this.#pushRecords(metadata, topicIds, response, requestedOffsets, offsetsEpoch);
         });
     }
 }

package/dist/clients/consumer/options.d.ts

@@ -63,6 +63,9 @@ export declare const groupOptionsProperties: {
     partitionAssigner: {
         function: boolean;
     };
+    partitionAssignerTopicsSelector: {
+        function: boolean;
+    };
 };
 export declare const groupOptionsAdditionalValidations: {
     rebalanceTimeout: {
@@ -212,6 +215,9 @@ export declare const groupOptionsSchema: {
         partitionAssigner: {
             function: boolean;
         };
+        partitionAssignerTopicsSelector: {
+            function: boolean;
+        };
     };
     additionalProperties: boolean;
 };
@@ -339,6 +345,9 @@ export declare const consumeOptionsSchema: {
         partitionAssigner: {
             function: boolean;
         };
+        partitionAssignerTopicsSelector: {
+            function: boolean;
+        };
         topics: {
             type: string;
             items: {
@@ -515,6 +524,9 @@ export declare const consumerOptionsSchema: {
         partitionAssigner: {
             function: boolean;
         };
+        partitionAssignerTopicsSelector: {
+            function: boolean;
+        };
         groupId: {
             type: string;
             pattern: string;
@@ -649,6 +661,9 @@ export declare const fetchOptionsSchema: {
         partitionAssigner: {
             function: boolean;
         };
+        partitionAssignerTopicsSelector: {
+            function: boolean;
+        };
         node: {
             type: string;
             minimum: number;

package/dist/clients/consumer/options.js

@@ -27,7 +27,8 @@ export const groupOptionsProperties = {
         }
     },
     protocolsMetadata: { function: true },
-    partitionAssigner: { function: true }
+    partitionAssigner: { function: true },
+    partitionAssignerTopicsSelector: { function: true }
 };
 export const groupOptionsAdditionalValidations = {
     rebalanceTimeout: {
@@ -200,7 +201,7 @@ export const listOffsetsOptionsSchema = {
                 items: { type: 'number', minimum: 0 }
             }
         },
-        isolationLevel: { type: 'string', enum: allowedFetchIsolationLevels },
+        isolationLevel: { type: 'number', enum: allowedFetchIsolationLevels },
         timestamp: { bigint: true }
     },
     required: ['topics'],

package/dist/clients/consumer/types.d.ts

@@ -1,5 +1,5 @@
-import { type Callback } from '../../apis/definitions.ts';
 import { type FetchRequestTopic } from '../../apis/consumer/fetch-v17.ts';
+import { type Callback } from '../../apis/definitions.ts';
 import { type GroupProtocols } from '../../apis/enumerations.ts';
 import { type ConnectionPool } from '../../network/connection-pool.ts';
 import { type KafkaRecord, type Message } from '../../protocol/records.ts';
@@ -65,6 +65,7 @@ export type DeserializationErrorHandler = (context: DeserializationErrorContext)
  */
 export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit'], error: unknown) => boolean;
 export type GroupPartitionsAssigner = (current: string, members: Map<string, ExtendedGroupProtocolSubscription>, topics: Set<string>, metadata: ClusterMetadata) => GroupPartitionsAssignments[];
+export type GroupPartitionsAssignerTopicsSelector = (leader: ExtendedGroupProtocolSubscription, members: Map<string, ExtendedGroupProtocolSubscription>) => string[];
 export type GroupProtocolsMetadataCallback = (protocols: GroupProtocolSubscription[], topics: string[], metadata: ClusterMetadata, callback: Callback<Buffer>) => Buffer | Promise<Buffer> | undefined;
 export declare const MessagesStreamModes: {
     readonly LATEST: "latest";
@@ -93,6 +94,7 @@ export interface GroupOptions {
     protocols?: GroupProtocolSubscription[];
     protocolsMetadata?: GroupProtocolsMetadataCallback;
     partitionAssigner?: GroupPartitionsAssigner;
+    partitionAssignerTopicsSelector?: GroupPartitionsAssignerTopicsSelector;
     assignmentUserData?: Buffer;
 }
 export interface ConsumerGroupOptions {

package/dist/clients/producer/producer.js

@@ -295,7 +295,7 @@ export class Producer extends Base {
             const brokers = {};
             for (const [topic, partitions] of topicsPartitions) {
                 for (const rawPartition of partitions) {
-                    const partition = rawPartition & metadata.topics.get(topic).partitionsCount;
+                    const partition = rawPartition % metadata.topics.get(topic).partitionsCount;
                     const leader = metadata.topics.get(topic).partitions[partition].leader;
                     const broker = metadata.brokers.get(leader);
                     brokers[`${topic}:${partition}`] = broker;

package/dist/errors.d.ts

@@ -7,6 +7,7 @@ export type ErrorCode = (typeof errorCodes)[number];
 export type ErrorProperties = {
     cause?: Error;
 } & Record<string, any>;
+export declare function findNestedErrorBy(error: Error, property: string, value: unknown, visited: Set<Error>): Error | null;
 export declare class GenericError extends Error {
     code: string;
     [index: string]: any;

package/dist/errors.js

@@ -19,6 +19,30 @@ export const errorCodes = [
     'PLT_KFK_UNSUPPORTED',
     'PLT_KFK_USER'
 ];
+export function findNestedErrorBy(error, property, value, visited) {
+    if (!error || visited.has(error)) {
+        return null;
+    }
+    visited.add(error);
+    if (error[property] === value) {
+        return error;
+    }
+    // Visit aggregate entries before the cause, preserving the existing search order.
+    if (MultipleErrors.isMultipleErrors(error)) {
+        for (const nested of error.errors) {
+            if (nested && typeof nested === 'object') {
+                const found = findNestedErrorBy(nested, property, value, visited);
+                if (found) {
+                    return found;
+                }
+            }
+        }
+    }
+    if (error.cause && typeof error.cause === 'object') {
+        return findNestedErrorBy(error.cause, property, value, visited);
+    }
+    return null;
+}
 export class GenericError extends Error {
     code;
     [kGenericError];
@@ -40,10 +64,7 @@ export class GenericError extends Error {
         Reflect.defineProperty(this, kGenericError, { value: true, enumerable: false });
     }
     findBy(property, value) {
-        if (this[property] === value) {
-            return this;
-        }
-        return null;
+        return findNestedErrorBy(this, property, value, new Set());
     }
     toString() {
         return `Error ${this.code}: ${this.message}`;
@@ -77,22 +98,7 @@ export class MultipleErrors extends AggregateError {
         Reflect.defineProperty(this, kMultipleErrors, { value: true, enumerable: false });
     }
     findBy(property, value) {
-        if (this[property] === value) {
-            return this;
-        }
-        for (const error of this.errors) {
-            if (!error) {
-                continue;
-            }
-            if (error[property] === value) {
-                return error;
-            }
-            const found = error[kGenericError] ? error.findBy(property, value) : undefined;
-            if (found) {
-                return found;
-            }
-        }
-        return null;
+        return findNestedErrorBy(this, property, value, new Set());
     }
     toString() {
         return `Error ${this.code}: ${this.message} (caused by: ${this.errors.map(error => error.toString()).join(', ')})`;
@@ -121,7 +127,8 @@ export class NetworkError extends GenericError {
     static code = 'PLT_KFK_NETWORK';
     static isRetryable(error) {
         const networkError = findErrorBy(error, 'code', NetworkError.code);
-        return networkError?.canRetry === true;
+        // Require a retryable network error, but reject any non-retriable error in the whole tree.
+        return networkError?.canRetry === true && !findErrorBy(error, 'canRetry', false);
     }
     constructor(message, properties = {}) {
         super(NetworkError.code, message, { canRetry: true, ...properties });

package/dist/network/connection.d.ts

@@ -35,6 +35,9 @@ export interface SASLOptions {
     oauthBearerExtensions?: Record<string, string> | CredentialProvider<Record<string, string>>;
     authenticate?: SASLCustomAuthenticator;
     authBytesValidator?: (authBytes: Buffer, callback: CallbackWithPromise<Buffer>) => void;
+    reauthFraction?: number;
+    reauthLeadTime?: number;
+    lazyReauthentication?: boolean;
 }
 export interface ConnectionOptions {
     connectTimeout?: number;

package/dist/network/connection.js

@@ -4,8 +4,8 @@ import { createConnection } from 'node:net';
 import { connect as createTLSConnection } from 'node:tls';
 import { createPromisifiedCallback, kCallbackPromise } from "../apis/callbacks.js";
 import { allowedSASLMechanisms, SASLMechanisms } from "../apis/enumerations.js";
-import { clientSoftwareName, clientSoftwareVersion } from "../clients/base/options.js";
 import { apiVersionsV1, saslAuthenticateV0, saslAuthenticateV1, saslAuthenticateV2, saslHandshakeV1 } from "../apis/index.js";
+import { clientSoftwareName, clientSoftwareVersion } from "../clients/base/options.js";
 import { connectionsApiChannel, connectionsConnectsChannel, createDiagnosticContext, notifyCreation } from "../diagnostic.js";
 import { AuthenticationError, NetworkError, TimeoutError, UnexpectedCorrelationIdError, UserError } from "../errors.js";
 import { TypedEventEmitter } from "../events.js";
@@ -54,6 +54,7 @@ export class Connection extends TypedEventEmitter {
     #socketMustBeDrained;
     #detectMissingTLS;
     #reauthenticationTimeout;
+    #reauthenticationDue = false;
     // Negotiated once per connection: reauthentication runs the same path again.
     #saslAuthenticateApi;
     constructor(clientId, options = {}) {
@@ -239,7 +240,7 @@ export class Connection extends TypedEventEmitter {
         if (!callback) {
             callback = createPromisifiedCallback();
         }
-        clearInterval(this.#reauthenticationTimeout);
+        clearTimeout(this.#reauthenticationTimeout);
         if (this.#status === ConnectionStatuses.CLOSED ||
             this.#status === ConnectionStatuses.ERROR ||
             this.#status === ConnectionStatuses.NONE) {
@@ -266,6 +267,25 @@ export class Connection extends TypedEventEmitter {
         return callback[kCallbackPromise];
     }
     send(apiKey, apiVersion, createPayload, responseParser, hasRequestHeaderTaggedFields, hasResponseHeaderTaggedFields, callback) {
+        if ((this.#status === ConnectionStatuses.REAUTHENTICATING ||
+            (this.#status === ConnectionStatuses.CONNECTED && this.#reauthenticationDue)) &&
+            // SASL requests must still pass through send() while reauthentication is in progress.
+            apiKey !== apiVersionsV1.api.key &&
+            apiKey !== saslHandshakeV1.api.key &&
+            apiKey !== saslAuthenticateV2.api.key) {
+            this.ready(error => {
+                if (error) {
+                    callback(error);
+                }
+                else {
+                    this.send(apiKey, apiVersion, createPayload, responseParser, hasRequestHeaderTaggedFields, hasResponseHeaderTaggedFields, callback);
+                }
+            });
+            if (this.#reauthenticationDue) {
+                this.reauthenticate();
+            }
+            return;
+        }
         // Correlation ID is a 32-bit integer in the protocol, so we need to wrap around after 2^31 - 1
         const correlationId = (this.#correlationId + 1) & 0x7fffffff;
         this.#correlationId = correlationId;
@@ -358,6 +378,7 @@ export class Connection extends TypedEventEmitter {
         const host = this.#host;
         const port = this.#port;
         const diagnosticContext = createDiagnosticContext({ connection: this, operation: 'reauthenticate', host, port });
+        this.#reauthenticationDue = false;
         this.#status = ConnectionStatuses.REAUTHENTICATING;
         clearTimeout(this.#reauthenticationTimeout);
         this.#authenticate(host, port, diagnosticContext);
@@ -537,7 +558,18 @@ export class Connection extends TypedEventEmitter {
             return;
         }
         if (sessionLifetimeMs > 0) {
-            this.#reauthenticationTimeout = setTimeout(this.reauthenticate.bind(this), Number(sessionLifetimeMs) * 0.8);
+            const lifetime = Number(sessionLifetimeMs);
+            const leadTime = lifetime - (this.#options.sasl?.reauthLeadTime ?? 0);
+            const fractionTime = lifetime * (this.#options.sasl?.reauthFraction ?? 0.8);
+            // Ignore a non-positive lead-time delay to avoid an immediate reauthentication loop.
+            this.#reauthenticationTimeout = setTimeout(() => {
+                if (this.#options.sasl?.lazyReauthentication) {
+                    this.#reauthenticationDue = true;
+                }
+                else {
+                    this.reauthenticate();
+                }
+            }, Math.min(fractionTime, leadTime > 0 ? leadTime : fractionTime));
         }
         const isReauthenticating = this.#status === ConnectionStatuses.REAUTHENTICATING;
         if (this.#status === ConnectionStatuses.CONNECTED || isReauthenticating) {

package/dist/typescript-4/dist/clients/admin/types.d.ts

@@ -40,6 +40,7 @@ export interface Group extends Omit<GroupBase, 'groupType'> {
   protocol: string;
   members: Map<string, GroupMember>;
   authorizedOperations: number;
+  error?: Error;
 }
 export interface AdminOptions extends BaseOptions {}
 export interface CreateTopicsTopicOptions {

package/dist/typescript-4/dist/clients/base/options.d.ts

@@ -108,6 +108,19 @@ export declare const baseOptionsSchema: {
           type: string;
           enum: ("PLAIN" | "SCRAM-SHA-256" | "SCRAM-SHA-512" | "OAUTHBEARER" | "GSSAPI")[];
         };
+        reauthFraction: {
+          type: string;
+          exclusiveMinimum: number;
+          maximum: number;
+          default: number;
+        };
+        reauthLeadTime: {
+          type: string;
+          minimum: number;
+        };
+        lazyReauthentication: {
+          type: string;
+        };
         username: {
           oneOf: ({
             type: string;

package/dist/typescript-4/dist/clients/consumer/consumer.d.ts

@@ -6,7 +6,7 @@ import { Base, type BaseEvents, kCreateConnectionPool } from "../base/base";
 import { type ClusterMetadata } from "../base/types";
 import { MessagesStream } from "./messages-stream";
 import { TopicsMap } from "./topics-map";
-import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupAutocommitErrorPayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerHeartbeatStalledPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from "./types";
+import { type CommitOptions, type ConsumeOptions, type ConsumerGroupAutocommitErrorPayload, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerHeartbeatStalledPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from "./types";
 export interface ConsumerEvents extends BaseEvents {
   'consumer:group:join': (payload: ConsumerGroupJoinPayload) => void;
   'consumer:group:leave': (payload: ConsumerGroupLeavePayload) => void;

package/dist/typescript-4/dist/clients/consumer/options.d.ts

@@ -63,6 +63,9 @@ export declare const groupOptionsProperties: {
   partitionAssigner: {
     function: boolean;
   };
+  partitionAssignerTopicsSelector: {
+    function: boolean;
+  };
 };
 export declare const groupOptionsAdditionalValidations: {
   rebalanceTimeout: {
@@ -212,6 +215,9 @@ export declare const groupOptionsSchema: {
     partitionAssigner: {
       function: boolean;
     };
+    partitionAssignerTopicsSelector: {
+      function: boolean;
+    };
   };
   additionalProperties: boolean;
 };
@@ -339,6 +345,9 @@ export declare const consumeOptionsSchema: {
     partitionAssigner: {
       function: boolean;
     };
+    partitionAssignerTopicsSelector: {
+      function: boolean;
+    };
     topics: {
       type: string;
       items: {
@@ -515,6 +524,9 @@ export declare const consumerOptionsSchema: {
     partitionAssigner: {
       function: boolean;
     };
+    partitionAssignerTopicsSelector: {
+      function: boolean;
+    };
     groupId: {
       type: string;
       pattern: string;
@@ -649,6 +661,9 @@ export declare const fetchOptionsSchema: {
     partitionAssigner: {
       function: boolean;
     };
+    partitionAssignerTopicsSelector: {
+      function: boolean;
+    };
     node: {
       type: string;
       minimum: number;

package/dist/typescript-4/dist/clients/consumer/types.d.ts

@@ -1,5 +1,5 @@
-import { type Callback } from "../../apis/definitions";
 import { type FetchRequestTopic } from "../../apis/consumer/fetch-v17";
+import { type Callback } from "../../apis/definitions";
 import { type GroupProtocols } from "../../apis/enumerations";
 import { type ConnectionPool } from "../../network/connection-pool";
 import { type KafkaRecord, type Message } from "../../protocol/records";
@@ -65,6 +65,7 @@ export type DeserializationErrorHandler = (context: DeserializationErrorContext)
  */
 export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit'], error: unknown) => boolean;
 export type GroupPartitionsAssigner = (current: string, members: Map<string, ExtendedGroupProtocolSubscription>, topics: Set<string>, metadata: ClusterMetadata) => GroupPartitionsAssignments[];
+export type GroupPartitionsAssignerTopicsSelector = (leader: ExtendedGroupProtocolSubscription, members: Map<string, ExtendedGroupProtocolSubscription>) => string[];
 export type GroupProtocolsMetadataCallback = (protocols: GroupProtocolSubscription[], topics: string[], metadata: ClusterMetadata, callback: Callback<Buffer>) => Buffer | Promise<Buffer> | undefined;
 export declare const MessagesStreamModes: {
   readonly LATEST: "latest";
@@ -93,6 +94,7 @@ export interface GroupOptions {
   protocols?: GroupProtocolSubscription[];
   protocolsMetadata?: GroupProtocolsMetadataCallback;
   partitionAssigner?: GroupPartitionsAssigner;
+  partitionAssignerTopicsSelector?: GroupPartitionsAssignerTopicsSelector;
   assignmentUserData?: Buffer;
 }
 export interface ConsumerGroupOptions {

package/dist/typescript-4/dist/errors.d.ts

@@ -7,6 +7,7 @@ export type ErrorCode = (typeof errorCodes)[number];
 export type ErrorProperties = {
   cause?: Error;
 } & Record<string, any>;
+export declare function findNestedErrorBy(error: Error, property: string, value: unknown, visited: Set<Error>): Error | null;
 export declare class GenericError extends Error {
   code: string;
   [index: string]: any;

package/dist/typescript-4/dist/network/connection.d.ts

@@ -35,6 +35,9 @@ export interface SASLOptions {
   oauthBearerExtensions?: Record<string, string> | CredentialProvider<Record<string, string>>;
   authenticate?: SASLCustomAuthenticator;
   authBytesValidator?: (authBytes: Buffer, callback: CallbackWithPromise<Buffer>) => void;
+  reauthFraction?: number;
+  reauthLeadTime?: number;
+  lazyReauthentication?: boolean;
 }
 export interface ConnectionOptions {
   connectTimeout?: number;

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "2.12.0";
+export const version = "2.13.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "2.12.0",
+  "version": "2.13.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
@@ -95,12 +95,13 @@
     "test": "c8 -c test/config/c8-local.json ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
     "test:ci": "./scripts/node --test --experimental-test-coverage --test-coverage-include='src/**' --test-coverage-lines=90 --test-coverage-branches=90 --test-coverage-functions=90 --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
     "test:integrity": "./scripts/node --test --test-reporter=cleaner-spec-reporter 'regression/integrity/*.test.ts'",
-    "test:performance": "./scripts/node --test --test-reporter=cleaner-spec-reporter 'regression/performance/*.test.ts'",
+    "test:performance": "./scripts/node --test --test-concurrency=1 --test-reporter=cleaner-spec-reporter 'regression/performance/*.test.ts'",
     "test:regression": "npm run test:integrity",
     "test:regression:all": "npm run test:integrity && npm run test:performance",
     "test:all": "npm test && npm run test:regression:all",
     "test:compat": "./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/integration/*.compat-test.ts'",
     "test:memory": "NODE_OPTIONS='--expose-gc' ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/**/*.memory-test.ts'",
+    "test:e2e:upgrade": "./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/e2e/client-upgrade.e2e-test.ts'",
     "test:e2e:redpanda": "./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/e2e/redpanda.e2e-test.ts'",
     "test:e2e:eventhubs": "./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/e2e/eventhubs.e2e-test.ts'",
     "test:docker:up": "docker-compose up -d --wait",