@platformatic/kafka 1.7.0 → 1.9.0

package/dist/apis/callbacks.js

@@ -37,6 +37,9 @@ export function runConcurrentCallbacks(errorMessage, collection, operation, call
             callback(hasErrors ? new MultipleErrors(errorMessage, errors) : null, results);
         }
     }
+    if (remaining === 0) {
+        callback(null, results);
+    }
     for (const item of collection) {
         operation(item, operationCallback.bind(null, i++));
     }

package/dist/apis/enumerations.d.ts

@@ -1,4 +1,4 @@
-export declare const SASLMechanisms: readonly ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512"];
+export declare const SASLMechanisms: readonly ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512", "OAUTHBEARER"];
 export type SASLMechanism = (typeof SASLMechanisms)[number];
 export declare const FindCoordinatorKeyTypes: {
     readonly GROUP: 0;

package/dist/apis/enumerations.js

@@ -1,5 +1,5 @@
 // SASL Authentication
-export const SASLMechanisms = ['PLAIN', 'SCRAM-SHA-256', 'SCRAM-SHA-512'];
+export const SASLMechanisms = ['PLAIN', 'SCRAM-SHA-256', 'SCRAM-SHA-512', 'OAUTHBEARER'];
 // Metadata API
 // ./metadata/find-coordinator.ts
 export const FindCoordinatorKeyTypes = { GROUP: 0, TRANSACTION: 1, SHARE: 2 };

package/dist/clients/admin/admin.js

@@ -144,6 +144,7 @@ export class Admin extends Base {
         const numPartitions = options.partitions ?? 1;
         const replicationFactor = options.replicas ?? 1;
         const assignments = [];
+        const configs = options.configs ?? [];
         for (const { partition, brokers } of options.assignments ?? []) {
             assignments.push({ partitionIndex: partition, brokerIds: brokers });
         }
@@ -154,7 +155,7 @@ export class Admin extends Base {
                 numPartitions,
                 replicationFactor,
                 assignments,
-                configs: []
+                configs
             });
         }
         this[kPerformDeduplicated]('createTopics', deduplicateCallback => {
@@ -322,10 +323,12 @@ export class Admin extends Base {
                                     metadata: reader.readBytes(false)
                                 };
                                 reader.reset(member.memberAssignment);
+                                reader.skip(2); // Ignore Version information
                                 const memberAssignments = reader.readMap(r => {
-                                    const topic = r.readString();
-                                    return [topic, { topic, partitions: reader.readArray(r => r.readInt32(), true, false) }];
-                                }, true, false);
+                                    const topic = r.readString(false);
+                                    return [topic, { topic, partitions: reader.readArray(r => r.readInt32(), false, false) }];
+                                }, false, false);
+                                reader.readBytes(); // Ignore the user data
                                 group.members.set(member.memberId, {
                                     id: member.memberId,
                                     groupInstanceId: member.groupInstanceId,

package/dist/clients/admin/types.d.ts

@@ -1,3 +1,4 @@
+import { type CreateTopicsRequestTopicConfig } from '../../apis/admin/create-topics-v7.ts';
 import { type ConsumerGroupState } from '../../apis/enumerations.ts';
 import { type NullableString } from '../../protocol/definitions.ts';
 import { type BaseOptions } from '../base/types.ts';
@@ -39,6 +40,7 @@ export interface CreateTopicsOptions {
     partitions?: number;
     replicas?: number;
     assignments?: BrokerAssignment[];
+    configs?: CreateTopicsRequestTopicConfig[];
 }
 export interface ListTopicsOptions {
     includeInternals?: boolean;

package/dist/clients/base/base.d.ts

@@ -56,6 +56,10 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions> extends
     listApis(): Promise<ApiVersionsResponseApi[]>;
     metadata(options: MetadataOptions, callback: CallbackWithPromise<ClusterMetadata>): void;
     metadata(options: MetadataOptions): Promise<ClusterMetadata>;
+    connectToBrokers(nodeIds: number[] | null, callback: CallbackWithPromise<Map<number, Connection>>): void;
+    connectToBrokers(nodeIds?: number[] | null): Promise<Map<number, Connection>>;
+    isActive(): boolean;
+    isConnected(): boolean;
     [kCreateConnectionPool](): ConnectionPool;
     [kListApis](callback: CallbackWithPromise<ApiVersionsResponseApi[]>): void;
     [kMetadata](options: MetadataOptions, callback: CallbackWithPromise<ClusterMetadata>): void;

package/dist/clients/base/base.js

@@ -1,5 +1,5 @@
 import { EventEmitter } from 'node:events';
-import { createPromisifiedCallback, kCallbackPromise } from "../../apis/callbacks.js";
+import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
 import * as apis from "../../apis/index.js";
 import { api as apiVersionsV3 } from "../../apis/metadata/api-versions-v3.js";
 import { baseApisChannel, baseMetadataChannel, createDiagnosticContext, notifyCreation } from "../../diagnostic.js";
@@ -49,6 +49,7 @@ export class Base extends EventEmitter {
     #inflightDeduplications;
     constructor(options) {
         super();
+        this.setMaxListeners(0);
         this[kClientType] = 'base';
         this[kInstance] = currentInstance++;
         this[kApis] = [];
@@ -97,6 +98,7 @@ export class Base extends EventEmitter {
             callback = createPromisifiedCallback();
         }
         this[kClosed] = true;
+        this.emitWithDebug('client', 'close');
         this[kConnections].close(callback);
         return callback[kCallbackPromise];
     }
@@ -119,6 +121,57 @@ export class Base extends EventEmitter {
         baseMetadataChannel.traceCallback(this[kMetadata], 1, createDiagnosticContext({ client: this, operation: 'metadata' }), this, options, callback);
         return callback[kCallbackPromise];
     }
+    connectToBrokers(nodeIds, callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        // Fetch the metadata
+        this[kMetadata]({ topics: [] }, (error, metadata) => {
+            if (error) {
+                callback(error, undefined);
+                return;
+            }
+            let nodes = [];
+            if (nodeIds?.length) {
+                for (const node of nodeIds) {
+                    if (metadata.brokers.has(node)) {
+                        nodes.push(node);
+                    }
+                }
+            }
+            else {
+                nodes = Array.from(metadata.brokers.keys());
+            }
+            runConcurrentCallbacks('Connecting to brokers failed.', nodes, (nodeId, concurrentCallback) => {
+                this[kGetConnection](metadata.brokers.get(nodeId), (error, connection) => {
+                    if (error) {
+                        concurrentCallback(error, undefined);
+                        return;
+                    }
+                    concurrentCallback(null, [nodeId, connection]);
+                });
+            }, (error, connections) => {
+                if (error) {
+                    callback(error, undefined);
+                    return;
+                }
+                return callback(null, new Map(connections));
+            });
+        });
+        return callback[kCallbackPromise];
+    }
+    isActive() {
+        if (this[kClosed]) {
+            return false;
+        }
+        return true;
+    }
+    isConnected() {
+        if (this[kClosed]) {
+            return false;
+        }
+        return this[kConnections].isConnected();
+    }
     [kCreateConnectionPool]() {
         const pool = new ConnectionPool(this[kClientId], {
             ownerId: this[kInstance],
@@ -148,13 +201,26 @@ export class Base extends EventEmitter {
         }, callback);
     }
     [kMetadata](options, callback) {
-        const metadataMaxAge = options.metadataMaxAge ?? this[kOptions].metadataMaxAge;
-        const isStale = options.forceUpdate ||
-            !this.#metadata ||
-            Date.now() > this.#metadata.lastUpdate + metadataMaxAge ||
-            options.topics.some(topic => !this.#metadata?.topics.has(topic));
-        if (!isStale) {
-            callback(null, this.#metadata);
+        const expiralDate = Date.now() - (options.metadataMaxAge ?? this[kOptions].metadataMaxAge);
+        let topicsToFetch = [];
+        // Determine which topics we need to fetch
+        if (!this.#metadata || options.forceUpdate) {
+            topicsToFetch = options.topics;
+        }
+        else {
+            for (const topic of options.topics) {
+                const existingTopic = this.#metadata.topics.get(topic);
+                if (!existingTopic || existingTopic.lastUpdate < expiralDate) {
+                    topicsToFetch.push(topic);
+                }
+            }
+        }
+        // All topics are already up-to-date, simply return them
+        if (this.#metadata && !topicsToFetch.length) {
+            callback(null, {
+                ...this.#metadata,
+                topics: new Map(options.topics.map(topic => [topic, this.#metadata.topics.get(topic)]))
+            });
             return;
         }
         const autocreateTopics = options.autocreateTopics ?? this[kOptions].autocreateTopics;
@@ -170,20 +236,40 @@ export class Base extends EventEmitter {
                             retryCallback(error, undefined);
                             return;
                         }
-                        api(connection, options.topics, autocreateTopics, true, retryCallback);
+                        api(connection, topicsToFetch, autocreateTopics, true, retryCallback);
                     });
                 });
             }, (error, metadata) => {
                 if (error) {
+                    const hasStaleMetadata = error.findBy('hasStaleMetadata', true);
+                    // Stale metadata, we need to fetch everything again
+                    if (hasStaleMetadata) {
+                        this[kClearMetadata]();
+                        topicsToFetch = options.topics;
+                    }
                     deduplicateCallback(error, undefined);
                     return;
                 }
+                const lastUpdate = Date.now();
+                if (!this.#metadata) {
+                    this.#metadata = {
+                        id: metadata.clusterId,
+                        brokers: new Map(),
+                        topics: new Map(),
+                        lastUpdate
+                    };
+                }
+                else {
+                    this.#metadata.lastUpdate = lastUpdate;
+                }
                 const brokers = new Map();
-                const topics = new Map();
+                // This should never change, but we act defensively here
                 for (const broker of metadata.brokers) {
                     const { host, port } = broker;
                     brokers.set(broker.nodeId, { host, port });
                 }
+                this.#metadata.brokers = brokers;
+                // Update all the topics in the cache
                 for (const { name, topicId: id, partitions: rawPartitions, isInternal } of metadata.topics) {
                     /* c8 ignore next 3 - Sometimes internal topics might be returned by Kafka */
                     if (isInternal) {
@@ -197,16 +283,15 @@ export class Base extends EventEmitter {
                             replicas: rawPartition.replicaNodes
                         };
                     }
-                    topics.set(name, { id, partitions, partitionsCount: rawPartitions.length });
+                    this.#metadata.topics.set(name, { id, partitions, partitionsCount: rawPartitions.length, lastUpdate });
                 }
-                this.#metadata = {
-                    id: metadata.clusterId,
-                    brokers,
-                    topics,
-                    lastUpdate: Date.now()
+                // Now build the object to return
+                const updatedMetadata = {
+                    ...this.#metadata,
+                    topics: new Map(options.topics.map(topic => [topic, this.#metadata.topics.get(topic)]))
                 };
-                this.emitWithDebug('client', 'metadata', this.#metadata);
-                deduplicateCallback(null, this.#metadata);
+                this.emitWithDebug('client', 'metadata', updatedMetadata);
+                deduplicateCallback(null, updatedMetadata);
             }, 0);
         }, callback);
     }
@@ -242,9 +327,17 @@ export class Base extends EventEmitter {
                 const retriable = genericError.findBy?.('code', NetworkError.code) || genericError.findBy?.('canRetry', true);
                 errors.push(error);
                 if (attempt < retries && retriable && !shouldSkipRetry?.(error)) {
-                    setTimeout(() => {
+                    this.emitWithDebug('client', 'performWithRetry:retry', operationId, attempt, retries);
+                    function onClose() {
+                        clearTimeout(timeout);
+                        errors.push(new UserError(`Client closed while retrying ${operationId}.`));
+                        callback(new MultipleErrors(`${operationId} failed ${attempt + 1} times.`, errors), undefined);
+                    }
+                    const timeout = setTimeout(() => {
+                        this.removeListener('client:close', onClose);
                         this[kPerformWithRetry](operationId, operation, callback, attempt + 1, errors, shouldSkipRetry);
                     }, this[kOptions].retryDelay);
+                    this.once('client:close', onClose);
                 }
                 else {
                     if (attempt === 0) {

package/dist/clients/base/options.d.ts

@@ -88,7 +88,7 @@ export declare const baseOptionsSchema: {
             properties: {
                 mechanism: {
                     type: string;
-                    enum: readonly ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512"];
+                    enum: readonly ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512", "OAUTHBEARER"];
                 };
                 username: {
                     type: string;
@@ -96,6 +96,9 @@ export declare const baseOptionsSchema: {
                 password: {
                     type: string;
                 };
+                token: {
+                    type: string;
+                };
             };
             required: string[];
             additionalProperties: boolean;

package/dist/clients/base/options.js

@@ -1,10 +1,9 @@
-import { readFileSync } from 'node:fs';
 import { SASLMechanisms } from "../../apis/enumerations.js";
 import { ajv } from "../../utils.js";
-const packageJson = JSON.parse(readFileSync(new URL('../../../package.json', import.meta.url), 'utf-8'));
+import { version } from "../../version.js";
 // Note: clientSoftwareName can only contain alphanumeric characters, hyphens and dots
 export const clientSoftwareName = 'platformatic-kafka';
-export const clientSoftwareVersion = packageJson.version;
+export const clientSoftwareVersion = version;
 export const idProperty = { type: 'string', pattern: '^\\S+$' };
 export const topicWithPartitionAndOffsetProperties = {
     topic: idProperty,
@@ -39,9 +38,10 @@ export const baseOptionsSchema = {
             properties: {
                 mechanism: { type: 'string', enum: SASLMechanisms },
                 username: { type: 'string' },
-                password: { type: 'string' }
+                password: { type: 'string' },
+                token: { type: 'string' }
             },
-            required: ['mechanism', 'username', 'password'],
+            required: ['mechanism'],
             additionalProperties: false
         },
         metadataMaxAge: { type: 'number', minimum: 0 },
@@ -72,7 +72,7 @@ export const defaultBaseOptions = {
     timeout: 5000,
     retries: 3,
     retryDelay: 1000,
-    metadataMaxAge: 5000, // 5 minutes
+    metadataMaxAge: 5000, // 5 seconds
     autocreateTopics: false,
     strict: false
 };

package/dist/clients/base/types.d.ts

@@ -14,6 +14,7 @@ export interface ClusterTopicMetadata {
     id: string;
     partitions: ClusterPartitionMetadata[];
     partitionsCount: number;
+    lastUpdate: number;
 }
 export interface ClusterMetadata {
     id: string;

package/dist/clients/consumer/consumer.d.ts

@@ -15,8 +15,10 @@ export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
     [kFetchConnections]: ConnectionPool;
     constructor(options: ConsumerOptions<Key, Value, HeaderKey, HeaderValue>);
     get streamsCount(): number;
+    get lastHeartbeat(): Date | null;
     close(force: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
     close(force?: boolean): Promise<void>;
+    isActive(): boolean;
     consume(options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<MessagesStream<Key, Value, HeaderKey, HeaderValue>>): void;
     consume(options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>): Promise<MessagesStream<Key, Value, HeaderKey, HeaderValue>>;
     fetch(options: FetchOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<FetchResponse>): void;

package/dist/clients/consumer/consumer.js

@@ -4,7 +4,7 @@ import { consumerCommitsChannel, consumerConsumesChannel, consumerFetchesChannel
 import { UserError } from "../../errors.js";
 import { Reader } from "../../protocol/reader.js";
 import { Writer } from "../../protocol/writer.js";
-import { Base, kAfterCreate, kCheckNotClosed, kClosed, kCreateConnectionPool, kFetchConnections, kFormatValidationErrors, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
+import { Base, kAfterCreate, kCheckNotClosed, kClearMetadata, kClosed, kCreateConnectionPool, kFetchConnections, kFormatValidationErrors, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
 import { defaultBaseOptions } from "../base/options.js";
 import { ensureMetric } from "../metrics.js";
 import { MessagesStream } from "./messages-stream.js";
@@ -23,6 +23,7 @@ export class Consumer extends Base {
     #protocol;
     #coordinatorId;
     #heartbeatInterval;
+    #lastHeartbeat;
     #streams;
     #partitionsAssigner;
     /*
@@ -57,6 +58,7 @@ export class Consumer extends Base {
         this.#protocol = null;
         this.#coordinatorId = null;
         this.#heartbeatInterval = null;
+        this.#lastHeartbeat = null;
         this.#streams = new Set();
         this.#partitionsAssigner = this[kOptions].partitionAssigner ?? roundRobinAssigner;
         this.#validateGroupOptions(this[kOptions], groupIdAndOptionsValidator);
@@ -72,6 +74,9 @@ export class Consumer extends Base {
     get streamsCount() {
         return this.#streams.size;
     }
+    get lastHeartbeat() {
+        return this.#lastHeartbeat;
+    }
     close(force, callback) {
         if (typeof force === 'function') {
             callback = force;
@@ -118,6 +123,14 @@ export class Consumer extends Base {
         });
         return callback[kCallbackPromise];
     }
+    isActive() {
+        const baseReady = super.isActive();
+        if (!baseReady) {
+            return false;
+        }
+        // We consider the group ready if we have a groupId, a memberId and heartbeat interval
+        return this.#membershipActive && Boolean(this.groupId) && Boolean(this.memberId) && this.#heartbeatInterval !== null;
+    }
     consume(options, callback) {
         if (!callback) {
             callback = createPromisifiedCallback();
@@ -265,6 +278,7 @@ export class Consumer extends Base {
                 callback(error);
                 return;
             }
+            this.#lastHeartbeat = null;
             callback(null);
         });
         return callback[kCallbackPromise];
@@ -336,10 +350,10 @@ export class Consumer extends Base {
             const requests = new Map();
             for (const name of options.topics) {
                 const topic = metadata.topics.get(name);
-                const toInclude = options.partitions?.[name] ?? [];
-                const hasPartitionsFilter = toInclude.length > 0;
+                const toInclude = new Set(options.partitions?.[name] ?? []);
+                const hasPartitionsFilter = toInclude.size > 0;
                 for (let i = 0; i < topic.partitionsCount; i++) {
-                    if (hasPartitionsFilter && !toInclude.includes(i)) {
+                    if (hasPartitionsFilter && !toInclude.delete(i)) {
                         continue;
                     }
                     const partition = topic.partitions[i];
@@ -360,6 +374,10 @@ export class Consumer extends Base {
                         timestamp: options.timestamp ?? -1n
                     });
                 }
+                if (toInclude.size > 0) {
+                    callback(new UserError(`Specified partition(s) not found in topic ${name}`), undefined);
+                    return;
+                }
             }
             runConcurrentCallbacks('Listing offsets failed.', requests, ([leader, requests], concurrentCallback) => {
                 this[kPerformWithRetry]('listOffsets', retryCallback => {
@@ -379,7 +397,7 @@ export class Consumer extends Base {
                 }, concurrentCallback, 0);
             }, (error, responses) => {
                 if (error) {
-                    callback(error, undefined);
+                    callback(this.#handleMetadataError(error), undefined);
                     return;
                 }
                 let offsets = new Map();
@@ -434,7 +452,7 @@ export class Consumer extends Base {
             });
         }, (error, response) => {
             if (error) {
-                callback(error, undefined);
+                callback(this.#handleMetadataError(error), undefined);
                 return;
             }
             const committed = new Map();
@@ -506,6 +524,7 @@ export class Consumer extends Base {
                 // Note that here we purposely do not return, since it was not a group related problem we schedule another heartbeat
             }
             else {
+                this.#lastHeartbeat = new Date();
                 this.emitWithDebug('consumer:heartbeat', 'end', eventPayload);
             }
             this.#heartbeatInterval?.refresh();
@@ -542,6 +561,7 @@ export class Consumer extends Base {
         this.#metricActiveStreams?.inc();
         stream.once('close', () => {
             this.#metricActiveStreams?.dec();
+            this.topics.untrackAll(...options.topics);
             this.#streams.delete(stream);
         });
         callback(null, stream);
@@ -722,7 +742,7 @@ export class Consumer extends Base {
                 }
                 this[kMetadata]({ topics: Array.from(topicsSubscriptions.keys()) }, (error, metadata) => {
                     if (error) {
-                        callback(error, undefined);
+                        callback(this.#handleMetadataError(error), undefined);
                         return;
                     }
                     this.#performSyncGroup(this.#createAssignments(metadata), callback);
@@ -751,15 +771,7 @@ export class Consumer extends Base {
                 callback(error, undefined);
                 return;
             }
-            // Read the assignment back
-            const reader = Reader.from(response.assignment);
-            const assignments = reader.readArray(r => {
-                return {
-                    topic: r.readString(),
-                    partitions: r.readArray(r => r.readInt32(), true, false)
-                };
-            }, true, false);
-            callback(error, assignments);
+            callback(error, this.#decodeProtocolAssignment(response.assignment));
         });
     }
     #performDeduplicateGroupOperaton(operationId, operation, callback) {
@@ -775,7 +787,7 @@ export class Consumer extends Base {
             }
             this[kMetadata]({ topics: this.topics.current }, (error, metadata) => {
                 if (error) {
-                    callback(error, undefined);
+                    callback(this.#handleMetadataError(error), undefined);
                     return;
                 }
                 this[kPerformWithRetry](operationId, retryCallback => {
@@ -818,13 +830,26 @@ export class Consumer extends Base {
         };
     }
     /*
-      This follows:
+      The following two methods follow:
       https://github.com/apache/kafka/blob/trunk/clients/src/main/resources/common/message/ConsumerProtocolAssignment.json
     */
     #encodeProtocolAssignment(assignments) {
-        return Writer.create().appendArray(assignments, (w, { topic, partitions }) => {
-            w.appendString(topic).appendArray(partitions, (w, a) => w.appendInt32(a), true, false);
-        }, true, false).buffer;
+        return Writer.create()
+            .appendInt16(0) // Version information
+            .appendArray(assignments, (w, { topic, partitions }) => {
+            w.appendString(topic, false).appendArray(partitions, (w, a) => w.appendInt32(a), false, false);
+        }, false, false)
+            .appendInt32(0).buffer; // No user data
+    }
+    #decodeProtocolAssignment(buffer) {
+        const reader = Reader.from(buffer);
+        reader.skip(2); // Ignore Version information
+        return reader.readArray(r => {
+            return {
+                topic: r.readString(false),
+                partitions: r.readArray(r => r.readInt32(), false, false)
+            };
+        }, false, false);
     }
     #createAssignments(metadata) {
         const partitionTracker = new Map();
@@ -875,4 +900,10 @@ export class Consumer extends Base {
         }
         return protocolError;
     }
+    #handleMetadataError(error) {
+        if (error && error?.findBy('hasStaleMetadata', true)) {
+            this[kClearMetadata]();
+        }
+        return error;
+    }
 }

package/dist/clients/consumer/messages-stream.d.ts

@@ -11,6 +11,8 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     constructor(consumer: Consumer<Key, Value, HeaderKey, HeaderValue>, options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>);
     close(callback?: CallbackWithPromise<void>): void;
     close(): Promise<void>;
+    isActive(): boolean;
+    isConnected(): boolean;
     addListener(event: 'autocommit', listener: (err: Error, offsets: CommitOptionsPartition[]) => void): this;
     addListener(event: 'data', listener: (message: Message<Key, Value, HeaderKey, HeaderValue>) => void): this;
     addListener(event: 'close', listener: () => void): this;

package/dist/clients/consumer/messages-stream.js

@@ -20,6 +20,8 @@ export class MessagesStream extends Readable {
     #consumer;
     #mode;
     #fallbackMode;
+    #fetches;
+    #maxFetches;
     #options;
     #topics;
     #offsetsToFetch;
@@ -37,7 +39,7 @@ export class MessagesStream extends Readable {
     #metricsConsumedMessages;
     #corruptedMessageHandler;
     constructor(consumer, options) {
-        const { autocommit, mode, fallbackMode, offsets, deserializers, onCorruptedMessage, ...otherOptions } = options;
+        const { autocommit, mode, fallbackMode, maxFetches, offsets, deserializers, onCorruptedMessage, ...otherOptions } = options;
         if (offsets && mode !== MessagesStreamModes.MANUAL) {
             throw new UserError('Cannot specify offsets when the stream mode is not MANUAL.');
         }
@@ -50,6 +52,8 @@ export class MessagesStream extends Readable {
         this.#mode = mode ?? MessagesStreamModes.LATEST;
         this.#fallbackMode = fallbackMode ?? MessagesStreamFallbackModes.LATEST;
         this.#offsetsToCommit = new Map();
+        this.#fetches = 0;
+        this.#maxFetches = maxFetches ?? 0;
         this.#topics = structuredClone(options.topics);
         this.#inflightNodes = new Set();
         this.#keyDeserializer = deserializers?.key ?? noopDeserializer;
@@ -138,6 +142,18 @@ export class MessagesStream extends Readable {
         });
         return callback[kCallbackPromise];
     }
+    isActive() {
+        if (this.#shouldClose || this.closed || this.destroyed) {
+            return false;
+        }
+        return this.#consumer.isActive();
+    }
+    isConnected() {
+        if (this.#shouldClose || this.closed || this.destroyed) {
+            return false;
+        }
+        return this.#consumer.isConnected();
+    }
     /* c8 ignore next 3 - Only forwards to Node.js implementation - Inserted here to please Typescript */
     addListener(event, listener) {
         return super.addListener(event, listener);
@@ -183,7 +199,7 @@ export class MessagesStream extends Readable {
             if (error) {
                 // The stream has been closed, ignore any error
                 /* c8 ignore next 4 - Hard to test */
-                if (this.#shouldClose) {
+                if (this.#shouldClose || this.closed || this.destroyed) {
                     this.push(null);
                     return;
                 }
@@ -241,7 +257,7 @@ export class MessagesStream extends Readable {
                     if (error) {
                         // The stream has been closed, ignore the error
                         /* c8 ignore next 4 - Hard to test */
-                        if (this.#shouldClose) {
+                        if (this.#shouldClose || this.closed || this.destroyed) {
                             this.push(null);
                             return;
                         }
@@ -257,6 +273,9 @@ export class MessagesStream extends Readable {
                         return;
                     }
                     this.#pushRecords(metadata, topicIds, response, requestedOffsets);
+                    if (this.#maxFetches > 0 && ++this.#fetches >= this.#maxFetches) {
+                        this.push(null);
+                    }
                 });
             }
         });
@@ -387,6 +406,11 @@ export class MessagesStream extends Readable {
                 : ListOffsetTimestamps.LATEST
         }, (error, offsets) => {
             if (error) {
+                /* c8 ignore next 4 - Hard to test */
+                if (this.#shouldClose || this.closed || this.destroyed) {
+                    callback(null);
+                    return;
+                }
                 callback(error);
                 return;
             }
@@ -409,6 +433,11 @@ export class MessagesStream extends Readable {
             }
             this.#consumer.listCommittedOffsets({ topics }, (error, commits) => {
                 if (error) {
+                    /* c8 ignore next 4 - Hard to test */
+                    if (this.#shouldClose || this.closed || this.destroyed) {
+                        callback(null);
+                        return;
+                    }
                     callback(error);
                     return;
                 }

package/dist/clients/consumer/options.d.ts

@@ -281,6 +281,11 @@ export declare const consumeOptionsSchema: {
             type: string;
             enum: ("latest" | "earliest" | "fail")[];
         };
+        maxFetches: {
+            type: string;
+            minimum: number;
+            default: number;
+        };
         offsets: {
             type: string;
             items: {

package/dist/clients/consumer/options.js

@@ -71,6 +71,7 @@ export const consumeOptionsSchema = {
         topics: { type: 'array', items: idProperty },
         mode: { type: 'string', enum: Object.values(MessagesStreamModes) },
         fallbackMode: { type: 'string', enum: Object.values(MessagesStreamFallbackModes) },
+        maxFetches: { type: 'number', minimum: 0, default: 0 },
         offsets: {
             type: 'array',
             items: {

package/dist/clients/consumer/types.d.ts

@@ -62,6 +62,7 @@ export interface StreamOptions {
     topics: string[];
     mode?: MessagesStreamModeValue;
     fallbackMode?: MessagesStreamFallbackModeValue;
+    maxFetches?: number;
     offsets?: TopicWithPartitionAndOffset[];
     onCorruptedMessage?: CorruptedMessageHandler;
 }

package/dist/clients/producer/producer.js

@@ -198,7 +198,7 @@ export class Producer extends Base {
                 }
                 else {
                     // Use the roundrobin
-                    partition = this.#partitionsRoundRobin.postIncrement(topic, 1, -1);
+                    partition = this.#partitionsRoundRobin.postIncrement(topic, 1, 0);
                 }
             }
             else {

package/dist/errors.d.ts

@@ -35,6 +35,7 @@ export declare class NetworkError extends GenericError {
     constructor(message: string, properties?: ErrorProperties);
 }
 export declare class ProtocolError extends GenericError {
+    static code: ErrorCode;
     constructor(codeOrId: string | number, properties?: ErrorProperties, response?: unknown);
 }
 export declare class ResponseError extends MultipleErrors {

package/dist/errors.js

@@ -76,9 +76,13 @@ export class MultipleErrors extends AggregateError {
             return this;
         }
         for (const error of this.errors) {
-            if (error[kGenericError] ? error.findBy(property, value) : error[property] === value) {
+            if (error[property] === value) {
                 return error;
             }
+            const found = error[kGenericError] ? error.findBy(property, value) : undefined;
+            if (found) {
+                return found;
+            }
         }
         return null;
     }
@@ -97,9 +101,10 @@ export class NetworkError extends GenericError {
     }
 }
 export class ProtocolError extends GenericError {
+    static code = 'PLT_KFK_PROTOCOL';
     constructor(codeOrId, properties = {}, response = undefined) {
         const { id, code, message, canRetry } = protocolErrors[typeof codeOrId === 'number' ? protocolErrorsCodesById[codeOrId] : codeOrId];
-        super('PLT_KFK_PROTOCOL', message, {
+        super(ProtocolError.code, message, {
             apiId: id,
             apiCode: code,
             canRetry,

package/dist/network/connection-pool.d.ts

@@ -11,4 +11,6 @@ export declare class ConnectionPool extends EventEmitter {
     getFirstAvailable(brokers: Broker[]): Promise<Connection>;
     close(callback: CallbackWithPromise<void>): void;
     close(): Promise<void>;
+    isActive(): boolean;
+    isConnected(): boolean;
 }

package/dist/network/connection-pool.js

@@ -7,12 +7,14 @@ let currentInstance = 0;
 export class ConnectionPool extends EventEmitter {
     #instanceId;
     #clientId;
+    #closed;
     // @ts-ignore This is used just for debugging
     #ownerId;
     #connections;
     #connectionOptions;
     constructor(clientId, connectionOptions = {}) {
         super();
+        this.#closed = false;
         this.#instanceId = currentInstance++;
         this.#clientId = clientId;
         this.#ownerId = connectionOptions.ownerId;
@@ -41,17 +43,40 @@ export class ConnectionPool extends EventEmitter {
         if (!callback) {
             callback = createPromisifiedCallback();
         }
-        if (this.#connections.size === 0) {
+        if (this.#closed || this.#connections.size === 0) {
+            this.#closed = true;
             callback(null);
             return callback[kCallbackPromise];
         }
+        this.#closed = true;
         runConcurrentCallbacks('Closing connections failed.', this.#connections, ([key, connection], cb) => {
             connection.close(cb);
             this.#connections.delete(key);
         }, error => callback(error));
         return callback[kCallbackPromise];
     }
+    isActive() {
+        if (this.#connections.size === 0) {
+            return false;
+        }
+        return true;
+    }
+    isConnected() {
+        if (this.#connections.size === 0) {
+            return false;
+        }
+        for (const connection of this.#connections.values()) {
+            if (!connection.isConnected()) {
+                return false;
+            }
+        }
+        return true;
+    }
     #get(broker, callback) {
+        if (this.#closed) {
+            callback(new Error('Connection pool is closed.'), undefined);
+            return;
+        }
         const key = `${broker.host}:${broker.port}`;
         const existing = this.#connections.get(key);
         if (existing) {

package/dist/network/connection.d.ts

@@ -11,8 +11,9 @@ export interface Broker {
 }
 export interface SASLOptions {
     mechanism: SASLMechanism;
-    username: string;
-    password: string;
+    username?: string;
+    password?: string;
+    token?: string;
 }
 export interface ConnectionOptions {
     connectTimeout?: number;
@@ -53,6 +54,7 @@ export declare class Connection extends EventEmitter {
     get instanceId(): number;
     get status(): ConnectionStatusValue;
     get socket(): Socket;
+    isConnected(): boolean;
     connect(host: string, port: number, callback?: CallbackWithPromise<void>): void | Promise<void>;
     ready(callback: CallbackWithPromise<void>): void;
     ready(): Promise<void>;

package/dist/network/connection.js

@@ -10,7 +10,7 @@ import { AuthenticationError, NetworkError, TimeoutError, UnexpectedCorrelationI
 import { protocolAPIsById } from "../protocol/apis.js";
 import { EMPTY_OR_SINGLE_COMPACT_LENGTH_SIZE, INT32_SIZE } from "../protocol/definitions.js";
 import { DynamicBuffer } from "../protocol/dynamic-buffer.js";
-import { saslPlain, saslScramSha } from "../protocol/index.js";
+import { saslOAuthBearer, saslPlain, saslScramSha } from "../protocol/index.js";
 import { Reader } from "../protocol/reader.js";
 import { defaultCrypto } from "../protocol/sasl/scram-sha.js";
 import { Writer } from "../protocol/writer.js";
@@ -80,6 +80,9 @@ export class Connection extends EventEmitter {
     get socket() {
         return this.#socket;
     }
+    isConnected() {
+        return this.#status === ConnectionStatuses.CONNECTED;
+    }
     connect(host, port, callback) {
         if (!callback) {
             callback = createPromisifiedCallback();
@@ -229,7 +232,7 @@ export class Connection extends EventEmitter {
     }
     #authenticate(host, port, diagnosticContext) {
         this.#status = ConnectionStatuses.AUTHENTICATING;
-        const { mechanism, username, password } = this.#options.sasl;
+        const { mechanism, username, password, token } = this.#options.sasl;
         if (!SASLMechanisms.includes(mechanism)) {
             this.#onConnectionError(host, port, diagnosticContext, new UserError(`SASL mechanism ${mechanism} not supported.`));
             return;
@@ -243,6 +246,9 @@ export class Connection extends EventEmitter {
             if (mechanism === 'PLAIN') {
                 saslPlain.authenticate(saslAuthenticateV2.api, this, username, password, this.#onSaslAuthenticate.bind(this, host, port, diagnosticContext));
             }
+            else if (mechanism === 'OAUTHBEARER') {
+                saslOAuthBearer.authenticate(saslAuthenticateV2.api, this, token, this.#onSaslAuthenticate.bind(this, host, port, diagnosticContext));
+            }
             else {
                 saslScramSha.authenticate(saslAuthenticateV2.api, this, mechanism.substring(6), username, password, defaultCrypto, this.#onSaslAuthenticate.bind(this, host, port, diagnosticContext));
             }

package/dist/protocol/crc32c.d.ts

@@ -1,2 +1,2 @@
 import { DynamicBuffer } from './dynamic-buffer.ts';
-export declare function crc32c(data: Buffer | DynamicBuffer): number;
+export declare function crc32c(data: Buffer | Uint8Array | DynamicBuffer): number;

package/dist/protocol/crc32c.js

@@ -68,16 +68,12 @@ const CRC = [
     0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351
 ];
 export function crc32c(data) {
-    let crc = 0 ^ -1;
-    if (DynamicBuffer.isDynamicBuffer(data)) {
-        for (let i = 0; i < data.length; i++) {
-            crc = CRC[(crc ^ data.get(i)) & 0xff] ^ (crc >>> 8);
-        }
+    const bytes = DynamicBuffer.isDynamicBuffer(data)
+        ? data.buffer
+        : new Uint8Array(data);
+    let crc = 0xffffffff;
+    for (let i = 0, len = bytes.length; i < len; ++i) {
+        crc = CRC[(crc ^ bytes[i]) & 0xff] ^ (crc >>> 8);
     }
-    else {
-        for (let i = 0; i < data.length; i++) {
-            crc = CRC[(crc ^ data[i]) & 0xff] ^ (crc >>> 8);
-        }
-    }
-    return Uint32Array.from([(crc ^ -1) >>> 0])[0];
+    return (crc ^ 0xffffffff) >>> 0;
 }

package/dist/protocol/index.d.ts

@@ -8,6 +8,7 @@ export * from './index.ts';
 export * from './murmur2.ts';
 export * from './reader.ts';
 export * from './records.ts';
+export * as saslOAuthBearer from './sasl/oauth-bearer.ts';
 export * as saslPlain from './sasl/plain.ts';
 export * as saslScramSha from './sasl/scram-sha.ts';
 export * from './varint.ts';

package/dist/protocol/index.js

@@ -8,6 +8,7 @@ export * from "./index.js";
 export * from "./murmur2.js";
 export * from "./reader.js";
 export * from "./records.js";
+export * as saslOAuthBearer from "./sasl/oauth-bearer.js";
 export * as saslPlain from "./sasl/plain.js";
 export * as saslScramSha from "./sasl/scram-sha.js";
 export * from "./varint.js";

package/dist/protocol/sasl/oauth-bearer.d.ts

@@ -0,0 +1,5 @@
+import { type CallbackWithPromise } from '../../apis/callbacks.ts';
+import { type SASLAuthenticationAPI, type SaslAuthenticateResponse } from '../../apis/security/sasl-authenticate-v2.ts';
+import { type Connection } from '../../network/connection.ts';
+export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, token: string, callback: CallbackWithPromise<SaslAuthenticateResponse>): void;
+export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, token: string): Promise<SaslAuthenticateResponse>;

package/dist/protocol/sasl/oauth-bearer.js

@@ -0,0 +1,8 @@
+import { createPromisifiedCallback, kCallbackPromise } from "../../apis/callbacks.js";
+export function authenticate(authenticateAPI, connection, token, callback) {
+    if (!callback) {
+        callback = createPromisifiedCallback();
+    }
+    authenticateAPI(connection, Buffer.from(`n,,\x01auth=Bearer ${token}\x01\x01`), callback);
+    return callback[kCallbackPromise];
+}

package/dist/protocol/writer.js

@@ -152,7 +152,7 @@ export class Writer {
     }
     appendArray(value, entryWriter, compact = true, appendTrailingTaggedFields = true) {
         if (value == null) {
-            return compact ? this.appendUnsignedVarInt(0) : this.appendInt32(0);
+            return compact ? this.appendUnsignedVarInt(0) : this.appendInt32(-1);
         }
         const length = value.length;
         if (compact) {
@@ -171,7 +171,7 @@ export class Writer {
     }
     appendMap(value, entryWriter, compact = true, appendTrailingTaggedFields = true) {
         if (value == null) {
-            return compact ? this.appendUnsignedVarInt(0) : this.appendInt32(0);
+            return compact ? this.appendUnsignedVarInt(0) : this.appendInt32(-1);
         }
         const length = value.size;
         if (compact) {

package/dist/version.d.ts

@@ -0,0 +1,2 @@
+export declare const name: any;
+export declare const version: any;

package/dist/version.js

@@ -0,0 +1,2 @@
+export const name = "@platformatic/kafka";
+export const version = "1.9.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "1.7.0",
+  "version": "1.9.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
@@ -46,6 +46,7 @@
     "cleaner-spec-reporter": "^0.5.0",
     "cronometro": "^5.3.0",
     "eslint": "^9.21.0",
+    "fast-jwt": "^6.0.2",
     "hwp": "^0.4.1",
     "json5": "^2.2.3",
     "kafkajs": "^2.2.4",
@@ -64,11 +65,14 @@
   },
   "scripts": {
     "build": "rm -rf dist && tsc -p tsconfig.base.json",
+    "postbuild": "node --experimental-strip-types scripts/postbuild.ts",
     "lint": "eslint --cache",
     "typecheck": "tsc -p . --noEmit",
     "format": "prettier -w benchmarks playground src test",
     "test": "c8 -c test/config/c8-local.json node --env-file=test/config/env --no-warnings --test --test-timeout=300000 'test/**/*.test.ts'",
     "test:ci": "c8 -c test/config/c8-ci.json node --env-file=test/config/env --no-warnings --test --test-timeout=300000 'test/**/*.test.ts'",
+    "test:docker:up": "node scripts/docker.ts up -d --wait",
+    "test:docker:down": "node scripts/docker.ts down",
     "ci": "npm run build && npm run lint && npm run test:ci",
     "generate:apis": "node --experimental-strip-types scripts/generate-apis.ts",
     "generate:errors": "node --experimental-strip-types scripts/generate-errors.ts",