@platformatic/kafka 1.8.0 → 1.10.0

package/dist/apis/callbacks.js

@@ -37,6 +37,9 @@ export function runConcurrentCallbacks(errorMessage, collection, operation, call
             callback(hasErrors ? new MultipleErrors(errorMessage, errors) : null, results);
         }
     }
+    if (remaining === 0) {
+        callback(null, results);
+    }
     for (const item of collection) {
         operation(item, operationCallback.bind(null, i++));
     }

package/dist/apis/consumer/fetch-v15.d.ts

@@ -29,7 +29,7 @@ export interface FetchResponsePartition {
     logStartOffset: bigint;
     abortedTransactions: FetchResponsePartitionAbortedTransaction[];
     preferredReadReplica: number;
-    records?: RecordsBatch;
+    records?: RecordsBatch[];
 }
 export interface FetchResponseTopic {
     topicId: string;

package/dist/apis/consumer/fetch-v15.js

@@ -105,7 +105,11 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
                     }
                     if (recordsSize > 1) {
                         recordsSize--;
-                        partition.records = readRecordsBatch(Reader.from(r.buffer.subarray(r.position, r.position + recordsSize)));
+                        const recordsBatchesReader = Reader.from(r.buffer.subarray(r.position, r.position + recordsSize));
+                        partition.records = [];
+                        do {
+                            partition.records.push(readRecordsBatch(recordsBatchesReader));
+                        } while (recordsBatchesReader.position < recordsSize);
                         r.skip(recordsSize);
                     }
                     return partition;

package/dist/apis/consumer/fetch-v16.d.ts

@@ -29,7 +29,7 @@ export interface FetchResponsePartition {
     logStartOffset: bigint;
     abortedTransactions: FetchResponsePartitionAbortedTransaction[];
     preferredReadReplica: number;
-    records?: RecordsBatch;
+    records?: RecordsBatch[];
 }
 export interface FetchResponseTopic {
     topicId: string;

package/dist/apis/consumer/fetch-v16.js

@@ -105,7 +105,11 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
                     }
                     if (recordsSize > 1) {
                         recordsSize--;
-                        partition.records = readRecordsBatch(Reader.from(r.buffer.subarray(r.position, r.position + recordsSize)));
+                        const recordsBatchesReader = Reader.from(r.buffer.subarray(r.position, r.position + recordsSize));
+                        partition.records = [];
+                        do {
+                            partition.records.push(readRecordsBatch(recordsBatchesReader));
+                        } while (recordsBatchesReader.position < recordsSize);
                         r.skip(recordsSize);
                     }
                     return partition;

package/dist/apis/consumer/fetch-v17.d.ts

@@ -29,7 +29,7 @@ export interface FetchResponsePartition {
     logStartOffset: bigint;
     abortedTransactions: FetchResponsePartitionAbortedTransaction[];
     preferredReadReplica: number;
-    records?: RecordsBatch;
+    records?: RecordsBatch[];
 }
 export interface FetchResponseTopic {
     topicId: string;

package/dist/apis/consumer/fetch-v17.js

@@ -105,7 +105,11 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
                     }
                     if (recordsSize > 1) {
                         recordsSize--;
-                        partition.records = readRecordsBatch(Reader.from(r.buffer.subarray(r.position, r.position + recordsSize)));
+                        const recordsBatchesReader = Reader.from(r.buffer.subarray(r.position, r.position + recordsSize));
+                        partition.records = [];
+                        do {
+                            partition.records.push(readRecordsBatch(recordsBatchesReader));
+                        } while (recordsBatchesReader.position < recordsSize);
                         r.skip(recordsSize);
                     }
                     return partition;

package/dist/apis/enumerations.d.ts

@@ -1,4 +1,4 @@
-export declare const SASLMechanisms: readonly ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512"];
+export declare const SASLMechanisms: readonly ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512", "OAUTHBEARER"];
 export type SASLMechanism = (typeof SASLMechanisms)[number];
 export declare const FindCoordinatorKeyTypes: {
     readonly GROUP: 0;

package/dist/apis/enumerations.js

@@ -1,5 +1,5 @@
 // SASL Authentication
-export const SASLMechanisms = ['PLAIN', 'SCRAM-SHA-256', 'SCRAM-SHA-512'];
+export const SASLMechanisms = ['PLAIN', 'SCRAM-SHA-256', 'SCRAM-SHA-512', 'OAUTHBEARER'];
 // Metadata API
 // ./metadata/find-coordinator.ts
 export const FindCoordinatorKeyTypes = { GROUP: 0, TRANSACTION: 1, SHARE: 2 };

package/dist/clients/admin/admin.js

@@ -144,6 +144,7 @@ export class Admin extends Base {
         const numPartitions = options.partitions ?? 1;
         const replicationFactor = options.replicas ?? 1;
         const assignments = [];
+        const configs = options.configs ?? [];
         for (const { partition, brokers } of options.assignments ?? []) {
             assignments.push({ partitionIndex: partition, brokerIds: brokers });
         }
@@ -154,7 +155,7 @@ export class Admin extends Base {
                 numPartitions,
                 replicationFactor,
                 assignments,
-                configs: []
+                configs
             });
         }
         this[kPerformDeduplicated]('createTopics', deduplicateCallback => {
@@ -322,10 +323,12 @@ export class Admin extends Base {
                                     metadata: reader.readBytes(false)
                                 };
                                 reader.reset(member.memberAssignment);
+                                reader.skip(2); // Ignore Version information
                                 const memberAssignments = reader.readMap(r => {
-                                    const topic = r.readString();
-                                    return [topic, { topic, partitions: reader.readArray(r => r.readInt32(), true, false) }];
-                                }, true, false);
+                                    const topic = r.readString(false);
+                                    return [topic, { topic, partitions: reader.readArray(r => r.readInt32(), false, false) }];
+                                }, false, false);
+                                reader.readBytes(); // Ignore the user data
                                 group.members.set(member.memberId, {
                                     id: member.memberId,
                                     groupInstanceId: member.groupInstanceId,

package/dist/clients/admin/types.d.ts

@@ -1,3 +1,4 @@
+import { type CreateTopicsRequestTopicConfig } from '../../apis/admin/create-topics-v7.ts';
 import { type ConsumerGroupState } from '../../apis/enumerations.ts';
 import { type NullableString } from '../../protocol/definitions.ts';
 import { type BaseOptions } from '../base/types.ts';
@@ -39,6 +40,7 @@ export interface CreateTopicsOptions {
     partitions?: number;
     replicas?: number;
     assignments?: BrokerAssignment[];
+    configs?: CreateTopicsRequestTopicConfig[];
 }
 export interface ListTopicsOptions {
     includeInternals?: boolean;

package/dist/clients/base/base.d.ts

@@ -56,6 +56,10 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions> extends
     listApis(): Promise<ApiVersionsResponseApi[]>;
     metadata(options: MetadataOptions, callback: CallbackWithPromise<ClusterMetadata>): void;
     metadata(options: MetadataOptions): Promise<ClusterMetadata>;
+    connectToBrokers(nodeIds: number[] | null, callback: CallbackWithPromise<Map<number, Connection>>): void;
+    connectToBrokers(nodeIds?: number[] | null): Promise<Map<number, Connection>>;
+    isActive(): boolean;
+    isConnected(): boolean;
     [kCreateConnectionPool](): ConnectionPool;
     [kListApis](callback: CallbackWithPromise<ApiVersionsResponseApi[]>): void;
     [kMetadata](options: MetadataOptions, callback: CallbackWithPromise<ClusterMetadata>): void;

package/dist/clients/base/base.js

@@ -1,5 +1,5 @@
 import { EventEmitter } from 'node:events';
-import { createPromisifiedCallback, kCallbackPromise } from "../../apis/callbacks.js";
+import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
 import * as apis from "../../apis/index.js";
 import { api as apiVersionsV3 } from "../../apis/metadata/api-versions-v3.js";
 import { baseApisChannel, baseMetadataChannel, createDiagnosticContext, notifyCreation } from "../../diagnostic.js";
@@ -49,6 +49,7 @@ export class Base extends EventEmitter {
     #inflightDeduplications;
     constructor(options) {
         super();
+        this.setMaxListeners(0);
         this[kClientType] = 'base';
         this[kInstance] = currentInstance++;
         this[kApis] = [];
@@ -97,6 +98,7 @@ export class Base extends EventEmitter {
             callback = createPromisifiedCallback();
         }
         this[kClosed] = true;
+        this.emitWithDebug('client', 'close');
         this[kConnections].close(callback);
         return callback[kCallbackPromise];
     }
@@ -119,6 +121,57 @@ export class Base extends EventEmitter {
         baseMetadataChannel.traceCallback(this[kMetadata], 1, createDiagnosticContext({ client: this, operation: 'metadata' }), this, options, callback);
         return callback[kCallbackPromise];
     }
+    connectToBrokers(nodeIds, callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        // Fetch the metadata
+        this[kMetadata]({ topics: [] }, (error, metadata) => {
+            if (error) {
+                callback(error, undefined);
+                return;
+            }
+            let nodes = [];
+            if (nodeIds?.length) {
+                for (const node of nodeIds) {
+                    if (metadata.brokers.has(node)) {
+                        nodes.push(node);
+                    }
+                }
+            }
+            else {
+                nodes = Array.from(metadata.brokers.keys());
+            }
+            runConcurrentCallbacks('Connecting to brokers failed.', nodes, (nodeId, concurrentCallback) => {
+                this[kGetConnection](metadata.brokers.get(nodeId), (error, connection) => {
+                    if (error) {
+                        concurrentCallback(error, undefined);
+                        return;
+                    }
+                    concurrentCallback(null, [nodeId, connection]);
+                });
+            }, (error, connections) => {
+                if (error) {
+                    callback(error, undefined);
+                    return;
+                }
+                return callback(null, new Map(connections));
+            });
+        });
+        return callback[kCallbackPromise];
+    }
+    isActive() {
+        if (this[kClosed]) {
+            return false;
+        }
+        return true;
+    }
+    isConnected() {
+        if (this[kClosed]) {
+            return false;
+        }
+        return this[kConnections].isConnected();
+    }
     [kCreateConnectionPool]() {
         const pool = new ConnectionPool(this[kClientId], {
             ownerId: this[kInstance],
@@ -183,11 +236,17 @@ export class Base extends EventEmitter {
                             retryCallback(error, undefined);
                             return;
                         }
-                        api(connection, options.topics, autocreateTopics, true, retryCallback);
+                        api(connection, topicsToFetch, autocreateTopics, true, retryCallback);
                     });
                 });
             }, (error, metadata) => {
                 if (error) {
+                    const hasStaleMetadata = error.findBy('hasStaleMetadata', true);
+                    // Stale metadata, we need to fetch everything again
+                    if (hasStaleMetadata) {
+                        this[kClearMetadata]();
+                        topicsToFetch = options.topics;
+                    }
                     deduplicateCallback(error, undefined);
                     return;
                 }
@@ -268,9 +327,17 @@ export class Base extends EventEmitter {
                 const retriable = genericError.findBy?.('code', NetworkError.code) || genericError.findBy?.('canRetry', true);
                 errors.push(error);
                 if (attempt < retries && retriable && !shouldSkipRetry?.(error)) {
-                    setTimeout(() => {
+                    this.emitWithDebug('client', 'performWithRetry:retry', operationId, attempt, retries);
+                    function onClose() {
+                        clearTimeout(timeout);
+                        errors.push(new UserError(`Client closed while retrying ${operationId}.`));
+                        callback(new MultipleErrors(`${operationId} failed ${attempt + 1} times.`, errors), undefined);
+                    }
+                    const timeout = setTimeout(() => {
+                        this.removeListener('client:close', onClose);
                         this[kPerformWithRetry](operationId, operation, callback, attempt + 1, errors, shouldSkipRetry);
                     }, this[kOptions].retryDelay);
+                    this.once('client:close', onClose);
                 }
                 else {
                     if (attempt === 0) {

package/dist/clients/base/options.d.ts

@@ -88,7 +88,7 @@ export declare const baseOptionsSchema: {
             properties: {
                 mechanism: {
                     type: string;
-                    enum: readonly ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512"];
+                    enum: readonly ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512", "OAUTHBEARER"];
                 };
                 username: {
                     type: string;
@@ -96,6 +96,9 @@ export declare const baseOptionsSchema: {
                 password: {
                     type: string;
                 };
+                token: {
+                    type: string;
+                };
             };
             required: string[];
             additionalProperties: boolean;

package/dist/clients/base/options.js

@@ -1,10 +1,9 @@
-import { readFileSync } from 'node:fs';
 import { SASLMechanisms } from "../../apis/enumerations.js";
 import { ajv } from "../../utils.js";
-const packageJson = JSON.parse(readFileSync(new URL('../../../package.json', import.meta.url), 'utf-8'));
+import { version } from "../../version.js";
 // Note: clientSoftwareName can only contain alphanumeric characters, hyphens and dots
 export const clientSoftwareName = 'platformatic-kafka';
-export const clientSoftwareVersion = packageJson.version;
+export const clientSoftwareVersion = version;
 export const idProperty = { type: 'string', pattern: '^\\S+$' };
 export const topicWithPartitionAndOffsetProperties = {
     topic: idProperty,
@@ -39,9 +38,10 @@ export const baseOptionsSchema = {
             properties: {
                 mechanism: { type: 'string', enum: SASLMechanisms },
                 username: { type: 'string' },
-                password: { type: 'string' }
+                password: { type: 'string' },
+                token: { type: 'string' }
             },
-            required: ['mechanism', 'username', 'password'],
+            required: ['mechanism'],
             additionalProperties: false
         },
         metadataMaxAge: { type: 'number', minimum: 0 },
@@ -72,7 +72,7 @@ export const defaultBaseOptions = {
     timeout: 5000,
     retries: 3,
     retryDelay: 1000,
-    metadataMaxAge: 5000, // 5 minutes
+    metadataMaxAge: 5000, // 5 seconds
     autocreateTopics: false,
     strict: false
 };

package/dist/clients/consumer/consumer.d.ts

@@ -15,8 +15,10 @@ export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
     [kFetchConnections]: ConnectionPool;
     constructor(options: ConsumerOptions<Key, Value, HeaderKey, HeaderValue>);
     get streamsCount(): number;
+    get lastHeartbeat(): Date | null;
     close(force: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
     close(force?: boolean): Promise<void>;
+    isActive(): boolean;
     consume(options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<MessagesStream<Key, Value, HeaderKey, HeaderValue>>): void;
     consume(options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>): Promise<MessagesStream<Key, Value, HeaderKey, HeaderValue>>;
     fetch(options: FetchOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<FetchResponse>): void;

package/dist/clients/consumer/consumer.js

@@ -4,7 +4,7 @@ import { consumerCommitsChannel, consumerConsumesChannel, consumerFetchesChannel
 import { UserError } from "../../errors.js";
 import { Reader } from "../../protocol/reader.js";
 import { Writer } from "../../protocol/writer.js";
-import { Base, kAfterCreate, kCheckNotClosed, kClosed, kCreateConnectionPool, kFetchConnections, kFormatValidationErrors, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
+import { Base, kAfterCreate, kCheckNotClosed, kClearMetadata, kClosed, kCreateConnectionPool, kFetchConnections, kFormatValidationErrors, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
 import { defaultBaseOptions } from "../base/options.js";
 import { ensureMetric } from "../metrics.js";
 import { MessagesStream } from "./messages-stream.js";
@@ -23,6 +23,7 @@ export class Consumer extends Base {
     #protocol;
     #coordinatorId;
     #heartbeatInterval;
+    #lastHeartbeat;
     #streams;
     #partitionsAssigner;
     /*
@@ -57,6 +58,7 @@ export class Consumer extends Base {
         this.#protocol = null;
         this.#coordinatorId = null;
         this.#heartbeatInterval = null;
+        this.#lastHeartbeat = null;
         this.#streams = new Set();
         this.#partitionsAssigner = this[kOptions].partitionAssigner ?? roundRobinAssigner;
         this.#validateGroupOptions(this[kOptions], groupIdAndOptionsValidator);
@@ -72,6 +74,9 @@ export class Consumer extends Base {
     get streamsCount() {
         return this.#streams.size;
     }
+    get lastHeartbeat() {
+        return this.#lastHeartbeat;
+    }
     close(force, callback) {
         if (typeof force === 'function') {
             callback = force;
@@ -118,6 +123,14 @@ export class Consumer extends Base {
         });
         return callback[kCallbackPromise];
     }
+    isActive() {
+        const baseReady = super.isActive();
+        if (!baseReady) {
+            return false;
+        }
+        // We consider the group ready if we have a groupId, a memberId and heartbeat interval
+        return this.#membershipActive && Boolean(this.groupId) && Boolean(this.memberId) && this.#heartbeatInterval !== null;
+    }
     consume(options, callback) {
         if (!callback) {
             callback = createPromisifiedCallback();
@@ -265,6 +278,7 @@ export class Consumer extends Base {
                 callback(error);
                 return;
             }
+            this.#lastHeartbeat = null;
             callback(null);
         });
         return callback[kCallbackPromise];
@@ -336,10 +350,10 @@ export class Consumer extends Base {
             const requests = new Map();
             for (const name of options.topics) {
                 const topic = metadata.topics.get(name);
-                const toInclude = options.partitions?.[name] ?? [];
-                const hasPartitionsFilter = toInclude.length > 0;
+                const toInclude = new Set(options.partitions?.[name] ?? []);
+                const hasPartitionsFilter = toInclude.size > 0;
                 for (let i = 0; i < topic.partitionsCount; i++) {
-                    if (hasPartitionsFilter && !toInclude.includes(i)) {
+                    if (hasPartitionsFilter && !toInclude.delete(i)) {
                         continue;
                     }
                     const partition = topic.partitions[i];
@@ -360,6 +374,10 @@ export class Consumer extends Base {
                         timestamp: options.timestamp ?? -1n
                     });
                 }
+                if (toInclude.size > 0) {
+                    callback(new UserError(`Specified partition(s) not found in topic ${name}`), undefined);
+                    return;
+                }
             }
             runConcurrentCallbacks('Listing offsets failed.', requests, ([leader, requests], concurrentCallback) => {
                 this[kPerformWithRetry]('listOffsets', retryCallback => {
@@ -379,7 +397,7 @@ export class Consumer extends Base {
                 }, concurrentCallback, 0);
             }, (error, responses) => {
                 if (error) {
-                    callback(error, undefined);
+                    callback(this.#handleMetadataError(error), undefined);
                     return;
                 }
                 let offsets = new Map();
@@ -434,7 +452,7 @@ export class Consumer extends Base {
             });
         }, (error, response) => {
             if (error) {
-                callback(error, undefined);
+                callback(this.#handleMetadataError(error), undefined);
                 return;
             }
             const committed = new Map();
@@ -506,6 +524,7 @@ export class Consumer extends Base {
                 // Note that here we purposely do not return, since it was not a group related problem we schedule another heartbeat
             }
             else {
+                this.#lastHeartbeat = new Date();
                 this.emitWithDebug('consumer:heartbeat', 'end', eventPayload);
             }
             this.#heartbeatInterval?.refresh();
@@ -542,6 +561,7 @@ export class Consumer extends Base {
         this.#metricActiveStreams?.inc();
         stream.once('close', () => {
             this.#metricActiveStreams?.dec();
+            this.topics.untrackAll(...options.topics);
             this.#streams.delete(stream);
         });
         callback(null, stream);
@@ -722,7 +742,7 @@ export class Consumer extends Base {
                 }
                 this[kMetadata]({ topics: Array.from(topicsSubscriptions.keys()) }, (error, metadata) => {
                     if (error) {
-                        callback(error, undefined);
+                        callback(this.#handleMetadataError(error), undefined);
                         return;
                     }
                     this.#performSyncGroup(this.#createAssignments(metadata), callback);
@@ -751,15 +771,7 @@ export class Consumer extends Base {
                 callback(error, undefined);
                 return;
             }
-            // Read the assignment back
-            const reader = Reader.from(response.assignment);
-            const assignments = reader.readArray(r => {
-                return {
-                    topic: r.readString(),
-                    partitions: r.readArray(r => r.readInt32(), true, false)
-                };
-            }, true, false);
-            callback(error, assignments);
+            callback(error, this.#decodeProtocolAssignment(response.assignment));
         });
     }
     #performDeduplicateGroupOperaton(operationId, operation, callback) {
@@ -775,7 +787,7 @@ export class Consumer extends Base {
             }
             this[kMetadata]({ topics: this.topics.current }, (error, metadata) => {
                 if (error) {
-                    callback(error, undefined);
+                    callback(this.#handleMetadataError(error), undefined);
                     return;
                 }
                 this[kPerformWithRetry](operationId, retryCallback => {
@@ -818,13 +830,26 @@ export class Consumer extends Base {
         };
     }
     /*
-      This follows:
+      The following two methods follow:
       https://github.com/apache/kafka/blob/trunk/clients/src/main/resources/common/message/ConsumerProtocolAssignment.json
     */
     #encodeProtocolAssignment(assignments) {
-        return Writer.create().appendArray(assignments, (w, { topic, partitions }) => {
-            w.appendString(topic).appendArray(partitions, (w, a) => w.appendInt32(a), true, false);
-        }, true, false).buffer;
+        return Writer.create()
+            .appendInt16(0) // Version information
+            .appendArray(assignments, (w, { topic, partitions }) => {
+            w.appendString(topic, false).appendArray(partitions, (w, a) => w.appendInt32(a), false, false);
+        }, false, false)
+            .appendInt32(0).buffer; // No user data
+    }
+    #decodeProtocolAssignment(buffer) {
+        const reader = Reader.from(buffer);
+        reader.skip(2); // Ignore Version information
+        return reader.readArray(r => {
+            return {
+                topic: r.readString(false),
+                partitions: r.readArray(r => r.readInt32(), false, false)
+            };
+        }, false, false);
     }
     #createAssignments(metadata) {
         const partitionTracker = new Map();
@@ -875,4 +900,10 @@ export class Consumer extends Base {
         }
         return protocolError;
     }
+    #handleMetadataError(error) {
+        if (error && error?.findBy('hasStaleMetadata', true)) {
+            this[kClearMetadata]();
+        }
+        return error;
+    }
 }

package/dist/clients/consumer/messages-stream.d.ts

@@ -9,8 +9,10 @@ export declare function defaultCorruptedMessageHandler(): boolean;
 export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends Readable {
     #private;
     constructor(consumer: Consumer<Key, Value, HeaderKey, HeaderValue>, options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>);
-    close(callback?: CallbackWithPromise<void>): void;
+    close(callback: CallbackWithPromise<void>): void;
     close(): Promise<void>;
+    isActive(): boolean;
+    isConnected(): boolean;
     addListener(event: 'autocommit', listener: (err: Error, offsets: CommitOptionsPartition[]) => void): this;
     addListener(event: 'data', listener: (message: Message<Key, Value, HeaderKey, HeaderValue>) => void): this;
     addListener(event: 'close', listener: () => void): this;

package/dist/clients/consumer/messages-stream.js

@@ -20,6 +20,8 @@ export class MessagesStream extends Readable {
     #consumer;
     #mode;
     #fallbackMode;
+    #fetches;
+    #maxFetches;
     #options;
     #topics;
     #offsetsToFetch;
@@ -37,7 +39,7 @@ export class MessagesStream extends Readable {
     #metricsConsumedMessages;
     #corruptedMessageHandler;
     constructor(consumer, options) {
-        const { autocommit, mode, fallbackMode, offsets, deserializers, onCorruptedMessage, ...otherOptions } = options;
+        const { autocommit, mode, fallbackMode, maxFetches, offsets, deserializers, onCorruptedMessage, ...otherOptions } = options;
         if (offsets && mode !== MessagesStreamModes.MANUAL) {
             throw new UserError('Cannot specify offsets when the stream mode is not MANUAL.');
         }
@@ -50,6 +52,8 @@ export class MessagesStream extends Readable {
         this.#mode = mode ?? MessagesStreamModes.LATEST;
         this.#fallbackMode = fallbackMode ?? MessagesStreamFallbackModes.LATEST;
         this.#offsetsToCommit = new Map();
+        this.#fetches = 0;
+        this.#maxFetches = maxFetches ?? 0;
         this.#topics = structuredClone(options.topics);
         this.#inflightNodes = new Set();
         this.#keyDeserializer = deserializers?.key ?? noopDeserializer;
@@ -138,6 +142,18 @@ export class MessagesStream extends Readable {
         });
         return callback[kCallbackPromise];
     }
+    isActive() {
+        if (this.#shouldClose || this.closed || this.destroyed) {
+            return false;
+        }
+        return this.#consumer.isActive();
+    }
+    isConnected() {
+        if (this.#shouldClose || this.closed || this.destroyed) {
+            return false;
+        }
+        return this.#consumer.isConnected();
+    }
     /* c8 ignore next 3 - Only forwards to Node.js implementation - Inserted here to please Typescript */
     addListener(event, listener) {
         return super.addListener(event, listener);
@@ -183,7 +199,7 @@ export class MessagesStream extends Readable {
             if (error) {
                 // The stream has been closed, ignore any error
                 /* c8 ignore next 4 - Hard to test */
-                if (this.#shouldClose) {
+                if (this.#shouldClose || this.closed || this.destroyed) {
                     this.push(null);
                     return;
                 }
@@ -241,7 +257,7 @@ export class MessagesStream extends Readable {
                     if (error) {
                         // The stream has been closed, ignore the error
                         /* c8 ignore next 4 - Hard to test */
-                        if (this.#shouldClose) {
+                        if (this.#shouldClose || this.closed || this.destroyed) {
                             this.push(null);
                             return;
                         }
@@ -257,6 +273,9 @@ export class MessagesStream extends Readable {
                         return;
                     }
                     this.#pushRecords(metadata, topicIds, response, requestedOffsets);
+                    if (this.#maxFetches > 0 && ++this.#fetches >= this.#maxFetches) {
+                        this.push(null);
+                    }
                 });
             }
         });
@@ -272,70 +291,74 @@ export class MessagesStream extends Readable {
         // Parse results
         for (const topicResponse of response.responses) {
             const topic = topicIds.get(topicResponse.topicId);
-            for (const { records, partitionIndex: partition } of topicResponse.partitions) {
-                if (!records) {
+            for (const { records: recordsBatches, partitionIndex: partition } of topicResponse.partitions) {
+                if (!recordsBatches) {
                     continue;
                 }
-                const firstTimestamp = records.firstTimestamp;
-                const firstOffset = records.firstOffset;
-                const leaderEpoch = metadata.topics.get(topic).partitions[partition].leaderEpoch;
-                for (const record of records.records) {
-                    const offset = records.firstOffset + BigInt(record.offsetDelta);
-                    if (offset < requestedOffsets.get(`${topic}:${partition}`)) {
-                        // Thi is a duplicate message, ignore it
-                        continue;
-                    }
-                    diagnosticContext = createDiagnosticContext({
-                        client: this.#consumer,
-                        stream: this,
-                        operation: 'receive',
-                        raw: record
-                    });
-                    consumerReceivesChannel.start.publish(diagnosticContext);
-                    const commit = autocommit ? noopCallback : this.#commit.bind(this, topic, partition, offset, leaderEpoch);
-                    try {
-                        const headers = new Map();
-                        for (const [headerKey, headerValue] of record.headers) {
-                            headers.set(headerKeyDeserializer(headerKey), headerValueDeserializer(headerValue));
+                for (const batch of recordsBatches) {
+                    const firstTimestamp = batch.firstTimestamp;
+                    const firstOffset = batch.firstOffset;
+                    const leaderEpoch = metadata.topics.get(topic).partitions[partition].leaderEpoch;
+                    for (const record of batch.records) {
+                        const offset = batch.firstOffset + BigInt(record.offsetDelta);
+                        if (offset < requestedOffsets.get(`${topic}:${partition}`)) {
+                            // Thi is a duplicate message, ignore it
+                            continue;
                         }
-                        const key = keyDeserializer(record.key, headers);
-                        const value = valueDeserializer(record.value, headers);
-                        this.#metricsConsumedMessages?.inc();
-                        const message = {
-                            key,
-                            value,
-                            headers,
-                            topic,
-                            partition,
-                            timestamp: firstTimestamp + record.timestampDelta,
-                            offset,
-                            commit
-                        };
-                        diagnosticContext.result = message;
-                        consumerReceivesChannel.asyncStart.publish(diagnosticContext);
-                        canPush = this.push(message);
-                        consumerReceivesChannel.asyncEnd.publish(diagnosticContext);
-                    }
-                    catch (error) {
-                        const shouldDestroy = this.#corruptedMessageHandler(record, topic, partition, firstTimestamp, firstOffset, commit);
-                        if (shouldDestroy) {
-                            diagnosticContext.error = error;
-                            consumerReceivesChannel.error.publish(diagnosticContext);
-                            this.destroy(new UserError('Failed to deserialize a message.', { cause: error }));
-                            return;
+                        diagnosticContext = createDiagnosticContext({
+                            client: this.#consumer,
+                            stream: this,
+                            operation: 'receive',
+                            raw: record
+                        });
+                        consumerReceivesChannel.start.publish(diagnosticContext);
+                        const commit = autocommit ? noopCallback : this.#commit.bind(this, topic, partition, offset, leaderEpoch);
+                        try {
+                            const headers = new Map();
+                            for (const [headerKey, headerValue] of record.headers) {
+                                headers.set(headerKeyDeserializer(headerKey), headerValueDeserializer(headerValue));
+                            }
+                            const key = keyDeserializer(record.key, headers);
+                            const value = valueDeserializer(record.value, headers);
+                            this.#metricsConsumedMessages?.inc();
+                            const message = {
+                                key,
+                                value,
+                                headers,
+                                topic,
+                                partition,
+                                timestamp: firstTimestamp + record.timestampDelta,
+                                offset,
+                                commit
+                            };
+                            diagnosticContext.result = message;
+                            consumerReceivesChannel.asyncStart.publish(diagnosticContext);
+                            canPush = this.push(message);
+                            consumerReceivesChannel.asyncEnd.publish(diagnosticContext);
+                        }
+                        catch (error) {
+                            const shouldDestroy = this.#corruptedMessageHandler(record, topic, partition, firstTimestamp, firstOffset, commit);
+                            if (shouldDestroy) {
+                                diagnosticContext.error = error;
+                                consumerReceivesChannel.error.publish(diagnosticContext);
+                                this.destroy(new UserError('Failed to deserialize a message.', { cause: error }));
+                                return;
+                            }
+                        }
+                        finally {
+                            consumerReceivesChannel.end.publish(diagnosticContext);
                         }
                     }
-                    finally {
-                        consumerReceivesChannel.end.publish(diagnosticContext);
+                    if (batch === recordsBatches[recordsBatches.length - 1]) {
+                        // Track the last read offset
+                        const lastOffset = batch.firstOffset + BigInt(batch.lastOffsetDelta);
+                        this.#offsetsToFetch.set(`${topic}:${partition}`, lastOffset + 1n);
+                        // Autocommit if needed
+                        if (autocommit) {
+                            this.#offsetsToCommit.set(`${topic}:${partition}`, { topic, partition, offset: lastOffset, leaderEpoch });
+                        }
                     }
                 }
-                // Track the last read offset
-                const lastOffset = records.firstOffset + BigInt(records.lastOffsetDelta);
-                this.#offsetsToFetch.set(`${topic}:${partition}`, lastOffset + 1n);
-                // Autocommit if needed
-                if (autocommit) {
-                    this.#offsetsToCommit.set(`${topic}:${partition}`, { topic, partition, offset: lastOffset, leaderEpoch });
-                }
             }
         }
         if (this.#autocommitEnabled && !this.#autocommitInterval) {
@@ -387,6 +410,11 @@ export class MessagesStream extends Readable {
                 : ListOffsetTimestamps.LATEST
         }, (error, offsets) => {
             if (error) {
+                /* c8 ignore next 4 - Hard to test */
+                if (this.#shouldClose || this.closed || this.destroyed) {
+                    callback(null);
+                    return;
+                }
                 callback(error);
                 return;
             }
@@ -409,6 +437,11 @@ export class MessagesStream extends Readable {
             }
             this.#consumer.listCommittedOffsets({ topics }, (error, commits) => {
                 if (error) {
+                    /* c8 ignore next 4 - Hard to test */
+                    if (this.#shouldClose || this.closed || this.destroyed) {
+                        callback(null);
+                        return;
+                    }
                     callback(error);
                     return;
                 }

package/dist/clients/consumer/options.d.ts

@@ -281,6 +281,11 @@ export declare const consumeOptionsSchema: {
             type: string;
             enum: ("latest" | "earliest" | "fail")[];
         };
+        maxFetches: {
+            type: string;
+            minimum: number;
+            default: number;
+        };
         offsets: {
             type: string;
             items: {

package/dist/clients/consumer/options.js

@@ -71,6 +71,7 @@ export const consumeOptionsSchema = {
         topics: { type: 'array', items: idProperty },
         mode: { type: 'string', enum: Object.values(MessagesStreamModes) },
         fallbackMode: { type: 'string', enum: Object.values(MessagesStreamFallbackModes) },
+        maxFetches: { type: 'number', minimum: 0, default: 0 },
         offsets: {
             type: 'array',
             items: {

package/dist/clients/consumer/types.d.ts

@@ -62,6 +62,7 @@ export interface StreamOptions {
     topics: string[];
     mode?: MessagesStreamModeValue;
     fallbackMode?: MessagesStreamFallbackModeValue;
+    maxFetches?: number;
     offsets?: TopicWithPartitionAndOffset[];
     onCorruptedMessage?: CorruptedMessageHandler;
 }

package/dist/clients/producer/producer.js

@@ -198,7 +198,7 @@ export class Producer extends Base {
                 }
                 else {
                     // Use the roundrobin
-                    partition = this.#partitionsRoundRobin.postIncrement(topic, 1, -1);
+                    partition = this.#partitionsRoundRobin.postIncrement(topic, 1, 0);
                 }
             }
             else {

package/dist/errors.d.ts

@@ -35,6 +35,7 @@ export declare class NetworkError extends GenericError {
     constructor(message: string, properties?: ErrorProperties);
 }
 export declare class ProtocolError extends GenericError {
+    static code: ErrorCode;
     constructor(codeOrId: string | number, properties?: ErrorProperties, response?: unknown);
 }
 export declare class ResponseError extends MultipleErrors {

package/dist/errors.js

@@ -76,9 +76,13 @@ export class MultipleErrors extends AggregateError {
             return this;
         }
         for (const error of this.errors) {
-            if (error[kGenericError] ? error.findBy(property, value) : error[property] === value) {
+            if (error[property] === value) {
                 return error;
             }
+            const found = error[kGenericError] ? error.findBy(property, value) : undefined;
+            if (found) {
+                return found;
+            }
         }
         return null;
     }
@@ -97,9 +101,10 @@ export class NetworkError extends GenericError {
     }
 }
 export class ProtocolError extends GenericError {
+    static code = 'PLT_KFK_PROTOCOL';
     constructor(codeOrId, properties = {}, response = undefined) {
         const { id, code, message, canRetry } = protocolErrors[typeof codeOrId === 'number' ? protocolErrorsCodesById[codeOrId] : codeOrId];
-        super('PLT_KFK_PROTOCOL', message, {
+        super(ProtocolError.code, message, {
             apiId: id,
             apiCode: code,
             canRetry,

package/dist/network/connection-pool.d.ts

@@ -11,4 +11,6 @@ export declare class ConnectionPool extends EventEmitter {
     getFirstAvailable(brokers: Broker[]): Promise<Connection>;
     close(callback: CallbackWithPromise<void>): void;
     close(): Promise<void>;
+    isActive(): boolean;
+    isConnected(): boolean;
 }

package/dist/network/connection-pool.js

@@ -7,12 +7,14 @@ let currentInstance = 0;
 export class ConnectionPool extends EventEmitter {
     #instanceId;
     #clientId;
+    #closed;
     // @ts-ignore This is used just for debugging
     #ownerId;
     #connections;
     #connectionOptions;
     constructor(clientId, connectionOptions = {}) {
         super();
+        this.#closed = false;
         this.#instanceId = currentInstance++;
         this.#clientId = clientId;
         this.#ownerId = connectionOptions.ownerId;
@@ -41,17 +43,40 @@ export class ConnectionPool extends EventEmitter {
         if (!callback) {
             callback = createPromisifiedCallback();
         }
-        if (this.#connections.size === 0) {
+        if (this.#closed || this.#connections.size === 0) {
+            this.#closed = true;
             callback(null);
             return callback[kCallbackPromise];
         }
+        this.#closed = true;
         runConcurrentCallbacks('Closing connections failed.', this.#connections, ([key, connection], cb) => {
             connection.close(cb);
             this.#connections.delete(key);
         }, error => callback(error));
         return callback[kCallbackPromise];
     }
+    isActive() {
+        if (this.#connections.size === 0) {
+            return false;
+        }
+        return true;
+    }
+    isConnected() {
+        if (this.#connections.size === 0) {
+            return false;
+        }
+        for (const connection of this.#connections.values()) {
+            if (!connection.isConnected()) {
+                return false;
+            }
+        }
+        return true;
+    }
     #get(broker, callback) {
+        if (this.#closed) {
+            callback(new Error('Connection pool is closed.'), undefined);
+            return;
+        }
         const key = `${broker.host}:${broker.port}`;
         const existing = this.#connections.get(key);
         if (existing) {

package/dist/network/connection.d.ts

@@ -11,8 +11,9 @@ export interface Broker {
 }
 export interface SASLOptions {
     mechanism: SASLMechanism;
-    username: string;
-    password: string;
+    username?: string;
+    password?: string;
+    token?: string;
 }
 export interface ConnectionOptions {
     connectTimeout?: number;
@@ -53,6 +54,7 @@ export declare class Connection extends EventEmitter {
     get instanceId(): number;
     get status(): ConnectionStatusValue;
     get socket(): Socket;
+    isConnected(): boolean;
     connect(host: string, port: number, callback?: CallbackWithPromise<void>): void | Promise<void>;
     ready(callback: CallbackWithPromise<void>): void;
     ready(): Promise<void>;

package/dist/network/connection.js

@@ -10,7 +10,7 @@ import { AuthenticationError, NetworkError, TimeoutError, UnexpectedCorrelationI
 import { protocolAPIsById } from "../protocol/apis.js";
 import { EMPTY_OR_SINGLE_COMPACT_LENGTH_SIZE, INT32_SIZE } from "../protocol/definitions.js";
 import { DynamicBuffer } from "../protocol/dynamic-buffer.js";
-import { saslPlain, saslScramSha } from "../protocol/index.js";
+import { saslOAuthBearer, saslPlain, saslScramSha } from "../protocol/index.js";
 import { Reader } from "../protocol/reader.js";
 import { defaultCrypto } from "../protocol/sasl/scram-sha.js";
 import { Writer } from "../protocol/writer.js";
@@ -80,6 +80,9 @@ export class Connection extends EventEmitter {
     get socket() {
         return this.#socket;
     }
+    isConnected() {
+        return this.#status === ConnectionStatuses.CONNECTED;
+    }
     connect(host, port, callback) {
         if (!callback) {
             callback = createPromisifiedCallback();
@@ -229,7 +232,7 @@ export class Connection extends EventEmitter {
     }
     #authenticate(host, port, diagnosticContext) {
         this.#status = ConnectionStatuses.AUTHENTICATING;
-        const { mechanism, username, password } = this.#options.sasl;
+        const { mechanism, username, password, token } = this.#options.sasl;
         if (!SASLMechanisms.includes(mechanism)) {
             this.#onConnectionError(host, port, diagnosticContext, new UserError(`SASL mechanism ${mechanism} not supported.`));
             return;
@@ -243,6 +246,9 @@ export class Connection extends EventEmitter {
             if (mechanism === 'PLAIN') {
                 saslPlain.authenticate(saslAuthenticateV2.api, this, username, password, this.#onSaslAuthenticate.bind(this, host, port, diagnosticContext));
             }
+            else if (mechanism === 'OAUTHBEARER') {
+                saslOAuthBearer.authenticate(saslAuthenticateV2.api, this, token, this.#onSaslAuthenticate.bind(this, host, port, diagnosticContext));
+            }
             else {
                 saslScramSha.authenticate(saslAuthenticateV2.api, this, mechanism.substring(6), username, password, defaultCrypto, this.#onSaslAuthenticate.bind(this, host, port, diagnosticContext));
             }
@@ -263,46 +269,36 @@ export class Connection extends EventEmitter {
                 request.callback(new NetworkError('Connection closed'), undefined);
                 return false;
             }
-            let canWrite = true;
-            const { correlationId, apiKey, apiVersion, payload: payloadFn, hasRequestHeaderTaggedFields } = request;
-            const writer = Writer.create()
-                .appendInt16(apiKey)
-                .appendInt16(apiVersion)
-                .appendInt32(correlationId)
+            const writer = Writer.create();
+            writer
+                .appendInt16(request.apiKey)
+                .appendInt16(request.apiVersion)
+                .appendInt32(request.correlationId)
                 .appendString(this.#clientId, false);
-            if (hasRequestHeaderTaggedFields) {
+            if (request.hasRequestHeaderTaggedFields) {
                 writer.appendTaggedFields();
             }
-            const payload = payloadFn();
-            writer.appendFrom(payload);
-            writer.prependLength();
-            // Write the header
-            this.#socket.cork();
-            if (!payload.context.noResponse) {
-                this.#inflightRequests.set(correlationId, request);
-            }
-            loggers.protocol('Sending request.', { apiKey: protocolAPIsById[apiKey], correlationId, request });
-            for (const buf of writer.buffers) {
-                if (!this.#socket.write(buf)) {
-                    canWrite = false;
-                }
-            }
-            if (!canWrite) {
+            const payload = request.payload();
+            writer.appendFrom(payload).prependLength();
+            const expectResponse = !payload.context.noResponse;
+            if (expectResponse)
+                this.#inflightRequests.set(request.correlationId, request);
+            const canWrite = this.#socket.write(writer.buffer);
+            if (!canWrite)
                 this.#socketMustBeDrained = true;
-            }
-            this.#socket.uncork();
-            if (payload.context.noResponse) {
+            if (!expectResponse)
                 request.callback(null, canWrite);
-            }
-            // debugDump(Date.now() % 100000, 'send', { owner: this.#ownerId, apiKey: protocolAPIsById[apiKey], correlationId })
+            loggers.protocol('Sending request.', {
+                apiKey: protocolAPIsById[request.apiKey],
+                correlationId: request.correlationId,
+                request
+            });
             return canWrite;
         }
-        catch (error) {
-            request.diagnostic.error = error;
+        catch (err) {
+            request.diagnostic.error = err;
             connectionsApiChannel.error.publish(request.diagnostic);
-            connectionsApiChannel.end.publish(request.diagnostic);
-            throw error;
-            /* c8 ignore next 3 - C8 does not detect these as covered */
+            throw err;
         }
         finally {
             connectionsApiChannel.end.publish(request.diagnostic);

package/dist/protocol/crc32c.d.ts

@@ -1,2 +1,2 @@
 import { DynamicBuffer } from './dynamic-buffer.ts';
-export declare function crc32c(data: Buffer | DynamicBuffer): number;
+export declare function crc32c(data: Buffer | Uint8Array | DynamicBuffer): number;

package/dist/protocol/crc32c.js

@@ -68,16 +68,12 @@ const CRC = [
     0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351
 ];
 export function crc32c(data) {
-    let crc = 0 ^ -1;
-    if (DynamicBuffer.isDynamicBuffer(data)) {
-        for (let i = 0; i < data.length; i++) {
-            crc = CRC[(crc ^ data.get(i)) & 0xff] ^ (crc >>> 8);
-        }
+    const bytes = DynamicBuffer.isDynamicBuffer(data)
+        ? data.buffer
+        : new Uint8Array(data);
+    let crc = 0xffffffff;
+    for (let i = 0, len = bytes.length; i < len; ++i) {
+        crc = CRC[(crc ^ bytes[i]) & 0xff] ^ (crc >>> 8);
     }
-    else {
-        for (let i = 0; i < data.length; i++) {
-            crc = CRC[(crc ^ data[i]) & 0xff] ^ (crc >>> 8);
-        }
-    }
-    return Uint32Array.from([(crc ^ -1) >>> 0])[0];
+    return (crc ^ 0xffffffff) >>> 0;
 }

package/dist/protocol/index.d.ts

@@ -8,6 +8,7 @@ export * from './index.ts';
 export * from './murmur2.ts';
 export * from './reader.ts';
 export * from './records.ts';
+export * as saslOAuthBearer from './sasl/oauth-bearer.ts';
 export * as saslPlain from './sasl/plain.ts';
 export * as saslScramSha from './sasl/scram-sha.ts';
 export * from './varint.ts';

package/dist/protocol/index.js

@@ -8,6 +8,7 @@ export * from "./index.js";
 export * from "./murmur2.js";
 export * from "./reader.js";
 export * from "./records.js";
+export * as saslOAuthBearer from "./sasl/oauth-bearer.js";
 export * as saslPlain from "./sasl/plain.js";
 export * as saslScramSha from "./sasl/scram-sha.js";
 export * from "./varint.js";

package/dist/protocol/records.js

@@ -60,15 +60,16 @@ export function readRecord(reader) {
 }
 export function createRecordsBatch(messages, options = {}) {
     const now = BigInt(Date.now());
-    const timestamps = [];
-    for (let i = 0; i < messages.length; i++) {
-        timestamps.push(messages[i].timestamp ?? now);
-    }
-    messages.sort();
-    const firstTimestamp = timestamps[0];
-    const maxTimestamp = timestamps[timestamps.length - 1];
+    const firstTimestamp = messages[0].timestamp ?? now;
+    let maxTimestamp = firstTimestamp;
     let buffer = new DynamicBuffer();
     for (let i = 0; i < messages.length; i++) {
+        let ts = messages[i].timestamp ?? now;
+        if (typeof ts === 'number')
+            ts = BigInt(ts);
+        messages[i].timestamp = ts;
+        if (ts > maxTimestamp)
+            maxTimestamp = ts;
         const record = createRecord(messages[i], i, firstTimestamp);
         buffer.appendFrom(record.dynamicBuffer);
     }

package/dist/protocol/sasl/oauth-bearer.d.ts

@@ -0,0 +1,5 @@
+import { type CallbackWithPromise } from '../../apis/callbacks.ts';
+import { type SASLAuthenticationAPI, type SaslAuthenticateResponse } from '../../apis/security/sasl-authenticate-v2.ts';
+import { type Connection } from '../../network/connection.ts';
+export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, token: string, callback: CallbackWithPromise<SaslAuthenticateResponse>): void;
+export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, token: string): Promise<SaslAuthenticateResponse>;

package/dist/protocol/sasl/oauth-bearer.js

@@ -0,0 +1,8 @@
+import { createPromisifiedCallback, kCallbackPromise } from "../../apis/callbacks.js";
+export function authenticate(authenticateAPI, connection, token, callback) {
+    if (!callback) {
+        callback = createPromisifiedCallback();
+    }
+    authenticateAPI(connection, Buffer.from(`n,,\x01auth=Bearer ${token}\x01\x01`), callback);
+    return callback[kCallbackPromise];
+}

package/dist/protocol/writer.js

@@ -152,7 +152,7 @@ export class Writer {
     }
     appendArray(value, entryWriter, compact = true, appendTrailingTaggedFields = true) {
         if (value == null) {
-            return compact ? this.appendUnsignedVarInt(0) : this.appendInt32(0);
+            return compact ? this.appendUnsignedVarInt(0) : this.appendInt32(-1);
         }
         const length = value.length;
         if (compact) {
@@ -171,7 +171,7 @@ export class Writer {
     }
     appendMap(value, entryWriter, compact = true, appendTrailingTaggedFields = true) {
         if (value == null) {
-            return compact ? this.appendUnsignedVarInt(0) : this.appendInt32(0);
+            return compact ? this.appendUnsignedVarInt(0) : this.appendInt32(-1);
         }
         const length = value.size;
         if (compact) {

package/dist/utils.d.ts

@@ -25,7 +25,7 @@ export declare class NumericMap extends Map<string, number> {
 export declare function niceJoin(array: string[], lastSeparator?: string, separator?: string): string;
 export declare function listErrorMessage(type: string[]): string;
 export declare function enumErrorMessage(type: Record<string, unknown>, keysOnly?: boolean): string;
-export declare function groupByProperty<Key, Value>(entries: Value[], property: keyof Value): [Key, Value[]][];
+export declare function groupByProperty<Key extends PropertyKey, Value>(entries: readonly Value[], property: keyof Value): [Key, Value[]][];
 export declare function humanize(label: string, buffer: Buffer | DynamicBuffer): string;
 export declare function setDebugDumpLogger(logger: DebugDumpLogger): void;
 export declare function debugDump(...values: unknown[]): void;

package/dist/utils.js

@@ -121,19 +121,13 @@ export function enumErrorMessage(type, keysOnly = false) {
     return `should be one of ${niceJoin(Object.entries(type).map(([k, v]) => `${v} (${k})`), ' or ')}`;
 }
 export function groupByProperty(entries, property) {
-    const grouped = new Map();
-    const result = [];
-    for (const entry of entries) {
-        const value = entry[property];
-        let values = grouped.get(value);
-        if (!values) {
-            values = [];
-            grouped.set(value, values);
-            result.push([value, values]);
-        }
-        values.push(entry);
+    const buckets = Object.create(null);
+    for (let i = 0, len = entries.length; i < len; ++i) {
+        const e = entries[i];
+        const key = e[property];
+        (buckets[key] ||= []).push(e);
     }
-    return result;
+    return Object.entries(buckets);
 }
 export function humanize(label, buffer) {
     const formatted = buffer

package/dist/version.d.ts

@@ -0,0 +1,2 @@
+export declare const name: any;
+export declare const version: any;

package/dist/version.js

@@ -0,0 +1,2 @@
+export const name = "@platformatic/kafka";
+export const version = "1.10.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "1.8.0",
+  "version": "1.10.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
@@ -46,6 +46,7 @@
     "cleaner-spec-reporter": "^0.5.0",
     "cronometro": "^5.3.0",
     "eslint": "^9.21.0",
+    "fast-jwt": "^6.0.2",
     "hwp": "^0.4.1",
     "json5": "^2.2.3",
     "kafkajs": "^2.2.4",
@@ -64,11 +65,14 @@
   },
   "scripts": {
     "build": "rm -rf dist && tsc -p tsconfig.base.json",
+    "postbuild": "node --experimental-strip-types scripts/postbuild.ts",
     "lint": "eslint --cache",
     "typecheck": "tsc -p . --noEmit",
     "format": "prettier -w benchmarks playground src test",
     "test": "c8 -c test/config/c8-local.json node --env-file=test/config/env --no-warnings --test --test-timeout=300000 'test/**/*.test.ts'",
     "test:ci": "c8 -c test/config/c8-ci.json node --env-file=test/config/env --no-warnings --test --test-timeout=300000 'test/**/*.test.ts'",
+    "test:docker:up": "node scripts/docker.ts up -d --wait",
+    "test:docker:down": "node scripts/docker.ts down",
     "ci": "npm run build && npm run lint && npm run test:ci",
     "generate:apis": "node --experimental-strip-types scripts/generate-apis.ts",
     "generate:errors": "node --experimental-strip-types scripts/generate-errors.ts",