@platformatic/kafka 2.11.0 → 2.12.1

package/README.md

@@ -108,6 +108,12 @@ await forEach(
 await consumer.close()
 ```
 
+### Consuming messages in batches
+
+The [`examples/batching`](./examples/batching) example uses the [`hwp`](https://www.npmjs.com/package/hwp)
+`batchIterator` operator to process messages in batches and commit offsets after each batch. The batch is emitted
+when it reaches the configured size or when its timeout expires, so the final batch can contain fewer messages.
+
 ### Admin
 
 ```typescript

package/dist/apis/definitions.d.ts

@@ -4,7 +4,7 @@ import { type Reader } from '../protocol/reader.ts';
 import { type Writer } from '../protocol/writer.ts';
 export type Callback<ReturnType> = (error: Error | null, payload?: ReturnType) => void;
 export type CallbackArguments<ReturnType> = [cb: Callback<ReturnType>];
-export type RequestCreator = (...args: any[]) => Writer;
+export type RequestCreator = (...args: any[]) => Writer | Promise<Writer>;
 export type ResponseParser<ReturnType> = (correlationId: number, apiKey: number, apiVersion: number, reader: Reader) => ReturnType | Promise<ReturnType>;
 export type ResponseErrorWithLocation = [string, [number, NullableString]];
 export type APIWithCallback<RequestArguments extends Array<unknown>, ResponseType> = (connection: Connection, ...args: [...RequestArguments, ...Partial<CallbackArguments<ResponseType>>]) => void;

package/dist/apis/producer/produce-v10.d.ts

@@ -37,5 +37,6 @@ export interface ProduceResponse {
     nodeEndpoints?: ProduceResponseNodeEndpoint[];
 }
 export declare function createRequest(acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Writer;
+export declare function createRequestAsync(acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Promise<Writer>;
 export declare function parseResponse(_correlationId: number, apiKey: number, apiVersion: number, reader: Reader): ProduceResponse;
 export declare const api: import("../definitions.ts").API<[acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions> | undefined], boolean | ProduceResponse>;

package/dist/apis/producer/produce-v10.js

@@ -5,6 +5,7 @@ import { groupByProperty } from "../../utils.js";
 import { createAPI } from "../definitions.js";
 import { ProduceAcks } from "../enumerations.js";
 import { readKnownTaggedFields } from "../tagged-fields.js";
+import { encodeProduceTopicData } from "./utils.js";
 /*
   Produce Request (Version: 10) => transactional_id acks timeout_ms [topic_data] TAG_BUFFER
     transactional_id => COMPACT_NULLABLE_STRING
@@ -45,6 +46,30 @@ export function createRequest(acks = 1, timeout = 0, topicData, options = {}) {
     }
     return writer;
 }
+export async function createRequestAsync(acks = 1, timeout = 0, topicData, options = {}) {
+    const topics = await encodeProduceTopicData(topicData, options);
+    const writer = Writer.create()
+        .appendString(options.transactionalId)
+        .appendInt16(acks)
+        .appendInt32(timeout)
+        .appendArray(topics, (w, { topic, partitions }) => {
+        w.appendString(topic).appendArray(partitions, (w, { partition, records }) => {
+            w.appendInt32(partition)
+                .appendUnsignedVarInt(records.length + 1)
+                .appendFrom(records);
+        });
+    })
+        .appendTaggedFields();
+    if (acks === ProduceAcks.NO_RESPONSE) {
+        writer.context.noResponse = true;
+    }
+    return writer;
+}
+function createApiRequest(acks = 1, timeout = 0, topicData, options = {}) {
+    return options.compression === 'gzip'
+        ? createRequestAsync(acks, timeout, topicData, options)
+        : createRequest(acks, timeout, topicData, options);
+}
 /*
   Produce Response (Version: 10) => [responses] throttle_time_ms TAG_BUFFER
     responses => name [partition_responses] TAG_BUFFER
@@ -143,4 +168,4 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
     }
     return response;
 }
-export const api = createAPI(0, 10, createRequest, parseResponse);
+export const api = createAPI(0, 10, createApiRequest, parseResponse);

package/dist/apis/producer/produce-v11.d.ts

@@ -37,5 +37,6 @@ export interface ProduceResponse {
     nodeEndpoints?: ProduceResponseNodeEndpoint[];
 }
 export declare function createRequest(acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Writer;
+export declare function createRequestAsync(acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Promise<Writer>;
 export declare function parseResponse(_correlationId: number, apiKey: number, apiVersion: number, reader: Reader): ProduceResponse;
 export declare const api: import("../definitions.ts").API<[acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions> | undefined], boolean | ProduceResponse>;

package/dist/apis/producer/produce-v11.js

@@ -5,6 +5,7 @@ import { groupByProperty } from "../../utils.js";
 import { createAPI } from "../definitions.js";
 import { ProduceAcks } from "../enumerations.js";
 import { readKnownTaggedFields } from "../tagged-fields.js";
+import { encodeProduceTopicData } from "./utils.js";
 /*
   Produce Request (Version: 11) => transactional_id acks timeout_ms [topic_data] TAG_BUFFER
     transactional_id => COMPACT_NULLABLE_STRING
@@ -45,6 +46,30 @@ export function createRequest(acks = 1, timeout = 0, topicData, options = {}) {
     }
     return writer;
 }
+export async function createRequestAsync(acks = 1, timeout = 0, topicData, options = {}) {
+    const topics = await encodeProduceTopicData(topicData, options);
+    const writer = Writer.create()
+        .appendString(options.transactionalId)
+        .appendInt16(acks)
+        .appendInt32(timeout)
+        .appendArray(topics, (w, { topic, partitions }) => {
+        w.appendString(topic).appendArray(partitions, (w, { partition, records }) => {
+            w.appendInt32(partition)
+                .appendUnsignedVarInt(records.length + 1)
+                .appendFrom(records);
+        });
+    })
+        .appendTaggedFields();
+    if (acks === ProduceAcks.NO_RESPONSE) {
+        writer.context.noResponse = true;
+    }
+    return writer;
+}
+function createApiRequest(acks = 1, timeout = 0, topicData, options = {}) {
+    return options.compression === 'gzip'
+        ? createRequestAsync(acks, timeout, topicData, options)
+        : createRequest(acks, timeout, topicData, options);
+}
 /*
   Produce Response (Version: 11) => [responses] throttle_time_ms TAG_BUFFER
     responses => name [partition_responses] TAG_BUFFER
@@ -143,4 +168,4 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
     }
     return response;
 }
-export const api = createAPI(0, 11, createRequest, parseResponse);
+export const api = createAPI(0, 11, createApiRequest, parseResponse);

package/dist/apis/producer/produce-v7.d.ts

@@ -25,5 +25,6 @@ export interface ProduceResponse {
     throttleTimeMs: number;
 }
 export declare function createRequest(acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Writer;
+export declare function createRequestAsync(acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Promise<Writer>;
 export declare function parseResponse(_correlationId: number, apiKey: number, apiVersion: number, reader: Reader): ProduceResponse;
 export declare const api: import("../definitions.ts").API<[acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions> | undefined], boolean | ProduceResponse>;

package/dist/apis/producer/produce-v7.js

@@ -4,6 +4,7 @@ import { Writer } from "../../protocol/writer.js";
 import { groupByProperty } from "../../utils.js";
 import { createAPI } from "../definitions.js";
 import { ProduceAcks } from "../enumerations.js";
+import { encodeProduceTopicData } from "./utils.js";
 /*
   Produce Request (Version: 7) => transactional_id acks timeout_ms [topic_data]
     transactional_id => NULLABLE_STRING
@@ -41,6 +42,27 @@ export function createRequest(acks = 1, timeout = 0, topicData, options = {}) {
     }
     return writer;
 }
+export async function createRequestAsync(acks = 1, timeout = 0, topicData, options = {}) {
+    const topics = await encodeProduceTopicData(topicData, options);
+    const writer = Writer.create()
+        .appendString(options.transactionalId, false)
+        .appendInt16(acks)
+        .appendInt32(timeout)
+        .appendArray(topics, (w, { topic, partitions }) => {
+        w.appendString(topic, false).appendArray(partitions, (w, { partition, records }) => {
+            w.appendInt32(partition).appendInt32(records.length).appendFrom(records);
+        }, false, false);
+    }, false, false);
+    if (acks === ProduceAcks.NO_RESPONSE) {
+        writer.context.noResponse = true;
+    }
+    return writer;
+}
+function createApiRequest(acks = 1, timeout = 0, topicData, options = {}) {
+    return options.compression === 'gzip'
+        ? createRequestAsync(acks, timeout, topicData, options)
+        : createRequest(acks, timeout, topicData, options);
+}
 /*
   Produce Response (Version: 7) => [responses] throttle_time_ms
     responses => name [partition_responses]
@@ -85,4 +107,4 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
     }
     return response;
 }
-export const api = createAPI(0, 7, createRequest, parseResponse, false, false);
+export const api = createAPI(0, 7, createApiRequest, parseResponse, false, false);

package/dist/apis/producer/produce-v8.d.ts

@@ -25,5 +25,6 @@ export interface ProduceResponse {
     throttleTimeMs: number;
 }
 export declare function createRequest(acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Writer;
+export declare function createRequestAsync(acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Promise<Writer>;
 export declare function parseResponse(_correlationId: number, apiKey: number, apiVersion: number, reader: Reader): ProduceResponse;
 export declare const api: import("../definitions.ts").API<[acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions> | undefined], boolean | ProduceResponse>;

package/dist/apis/producer/produce-v8.js

@@ -4,6 +4,7 @@ import { Writer } from "../../protocol/writer.js";
 import { groupByProperty } from "../../utils.js";
 import { createAPI } from "../definitions.js";
 import { ProduceAcks } from "../enumerations.js";
+import { encodeProduceTopicData } from "./utils.js";
 /*
   Produce Request (Version: 8) => transactional_id acks timeout_ms [topic_data]
     transactional_id => NULLABLE_STRING
@@ -41,6 +42,27 @@ export function createRequest(acks = 1, timeout = 0, topicData, options = {}) {
     }
     return writer;
 }
+export async function createRequestAsync(acks = 1, timeout = 0, topicData, options = {}) {
+    const topics = await encodeProduceTopicData(topicData, options);
+    const writer = Writer.create()
+        .appendString(options.transactionalId, false)
+        .appendInt16(acks)
+        .appendInt32(timeout)
+        .appendArray(topics, (w, { topic, partitions }) => {
+        w.appendString(topic, false).appendArray(partitions, (w, { partition, records }) => {
+            w.appendInt32(partition).appendInt32(records.length).appendFrom(records);
+        }, false, false);
+    }, false, false);
+    if (acks === ProduceAcks.NO_RESPONSE) {
+        writer.context.noResponse = true;
+    }
+    return writer;
+}
+function createApiRequest(acks = 1, timeout = 0, topicData, options = {}) {
+    return options.compression === 'gzip'
+        ? createRequestAsync(acks, timeout, topicData, options)
+        : createRequest(acks, timeout, topicData, options);
+}
 /*
   Produce Response (Version: 8) => [responses] throttle_time_ms
     responses => name [partition_responses]
@@ -103,4 +125,4 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
     }
     return response;
 }
-export const api = createAPI(0, 8, createRequest, parseResponse, false, false);
+export const api = createAPI(0, 8, createApiRequest, parseResponse, false, false);

package/dist/apis/producer/produce-v9.d.ts

@@ -25,5 +25,6 @@ export interface ProduceResponse {
     throttleTimeMs: number;
 }
 export declare function createRequest(acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Writer;
+export declare function createRequestAsync(acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Promise<Writer>;
 export declare function parseResponse(_correlationId: number, apiKey: number, apiVersion: number, reader: Reader): ProduceResponse;
 export declare const api: import("../definitions.ts").API<[acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions> | undefined], boolean | ProduceResponse>;

package/dist/apis/producer/produce-v9.js

@@ -4,6 +4,7 @@ import { Writer } from "../../protocol/writer.js";
 import { groupByProperty } from "../../utils.js";
 import { createAPI } from "../definitions.js";
 import { ProduceAcks } from "../enumerations.js";
+import { encodeProduceTopicData } from "./utils.js";
 /*
   Produce Request (Version: 9) => transactional_id acks timeout_ms [topic_data] TAG_BUFFER
     transactional_id => COMPACT_NULLABLE_STRING
@@ -44,6 +45,30 @@ export function createRequest(acks = 1, timeout = 0, topicData, options = {}) {
     }
     return writer;
 }
+export async function createRequestAsync(acks = 1, timeout = 0, topicData, options = {}) {
+    const topics = await encodeProduceTopicData(topicData, options);
+    const writer = Writer.create()
+        .appendString(options.transactionalId)
+        .appendInt16(acks)
+        .appendInt32(timeout)
+        .appendArray(topics, (w, { topic, partitions }) => {
+        w.appendString(topic).appendArray(partitions, (w, { partition, records }) => {
+            w.appendInt32(partition)
+                .appendUnsignedVarInt(records.length + 1)
+                .appendFrom(records);
+        });
+    })
+        .appendTaggedFields();
+    if (acks === ProduceAcks.NO_RESPONSE) {
+        writer.context.noResponse = true;
+    }
+    return writer;
+}
+function createApiRequest(acks = 1, timeout = 0, topicData, options = {}) {
+    return options.compression === 'gzip'
+        ? createRequestAsync(acks, timeout, topicData, options)
+        : createRequest(acks, timeout, topicData, options);
+}
 /*
   Produce Response (Version: 9) => [responses] throttle_time_ms TAG_BUFFER
     responses => name [partition_responses] TAG_BUFFER
@@ -110,4 +135,4 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
     }
     return response;
 }
-export const api = createAPI(0, 9, createRequest, parseResponse);
+export const api = createAPI(0, 9, createApiRequest, parseResponse);

package/dist/apis/producer/utils.d.ts

@@ -0,0 +1,11 @@
+import { type CreateRecordsBatchOptions, type MessageRecord } from '../../protocol/records.ts';
+import { type Writer } from '../../protocol/writer.ts';
+export interface EncodedProducePartition {
+    partition: number;
+    records: Writer;
+}
+export interface EncodedProduceTopic {
+    topic: string;
+    partitions: EncodedProducePartition[];
+}
+export declare function encodeProduceTopicData(topicData: MessageRecord[], options: Partial<CreateRecordsBatchOptions>): Promise<EncodedProduceTopic[]>;

package/dist/apis/producer/utils.js

@@ -0,0 +1,31 @@
+import { createRecordsBatchAsync } from "../../protocol/records.js";
+import { groupByProperty } from "../../utils.js";
+const gzipConcurrency = 4;
+export async function encodeProduceTopicData(topicData, options) {
+    const now = BigInt(Date.now());
+    for (const message of topicData) {
+        message.partition ??= 0;
+        message.timestamp ??= now;
+    }
+    const topics = groupByProperty(topicData, 'topic').map(([topic, messages]) => ({
+        topic,
+        partitions: groupByProperty(messages, 'partition').map(([partition, messages]) => ({
+            partition: Number(partition),
+            messages
+        }))
+    }));
+    const tasks = topics.flatMap(topic => topic.partitions);
+    let nextTask = 0;
+    async function worker() {
+        while (nextTask < tasks.length) {
+            const task = tasks[nextTask++];
+            task.records = await createRecordsBatchAsync(task.messages, options);
+        }
+    }
+    const workers = Array.from({ length: Math.min(gzipConcurrency, tasks.length) }, worker);
+    await Promise.all(workers);
+    return topics.map(({ topic, partitions }) => ({
+        topic,
+        partitions: partitions.map(({ partition, records }) => ({ partition, records: records }))
+    }));
+}

package/dist/clients/admin/admin.js

@@ -2,8 +2,8 @@ import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } f
 import { ConfigResourceTypes, FetchIsolationLevels, FindCoordinatorKeyTypes, legacyConsumerGroupStates } from "../../apis/enumerations.js";
 import { adminAclsChannel, adminClientQuotasChannel, adminConfigsChannel, adminConsumerGroupOffsetsChannel, adminGroupsChannel, adminLogDirsChannel, adminOffsetsChannel, adminTopicsChannel, createDiagnosticContext } from "../../diagnostic.js";
 import { findErrorBy, MultipleErrors, NetworkError, UserError } from "../../errors.js";
-import { Reader } from "../../protocol/reader.js";
 import { Base, kAfterCreate, kCheckNotClosed, kConnections, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from "../base/base.js";
+import { decodeConsumerProtocolAssignment, decodeConsumerProtocolSubscription } from "../consumer/consumer-protocol.js";
 import { adminListOffsetsOptionsValidator, alterClientQuotasOptionsValidator, alterConfigsOptionsValidator, alterConsumerGroupOffsetsOptionsValidator, createAclsOptionsValidator, createPartitionsOptionsValidator, createTopicsOptionsValidator, deleteAclsOptionsValidator, deleteConsumerGroupOffsetsOptionsValidator, deleteGroupsOptionsValidator, deleteRecordsOptionsValidator, deleteTopicsOptionsValidator, describeAclsOptionsValidator, describeClientQuotasOptionsValidator, describeConfigsOptionsValidator, describeGroupsOptionsValidator, describeLogDirsOptionsValidator, findCoordinatorOptionsValidator, incrementalAlterConfigsOptionsValidator, listConsumerGroupOffsetsOptionsValidator, listGroupsOptionsValidator, listTopicsOptionsValidator, removeMembersFromConsumerGroupOptionsValidator } from "./options.js";
 export class Admin extends Base {
     #controller = null;
@@ -649,22 +649,19 @@ export class Admin extends Base {
                                 authorizedOperations: raw.authorizedOperations
                             };
                             for (const member of raw.members) {
-                                const reader = Reader.from(member.memberMetadata);
                                 let memberMetadata;
                                 let memberAssignments;
-                                if (reader.remaining > 0) {
+                                if (member.memberMetadata.length > 0) {
+                                    const subscription = decodeConsumerProtocolSubscription(member.memberMetadata);
                                     memberMetadata = {
-                                        version: reader.readInt16(),
-                                        topics: reader.readArray(r => r.readString(false), false, false),
-                                        metadata: reader.readBytes(false)
+                                        version: subscription.version,
+                                        topics: subscription.topics,
+                                        metadata: subscription.userData,
+                                        ownedPartitions: subscription.ownedPartitions,
+                                        generationId: subscription.generationId,
+                                        rackId: subscription.rackId
                                     };
-                                    reader.reset(member.memberAssignment);
-                                    reader.skip(2); // Ignore Version information
-                                    memberAssignments = reader.readMap(r => {
-                                        const topic = r.readString(false);
-                                        return [topic, { topic, partitions: reader.readArray(r => r.readInt32(), false, false) }];
-                                    }, false, false);
-                                    // Ignore the user data
+                                    memberAssignments = new Map(decodeConsumerProtocolAssignment(member.memberAssignment).assignedPartitions.map(assignment => [assignment.topic, assignment]));
                                 }
                                 group.members.set(member.memberId, {
                                     id: member.memberId,

package/dist/clients/base/base.d.ts

@@ -84,7 +84,7 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions, EventsT
     [kMetadata](options: MetadataOptions, callback: CallbackWithPromise<ClusterMetadata>): void;
     [kCheckNotClosed](callback: CallbackWithPromise<any>): boolean;
     clearMetadata(): void;
-    [kPerformWithRetry]<ReturnType>(operationId: string, operation: (callback: Callback<ReturnType>, attempt: number) => void, callback: CallbackWithPromise<ReturnType>, attempt?: number, errors?: Error[], shouldSkipRetry?: (e: Error) => boolean): void | Promise<ReturnType>;
+    [kPerformWithRetry]<ReturnType>(operationId: string, operation: (callback: Callback<ReturnType>, attempt: number) => void, callback: CallbackWithPromise<ReturnType>, attempt?: number, errors?: Error[], shouldSkipRetry?: (e: Error) => boolean, retryLimit?: number): void | Promise<ReturnType>;
     [kPerformDeduplicated]<ReturnType>(operationId: string, operation: (callback: CallbackWithPromise<ReturnType>) => void, callback: CallbackWithPromise<ReturnType>): void | Promise<ReturnType>;
     [kGetApi]<RequestArguments extends Array<unknown>, ResponseType>(name: string, callback: Callback<API<RequestArguments, ResponseType>>): void;
     [kGetConnection](broker: Broker, callback: Callback<Connection>): void;

package/dist/clients/base/base.js

@@ -267,12 +267,12 @@ export class Base extends TypedEventEmitter {
     clearMetadata() {
         this.#metadata = undefined;
     }
-    [kPerformWithRetry](operationId, operation, callback, attempt = 0, errors = [], shouldSkipRetry) {
-        const retries = this[kOptions].retries;
+    [kPerformWithRetry](operationId, operation, callback, attempt = 0, errors = [], shouldSkipRetry, retryLimit) {
+        const retries = retryLimit ?? this[kOptions].retries;
         this.emitWithDebug('client', 'performWithRetry', operationId, attempt, retries);
         operation((error, result) => {
             if (error) {
-                // Only retry if all the errors in the chain are retriable
+                // Only retry if all the errors in the chain are retriable.
                 const retriable = !findErrorBy(error, 'canRetry', false);
                 errors.push(error);
                 if (attempt < retries && retriable && !shouldSkipRetry?.(error)) {
@@ -289,7 +289,7 @@ export class Base extends TypedEventEmitter {
                     const timeout = setTimeout(() => {
                         this.removeListener('client:close', onClose);
                         try {
-                            this[kPerformWithRetry](operationId, operation, callback, attempt + 1, errors, shouldSkipRetry);
+                            this[kPerformWithRetry](operationId, operation, callback, attempt + 1, errors, shouldSkipRetry, retries);
                         }
                         catch (error) {
                             errors.push(error);
@@ -435,7 +435,7 @@ export class Base extends TypedEventEmitter {
         const autocreateTopics = options.autocreateTopics ?? this[kOptions].autocreateTopics;
         this[kPerformDeduplicated](
         // Unique key to avoid mixing callbacks
-        `metadata-${topics.sort().join(',')}-${autocreateTopics}-${options.forceUpdate}`, deduplicateCallback => {
+        `metadata-${topics.sort().join(',')}-${autocreateTopics}-${options.forceUpdate}-${options.retries ?? 'default'}`, deduplicateCallback => {
             this[kPerformWithRetry]('metadata', (retryCallback, attempt) => {
                 this[kGetBootstrapConnection]((error, connection) => {
                     if (error) {
@@ -532,7 +532,7 @@ export class Base extends TypedEventEmitter {
                 };
                 this.emitWithDebug('client', 'metadata', updatedMetadata);
                 deduplicateCallback(null, updatedMetadata);
-            }, 0);
+            }, 0, [], undefined, options.retries);
         }, callback);
     }
     #forwardEvents(source, events) {

package/dist/clients/base/options.d.ts

@@ -199,6 +199,10 @@ export declare const metadataOptionsSchema: {
             type: string;
             minimum: number;
         };
+        retries: {
+            type: string;
+            minimum: number;
+        };
     };
     additionalProperties: boolean;
 };

package/dist/clients/base/options.js

@@ -67,7 +67,8 @@ export const metadataOptionsSchema = {
         topics: { type: 'array', items: idProperty },
         autocreateTopics: { type: 'boolean' },
         forceUpdate: { type: 'boolean' },
-        metadataMaxAge: { type: 'number', minimum: 0 }
+        metadataMaxAge: { type: 'number', minimum: 0 },
+        retries: { type: 'number', minimum: 0 }
     },
     additionalProperties: false
 };

package/dist/clients/base/types.d.ts

@@ -48,4 +48,5 @@ export interface MetadataOptions {
     autocreateTopics?: boolean;
     forceUpdate?: boolean;
     metadataMaxAge?: number;
+    retries?: number;
 }

package/dist/clients/consumer/consumer-protocol.d.ts

@@ -0,0 +1,32 @@
+import { type GroupAssignment } from './types.ts';
+export declare const CONSUMER_PROTOCOL_LOWEST_VERSION = 0;
+export declare const CONSUMER_PROTOCOL_HIGHEST_VERSION = 3;
+export interface ConsumerProtocolSubscriptionData {
+    version: number;
+    topics: string[];
+    userData: Buffer;
+    ownedPartitions: GroupAssignment[];
+    generationId: number;
+    rackId: string | null;
+}
+export interface ConsumerProtocolAssignmentData {
+    version: number;
+    assignedPartitions: GroupAssignment[];
+    userData: Buffer;
+}
+export declare function encodeConsumerProtocolSubscription(data: {
+    version: number;
+    topics: string[];
+    userData?: Buffer | string;
+    ownedPartitions?: GroupAssignment[];
+    generationId?: number;
+    rackId?: string | null;
+}): Buffer;
+export declare function decodeConsumerProtocolSubscription(buffer: Buffer): ConsumerProtocolSubscriptionData;
+export declare function decodeCooperativeStickyGeneration(userData: Buffer): number;
+export declare function encodeConsumerProtocolAssignment(data: {
+    version: number;
+    assignedPartitions: GroupAssignment[];
+    userData?: Buffer | string | null;
+}): Buffer;
+export declare function decodeConsumerProtocolAssignment(buffer: Buffer): ConsumerProtocolAssignmentData;

package/dist/clients/consumer/consumer-protocol.js

@@ -0,0 +1,116 @@
+import { Reader } from "../../protocol/reader.js";
+import { Writer } from "../../protocol/writer.js";
+export const CONSUMER_PROTOCOL_LOWEST_VERSION = 0;
+export const CONSUMER_PROTOCOL_HIGHEST_VERSION = 3;
+function supportedVersion(version) {
+    if (version < CONSUMER_PROTOCOL_LOWEST_VERSION) {
+        return CONSUMER_PROTOCOL_LOWEST_VERSION;
+    }
+    if (version > CONSUMER_PROTOCOL_HIGHEST_VERSION) {
+        return CONSUMER_PROTOCOL_HIGHEST_VERSION;
+    }
+    return version;
+}
+function sortedAssignments(assignments) {
+    return assignments
+        .filter(({ partitions }) => partitions.length > 0)
+        .map(({ topic, partitions }) => ({ topic, partitions: [...partitions].sort((a, b) => a - b) }))
+        .sort((a, b) => a.topic.localeCompare(b.topic));
+}
+function appendTopicPartitions(writer, assignments) {
+    return writer.appendArray(sortedAssignments(assignments), (w, { topic, partitions }) => {
+        w.appendString(topic, false).appendArray(partitions, (w, partition) => w.appendInt32(partition), false, false);
+    }, false, false);
+}
+function readTopicPartitions(reader) {
+    return reader.readArray(r => ({
+        topic: r.readString(false),
+        partitions: r.readArray(r => r.readInt32(), false, false)
+    }), false, false);
+}
+export function encodeConsumerProtocolSubscription(data) {
+    const version = supportedVersion(data.version);
+    const userData = typeof data.userData === 'string' ? Buffer.from(data.userData) : data.userData;
+    const writer = Writer.create()
+        .appendInt16(version)
+        .appendArray([...data.topics].sort(), (w, topic) => w.appendString(topic, false), false, false)
+        .appendBytes(userData, false);
+    if (version >= 1) {
+        appendTopicPartitions(writer, data.ownedPartitions ?? []);
+    }
+    if (version >= 2) {
+        writer.appendInt32(data.generationId ?? -1);
+    }
+    if (version >= 3) {
+        writer.appendString(data.rackId || null, false);
+    }
+    return writer.buffer;
+}
+export function decodeConsumerProtocolSubscription(buffer) {
+    const reader = Reader.from(buffer);
+    const encodedVersion = reader.readInt16();
+    const version = supportedVersion(encodedVersion);
+    const subscription = {
+        version: encodedVersion,
+        topics: reader.readArray(r => r.readString(false), false, false),
+        userData: reader.readBytes(false),
+        ownedPartitions: [],
+        generationId: -1,
+        rackId: null
+    };
+    // KafkaJS and Platformatic <= 2.11 may advertise newer versions with a v0 body.
+    // Default absent trailing fields, but still reject partially encoded fields.
+    if (version >= 1 && reader.remaining > 0) {
+        subscription.ownedPartitions = readTopicPartitions(reader);
+    }
+    if (version >= 2 && reader.remaining > 0) {
+        subscription.generationId = reader.readInt32();
+    }
+    if (version >= 3 && reader.remaining > 0) {
+        subscription.rackId = reader.readNullableString(false);
+    }
+    return subscription;
+}
+export function decodeCooperativeStickyGeneration(userData) {
+    if (userData.length < 4) {
+        return -1;
+    }
+    try {
+        return Reader.from(userData).readInt32();
+    }
+    catch {
+        return -1;
+    }
+}
+export function encodeConsumerProtocolAssignment(data) {
+    const userData = typeof data.userData === 'string' ? Buffer.from(data.userData) : data.userData;
+    const writer = Writer.create().appendInt16(supportedVersion(data.version));
+    appendTopicPartitions(writer, data.assignedPartitions);
+    writer.appendBytes(userData ?? null, false);
+    return writer.buffer;
+}
+export function decodeConsumerProtocolAssignment(buffer) {
+    if (buffer.length === 0) {
+        return {
+            version: 0,
+            assignedPartitions: [],
+            userData: Buffer.alloc(0)
+        };
+    }
+    const reader = Reader.from(buffer);
+    const encodedVersion = reader.readInt16();
+    if (reader.remaining === 0) {
+        return {
+            version: encodedVersion,
+            assignedPartitions: [],
+            userData: Buffer.alloc(0)
+        };
+    }
+    const assignedPartitions = readTopicPartitions(reader);
+    const userData = reader.remaining > 0 ? reader.readBytes(false) : Buffer.alloc(0);
+    return {
+        version: encodedVersion,
+        assignedPartitions,
+        userData
+    };
+}

package/dist/clients/consumer/consumer.d.ts

@@ -6,12 +6,13 @@ import { Base, type BaseEvents, kCreateConnectionPool } from '../base/base.ts';
 import { type ClusterMetadata } from '../base/types.ts';
 import { MessagesStream } from './messages-stream.ts';
 import { TopicsMap } from './topics-map.ts';
-import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerHeartbeatStalledPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from './types.ts';
+import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupAutocommitErrorPayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerHeartbeatStalledPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from './types.ts';
 export interface ConsumerEvents extends BaseEvents {
     'consumer:group:join': (payload: ConsumerGroupJoinPayload) => void;
     'consumer:group:leave': (payload: ConsumerGroupLeavePayload) => void;
     'consumer:group:rejoin': () => void;
     'consumer:group:rebalance': (payload: ConsumerGroupRebalancePayload) => void;
+    'consumer:group:autocommit:error': (payload: ConsumerGroupAutocommitErrorPayload) => void;
     'consumer:heartbeat:start': (payload?: ConsumerHeartbeatPayload) => void;
     'consumer:heartbeat:cancel': (payload: ConsumerHeartbeatPayload) => void;
     'consumer:heartbeat:end': (payload?: ConsumerHeartbeatPayload) => void;
@@ -33,8 +34,8 @@ export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
     get lastHeartbeat(): Date | null;
     get coordinatorId(): number | null;
     get streamContext(): unknown;
-    close(force: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
     close(force?: boolean): Promise<void>;
+    close(force: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
     isActive(): boolean;
     consume(options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<MessagesStream<Key, Value, HeaderKey, HeaderValue>>): void;
     consume(options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>): Promise<MessagesStream<Key, Value, HeaderKey, HeaderValue>>;