@platformatic/kafka 2.11.0 → 2.12.0

package/dist/network/connection.js

@@ -45,6 +45,7 @@ export class Connection extends TypedEventEmitter {
     #correlationId;
     #nextMessage;
     #afterDrainRequests;
+    #payloadQueue;
     #requestsQueue;
     #inflightRequests;
     #responseBuffer;
@@ -74,6 +75,7 @@ export class Connection extends TypedEventEmitter {
         this.#correlationId = 0;
         this.#nextMessage = 0;
         this.#afterDrainRequests = [];
+        this.#payloadQueue = fastq((op, cb) => op(cb), 1);
         this.#requestsQueue = fastq((op, cb) => op(cb), this.#options.maxInflights ?? defaultOptions.maxInflights);
         this.#inflightRequests = new Map();
         this.#responseBuffer = new DynamicBuffer();
@@ -274,45 +276,80 @@ export class Connection extends TypedEventEmitter {
             apiVersion,
             correlationId
         });
-        const writer = Writer.create();
-        writer.appendInt16(apiKey).appendInt16(apiVersion).appendInt32(correlationId).appendString(this.#clientId, false);
-        if (hasRequestHeaderTaggedFields) {
-            writer.appendTaggedFields();
-        }
-        let payload;
-        try {
-            payload = createPayload();
-        }
-        catch (err) {
-            diagnostic.error = err;
-            connectionsApiChannel.error.publish(diagnostic);
-            return callback(err);
-        }
-        writer.appendFrom(payload).prependLength();
-        const request = {
-            correlationId,
-            apiKey,
-            apiVersion,
-            parser: responseParser,
-            payload: writer.buffer,
-            callback: null, // Will be set later
-            hasResponseHeaderTaggedFields,
-            noResponse: payload.context.noResponse ?? false,
-            diagnostic,
-            timeoutHandle: null,
-            timedOut: false
-        };
-        this.#requestsQueue.push(fastQueueCallback => {
-            request.callback = fastQueueCallback;
-            if (!request.noResponse) {
-                request.timeoutHandle = setTimeout(this.#onRequestTimeout.bind(this, request), this.#options.requestTimeout);
+        this.#payloadQueue.push(payloadQueueCallback => {
+            let payloadQueueCompleted = false;
+            const completePayloadQueue = () => {
+                if (!payloadQueueCompleted) {
+                    payloadQueueCompleted = true;
+                    payloadQueueCallback(null);
+                }
+            };
+            const onPayloadError = (error) => {
+                diagnostic.error = error;
+                connectionsApiChannel.error.publish(diagnostic);
+                try {
+                    callback(error);
+                }
+                finally {
+                    completePayloadQueue();
+                }
+            };
+            const onPayload = (payload) => {
+                try {
+                    const writer = Writer.create();
+                    writer
+                        .appendInt16(apiKey)
+                        .appendInt16(apiVersion)
+                        .appendInt32(correlationId)
+                        .appendString(this.#clientId, false);
+                    if (hasRequestHeaderTaggedFields) {
+                        writer.appendTaggedFields();
+                    }
+                    writer.appendFrom(payload).prependLength();
+                    const request = {
+                        correlationId,
+                        apiKey,
+                        apiVersion,
+                        parser: responseParser,
+                        payload: writer.buffer,
+                        callback: null, // Will be set later
+                        hasResponseHeaderTaggedFields,
+                        noResponse: payload.context.noResponse ?? false,
+                        diagnostic,
+                        timeoutHandle: null,
+                        timedOut: false
+                    };
+                    this.#requestsQueue.push(fastQueueCallback => {
+                        request.callback = fastQueueCallback;
+                        if (!request.noResponse) {
+                            request.timeoutHandle = setTimeout(this.#onRequestTimeout.bind(this, request), this.#options.requestTimeout);
+                        }
+                        if (this.#socketMustBeDrained) {
+                            this.#afterDrainRequests.push(request);
+                            return false;
+                        }
+                        return this.#sendRequest(request);
+                    }, this.#onResponse.bind(this, request, callback));
+                }
+                catch (error) {
+                    onPayloadError(error);
+                    return;
+                }
+                completePayloadQueue();
+            };
+            try {
+                const payload = createPayload();
+                if (typeof payload.then === 'function') {
+                    Promise.resolve(payload).then(onPayload, onPayloadError);
+                }
+                else {
+                    onPayload(payload);
+                }
             }
-            if (this.#socketMustBeDrained) {
-                this.#afterDrainRequests.push(request);
-                return false;
+            catch (error) {
+                onPayloadError(error);
             }
-            return this.#sendRequest(request);
-        }, this.#onResponse.bind(this, request, callback));
+        }, () => { });
     }
     reauthenticate() {
         if (!this.#options.sasl) {
@@ -332,7 +369,11 @@ export class Connection extends TypedEventEmitter {
     }
     #onRequestTimeout(request) {
         request.timedOut = true;
-        request.callback(new TimeoutError('Request timed out'), null);
+        this.#inflightRequests.delete(request.correlationId);
+        this.#afterDrainRequests = this.#afterDrainRequests.filter(r => r !== request);
+        this.#status = ConnectionStatuses.CLOSING;
+        request.callback(new TimeoutError('Request timed out', { canRetry: true }), null);
+        this.#socket?.destroy();
     }
     #authenticate(host, port, diagnosticContext) {
         if (this.#status === ConnectionStatuses.CONNECTING) {
@@ -628,12 +669,14 @@ export class Connection extends TypedEventEmitter {
         this.emit('close');
         const error = new NetworkError('Connection closed');
         for (const request of this.#afterDrainRequests) {
-            if (!request.noResponse) {
+            if (!request.noResponse && !request.timedOut) {
                 request.callback(error, undefined);
             }
         }
         for (const inflight of this.#inflightRequests.values()) {
-            inflight.callback(error, undefined);
+            if (!inflight.timedOut) {
+                inflight.callback(error, undefined);
+            }
         }
     }
     #onError(error) {

package/dist/protocol/compression.d.ts

@@ -1,7 +1,9 @@
 import { DynamicBuffer } from '@platformatic/dynamic-buffer';
 export type SyncCompressionPhase = (data: Buffer | DynamicBuffer) => Buffer;
+export type AsyncCompressionPhase = (data: Buffer | DynamicBuffer) => Promise<Buffer>;
 export type CompressionOperation = (data: Buffer) => Buffer;
 export interface CompressionAlgorithmSpecification {
+    compress?: AsyncCompressionPhase;
     compressSync: SyncCompressionPhase;
     decompressSync: SyncCompressionPhase;
     bitmask: number;
@@ -17,6 +19,7 @@ export declare const CompressionAlgorithms: {
 export declare const allowedCompressionsAlgorithms: CompressionAlgorithmValue[];
 export type CompressionAlgorithm = keyof typeof CompressionAlgorithms;
 export type CompressionAlgorithmValue = (typeof CompressionAlgorithms)[keyof typeof CompressionAlgorithms];
+declare function gzipAsync(data: Buffer | DynamicBuffer): Promise<Buffer>;
 export declare const compressionsAlgorithms: {
     readonly none: {
         readonly compressSync: (data: Buffer | DynamicBuffer) => Buffer;
@@ -25,6 +28,7 @@ export declare const compressionsAlgorithms: {
         readonly available: true;
     };
     readonly gzip: {
+        readonly compress: typeof gzipAsync;
         readonly compressSync: (data: Buffer | DynamicBuffer) => Buffer;
         readonly decompressSync: (data: Buffer | DynamicBuffer) => Buffer;
         readonly bitmask: 1;
@@ -56,6 +60,7 @@ export declare const compressionsAlgorithmsByBitmask: {
         readonly bitmask: 0;
         readonly available: true;
     } | {
+        readonly compress: typeof gzipAsync;
         readonly compressSync: (data: Buffer | DynamicBuffer) => Buffer;
         readonly decompressSync: (data: Buffer | DynamicBuffer) => Buffer;
         readonly bitmask: 1;
@@ -77,3 +82,4 @@ export declare const compressionsAlgorithmsByBitmask: {
         readonly available: boolean;
     };
 };
+export {};

package/dist/protocol/compression.js

@@ -2,7 +2,7 @@ import { DynamicBuffer } from '@platformatic/dynamic-buffer';
 import { lz4Compress, lz4Decompress, snappyCompress, snappyDecompress } from '@platformatic/wasm-utils';
 import zlib from 'node:zlib';
 import { UnsupportedCompressionError } from "../errors.js";
-const { zstdCompressSync, zstdDecompressSync, gzipSync, gunzipSync } = zlib;
+const { gzip, zstdCompressSync, zstdDecompressSync, gzipSync, gunzipSync } = zlib;
 export const CompressionAlgorithms = {
     NONE: 'none',
     GZIP: 'gzip',
@@ -14,6 +14,18 @@ export const allowedCompressionsAlgorithms = Object.values(CompressionAlgorithms
 function ensureBuffer(data) {
     return DynamicBuffer.isDynamicBuffer(data) ? data.buffer : data;
 }
+function gzipAsync(data) {
+    const { promise, resolve, reject } = Promise.withResolvers();
+    gzip(ensureBuffer(data), (error, result) => {
+        if (error) {
+            reject(error);
+        }
+        else {
+            resolve(result);
+        }
+    });
+    return promise;
+}
 const snappyCompressSync = snappyCompress;
 const lz4CompressFrameSync = lz4Compress;
 const lz4DecompressFrameSync = lz4Decompress;
@@ -53,6 +65,7 @@ export const compressionsAlgorithms = {
         available: true
     },
     gzip: {
+        compress: gzipAsync,
         compressSync(data) {
             return gzipSync(ensureBuffer(data));
         },

package/dist/protocol/records.d.ts

@@ -123,5 +123,6 @@ export declare const messageSchema: {
 export declare function createRecord(message: MessageRecord, offsetDelta: number, firstTimestamp: bigint): Writer;
 export declare function readRecord(reader: Reader): KafkaRecord;
 export declare function createRecordsBatch(messages: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Writer;
+export declare function createRecordsBatchAsync(messages: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Promise<Writer>;
 export declare function readRecordsBatch(reader: Reader): RecordsBatch;
 export declare function readRecordsBatches(reader: Reader): RecordsBatch[];

package/dist/protocol/records.js

@@ -58,11 +58,11 @@ export function readRecord(reader) {
         headers: reader.readVarIntArray(r => [r.readVarIntBytes(), r.readVarIntBytes()])
     };
 }
-export function createRecordsBatch(messages, options = {}) {
+function prepareRecordsBatch(messages, options) {
     const now = BigInt(Date.now());
     const firstTimestamp = messages[0].timestamp ?? now;
     let maxTimestamp = firstTimestamp;
-    let buffer = new DynamicBuffer();
+    const buffer = new DynamicBuffer();
     for (let i = 0; i < messages.length; i++) {
         let ts = messages[i].timestamp ?? now;
         /* c8 ignore next 3 - Hard to test */
@@ -81,32 +81,34 @@ export function createRecordsBatch(messages, options = {}) {
         const firstMessage = messages[0];
         firstSequence = options.sequences.getWithDefault(`${firstMessage.topic}:${firstMessage.partition}`, 0);
     }
-    // Set the transaction
     if (options.transactionalId) {
         attributes |= IS_TRANSACTIONAL;
     }
-    // Set the compression, if any
-    if ((options.compression ?? 'none') !== 'none') {
-        const algorithm = compressionsAlgorithms[options.compression];
-        if (!algorithm) {
-            throw new UnsupportedCompressionError(`Unsupported compression algorithm ${options.compression}`);
-        }
-        attributes |= algorithm.bitmask;
-        const compressed = algorithm.compressSync(buffer.buffer);
-        buffer = new DynamicBuffer(compressed);
+    return { attributes, buffer, firstSequence, firstTimestamp, maxTimestamp };
+}
+function getCompressionAlgorithm(compression) {
+    if ((compression ?? 'none') === 'none') {
+        return undefined;
+    }
+    const algorithm = compressionsAlgorithms[compression];
+    if (!algorithm) {
+        throw new UnsupportedCompressionError(`Unsupported compression algorithm ${compression}`);
     }
+    return algorithm;
+}
+function finalizeRecordsBatch(messages, options, prepared) {
     const writer = Writer.create()
         // Phase 1: Prepare the message from Attributes (included) to the end
-        .appendInt16(attributes)
+        .appendInt16(prepared.attributes)
         // LastOffsetDelta, FirstTimestamp and MaxTimestamp are extracted from the messages
         .appendInt32(messages.length - 1)
-        .appendInt64(BigInt(firstTimestamp))
-        .appendInt64(BigInt(maxTimestamp))
+        .appendInt64(BigInt(prepared.firstTimestamp))
+        .appendInt64(BigInt(prepared.maxTimestamp))
         .appendInt64(options.producerId ?? -1n)
         .appendInt16(options.producerEpoch ?? 0)
-        .appendInt32(firstSequence)
+        .appendInt32(prepared.firstSequence)
         .appendInt32(messages.length) // Number of records
-        .appendFrom(buffer);
+        .appendFrom(prepared.buffer);
     // Phase 2: Prepend the PartitionLeaderEpoch, Magic and CRC, then the Length and firstOffset, in reverse order
     return (writer
         .appendUnsignedInt32(crc32c(writer.dynamicBuffer), false)
@@ -116,6 +118,27 @@ export function createRecordsBatch(messages, options = {}) {
         // FirstOffset is 0
         .appendInt64(0n, false));
 }
+export function createRecordsBatch(messages, options = {}) {
+    const prepared = prepareRecordsBatch(messages, options);
+    const algorithm = getCompressionAlgorithm(options.compression);
+    if (algorithm) {
+        prepared.attributes |= algorithm.bitmask;
+        prepared.buffer = new DynamicBuffer(algorithm.compressSync(prepared.buffer.buffer));
+    }
+    return finalizeRecordsBatch(messages, options, prepared);
+}
+export async function createRecordsBatchAsync(messages, options = {}) {
+    const prepared = prepareRecordsBatch(messages, options);
+    const algorithm = getCompressionAlgorithm(options.compression);
+    if (algorithm) {
+        prepared.attributes |= algorithm.bitmask;
+        const compressed = algorithm.compress
+            ? await algorithm.compress(prepared.buffer.buffer)
+            : algorithm.compressSync(prepared.buffer.buffer);
+        prepared.buffer = new DynamicBuffer(compressed);
+    }
+    return finalizeRecordsBatch(messages, options, prepared);
+}
 // TODO: Early bail out if there are not enough bytes to read all the records as it might be truncated
 export function readRecordsBatch(reader) {
     const initialPosition = reader.position;

package/dist/registries/confluent-schema-registry.d.ts

@@ -4,7 +4,8 @@ import { type ConnectionOptions as TLSConnectionOptions } from 'node:tls';
 import { type Root } from 'protobufjs';
 import { type Callback } from '../apis/definitions.ts';
 import { type BeforeDeserializationHook, type BeforeHookPayloadType, type BeforeSerializationHook, type Deserializers, type Serializers } from '../clients/serde.ts';
-import { type CredentialProvider, UserError } from '../index.ts';
+import { UserError } from '../errors.ts';
+import { type CredentialProvider } from '../network/connection.ts';
 import { type MessageToConsume, type MessageToProduce } from '../protocol/records.ts';
 import { AbstractSchemaRegistry } from './abstract.ts';
 type ConfluentSchemaRegistryMessageToProduce = MessageToProduce<unknown, unknown, unknown, unknown>;

package/dist/registries/confluent-schema-registry.js

@@ -6,7 +6,8 @@ import avro from 'avsc';
 import { createRequire } from 'node:module';
 import { setTimeout as sleep } from 'node:timers/promises';
 import { jsonDeserializer, jsonSerializer, stringDeserializer, stringSerializer } from "../clients/serde.js";
-import { EMPTY_BUFFER, MultipleErrors, NetworkError, TimeoutError, UnsupportedFormatError, UserError } from "../index.js";
+import { MultipleErrors, NetworkError, TimeoutError, UnsupportedFormatError, UserError } from "../errors.js";
+import { EMPTY_BUFFER } from "../protocol/definitions.js";
 import { getCredential } from "../protocol/sasl/utils.js";
 import { AbstractSchemaRegistry } from "./abstract.js";
 const require = createRequire(import.meta.url);

package/dist/symbols.d.ts

@@ -1,6 +1,7 @@
 export declare const kInstance: unique symbol;
 export declare const kRefreshOffsetsAndFetch: unique symbol;
 export declare const kAutocommit: unique symbol;
+export declare const kUpdateCommittedOffset: unique symbol;
 export declare const kGetFetchNode: unique symbol;
 export declare const kDeserializationError: unique symbol;
 export declare const kTransaction: unique symbol;

package/dist/symbols.js

@@ -2,6 +2,7 @@
 export const kInstance = Symbol('plt.kafka.base.instance');
 export const kRefreshOffsetsAndFetch = Symbol('plt.kafka.messagesStream.refreshOffsetsAndFetch');
 export const kAutocommit = Symbol('plt.kafka.messagesStream.autocommit');
+export const kUpdateCommittedOffset = Symbol('plt.kafka.messagesStream.updateCommittedOffset');
 export const kGetFetchNode = Symbol('plt.kafka.consumer.getFetchNode');
 export const kDeserializationError = Symbol('plt.kafka.messagesStream.deserializationError');
 export const kTransaction = Symbol('plt.kafka.producer.transactions');

package/dist/typescript-4/dist/apis/definitions.d.ts

@@ -4,7 +4,7 @@ import { type Reader } from "../protocol/reader";
 import { type Writer } from "../protocol/writer";
 export type Callback<ReturnType> = (error: Error | null, payload?: ReturnType) => void;
 export type CallbackArguments<ReturnType> = [cb: Callback<ReturnType>];
-export type RequestCreator = (...args: any[]) => Writer;
+export type RequestCreator = (...args: any[]) => Writer | Promise<Writer>;
 export type ResponseParser<ReturnType> = (correlationId: number, apiKey: number, apiVersion: number, reader: Reader) => ReturnType | Promise<ReturnType>;
 export type ResponseErrorWithLocation = [string, [number, NullableString]];
 export type APIWithCallback<RequestArguments extends Array<unknown>, ResponseType> = (connection: Connection, ...args: [...RequestArguments, ...Partial<CallbackArguments<ResponseType>>]) => void;

package/dist/typescript-4/dist/apis/producer/produce-v10.d.ts

@@ -37,5 +37,6 @@ export interface ProduceResponse {
   nodeEndpoints?: ProduceResponseNodeEndpoint[];
 }
 export declare function createRequest(acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Writer;
+export declare function createRequestAsync(acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Promise<Writer>;
 export declare function parseResponse(_correlationId: number, apiKey: number, apiVersion: number, reader: Reader): ProduceResponse;
 export declare const api: import("../definitions").API<[acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions> | undefined], boolean | ProduceResponse>;

package/dist/typescript-4/dist/apis/producer/produce-v11.d.ts

@@ -37,5 +37,6 @@ export interface ProduceResponse {
   nodeEndpoints?: ProduceResponseNodeEndpoint[];
 }
 export declare function createRequest(acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Writer;
+export declare function createRequestAsync(acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Promise<Writer>;
 export declare function parseResponse(_correlationId: number, apiKey: number, apiVersion: number, reader: Reader): ProduceResponse;
 export declare const api: import("../definitions").API<[acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions> | undefined], boolean | ProduceResponse>;

package/dist/typescript-4/dist/apis/producer/produce-v7.d.ts

@@ -25,5 +25,6 @@ export interface ProduceResponse {
   throttleTimeMs: number;
 }
 export declare function createRequest(acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Writer;
+export declare function createRequestAsync(acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Promise<Writer>;
 export declare function parseResponse(_correlationId: number, apiKey: number, apiVersion: number, reader: Reader): ProduceResponse;
 export declare const api: import("../definitions").API<[acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions> | undefined], boolean | ProduceResponse>;

package/dist/typescript-4/dist/apis/producer/produce-v8.d.ts

@@ -25,5 +25,6 @@ export interface ProduceResponse {
   throttleTimeMs: number;
 }
 export declare function createRequest(acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Writer;
+export declare function createRequestAsync(acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Promise<Writer>;
 export declare function parseResponse(_correlationId: number, apiKey: number, apiVersion: number, reader: Reader): ProduceResponse;
 export declare const api: import("../definitions").API<[acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions> | undefined], boolean | ProduceResponse>;

package/dist/typescript-4/dist/apis/producer/produce-v9.d.ts

@@ -25,5 +25,6 @@ export interface ProduceResponse {
   throttleTimeMs: number;
 }
 export declare function createRequest(acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Writer;
+export declare function createRequestAsync(acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Promise<Writer>;
 export declare function parseResponse(_correlationId: number, apiKey: number, apiVersion: number, reader: Reader): ProduceResponse;
 export declare const api: import("../definitions").API<[acks: number | undefined, timeout: number | undefined, topicData: MessageRecord[], options?: Partial<CreateRecordsBatchOptions> | undefined], boolean | ProduceResponse>;

package/dist/typescript-4/dist/apis/producer/utils.d.ts

@@ -0,0 +1,11 @@
+import { type CreateRecordsBatchOptions, type MessageRecord } from "../../protocol/records";
+import { type Writer } from "../../protocol/writer";
+export interface EncodedProducePartition {
+  partition: number;
+  records: Writer;
+}
+export interface EncodedProduceTopic {
+  topic: string;
+  partitions: EncodedProducePartition[];
+}
+export declare function encodeProduceTopicData(topicData: MessageRecord[], options: Partial<CreateRecordsBatchOptions>): Promise<EncodedProduceTopic[]>;

package/dist/typescript-4/dist/clients/base/base.d.ts

@@ -84,7 +84,7 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions, EventsT
   [kMetadata](options: MetadataOptions, callback: CallbackWithPromise<ClusterMetadata>): void;
   [kCheckNotClosed](callback: CallbackWithPromise<any>): boolean;
   clearMetadata(): void;
-  [kPerformWithRetry]<ReturnType>(operationId: string, operation: (callback: Callback<ReturnType>, attempt: number) => void, callback: CallbackWithPromise<ReturnType>, attempt?: number, errors?: Error[], shouldSkipRetry?: (e: Error) => boolean): void | Promise<ReturnType>;
+  [kPerformWithRetry]<ReturnType>(operationId: string, operation: (callback: Callback<ReturnType>, attempt: number) => void, callback: CallbackWithPromise<ReturnType>, attempt?: number, errors?: Error[], shouldSkipRetry?: (e: Error) => boolean, retryLimit?: number): void | Promise<ReturnType>;
   [kPerformDeduplicated]<ReturnType>(operationId: string, operation: (callback: CallbackWithPromise<ReturnType>) => void, callback: CallbackWithPromise<ReturnType>): void | Promise<ReturnType>;
   [kGetApi]<RequestArguments extends Array<unknown>, ResponseType>(name: string, callback: Callback<API<RequestArguments, ResponseType>>): void;
   [kGetConnection](broker: Broker, callback: Callback<Connection>): void;

package/dist/typescript-4/dist/clients/base/options.d.ts

@@ -199,6 +199,10 @@ export declare const metadataOptionsSchema: {
       type: string;
       minimum: number;
     };
+    retries: {
+      type: string;
+      minimum: number;
+    };
   };
   additionalProperties: boolean;
 };

package/dist/typescript-4/dist/clients/base/types.d.ts

@@ -48,4 +48,5 @@ export interface MetadataOptions {
   autocreateTopics?: boolean;
   forceUpdate?: boolean;
   metadataMaxAge?: number;
+  retries?: number;
 }

package/dist/typescript-4/dist/clients/consumer/consumer-protocol.d.ts

@@ -0,0 +1,32 @@
+import { type GroupAssignment } from "./types";
+export declare const CONSUMER_PROTOCOL_LOWEST_VERSION = 0;
+export declare const CONSUMER_PROTOCOL_HIGHEST_VERSION = 3;
+export interface ConsumerProtocolSubscriptionData {
+  version: number;
+  topics: string[];
+  userData: Buffer;
+  ownedPartitions: GroupAssignment[];
+  generationId: number;
+  rackId: string | null;
+}
+export interface ConsumerProtocolAssignmentData {
+  version: number;
+  assignedPartitions: GroupAssignment[];
+  userData: Buffer;
+}
+export declare function encodeConsumerProtocolSubscription(data: {
+  version: number;
+  topics: string[];
+  userData?: Buffer | string;
+  ownedPartitions?: GroupAssignment[];
+  generationId?: number;
+  rackId?: string | null;
+}): Buffer;
+export declare function decodeConsumerProtocolSubscription(buffer: Buffer): ConsumerProtocolSubscriptionData;
+export declare function decodeCooperativeStickyGeneration(userData: Buffer): number;
+export declare function encodeConsumerProtocolAssignment(data: {
+  version: number;
+  assignedPartitions: GroupAssignment[];
+  userData?: Buffer | string | null;
+}): Buffer;
+export declare function decodeConsumerProtocolAssignment(buffer: Buffer): ConsumerProtocolAssignmentData;

package/dist/typescript-4/dist/clients/consumer/consumer.d.ts

@@ -6,12 +6,13 @@ import { Base, type BaseEvents, kCreateConnectionPool } from "../base/base";
 import { type ClusterMetadata } from "../base/types";
 import { MessagesStream } from "./messages-stream";
 import { TopicsMap } from "./topics-map";
-import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerHeartbeatStalledPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from "./types";
+import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupAutocommitErrorPayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerHeartbeatStalledPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from "./types";
 export interface ConsumerEvents extends BaseEvents {
   'consumer:group:join': (payload: ConsumerGroupJoinPayload) => void;
   'consumer:group:leave': (payload: ConsumerGroupLeavePayload) => void;
   'consumer:group:rejoin': () => void;
   'consumer:group:rebalance': (payload: ConsumerGroupRebalancePayload) => void;
+  'consumer:group:autocommit:error': (payload: ConsumerGroupAutocommitErrorPayload) => void;
   'consumer:heartbeat:start': (payload?: ConsumerHeartbeatPayload) => void;
   'consumer:heartbeat:cancel': (payload: ConsumerHeartbeatPayload) => void;
   'consumer:heartbeat:end': (payload?: ConsumerHeartbeatPayload) => void;
@@ -33,8 +34,8 @@ export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
   get lastHeartbeat(): Date | null;
   get coordinatorId(): number | null;
   get streamContext(): unknown;
-  close(force: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
   close(force?: boolean): Promise<void>;
+  close(force: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
   isActive(): boolean;
   consume(options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<MessagesStream<Key, Value, HeaderKey, HeaderValue>>): void;
   consume(options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>): Promise<MessagesStream<Key, Value, HeaderKey, HeaderValue>>;

package/dist/typescript-4/dist/clients/consumer/index.d.ts

@@ -1,5 +1,7 @@
 export * from "./consumer";
+export * from "./consumer-protocol";
 export * from "./messages-stream";
 export * from "./options";
+export * from "./partitions-assigners";
 export * from "./topics-map";
 export * from "./types";

package/dist/typescript-4/dist/clients/consumer/messages-stream.d.ts

@@ -1,8 +1,9 @@
 import { Readable } from 'node:stream';
 import { type CallbackWithPromise } from "../../apis/callbacks";
+import type { Callback } from "../../apis/definitions";
 import type { ConnectionPool } from "../../network/connection-pool";
 import { type Message } from "../../protocol/records";
-import { kAutocommit, kInstance, kRefreshOffsetsAndFetch } from "../../symbols";
+import { kAutocommit, kInstance, kRefreshOffsetsAndFetch, kUpdateCommittedOffset } from "../../symbols";
 import { kConnections, kInspect } from "../base/base";
 import type { Consumer } from "./consumer";
 import { type CommitOptionsPartition, type ConsumeOptions } from "./types";
@@ -79,7 +80,8 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
   _construct(callback: (error?: Error) => void): void;
   _destroy(error: Error | null, callback: (error?: Error | null) => void): void;
   _read(): void;
-  [kAutocommit](): void;
+  [kUpdateCommittedOffset](topic: string, partition: number, offset: bigint): void;
+  [kAutocommit](callback?: Callback<void>): void;
   [kRefreshOffsetsAndFetch](): void;
   [kInspect](...args: unknown[]): void;
 }

package/dist/typescript-4/dist/clients/consumer/partitions-assigners.d.ts

@@ -1,3 +1,6 @@
 import { type ClusterMetadata } from "../base/types";
 import { type ExtendedGroupProtocolSubscription, type GroupPartitionsAssignments } from "./types";
-export declare function roundRobinAssigner(_current: string, members: Map<string, ExtendedGroupProtocolSubscription>, topics: Set<string>, metadata: ClusterMetadata): GroupPartitionsAssignments[];
+export declare const ROUNDROBIN_ASSIGNOR = "roundrobin";
+export declare const COOPERATIVE_STICKY_ASSIGNOR = "cooperative-sticky";
+export declare function roundRobinAssigner(_current: string, members: Map<string, ExtendedGroupProtocolSubscription>, topics: Set<string>, metadata: ClusterMetadata): GroupPartitionsAssignments[];
+export declare function cooperativeStickyAssigner(_current: string, members: Map<string, ExtendedGroupProtocolSubscription>, topics: Set<string>, metadata: ClusterMetadata): GroupPartitionsAssignments[];

package/dist/typescript-4/dist/clients/consumer/types.d.ts

@@ -11,6 +11,10 @@ export interface GroupProtocolSubscription {
   version: number;
   metadata?: Buffer | string;
 }
+export interface TopicPartitionAssignment {
+  topic: string;
+  partition: number;
+}
 export interface GroupAssignment {
   topic: string;
   partitions: number[];
@@ -21,6 +25,9 @@ export interface GroupPartitionsAssignments {
 }
 export interface ExtendedGroupProtocolSubscription extends Omit<GroupProtocolSubscription, 'name'> {
   topics?: string[];
+  ownedPartitions?: GroupAssignment[];
+  generationId?: number;
+  rackId?: string | null;
   memberId: string;
 }
 export type Offsets = Map<string, bigint[]>;
@@ -163,6 +170,10 @@ export interface ConsumerGroupLeavePayload {
 export interface ConsumerGroupRebalancePayload {
   groupId: string;
 }
+export interface ConsumerGroupAutocommitErrorPayload {
+  groupId: string;
+  error: Error;
+}
 export interface ConsumerHeartbeatPayload {
   groupId?: string;
   memberId?: string | null;

package/dist/typescript-4/dist/clients/producer/producer.d.ts

@@ -15,8 +15,8 @@ export declare class Producer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
   get transaction(): Transaction<Key, Value, HeaderKey, HeaderValue> | undefined;
   get coordinatorId(): number | undefined;
   get streamsCount(): number;
-  close(force: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
   close(force?: boolean): Promise<void>;
+  close(force: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
   initIdempotentProducer(options: ProduceOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<ProducerInfo>): void;
   initIdempotentProducer(options: ProduceOptions<Key, Value, HeaderKey, HeaderValue>): Promise<ProducerInfo>;
   send(options: SendOptions<Key, Value, HeaderKey, HeaderValue> & {

package/dist/typescript-4/dist/network/connection.d.ts

@@ -94,6 +94,6 @@ export declare class Connection extends TypedEventEmitter<ConnectionEvents> {
   ready(): Promise<void>;
   close(callback: CallbackWithPromise<void>): void;
   close(): Promise<void>;
-  send<ReturnType>(apiKey: number, apiVersion: number, createPayload: () => Writer, responseParser: ResponseParser<ReturnType>, hasRequestHeaderTaggedFields: boolean, hasResponseHeaderTaggedFields: boolean, callback: Callback<ReturnType>): void;
+  send<ReturnType>(apiKey: number, apiVersion: number, createPayload: () => Writer | Promise<Writer>, responseParser: ResponseParser<ReturnType>, hasRequestHeaderTaggedFields: boolean, hasResponseHeaderTaggedFields: boolean, callback: Callback<ReturnType>): void;
   reauthenticate(): void;
 }