@platformatic/kafka 1.30.0 → 1.31.0

package/dist/clients/consumer/messages-stream.js

@@ -60,6 +60,7 @@ export class MessagesStream extends Readable {
     #closeCallbacks;
     #metricsConsumedMessages;
     #corruptedMessageHandler;
+    #context;
     #pushRecordsOperation;
     [kInstance];
     /*
@@ -79,7 +80,7 @@ export class MessagesStream extends Readable {
     */
     [kConnections];
     constructor(consumer, options) {
-        const { autocommit, mode, fallbackMode, maxFetches, offsets, deserializers, onCorruptedMessage, registry, beforeDeserialization, 
+        const { autocommit, mode, fallbackMode, maxFetches, offsets, context, deserializers, onCorruptedMessage, registry, beforeDeserialization, 
         // The options below are only destructured to avoid being part of structuredClone below
         partitionAssigner: _partitionAssigner, ...otherOptions } = options;
         if (offsets && mode !== MessagesStreamModes.MANUAL) {
@@ -94,7 +95,7 @@ export class MessagesStream extends Readable {
             highWaterMark: maxFetches ?? options.highWaterMark ?? defaultConsumerOptions.highWaterMark
         });
         this[kInstance] = currentInstance++;
-        this[kConnections] = consumer[kCreateConnectionPool]();
+        this[kConnections] = consumer[kCreateConnectionPool](context);
         this.#consumer = consumer;
         this.#mode = mode ?? MessagesStreamModes.LATEST;
         this.#fallbackMode = fallbackMode ?? MessagesStreamFallbackModes.LATEST;
@@ -108,7 +109,7 @@ export class MessagesStream extends Readable {
         this.#fetches = 0;
         this.#maxFetches = maxFetches ?? 0;
         this.#topics = structuredClone(options.topics);
-        this.#inflightNodes = new Set();
+        this.#inflightNodes = new Map();
         this.#keyDeserializer =
             deserializers?.key ?? noopDeserializer;
         this.#valueDeserializer =
@@ -120,6 +121,7 @@ export class MessagesStream extends Readable {
         this.#closed = false;
         this.#closeCallbacks = [];
         this.#corruptedMessageHandler = onCorruptedMessage ?? defaultCorruptedMessageHandler;
+        this.#context = context;
         if (registry) {
             this.#pushRecordsOperation = this.#beforeDeserialization.bind(this, registry.getBeforeDeserializationHook());
         }
@@ -150,6 +152,7 @@ export class MessagesStream extends Readable {
         // having some.
         this.#consumer.on('consumer:group:join', () => {
             this.#offsetsCommitted.clear();
+            this.#partitionsEpochs.clear();
             this.#scheduleRefreshOffsetsAndFetch();
         });
         if (consumer[kPrometheus]) {
@@ -169,6 +172,9 @@ export class MessagesStream extends Readable {
     get offsetsToFetch() {
         return this.#offsetsToFetch;
     }
+    get context() {
+        return this.#context;
+    }
     /* c8 ignore next 3 - Simple getter */
     get offsetsToCommit() {
         return this.#offsetsToCommit;
@@ -239,8 +245,28 @@ export class MessagesStream extends Readable {
         return this.#consumer.isConnected();
     }
     resume() {
+        const wasPaused = this.#paused;
         this.#paused = false;
-        return super.resume();
+        const result = super.resume();
+        // Restart the fetch loop when transitioning from paused → unpaused.
+        //
+        // When a downstream Duplex (e.g. a batching stream) signals backpressure,
+        // pipeline()/pipe() calls pause() on this stream, setting #paused = true.
+        // If a previously scheduled process.nextTick(#fetch) fires while #paused is
+        // true, #fetch() returns early without scheduling another iteration — the
+        // loop is now dead. Later, pipe() calls resume() when the downstream drains,
+        // but super.resume() does not reliably trigger _read() when the readable
+        // buffer is already empty and the stream is in flowing mode (Node.js
+        // considers it "already flowing" and skips the _read() → #fetch() path).
+        //
+        // The wasPaused guard prevents a premature fetch during initial pipeline()
+        // setup, where resume() is called before _construct() completes.
+        if (wasPaused) {
+            process.nextTick(() => {
+                this.#fetch();
+            });
+        }
+        return result;
     }
     // We want to track if the stream is paused explicitly by the user, while isPaused from Node.js can also
     // be true if the stream is paused because there is no consumer.
@@ -278,7 +304,9 @@ export class MessagesStream extends Readable {
         if (this.#autocommitInterval) {
             clearInterval(this.#autocommitInterval);
         }
-        callback(error);
+        this[kConnections].close(closeError => {
+            callback(closeError ?? error);
+        });
     }
     _read() {
         this.#fetch();
@@ -311,6 +339,14 @@ export class MessagesStream extends Readable {
                 this.push(null);
                 return;
             }
+            // Remove stale inflight entries that have been pending for too long.
+            // This prevents permanent partition starvation if a fetch callback is never invoked.
+            const now = Date.now();
+            for (const [node, timestamp] of this.#inflightNodes) {
+                if (now - timestamp > 120_000) {
+                    this.#inflightNodes.delete(node);
+                }
+            }
             const requests = new Map();
             const topicIds = new Map();
             // Group topic-partitions by the destination broker
@@ -351,9 +387,12 @@ export class MessagesStream extends Readable {
                     });
                 }
             }
+            if (requests.size === 0) {
+                return;
+            }
             for (const [leader, leaderRequests] of requests) {
-                this.#inflightNodes.add(leader);
-                this.#consumer.fetch({ ...this.#options, node: leader, topics: leaderRequests }, (error, response) => {
+                this.#inflightNodes.set(leader, Date.now());
+                this.#consumer.fetch({ ...this.#options, node: leader, topics: leaderRequests, connectionPool: this[kConnections] }, (error, response) => {
                     this.#inflightNodes.delete(leader);
                     this.emit('fetch');
                     if (error) {
@@ -381,7 +420,6 @@ export class MessagesStream extends Readable {
     }
     #pushRecords(metadata, topicIds, response, requestedOffsets) {
         const autocommit = this.#autocommitEnabled;
-        let canPush = true;
         const keyDeserializer = this.#keyDeserializer;
         const valueDeserializer = this.#valueDeserializer;
         const headerKeyDeserializer = this.#headerKeyDeserializer;
@@ -406,7 +444,7 @@ export class MessagesStream extends Readable {
                     const firstTimestamp = batch.firstTimestamp;
                     const firstOffset = batch.firstOffset;
                     const leaderEpoch = metadata.topics.get(topic).partitions[partition].leaderEpoch;
-                    this.#partitionsEpochs.set(`${topic}:${partition}`, batch.partitionLeaderEpoch);
+                    this.#partitionsEpochs.set(`${topic}:${partition}`, leaderEpoch);
                     // Track offsets
                     if (batch === recordsBatches[recordsBatches.length - 1]) {
                         // Track the last read offset
@@ -469,7 +507,7 @@ export class MessagesStream extends Readable {
                             };
                             diagnosticContext.result = message;
                             consumerReceivesChannel.asyncStart.publish(diagnosticContext);
-                            canPush = this.push(message);
+                            this.push(message);
                             consumerReceivesChannel.asyncEnd.publish(diagnosticContext);
                         }
                         catch (error) {
@@ -491,11 +529,19 @@ export class MessagesStream extends Readable {
         if (this.#autocommitEnabled && !this.#autocommitInterval) {
             this[kAutocommit]();
         }
-        if (canPush) {
-            process.nextTick(() => {
-                this.#fetch();
-            });
-        }
+        // Always schedule the next fetch, even when push() returned false.
+        //
+        // In pull mode, _read() would restart the loop once the buffer drains.
+        // In flowing mode with pipeline(), however, _read() is not reliably called
+        // again when the buffer is already empty — Node.js considers the stream
+        // "already flowing" and does not re-invoke _read(). The fetch loop dies
+        // and unconsumed messages remain in Kafka.
+        //
+        // Unconditionally scheduling is safe because #fetch() checks #paused,
+        // #closed, and other guards before issuing a Kafka fetch request.
+        process.nextTick(() => {
+            this.#fetch();
+        });
         if (this.#maxFetches > 0 && ++this.#fetches >= this.#maxFetches) {
             this.push(null);
         }

package/dist/clients/consumer/options.d.ts

@@ -317,6 +317,7 @@ export declare const consumeOptionsSchema: {
                 pattern: string;
             };
         };
+        context: boolean;
         mode: {
             type: string;
             enum: import("./types.ts").MessagesStreamModeValue[];
@@ -475,6 +476,8 @@ export declare const consumerOptionsSchema: {
             type: string;
             pattern: string;
         };
+        context: boolean;
+        streamContext: boolean;
     };
     required: string[];
     additionalProperties: boolean;

package/dist/clients/consumer/options.js

@@ -74,6 +74,7 @@ export const consumeOptionsSchema = {
     type: 'object',
     properties: {
         topics: { type: 'array', items: idProperty },
+        context: true,
         mode: { type: 'string', enum: allowedMessagesStreamModes },
         fallbackMode: { type: 'string', enum: allowedMessagesStreamFallbackModes },
         maxFetches: { type: 'number', minimum: 0, default: 0 },
@@ -97,6 +98,8 @@ export const consumerOptionsSchema = {
     type: 'object',
     properties: {
         groupId: idProperty,
+        context: true,
+        streamContext: true,
         ...groupOptionsProperties,
         ...consumeOptionsProperties
     },

package/dist/clients/consumer/types.d.ts

@@ -74,6 +74,7 @@ export interface ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> {
 }
 export interface StreamOptions {
     topics: string[];
+    context?: unknown;
     mode?: MessagesStreamModeValue;
     fallbackMode?: MessagesStreamFallbackModeValue;
     maxFetches?: number;
@@ -83,6 +84,8 @@ export interface StreamOptions {
 export type ConsumeOptions<Key, Value, HeaderKey, HeaderValue> = StreamOptions & ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> & GroupOptions;
 export type ConsumerOptions<Key, Value, HeaderKey, HeaderValue> = BaseOptions & {
     groupId: string;
+    context?: unknown;
+    streamContext?: unknown;
 } & (GroupOptions | ConsumerGroupOptions) & ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue>;
 export type FetchOptions<Key, Value, HeaderKey, HeaderValue> = Pick<ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue>, 'minBytes' | 'maxBytes' | 'maxWaitTime' | 'isolationLevel'> & {
     node: number;

package/dist/clients/producer/producer.js

@@ -2,7 +2,7 @@ import { randomUUID } from 'crypto';
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
 import { FindCoordinatorKeyTypes, ProduceAcks } from "../../apis/enumerations.js";
 import { createDiagnosticContext, producerInitIdempotentChannel, producerSendsChannel, producerTransactionsChannel } from "../../diagnostic.js";
-import { UserError } from "../../errors.js";
+import { GenericError, UserError } from "../../errors.js";
 import { murmur2 } from "../../protocol/murmur2.js";
 import { runAsyncSeries } from "../../registries/abstract.js";
 import { kInstance, kTransaction, kTransactionAddOffsets, kTransactionAddPartitions, kTransactionCancel, kTransactionCommitOffset, kTransactionEnd, kTransactionFindCoordinator, kTransactionPrepare } from "../../symbols.js";
@@ -671,7 +671,10 @@ export class Producer extends Base {
                 if (error) {
                     // If the last error was due to stale metadata, we retry the operation with this set of messages
                     // since the partition is already set, it should attempt on the new destination
-                    const hasStaleMetadata = error.findBy('hasStaleMetadata', true);
+                    const kafkaError = GenericError.isGenericError(error)
+                        ? error
+                        : null;
+                    const hasStaleMetadata = kafkaError?.findBy('hasStaleMetadata', true);
                     if (hasStaleMetadata && repeatOnStaleMetadata) {
                         this.clearMetadata();
                         this.#performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, false, produceOptions, callback);
@@ -682,7 +685,10 @@ export class Producer extends Base {
                 }
                 callback(error, results);
             }, 0, [], error => {
-                return repeatOnStaleMetadata && !!error.findBy('hasStaleMetadata', true);
+                if (!repeatOnStaleMetadata || !GenericError.isGenericError(error)) {
+                    return false;
+                }
+                return !!error.findBy('hasStaleMetadata', true);
             });
         });
     }

package/dist/errors.js

@@ -78,6 +78,9 @@ export class MultipleErrors extends AggregateError {
             return this;
         }
         for (const error of this.errors) {
+            if (!error) {
+                continue;
+            }
             if (error[property] === value) {
                 return error;
             }
@@ -111,7 +114,12 @@ export class ProtocolError extends GenericError {
             apiCode: code,
             serverErrorMessage,
             canRetry,
-            hasStaleMetadata: ['UNKNOWN_TOPIC_OR_PARTITION', 'LEADER_NOT_AVAILABLE', 'NOT_LEADER_OR_FOLLOWER'].includes(id),
+            hasStaleMetadata: [
+                'UNKNOWN_TOPIC_OR_PARTITION',
+                'LEADER_NOT_AVAILABLE',
+                'NOT_LEADER_OR_FOLLOWER',
+                'FENCED_LEADER_EPOCH'
+            ].includes(id),
             needsRejoin: ['MEMBER_ID_REQUIRED', 'UNKNOWN_MEMBER_ID', 'REBALANCE_IN_PROGRESS'].includes(id),
             producerFenced: id === 'INVALID_PRODUCER_EPOCH',
             rebalanceInProgress: id === 'REBALANCE_IN_PROGRESS',

package/dist/network/connection-pool.d.ts

@@ -25,6 +25,8 @@ export declare class ConnectionPool extends TypedEventEmitter<ConnectionPoolEven
     #private;
     constructor(clientId: string, connectionOptions?: ConnectionOptions);
     get instanceId(): number;
+    get ownerId(): number | undefined;
+    get context(): unknown;
     get(broker: Broker, callback: CallbackWithPromise<Connection>): void;
     get(broker: Broker): Promise<Connection>;
     getFirstAvailable(brokers: Broker[], callback: CallbackWithPromise<Connection>): void;

package/dist/network/connection-pool.js

@@ -1,14 +1,14 @@
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../apis/callbacks.js";
 import { connectionsPoolGetsChannel, createDiagnosticContext, notifyCreation } from "../diagnostic.js";
-import { TypedEventEmitter } from "../events.js";
 import { MultipleErrors } from "../errors.js";
+import { TypedEventEmitter } from "../events.js";
 import { Connection, ConnectionStatuses } from "./connection.js";
 let currentInstance = 0;
 export class ConnectionPool extends TypedEventEmitter {
     #instanceId;
     #clientId;
     #closed;
-    // @ts-ignore This is used just for debugging
+    #context;
     #ownerId;
     #connections;
     #connectionOptions;
@@ -17,6 +17,7 @@ export class ConnectionPool extends TypedEventEmitter {
         this.#closed = false;
         this.#instanceId = currentInstance++;
         this.#clientId = clientId;
+        this.#context = connectionOptions.context;
         this.#ownerId = connectionOptions.ownerId;
         this.#connections = new Map();
         this.#connectionOptions = connectionOptions;
@@ -25,6 +26,14 @@ export class ConnectionPool extends TypedEventEmitter {
     get instanceId() {
         return this.#instanceId;
     }
+    /* c8 ignore next 3 - Simple getter */
+    get ownerId() {
+        return this.#ownerId;
+    }
+    /* c8 ignore next 3 - Simple getter */
+    get context() {
+        return this.#context;
+    }
     get(broker, callback) {
         if (!callback) {
             callback = createPromisifiedCallback();

package/dist/network/connection.d.ts

@@ -46,6 +46,7 @@ export interface ConnectionOptions {
     sasl?: SASLOptions;
     ownerId?: number;
     handleBackPressure?: boolean;
+    context?: unknown;
 }
 export interface Request {
     correlationId: number;
@@ -83,7 +84,9 @@ export declare class Connection extends TypedEventEmitter<ConnectionEvents> {
     get host(): string | undefined;
     get port(): number | undefined;
     get instanceId(): number;
+    get ownerId(): number | undefined;
     get status(): ConnectionStatusValue;
+    get context(): unknown;
     get socket(): Socket;
     isConnected(): boolean;
     connect(host: string, port: number, callback?: CallbackWithPromise<void>): void | Promise<void>;

package/dist/network/connection.js

@@ -38,7 +38,7 @@ export class Connection extends TypedEventEmitter {
     #status;
     #instanceId;
     #clientId;
-    // @ts-ignore This is used just for debugging
+    #context;
     #ownerId;
     #handleBackPressure;
     #correlationId;
@@ -59,6 +59,7 @@ export class Connection extends TypedEventEmitter {
         this.#options.tls ??= this.#options.ssl;
         this.#status = ConnectionStatuses.NONE;
         this.#clientId = clientId;
+        this.#context = options.context;
         this.#ownerId = options.ownerId;
         this.#handleBackPressure = options.handleBackPressure ?? false;
         this.#correlationId = 0;
@@ -82,9 +83,16 @@ export class Connection extends TypedEventEmitter {
     get instanceId() {
         return this.#instanceId;
     }
+    /* c8 ignore next 3 - Simple getter */
+    get ownerId() {
+        return this.#ownerId;
+    }
     get status() {
         return this.#status;
     }
+    get context() {
+        return this.#context;
+    }
     get socket() {
         return this.#socket;
     }
@@ -178,16 +186,38 @@ export class Connection extends TypedEventEmitter {
         if (!callback) {
             callback = createPromisifiedCallback();
         }
-        const onConnect = () => {
+        const cleanup = () => {
+            clearTimeout(timeout);
+            this.removeListener('connect', onConnect);
             this.removeListener('error', onError);
+            this.removeListener('close', onClose);
+        };
+        const onConnect = () => {
+            cleanup();
             callback(null);
         };
         const onError = (error) => {
-            this.removeListener('connect', onConnect);
+            cleanup();
             callback(error);
         };
+        const onClose = () => {
+            cleanup();
+            callback(new NetworkError('Connection closed while waiting for ready.'));
+        };
+        const timeout = setTimeout(() => {
+            cleanup();
+            // If the connection already failed (e.g. invalid port), don't double-invoke the callback
+            if (this.#status === ConnectionStatuses.ERROR || this.#status === ConnectionStatuses.CLOSED) {
+                return;
+            }
+            this.#socket?.destroy();
+            callback(new TimeoutError(this.#host
+                ? `Connection to ${this.#host}:${this.#port} timed out.`
+                : `Connection ready timed out after ${this.#options.connectTimeout}ms.`));
+        }, this.#options.connectTimeout);
         this.once('connect', onConnect);
         this.once('error', onError);
+        this.once('close', onClose);
         this.emit('ready');
         return callback[kCallbackPromise];
     }
@@ -218,7 +248,7 @@ export class Connection extends TypedEventEmitter {
         });
         this.#status = ConnectionStatuses.CLOSING;
         this.emit('closing');
-        this.#socket.end();
+        this.#socket.destroySoon();
         return callback[kCallbackPromise];
     }
     send(apiKey, apiVersion, createPayload, responseParser, hasRequestHeaderTaggedFields, hasResponseHeaderTaggedFields, callback) {

package/dist/typescript-4/dist/clients/admin/admin.d.ts

@@ -5,7 +5,7 @@ import { type CallbackWithPromise } from "../../apis/callbacks";
 import { type Callback } from "../../apis/definitions";
 import { type Acl } from "../../apis/types";
 import { Base } from "../base/base";
-import { type AdminListOffsetsOptions, type AdminOptions, type AlterClientQuotasOptions, type AlterConfigsOptions, type AlterConsumerGroupOffsetsOptions, type BrokerLogDirDescription, type ConfigDescription, type CreateAclsOptions, type CreatedTopic, type CreatePartitionsOptions, type CreateTopicsOptions, type DeleteAclsOptions, type DeleteConsumerGroupOffsetsOptions, type DeleteRecordsOptions, type DeleteGroupsOptions, type DeleteTopicsOptions, type DeletedRecordsTopic, type DescribeAclsOptions, type DescribeClientQuotasOptions, type DescribeConfigsOptions, type DescribeGroupsOptions, type DescribeLogDirsOptions, type Group, type GroupBase, type IncrementalAlterConfigsOptions, type ListConsumerGroupOffsetsGroup, type ListConsumerGroupOffsetsOptions, type ListedOffsetsTopic, type ListGroupsOptions, type ListTopicsOptions, type RemoveMembersFromConsumerGroupOptions } from "./types";
+import { type AdminListOffsetsOptions, type AdminOptions, type AlterClientQuotasOptions, type AlterConfigsOptions, type AlterConsumerGroupOffsetsOptions, type BrokerLogDirDescription, type ConfigDescription, type CreateAclsOptions, type CreatedTopic, type CreatePartitionsOptions, type CreateTopicsOptions, type DeleteAclsOptions, type DeleteConsumerGroupOffsetsOptions, type DeleteRecordsOptions, type DeleteGroupsOptions, type DeleteTopicsOptions, type DeletedRecordsTopic, type DescribeAclsOptions, type DescribeClientQuotasOptions, type DescribeConfigsOptions, type DescribeGroupsOptions, type DescribeLogDirsOptions, type FindCoordinatorOptions, type FindCoordinatorResult, type Group, type GroupBase, type IncrementalAlterConfigsOptions, type ListConsumerGroupOffsetsGroup, type ListConsumerGroupOffsetsOptions, type ListedOffsetsTopic, type ListGroupsOptions, type ListTopicsOptions, type RemoveMembersFromConsumerGroupOptions } from "./types";
 export declare class Admin extends Base<AdminOptions> {
   #private;
   constructor(options: AdminOptions);
@@ -21,6 +21,8 @@ export declare class Admin extends Base<AdminOptions> {
   listGroups(options?: ListGroupsOptions): Promise<Map<string, GroupBase>>;
   describeGroups(options: DescribeGroupsOptions, callback: CallbackWithPromise<Map<string, Group>>): void;
   describeGroups(options: DescribeGroupsOptions): Promise<Map<string, Group>>;
+  findCoordinator(options: FindCoordinatorOptions, callback: CallbackWithPromise<FindCoordinatorResult[]>): void;
+  findCoordinator(options: FindCoordinatorOptions): Promise<FindCoordinatorResult[]>;
   deleteGroups(options: DeleteGroupsOptions, callback: CallbackWithPromise<void>): void;
   deleteGroups(options: DeleteGroupsOptions): Promise<void>;
   removeMembersFromConsumerGroup(options: RemoveMembersFromConsumerGroupOptions, callback: CallbackWithPromise<void>): void;

package/dist/typescript-4/dist/clients/admin/options.d.ts

@@ -886,6 +886,25 @@ export declare const adminListOffsetsOptionsSchema: {
   required: string[];
   additionalProperties: boolean;
 };
+export declare const findCoordinatorOptionsSchema: {
+  type: string;
+  properties: {
+    keyType: {
+      type: string;
+      enum: (0 | 1 | 2)[];
+    };
+    keys: {
+      type: string;
+      items: {
+        type: string;
+        pattern: string;
+      };
+      minItems: number;
+    };
+  };
+  required: string[];
+  additionalProperties: boolean;
+};
 export declare const createTopicsOptionsValidator: import("ajv").ValidateFunction<{
   [x: string]: {};
 }>;
@@ -947,4 +966,7 @@ export declare const deleteAclsOptionsValidator: import("ajv").ValidateFunction<
 }>;
 export declare const adminListOffsetsOptionsValidator: import("ajv").ValidateFunction<{
   [x: string]: {};
+}>;
+export declare const findCoordinatorOptionsValidator: import("ajv").ValidateFunction<{
+  [x: string]: {};
 }>;

package/dist/typescript-4/dist/clients/admin/types.d.ts

@@ -6,7 +6,7 @@ import { type DescribeClientQuotasRequestComponent } from "../../apis/admin/desc
 import { type DescribeConfigsRequestResource, type DescribeConfigsResponseConfig } from "../../apis/admin/describe-configs-v4";
 import { type DescribeLogDirsRequestTopic, type DescribeLogDirsResponse, type DescribeLogDirsResponseResult } from "../../apis/admin/describe-log-dirs-v4";
 import { type IncrementalAlterConfigsRequestResource } from "../../apis/admin/incremental-alter-configs-v1";
-import { type ConfigResourceTypeValue, type ConsumerGroupStateValue, type FetchIsolationLevelValue } from "../../apis/enumerations";
+import { type ConfigResourceTypeValue, type ConsumerGroupStateValue, type FetchIsolationLevelValue, type FindCoordinatorKeyTypeValue } from "../../apis/enumerations";
 import { type Acl, type AclFilter } from "../../apis/types";
 import { type Nullable, type NullableString } from "../../protocol/definitions";
 import { type BaseOptions } from "../base/types";
@@ -67,6 +67,16 @@ export interface DescribeGroupsOptions {
   groups: string[];
   includeAuthorizedOperations?: boolean;
 }
+export interface FindCoordinatorOptions {
+  keyType: FindCoordinatorKeyTypeValue;
+  keys: string[];
+}
+export interface FindCoordinatorResult {
+  key: string;
+  nodeId: number;
+  host: string;
+  port: number;
+}
 export interface DeleteGroupsOptions {
   groups: string[];
 }

package/dist/typescript-4/dist/clients/base/base.d.ts

@@ -30,6 +30,7 @@ export declare const kFormatValidationErrors: unique symbol;
 export declare const kPrometheus: unique symbol;
 export declare const kClientType: unique symbol;
 export declare const kAfterCreate: unique symbol;
+export declare const kContext: unique symbol;
 export interface BaseEvents extends TypedEvents {
   'client:broker:connect': (payload: ConnectionPoolEventPayload) => void;
   'client:broker:disconnect': (payload: ConnectionPoolEventPayload) => void;
@@ -52,6 +53,7 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions, EventsT
   [kInstance]: number;
   [kClientId]: string;
   [kClientType]: ClientType;
+  [kContext]: unknown;
   [kBootstrapBrokers]: Broker[];
   [kApis]: ApiVersionsResponseApi[];
   [kOptions]: OptionsType;
@@ -63,6 +65,7 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions, EventsT
   get clientId(): string;
   get closed(): boolean;
   get type(): ClientType;
+  get context(): unknown;
   emitWithDebug(section: string | null, name: string, ...args: any[]): boolean;
   close(callback: CallbackWithPromise<void>): void;
   close(): Promise<void>;
@@ -74,7 +77,7 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions, EventsT
   connectToBrokers(nodeIds?: number[] | null): Promise<Map<number, Connection>>;
   isActive(): boolean;
   isConnected(): boolean;
-  [kCreateConnectionPool](): ConnectionPool;
+  [kCreateConnectionPool](context?: unknown): ConnectionPool;
   [kListApis](callback: CallbackWithPromise<ApiVersionsResponseApi[]>): void;
   [kMetadata](options: MetadataOptions, callback: CallbackWithPromise<ClusterMetadata>): void;
   [kCheckNotClosed](callback: CallbackWithPromise<any>): boolean;

package/dist/typescript-4/dist/clients/base/index.d.ts

@@ -1,3 +1,3 @@
-export { Base, kCheckNotClosed, kGetApi, kGetBootstrapConnection, kGetConnection, kListApis, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from "./base";
+export { Base, kContext, kCheckNotClosed, kGetApi, kGetBootstrapConnection, kGetConnection, kListApis, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from "./base";
 export * from "./options";
 export * from "./types";

package/dist/typescript-4/dist/clients/base/options.d.ts

@@ -157,6 +157,7 @@ export declare const baseOptionsSchema: {
       required: string[];
       additionalProperties: boolean;
     };
+    context: boolean;
     metadataMaxAge: {
       type: string;
       minimum: number;

package/dist/typescript-4/dist/clients/base/types.d.ts

@@ -33,6 +33,7 @@ export type RetryDelayGetter<Owner = object> = (client: Owner, operationId: stri
 export interface BaseOptions extends ConnectionOptions {
   clientId: string;
   bootstrapBrokers: Broker[] | string[];
+  context?: unknown;
   timeout?: number;
   retries?: number | boolean;
   retryDelay?: number | RetryDelayGetter;

package/dist/typescript-4/dist/clients/consumer/consumer.d.ts

@@ -1,6 +1,7 @@
 import { type CallbackWithPromise } from "../../apis/callbacks";
 import { type FetchResponse } from "../../apis/consumer/fetch-v17";
-import { Base, type BaseEvents } from "../base/base";
+import { type ConnectionPool } from "../../network/connection-pool";
+import { Base, type BaseEvents, kCreateConnectionPool } from "../base/base";
 import { MessagesStream } from "./messages-stream";
 import { TopicsMap } from "./topics-map";
 import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from "./types";
@@ -28,6 +29,7 @@ export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
   get streamsCount(): number;
   get lastHeartbeat(): Date | null;
   get coordinatorId(): number | null;
+  get streamContext(): unknown;
   close(force: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
   close(force?: boolean): Promise<void>;
   isActive(): boolean;
@@ -49,6 +51,7 @@ export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
   stopLagMonitoring(): void;
   findGroupCoordinator(callback: CallbackWithPromise<number>): void;
   findGroupCoordinator(): Promise<number>;
+  [kCreateConnectionPool](context?: unknown): ConnectionPool;
   joinGroup(options: GroupOptions, callback: CallbackWithPromise<string>): void;
   joinGroup(options?: GroupOptions): Promise<string>;
   leaveGroup(force?: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;

package/dist/typescript-4/dist/clients/consumer/messages-stream.d.ts

@@ -15,6 +15,7 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
   constructor(consumer: Consumer<Key, Value, HeaderKey, HeaderValue>, options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>);
   get consumer(): Consumer<Key, Value, HeaderKey, HeaderValue>;
   get offsetsToFetch(): Map<string, bigint>;
+  get context(): unknown;
   get offsetsToCommit(): Map<string, CommitOptionsPartition>;
   get offsetsCommitted(): Map<string, bigint>;
   get committedOffsets(): Map<string, bigint>;

package/dist/typescript-4/dist/clients/consumer/options.d.ts

@@ -317,6 +317,7 @@ export declare const consumeOptionsSchema: {
         pattern: string;
       };
     };
+    context: boolean;
     mode: {
       type: string;
       enum: import("./types").MessagesStreamModeValue[];
@@ -475,6 +476,8 @@ export declare const consumerOptionsSchema: {
       type: string;
       pattern: string;
     };
+    context: boolean;
+    streamContext: boolean;
   };
   required: string[];
   additionalProperties: boolean;

package/dist/typescript-4/dist/clients/consumer/types.d.ts

@@ -74,6 +74,7 @@ export interface ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> {
 }
 export interface StreamOptions {
   topics: string[];
+  context?: unknown;
   mode?: MessagesStreamModeValue;
   fallbackMode?: MessagesStreamFallbackModeValue;
   maxFetches?: number;
@@ -83,6 +84,8 @@ export interface StreamOptions {
 export type ConsumeOptions<Key, Value, HeaderKey, HeaderValue> = StreamOptions & ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> & GroupOptions;
 export type ConsumerOptions<Key, Value, HeaderKey, HeaderValue> = BaseOptions & {
   groupId: string;
+  context?: unknown;
+  streamContext?: unknown;
 } & (GroupOptions | ConsumerGroupOptions) & ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue>;
 export type FetchOptions<Key, Value, HeaderKey, HeaderValue> = Pick<ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue>, 'minBytes' | 'maxBytes' | 'maxWaitTime' | 'isolationLevel'> & {
   node: number;