@platformatic/kafka 1.29.0 → 1.31.0

package/dist/clients/admin/admin.js

@@ -1,10 +1,10 @@
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
 import { ConfigResourceTypes, FetchIsolationLevels, FindCoordinatorKeyTypes } from "../../apis/enumerations.js";
 import { adminAclsChannel, adminClientQuotasChannel, adminConfigsChannel, adminConsumerGroupOffsetsChannel, adminGroupsChannel, adminLogDirsChannel, adminOffsetsChannel, adminTopicsChannel, createDiagnosticContext } from "../../diagnostic.js";
-import { MultipleErrors } from "../../errors.js";
+import { MultipleErrors, UserError } from "../../errors.js";
 import { Reader } from "../../protocol/reader.js";
 import { Base, kAfterCreate, kCheckNotClosed, kConnections, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from "../base/base.js";
-import { adminListOffsetsOptionsValidator, alterClientQuotasOptionsValidator, alterConfigsOptionsValidator, alterConsumerGroupOffsetsOptionsValidator, createAclsOptionsValidator, createPartitionsOptionsValidator, createTopicsOptionsValidator, deleteAclsOptionsValidator, deleteConsumerGroupOffsetsOptionsValidator, deleteRecordsOptionsValidator, deleteGroupsOptionsValidator, deleteTopicsOptionsValidator, describeAclsOptionsValidator, describeClientQuotasOptionsValidator, describeConfigsOptionsValidator, describeGroupsOptionsValidator, describeLogDirsOptionsValidator, incrementalAlterConfigsOptionsValidator, listConsumerGroupOffsetsOptionsValidator, listGroupsOptionsValidator, listTopicsOptionsValidator, removeMembersFromConsumerGroupOptionsValidator } from "./options.js";
+import { adminListOffsetsOptionsValidator, alterClientQuotasOptionsValidator, alterConfigsOptionsValidator, alterConsumerGroupOffsetsOptionsValidator, createAclsOptionsValidator, createPartitionsOptionsValidator, createTopicsOptionsValidator, deleteAclsOptionsValidator, deleteConsumerGroupOffsetsOptionsValidator, deleteRecordsOptionsValidator, deleteGroupsOptionsValidator, deleteTopicsOptionsValidator, describeAclsOptionsValidator, describeClientQuotasOptionsValidator, describeConfigsOptionsValidator, describeGroupsOptionsValidator, describeLogDirsOptionsValidator, findCoordinatorOptionsValidator, incrementalAlterConfigsOptionsValidator, listConsumerGroupOffsetsOptionsValidator, listGroupsOptionsValidator, listTopicsOptionsValidator, removeMembersFromConsumerGroupOptionsValidator } from "./options.js";
 export class Admin extends Base {
     #controller = null;
     constructor(options) {
@@ -108,6 +108,21 @@ export class Admin extends Base {
         adminGroupsChannel.traceCallback(this.#describeGroups, 1, createDiagnosticContext({ client: this, operation: 'describeGroups', options }), this, options, callback);
         return callback[kCallbackPromise];
     }
+    findCoordinator(options, callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        if (this[kCheckNotClosed](callback)) {
+            return callback[kCallbackPromise];
+        }
+        const validationError = this[kValidateOptions](options, findCoordinatorOptionsValidator, '/options', false);
+        if (validationError) {
+            callback(validationError);
+            return callback[kCallbackPromise];
+        }
+        adminGroupsChannel.traceCallback(this.#findCoordinator, 1, createDiagnosticContext({ client: this, operation: 'findCoordinator', options }), this, options, callback);
+        return callback[kCallbackPromise];
+    }
     deleteGroups(options, callback) {
         if (!callback) {
             callback = createPromisifiedCallback();
@@ -563,22 +578,22 @@ export class Admin extends Base {
                 callback(error);
                 return;
             }
-            this.#findGroupCoordinator(options.groups, (error, response) => {
+            this.#findCoordinator({ keyType: FindCoordinatorKeyTypes.GROUP, keys: options.groups }, (error, coordinators) => {
                 if (error) {
                     callback(error);
                     return;
                 }
                 // Group the groups by coordinator
-                const coordinators = new Map();
-                for (const { key: group, nodeId: node } of response.coordinators) {
-                    let coordinator = coordinators.get(node);
+                const coordinatorsMap = new Map();
+                for (const { key: group, nodeId: node } of coordinators) {
+                    let coordinator = coordinatorsMap.get(node);
                     if (!coordinator) {
                         coordinator = [];
-                        coordinators.set(node, coordinator);
+                        coordinatorsMap.set(node, coordinator);
                     }
                     coordinator.push(group);
                 }
-                runConcurrentCallbacks('Describing groups failed.', coordinators, ([node, groups], concurrentCallback) => {
+                runConcurrentCallbacks('Describing groups failed.', coordinatorsMap, ([node, groups], concurrentCallback) => {
                     this[kGetConnection](metadata.brokers.get(node), (error, connection) => {
                         if (error) {
                             concurrentCallback(error);
@@ -651,22 +666,22 @@ export class Admin extends Base {
                 callback(error);
                 return;
             }
-            this.#findGroupCoordinator(options.groups, (error, response) => {
+            this.#findCoordinator({ keyType: FindCoordinatorKeyTypes.GROUP, keys: options.groups }, (error, coordinators) => {
                 if (error) {
                     callback(error);
                     return;
                 }
                 // Group the groups by coordinator
-                const coordinators = new Map();
-                for (const { key: group, nodeId: node } of response.coordinators) {
-                    let coordinator = coordinators.get(node);
+                const coordinatorsMap = new Map();
+                for (const { key: group, nodeId: node } of coordinators) {
+                    let coordinator = coordinatorsMap.get(node);
                     if (!coordinator) {
                         coordinator = [];
-                        coordinators.set(node, coordinator);
+                        coordinatorsMap.set(node, coordinator);
                     }
                     coordinator.push(group);
                 }
-                runConcurrentCallbacks('Deleting groups failed.', coordinators, ([node, groups], concurrentCallback) => {
+                runConcurrentCallbacks('Deleting groups failed.', coordinatorsMap, ([node, groups], concurrentCallback) => {
                     this[kGetConnection](metadata.brokers.get(node), (error, connection) => {
                         if (error) {
                             concurrentCallback(error);
@@ -714,12 +729,12 @@ export class Admin extends Base {
                 callback(error, undefined);
                 return;
             }
-            this.#findGroupCoordinator([options.groupId], (error, response) => {
+            this.#findCoordinator({ keyType: FindCoordinatorKeyTypes.GROUP, keys: [options.groupId] }, (error, coordinators) => {
                 if (error) {
                     callback(new MultipleErrors('Removing members from consumer group failed.', [error]));
                     return;
                 }
-                const coordinator = response.coordinators.find(c => c.key === options.groupId);
+                const coordinator = coordinators.find(c => c.key === options.groupId);
                 /* c8 ignore next 8 - Hard to test */
                 if (!coordinator) {
                     callback(new MultipleErrors('Removing members from consumer group failed.', [
@@ -765,8 +780,8 @@ export class Admin extends Base {
             });
         });
     }
-    #findGroupCoordinator(groups, callback) {
-        this[kPerformWithRetry]('findGroupCoordinator', retryCallback => {
+    #findCoordinator(options, callback) {
+        this[kPerformWithRetry]('findCoordinator', retryCallback => {
             this[kGetBootstrapConnection]((error, connection) => {
                 if (error) {
                     retryCallback(error);
@@ -777,7 +792,7 @@ export class Admin extends Base {
                         retryCallback(error);
                         return;
                     }
-                    api(connection, FindCoordinatorKeyTypes.GROUP, groups, retryCallback);
+                    api(connection, options.keyType, options.keys, retryCallback);
                 });
             });
         }, (error, response) => {
@@ -785,7 +800,12 @@ export class Admin extends Base {
                 callback(error);
                 return;
             }
-            callback(null, response);
+            callback(null, response.coordinators.map(coordinator => ({
+                key: coordinator.key,
+                nodeId: coordinator.nodeId,
+                host: coordinator.host,
+                port: coordinator.port
+            })));
         }, 0);
     }
     #describeClientQuotas(options, callback) {
@@ -883,13 +903,13 @@ export class Admin extends Base {
             }
             /* c8 ignore next - Hard to test */
             const groupIds = options.groups.map(group => (typeof group === 'string' ? group : group.groupId));
-            this.#findGroupCoordinator(groupIds, (error, response) => {
+            this.#findCoordinator({ keyType: FindCoordinatorKeyTypes.GROUP, keys: groupIds }, (error, coordinators) => {
                 if (error) {
                     callback(new MultipleErrors('Listing consumer group offsets failed.', [error]));
                     return;
                 }
-                const coordinators = new Map();
-                for (const { key: groupId, nodeId: node } of response.coordinators) {
+                const coordinatorsMap = new Map();
+                for (const { key: groupId, nodeId: node } of coordinators) {
                     const groupRequest = {
                         groupId,
                         memberId: null,
@@ -901,14 +921,14 @@ export class Admin extends Base {
                             break;
                         }
                     }
-                    let coordinator = coordinators.get(node);
+                    let coordinator = coordinatorsMap.get(node);
                     if (!coordinator) {
                         coordinator = [];
-                        coordinators.set(node, coordinator);
+                        coordinatorsMap.set(node, coordinator);
                     }
                     coordinator.push(groupRequest);
                 }
-                runConcurrentCallbacks('Listing consumer group offsets failed.', coordinators, ([node, groups], concurrentCallback) => {
+                runConcurrentCallbacks('Listing consumer group offsets failed.', coordinatorsMap, ([node, groups], concurrentCallback) => {
                     this[kGetConnection](metadata.brokers.get(node), (error, connection) => {
                         if (error) {
                             concurrentCallback(error);
@@ -952,12 +972,12 @@ export class Admin extends Base {
                 callback(error, undefined);
                 return;
             }
-            this.#findGroupCoordinator([options.groupId], (error, response) => {
+            this.#findCoordinator({ keyType: FindCoordinatorKeyTypes.GROUP, keys: [options.groupId] }, (error, coordinators) => {
                 if (error) {
                     callback(new MultipleErrors('Altering consumer group offsets failed.', [error]));
                     return;
                 }
-                const coordinator = response.coordinators.find(c => c.key === options.groupId);
+                const coordinator = coordinators.find(c => c.key === options.groupId);
                 /* c8 ignore next 9 - Hard to test */
                 if (!coordinator) {
                     callback(new MultipleErrors('Altering consumer group offsets failed.', [
@@ -1013,12 +1033,12 @@ export class Admin extends Base {
                 callback(error);
                 return;
             }
-            this.#findGroupCoordinator([options.groupId], (error, response) => {
+            this.#findCoordinator({ keyType: FindCoordinatorKeyTypes.GROUP, keys: [options.groupId] }, (error, coordinators) => {
                 if (error) {
                     callback(new MultipleErrors('Deleting consumer group offsets failed.', [error]));
                     return;
                 }
-                const coordinator = response.coordinators.find(c => c.key === options.groupId);
+                const coordinator = coordinators.find(c => c.key === options.groupId);
                 /* c8 ignore next 9 - Hard to test */
                 if (!coordinator) {
                     callback(new MultipleErrors('Deleting consumer group offsets failed.', [
@@ -1363,14 +1383,22 @@ export class Admin extends Base {
                 callback(error);
                 return;
             }
+            // metadata must be defined at this point
+            const { topics, brokers } = metadata;
             const requests = new Map();
             for (const topic of options.topics) {
                 for (const partition of topic.partitions) {
-                    const { leader, leaderEpoch } = metadata.topics.get(topic.name).partitions[partition.partitionIndex];
-                    let leaderRequests = requests.get(leader);
+                    // topics.get(topic.name) must be defined as the metadata request was successful
+                    const topicData = topics.get(topic.name);
+                    const targetPartitionData = topicData.partitions[partition.partitionIndex];
+                    if (!targetPartitionData) {
+                        callback(new UserError(`Unknown partition ${partition.partitionIndex} for topic ${topic.name}.`));
+                        return;
+                    }
+                    let leaderRequests = requests.get(targetPartitionData.leader);
                     if (!leaderRequests) {
                         leaderRequests = [];
-                        requests.set(leader, leaderRequests);
+                        requests.set(targetPartitionData.leader, leaderRequests);
                     }
                     let topicRequest = leaderRequests.find(t => t.name === topic.name);
                     if (!topicRequest) {
@@ -1379,14 +1407,14 @@ export class Admin extends Base {
                     }
                     topicRequest.partitions.push({
                         partitionIndex: partition.partitionIndex,
-                        currentLeaderEpoch: leaderEpoch,
+                        currentLeaderEpoch: targetPartitionData.leaderEpoch,
                         /* c8 ignore next - Hard to test */
                         timestamp: partition.timestamp ?? -1n
                     });
                 }
             }
             runConcurrentCallbacks('Listing offsets failed.', requests, ([leader, requests], concurrentCallback) => {
-                this[kGetConnection](metadata.brokers.get(leader), (error, connection) => {
+                this[kGetConnection](brokers.get(leader), (error, connection) => {
                     if (error) {
                         concurrentCallback(error);
                         return;

package/dist/clients/admin/options.d.ts

@@ -886,6 +886,25 @@ export declare const adminListOffsetsOptionsSchema: {
     required: string[];
     additionalProperties: boolean;
 };
+export declare const findCoordinatorOptionsSchema: {
+    type: string;
+    properties: {
+        keyType: {
+            type: string;
+            enum: (0 | 1 | 2)[];
+        };
+        keys: {
+            type: string;
+            items: {
+                type: string;
+                pattern: string;
+            };
+            minItems: number;
+        };
+    };
+    required: string[];
+    additionalProperties: boolean;
+};
 export declare const createTopicsOptionsValidator: import("ajv").ValidateFunction<{
     [x: string]: {};
 }>;
@@ -948,3 +967,6 @@ export declare const deleteAclsOptionsValidator: import("ajv").ValidateFunction<
 export declare const adminListOffsetsOptionsValidator: import("ajv").ValidateFunction<{
     [x: string]: {};
 }>;
+export declare const findCoordinatorOptionsValidator: import("ajv").ValidateFunction<{
+    [x: string]: {};
+}>;

package/dist/clients/admin/options.js

@@ -1,4 +1,4 @@
-import { allowedAclOperations, allowedAclPermissionTypes, allowedClientQuotaMatchTypes, allowedFetchIsolationLevels, allowedIncrementalAlterConfigOperationTypes, allowedResourcePatternTypes, allowedResourceTypes, ConsumerGroupStates, IncrementalAlterConfigOperationTypes } from "../../apis/enumerations.js";
+import { allowedAclOperations, allowedAclPermissionTypes, allowedClientQuotaMatchTypes, allowedFetchIsolationLevels, allowedFindCoordinatorKeyTypes, allowedIncrementalAlterConfigOperationTypes, allowedResourcePatternTypes, allowedResourceTypes, ConsumerGroupStates, IncrementalAlterConfigOperationTypes } from "../../apis/enumerations.js";
 import { ajv, listErrorMessage } from "../../utils.js";
 import { idProperty } from "../base/options.js";
 export const groupsProperties = {
@@ -587,6 +587,19 @@ export const adminListOffsetsOptionsSchema = {
     required: ['topics'],
     additionalProperties: false
 };
+export const findCoordinatorOptionsSchema = {
+    type: 'object',
+    properties: {
+        keyType: { type: 'number', enum: [...allowedFindCoordinatorKeyTypes] },
+        keys: {
+            type: 'array',
+            items: idProperty,
+            minItems: 1
+        }
+    },
+    required: ['keyType', 'keys'],
+    additionalProperties: false
+};
 export const createTopicsOptionsValidator = ajv.compile(createTopicOptionsSchema);
 export const createPartitionsOptionsValidator = ajv.compile(createPartitionsOptionsSchema);
 export const listTopicsOptionsValidator = ajv.compile(listTopicOptionsSchema);
@@ -609,3 +622,4 @@ export const createAclsOptionsValidator = ajv.compile(createAclsOptionsSchema);
 export const describeAclsOptionsValidator = ajv.compile(describeAclsOptionsSchema);
 export const deleteAclsOptionsValidator = ajv.compile(deleteAclsOptionsSchema);
 export const adminListOffsetsOptionsValidator = ajv.compile(adminListOffsetsOptionsSchema);
+export const findCoordinatorOptionsValidator = ajv.compile(findCoordinatorOptionsSchema);

package/dist/clients/admin/types.d.ts

@@ -6,7 +6,7 @@ import { type DescribeClientQuotasRequestComponent } from '../../apis/admin/desc
 import { type DescribeConfigsRequestResource, type DescribeConfigsResponseConfig } from '../../apis/admin/describe-configs-v4.ts';
 import { type DescribeLogDirsRequestTopic, type DescribeLogDirsResponse, type DescribeLogDirsResponseResult } from '../../apis/admin/describe-log-dirs-v4.ts';
 import { type IncrementalAlterConfigsRequestResource } from '../../apis/admin/incremental-alter-configs-v1.ts';
-import { type ConfigResourceTypeValue, type ConsumerGroupStateValue, type FetchIsolationLevelValue } from '../../apis/enumerations.ts';
+import { type ConfigResourceTypeValue, type ConsumerGroupStateValue, type FetchIsolationLevelValue, type FindCoordinatorKeyTypeValue } from '../../apis/enumerations.ts';
 import { type Acl, type AclFilter } from '../../apis/types.ts';
 import { type Nullable, type NullableString } from '../../protocol/definitions.ts';
 import { type BaseOptions } from '../base/types.ts';
@@ -68,6 +68,16 @@ export interface DescribeGroupsOptions {
     groups: string[];
     includeAuthorizedOperations?: boolean;
 }
+export interface FindCoordinatorOptions {
+    keyType: FindCoordinatorKeyTypeValue;
+    keys: string[];
+}
+export interface FindCoordinatorResult {
+    key: string;
+    nodeId: number;
+    host: string;
+    port: number;
+}
 export interface DeleteGroupsOptions {
     groups: string[];
 }

package/dist/clients/base/base.d.ts

@@ -30,6 +30,7 @@ export declare const kFormatValidationErrors: unique symbol;
 export declare const kPrometheus: unique symbol;
 export declare const kClientType: unique symbol;
 export declare const kAfterCreate: unique symbol;
+export declare const kContext: unique symbol;
 export interface BaseEvents extends TypedEvents {
     'client:broker:connect': (payload: ConnectionPoolEventPayload) => void;
     'client:broker:disconnect': (payload: ConnectionPoolEventPayload) => void;
@@ -52,6 +53,7 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions, EventsT
     [kInstance]: number;
     [kClientId]: string;
     [kClientType]: ClientType;
+    [kContext]: unknown;
     [kBootstrapBrokers]: Broker[];
     [kApis]: ApiVersionsResponseApi[];
     [kOptions]: OptionsType;
@@ -63,6 +65,7 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions, EventsT
     get clientId(): string;
     get closed(): boolean;
     get type(): ClientType;
+    get context(): unknown;
     emitWithDebug(section: string | null, name: string, ...args: any[]): boolean;
     close(callback: CallbackWithPromise<void>): void;
     close(): Promise<void>;
@@ -74,7 +77,7 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions, EventsT
     connectToBrokers(nodeIds?: number[] | null): Promise<Map<number, Connection>>;
     isActive(): boolean;
     isConnected(): boolean;
-    [kCreateConnectionPool](): ConnectionPool;
+    [kCreateConnectionPool](context?: unknown): ConnectionPool;
     [kListApis](callback: CallbackWithPromise<ApiVersionsResponseApi[]>): void;
     [kMetadata](options: MetadataOptions, callback: CallbackWithPromise<ClusterMetadata>): void;
     [kCheckNotClosed](callback: CallbackWithPromise<any>): boolean;

package/dist/clients/base/base.js

@@ -30,6 +30,7 @@ export const kFormatValidationErrors = Symbol('plt.kafka.base.formatValidationEr
 export const kPrometheus = Symbol('plt.kafka.base.prometheus');
 export const kClientType = Symbol('plt.kafka.base.clientType');
 export const kAfterCreate = Symbol('plt.kafka.base.afterCreate');
+export const kContext = Symbol('plt.kafka.base.context');
 let currentInstance = 0;
 export class Base extends TypedEventEmitter {
     // This is declared using a symbol (a.k.a protected/friend) to make it available in ConnectionPool and MessagesStream
@@ -37,6 +38,7 @@ export class Base extends TypedEventEmitter {
     // General status - Use symbols rather than JS private property to make them "protected" as in C++
     [kClientId];
     [kClientType];
+    [kContext];
     [kBootstrapBrokers];
     [kApis];
     [kOptions];
@@ -51,6 +53,7 @@ export class Base extends TypedEventEmitter {
         this[kClientType] = 'base';
         this[kInstance] = currentInstance++;
         this[kApis] = [];
+        this[kContext] = options.context;
         // Validate options
         this[kOptions] = Object.assign({}, defaultBaseOptions, options);
         this[kValidateOptions](this[kOptions], baseOptionsValidator, '/options');
@@ -84,6 +87,9 @@ export class Base extends TypedEventEmitter {
     get type() {
         return this[kClientType];
     }
+    get context() {
+        return this[kContext];
+    }
     emitWithDebug(section, name, ...args) {
         if (!section) {
             return this.emit(name, ...args);
@@ -170,10 +176,11 @@ export class Base extends TypedEventEmitter {
         }
         return this[kConnections].isConnected();
     }
-    [kCreateConnectionPool]() {
+    [kCreateConnectionPool](context) {
         const pool = new ConnectionPool(this[kClientId], {
+            ...this[kOptions],
             ownerId: this[kInstance],
-            ...this[kOptions]
+            context: context ?? this[kContext]
         });
         this.#forwardEvents(pool, [
             'connect',
@@ -388,6 +395,14 @@ export class Base extends TypedEventEmitter {
                 });
             }, (error, metadata) => {
                 if (error) {
+                    const unknownTopicError = error.findBy('apiCode', 3);
+                    if (unknownTopicError) {
+                        const topicIndexMatch = unknownTopicError.path?.match(/\/topics\/(\d+)/);
+                        const topicIndex = topicIndexMatch ? parseInt(topicIndexMatch[1]) : -1;
+                        const topicName = topicIndex >= 0 && topicIndex < topicsToFetch.length ? topicsToFetch[topicIndex] : 'unknown';
+                        deduplicateCallback(new UserError(`Unknown topic ${topicName}.`));
+                        return;
+                    }
                     const hasStaleMetadata = error.findBy('hasStaleMetadata', true);
                     // Stale metadata, we need to fetch everything again
                     if (hasStaleMetadata) {

package/dist/clients/base/index.d.ts

@@ -1,3 +1,3 @@
-export { Base, kCheckNotClosed, kGetApi, kGetBootstrapConnection, kGetConnection, kListApis, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from './base.ts';
+export { Base, kContext, kCheckNotClosed, kGetApi, kGetBootstrapConnection, kGetConnection, kListApis, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from './base.ts';
 export * from './options.ts';
 export * from './types.ts';

package/dist/clients/base/index.js

@@ -1,3 +1,3 @@
-export { Base, kCheckNotClosed, kGetApi, kGetBootstrapConnection, kGetConnection, kListApis, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from "./base.js";
+export { Base, kContext, kCheckNotClosed, kGetApi, kGetBootstrapConnection, kGetConnection, kListApis, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from "./base.js";
 export * from "./options.js";
 export * from "./types.js";

package/dist/clients/base/options.d.ts

@@ -157,6 +157,7 @@ export declare const baseOptionsSchema: {
             required: string[];
             additionalProperties: boolean;
         };
+        context: boolean;
         metadataMaxAge: {
             type: string;
             minimum: number;

package/dist/clients/base/options.js

@@ -51,6 +51,7 @@ export const baseOptionsSchema = {
             required: ['mechanism'],
             additionalProperties: false
         },
+        context: true,
         metadataMaxAge: { type: 'number', minimum: 0 },
         autocreateTopics: { type: 'boolean' },
         strict: { type: 'boolean' },

package/dist/clients/base/types.d.ts

@@ -33,6 +33,7 @@ export type RetryDelayGetter<Owner = object> = (client: Owner, operationId: stri
 export interface BaseOptions extends ConnectionOptions {
     clientId: string;
     bootstrapBrokers: Broker[] | string[];
+    context?: unknown;
     timeout?: number;
     retries?: number | boolean;
     retryDelay?: number | RetryDelayGetter;

package/dist/clients/consumer/consumer.d.ts

@@ -1,6 +1,7 @@
 import { type CallbackWithPromise } from '../../apis/callbacks.ts';
 import { type FetchResponse } from '../../apis/consumer/fetch-v17.ts';
-import { Base, type BaseEvents } from '../base/base.ts';
+import { type ConnectionPool } from '../../network/connection-pool.ts';
+import { Base, type BaseEvents, kCreateConnectionPool } from '../base/base.ts';
 import { MessagesStream } from './messages-stream.ts';
 import { TopicsMap } from './topics-map.ts';
 import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from './types.ts';
@@ -28,6 +29,7 @@ export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
     get streamsCount(): number;
     get lastHeartbeat(): Date | null;
     get coordinatorId(): number | null;
+    get streamContext(): unknown;
     close(force: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
     close(force?: boolean): Promise<void>;
     isActive(): boolean;
@@ -49,6 +51,7 @@ export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
     stopLagMonitoring(): void;
     findGroupCoordinator(callback: CallbackWithPromise<number>): void;
     findGroupCoordinator(): Promise<number>;
+    [kCreateConnectionPool](context?: unknown): ConnectionPool;
     joinGroup(options: GroupOptions, callback: CallbackWithPromise<string>): void;
     joinGroup(options?: GroupOptions): Promise<string>;
     leaveGroup(force?: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;

package/dist/clients/consumer/consumer.js

@@ -1,3 +1,4 @@
+import { randomUUID } from 'node:crypto';
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
 import { FetchIsolationLevels, FindCoordinatorKeyTypes } from "../../apis/enumerations.js";
 import { consumerCommitsChannel, consumerConsumesChannel, consumerFetchesChannel, consumerGroupChannel, consumerHeartbeatChannel, consumerLagChannel, consumerOffsetsChannel, createDiagnosticContext } from "../../diagnostic.js";
@@ -8,7 +9,7 @@ import { IS_CONTROL } from "../../protocol/records.js";
 import { Writer } from "../../protocol/writer.js";
 import { kAutocommit, kRefreshOffsetsAndFetch } from "../../symbols.js";
 import { emitExperimentalApiWarning } from "../../utils.js";
-import { Base, kAfterCreate, kCheckNotClosed, kClosed, kConnections, kFormatValidationErrors, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
+import { Base, kAfterCreate, kCheckNotClosed, kClosed, kConnections, kCreateConnectionPool, kFormatValidationErrors, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
 import { ensureMetric } from "../metrics.js";
 import { MessagesStream } from "./messages-stream.js";
 import { commitOptionsValidator, consumeOptionsValidator, consumerOptionsValidator, defaultConsumerOptions, fetchOptionsValidator, getLagOptionsValidator, groupIdAndOptionsValidator, groupOptionsValidator, listCommitsOptionsValidator, listOffsetsOptionsValidator } from "./options.js";
@@ -35,6 +36,7 @@ export class Consumer extends Base {
     #groupRemoteAssignor;
     #streams;
     #lagMonitoring;
+    #streamContext;
     // Metrics
     #metricActiveStreams;
     #metricLags;
@@ -76,6 +78,7 @@ export class Consumer extends Base {
         this.#memberEpoch = 0;
         this.#useConsumerGroupProtocol = this[kOptions].groupProtocol === 'consumer';
         this.#groupRemoteAssignor = this[kOptions].groupRemoteAssignor ?? null;
+        this.#streamContext = options.streamContext ?? options.context;
         this.#validateGroupOptions(this[kOptions], groupIdAndOptionsValidator);
         if (this[kPrometheus]) {
             ensureMetric(this[kPrometheus], 'Gauge', 'kafka_consumers', 'Number of active Kafka consumers').inc();
@@ -94,6 +97,9 @@ export class Consumer extends Base {
     get coordinatorId() {
         return this.#coordinatorId;
     }
+    get streamContext() {
+        return this.#streamContext;
+    }
     close(force, callback) {
         if (typeof force === 'function') {
             callback = force;
@@ -169,6 +175,7 @@ export class Consumer extends Base {
         options.highWaterMark ??= this[kOptions].highWaterMark;
         options.registry ??= this[kOptions].registry;
         options.beforeDeserialization ??= this[kOptions].beforeDeserialization;
+        options.context ??= this.#streamContext;
         if (options.beforeDeserialization) {
             emitExperimentalApiWarning('beforeDeserialization');
         }
@@ -353,6 +360,9 @@ export class Consumer extends Base {
         this.#findGroupCoordinator(callback);
         return callback[kCallbackPromise];
     }
+    [kCreateConnectionPool](context) {
+        return super[kCreateConnectionPool](context);
+    }
     joinGroup(options, callback) {
         if (!callback) {
             callback = createPromisifiedCallback();
@@ -443,7 +453,21 @@ export class Consumer extends Base {
                             retryCallback(error);
                             return;
                         }
-                        api(connection, options.maxWaitTime ?? this[kOptions].maxWaitTime, options.minBytes ?? this[kOptions].minBytes, options.maxBytes ?? this[kOptions].maxBytes, isolationLevel, 0, 0, options.topics, [], '', retryCallback);
+                        api(connection, options.maxWaitTime ?? this[kOptions].maxWaitTime, options.minBytes ?? this[kOptions].minBytes, options.maxBytes ?? this[kOptions].maxBytes, isolationLevel, 0, 0, options.topics, [], '', (error, result) => {
+                            if (error) {
+                                const genericError = error;
+                                if (genericError.findBy?.('apiId', 'FENCED_LEADER_EPOCH')) {
+                                    this.clearMetadata();
+                                    for (const topic of options.topics) {
+                                        for (const partition of topic.partitions) {
+                                            partition.currentLeaderEpoch = -1;
+                                            partition.lastFetchedEpoch = -1;
+                                        }
+                                    }
+                                }
+                            }
+                            retryCallback(error, result);
+                        });
                     });
                 });
             });
@@ -700,8 +724,9 @@ export class Consumer extends Base {
                     groupCallback(error);
                     return;
                 }
-                api(connection, this.groupId, this.memberId || '', this.#memberEpoch, this.groupInstanceId, null, // rackId
-                options.rebalanceTimeout, this.topics.current, this.#groupRemoteAssignor, this.#assignments, groupCallback);
+                const memberId = this.#getConsumerGroupHeartbeatMemberId(api.version);
+                api(connection, this.groupId, memberId, this.#memberEpoch, this.groupInstanceId, null, // rackId
+                options.rebalanceTimeout, this.topics.current, null, this.#groupRemoteAssignor, this.#assignments, groupCallback);
             });
         }, (error, response) => {
             if (this[kClosed]) {
@@ -812,6 +837,10 @@ export class Consumer extends Base {
     }
     #updateAssignments(newAssignments, callback) {
         this[kMetadata]({ topics: this.topics.current }, (error, metadata) => {
+            if (!this.#membershipActive) {
+                callback(null);
+                return;
+            }
             if (error) {
                 callback(error);
                 return;
@@ -849,10 +878,12 @@ export class Consumer extends Base {
                     groupCallback(error);
                     return;
                 }
-                api(connection, this.groupId, this.memberId, -1, // memberEpoch = -1 signals leave
+                const memberId = this.#getConsumerGroupHeartbeatMemberId(api.version);
+                api(connection, this.groupId, memberId, -1, // memberEpoch = -1 signals leave
                 this.groupInstanceId, null, // rackId
                 0, // rebalanceTimeout
                 [], // subscribedTopicNames
+                null, // subscribedTopicRegex
                 this.#groupRemoteAssignor, [], // topicPartitions
                 groupCallback);
             });
@@ -865,6 +896,16 @@ export class Consumer extends Base {
             callback(null);
         });
     }
+    #getConsumerGroupHeartbeatMemberId(apiVersion) {
+        if (this.memberId) {
+            return this.memberId;
+        }
+        if (apiVersion >= 1) {
+            this.memberId = randomUUID();
+            return this.memberId;
+        }
+        return '';
+    }
     #performConsume(options, trackTopics, callback) {
         // Subscribe all topics
         let joinNeeded = this.memberId === null;

package/dist/clients/consumer/messages-stream.d.ts

@@ -15,6 +15,7 @@ export declare class MessagesStream<Key, Value, HeaderKey, HeaderValue> extends
     constructor(consumer: Consumer<Key, Value, HeaderKey, HeaderValue>, options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>);
     get consumer(): Consumer<Key, Value, HeaderKey, HeaderValue>;
     get offsetsToFetch(): Map<string, bigint>;
+    get context(): unknown;
     get offsetsToCommit(): Map<string, CommitOptionsPartition>;
     get offsetsCommitted(): Map<string, bigint>;
     get committedOffsets(): Map<string, bigint>;