@platformatic/kafka 2.0.1 → 2.2.0

package/dist/clients/admin/options.d.ts

@@ -14,8 +14,48 @@ export declare const createTopicOptionsSchema: {
         topics: {
             type: string;
             items: {
-                type: string;
-                pattern: string;
+                oneOf: ({
+                    type: string;
+                    pattern: string;
+                } | {
+                    type: string;
+                    properties: {
+                        topic: {
+                            type: string;
+                            pattern: string;
+                        };
+                        partitions: {
+                            type: string;
+                        };
+                        replicas: {
+                            type: string;
+                        };
+                        assignments: {
+                            type: string;
+                            items: {
+                                type: string;
+                                properties: {
+                                    partition: {
+                                        type: string;
+                                        minimum: number;
+                                    };
+                                    brokers: {
+                                        type: string;
+                                        items: {
+                                            type: string;
+                                        };
+                                        minItems: number;
+                                    };
+                                };
+                                required: string[];
+                                additionalProperties: boolean;
+                            };
+                            minItems: number;
+                        };
+                    };
+                    required: string[];
+                    additionalProperties: boolean;
+                })[];
             };
         };
         partitions: {
@@ -46,6 +86,22 @@ export declare const createTopicOptionsSchema: {
             };
             minItems: number;
         };
+        configs: {
+            type: string;
+            items: {
+                type: string;
+                properties: {
+                    name: {
+                        type: string;
+                    };
+                    value: {
+                        type: string[];
+                    };
+                };
+                required: string[];
+                additionalProperties: boolean;
+            };
+        };
     };
     required: string[];
     additionalProperties: boolean;
@@ -129,7 +185,7 @@ export declare const listGroupsOptionsSchema: {
             items: {
                 type: string;
                 enumeration: {
-                    allowed: readonly ["PREPARING_REBALANCE", "COMPLETING_REBALANCE", "STABLE", "DEAD", "EMPTY"];
+                    allowed: ("Unknown" | "PreparingRebalance" | "CompletingRebalance" | "Stable" | "Dead" | "Empty" | "Assigning" | "Reconciling" | "NotReady" | "PREPARING_REBALANCE" | "COMPLETING_REBALANCE" | "STABLE" | "DEAD" | "EMPTY")[];
                     errorMessage: string;
                 };
             };
@@ -708,7 +764,7 @@ export declare const createAclsOptionsSchema: {
                     };
                     operation: {
                         type: string;
-                        enum: (0 | 1 | 2 | 4 | 8 | 7 | 10 | 3 | 5 | 6 | 9 | 11 | 12 | 13 | 14 | 15)[];
+                        enum: (0 | 1 | 2 | 4 | 8 | 7 | 10 | 12 | 3 | 5 | 6 | 9 | 11 | 13 | 14 | 15)[];
                     };
                     permissionType: {
                         type: string;
@@ -767,7 +823,7 @@ export declare const describeAclsOptionsSchema: {
                 };
                 operation: {
                     type: string;
-                    enum: (0 | 1 | 2 | 4 | 8 | 7 | 10 | 3 | 5 | 6 | 9 | 11 | 12 | 13 | 14 | 15)[];
+                    enum: (0 | 1 | 2 | 4 | 8 | 7 | 10 | 12 | 3 | 5 | 6 | 9 | 11 | 13 | 14 | 15)[];
                 };
                 permissionType: {
                     type: string;
@@ -826,7 +882,7 @@ export declare const deleteAclsOptionsSchema: {
                     };
                     operation: {
                         type: string;
-                        enum: (0 | 1 | 2 | 4 | 8 | 7 | 10 | 3 | 5 | 6 | 9 | 11 | 12 | 13 | 14 | 15)[];
+                        enum: (0 | 1 | 2 | 4 | 8 | 7 | 10 | 12 | 3 | 5 | 6 | 9 | 11 | 13 | 14 | 15)[];
                     };
                     permissionType: {
                         type: string;

package/dist/clients/admin/options.js

@@ -1,4 +1,4 @@
-import { allowedAclOperations, allowedAclPermissionTypes, allowedClientQuotaMatchTypes, allowedFetchIsolationLevels, allowedFindCoordinatorKeyTypes, allowedIncrementalAlterConfigOperationTypes, allowedResourcePatternTypes, allowedResourceTypes, ConsumerGroupStates, IncrementalAlterConfigOperationTypes } from "../../apis/enumerations.js";
+import { allowedAclOperations, allowedAclPermissionTypes, allowedClientQuotaMatchTypes, allowedFetchIsolationLevels, allowedFindCoordinatorKeyTypes, allowedIncrementalAlterConfigOperationTypes, allowedResourcePatternTypes, allowedResourceTypes, ConsumerGroupStates, KafkaConsumerGroupStates, IncrementalAlterConfigOperationTypes } from "../../apis/enumerations.js";
 import { ajv, listErrorMessage } from "../../utils.js";
 import { idProperty } from "../base/options.js";
 export const groupsProperties = {
@@ -8,24 +8,55 @@ export const groupsProperties = {
         minItems: 1
     }
 };
+const createTopicAssignmentsSchema = {
+    type: 'array',
+    items: {
+        type: 'object',
+        properties: {
+            partition: { type: 'number', minimum: 0 },
+            brokers: { type: 'array', items: { type: 'number' }, minItems: 1 }
+        },
+        required: ['partition', 'brokers'],
+        additionalProperties: false
+    },
+    minItems: 1
+};
 export const createTopicOptionsSchema = {
     type: 'object',
     properties: {
-        topics: { type: 'array', items: idProperty },
+        topics: {
+            type: 'array',
+            items: {
+                oneOf: [
+                    idProperty,
+                    {
+                        type: 'object',
+                        properties: {
+                            topic: idProperty,
+                            partitions: { type: 'number' },
+                            replicas: { type: 'number' },
+                            assignments: createTopicAssignmentsSchema
+                        },
+                        required: ['topic'],
+                        additionalProperties: false
+                    }
+                ]
+            }
+        },
         partitions: { type: 'number' },
         replicas: { type: 'number' },
-        assignments: {
+        assignments: createTopicAssignmentsSchema,
+        configs: {
             type: 'array',
             items: {
                 type: 'object',
                 properties: {
-                    partition: { type: 'number', minimum: 0 },
-                    brokers: { type: 'array', items: { type: 'number' }, minItems: 1 }
+                    name: { type: 'string' },
+                    value: { type: ['string', 'null'] }
                 },
-                required: ['partition', 'brokers'],
+                required: ['name'],
                 additionalProperties: false
-            },
-            minItems: 1
+            }
         }
     },
     required: ['topics'],
@@ -86,8 +117,8 @@ export const listGroupsOptionsSchema = {
             items: {
                 type: 'string',
                 enumeration: {
-                    allowed: ConsumerGroupStates,
-                    errorMessage: listErrorMessage(ConsumerGroupStates)
+                    allowed: [...ConsumerGroupStates, ...KafkaConsumerGroupStates],
+                    errorMessage: listErrorMessage([...ConsumerGroupStates, ...KafkaConsumerGroupStates])
                 }
             },
             minItems: 0

package/dist/clients/admin/types.d.ts

@@ -33,7 +33,7 @@ export interface GroupMember {
 export interface GroupBase {
     id: string;
     state: ConsumerGroupStateValue;
-    groupType: string;
+    groupType?: string;
     protocolType: string;
 }
 export interface Group extends Omit<GroupBase, 'groupType'> {
@@ -43,8 +43,14 @@ export interface Group extends Omit<GroupBase, 'groupType'> {
 }
 export interface AdminOptions extends BaseOptions {
 }
+export interface CreateTopicsTopicOptions {
+    topic: string;
+    partitions?: number;
+    replicas?: number;
+    assignments?: BrokerAssignment[];
+}
 export interface CreateTopicsOptions {
-    topics: string[];
+    topics: (string | CreateTopicsTopicOptions)[];
     partitions?: number;
     replicas?: number;
     assignments?: BrokerAssignment[];

package/dist/clients/base/base.d.ts

@@ -1,7 +1,7 @@
 import { type ValidateFunction } from 'ajv';
 import { type CallbackWithPromise } from '../../apis/callbacks.ts';
 import { type API, type Callback } from '../../apis/definitions.ts';
-import { type ApiVersionsResponseApi } from '../../apis/metadata/api-versions-v3.ts';
+import { type ApiVersionsResponseApi } from '../../apis/metadata/api-versions-v1.ts';
 import { type ClientType } from '../../diagnostic.ts';
 import { TypedEventEmitter, type TypedEvents } from '../../events.ts';
 import { ConnectionPool, type ConnectionPoolEventPayload } from '../../network/connection-pool.ts';
@@ -84,11 +84,11 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions, EventsT
     [kMetadata](options: MetadataOptions, callback: CallbackWithPromise<ClusterMetadata>): void;
     [kCheckNotClosed](callback: CallbackWithPromise<any>): boolean;
     clearMetadata(): void;
-    [kPerformWithRetry]<ReturnType>(operationId: string, operation: (callback: Callback<ReturnType>) => void, callback: CallbackWithPromise<ReturnType>, attempt?: number, errors?: Error[], shouldSkipRetry?: (e: Error) => boolean): void | Promise<ReturnType>;
+    [kPerformWithRetry]<ReturnType>(operationId: string, operation: (callback: Callback<ReturnType>, attempt: number) => void, callback: CallbackWithPromise<ReturnType>, attempt?: number, errors?: Error[], shouldSkipRetry?: (e: Error) => boolean): void | Promise<ReturnType>;
     [kPerformDeduplicated]<ReturnType>(operationId: string, operation: (callback: CallbackWithPromise<ReturnType>) => void, callback: CallbackWithPromise<ReturnType>): void | Promise<ReturnType>;
     [kGetApi]<RequestArguments extends Array<unknown>, ResponseType>(name: string, callback: Callback<API<RequestArguments, ResponseType>>): void;
     [kGetConnection](broker: Broker, callback: Callback<Connection>): void;
-    [kGetBootstrapConnection](callback: Callback<Connection>): void;
+    [kGetBootstrapConnection](callback: Callback<Connection>, attempt?: number): void;
     [kValidateOptions](target: unknown, validator: ValidateFunction<unknown>, targetName: string, throwOnErrors?: boolean): Error | null;
     [kInspect](...args: unknown[]): void;
     [kFormatValidationErrors](validator: ValidateFunction<unknown>, targetName: string): string;

package/dist/clients/base/base.js

@@ -1,8 +1,9 @@
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
 import * as apis from "../../apis/index.js";
+import { api as apiVersionsV1 } from "../../apis/metadata/api-versions-v1.js";
 import { api as apiVersionsV3 } from "../../apis/metadata/api-versions-v3.js";
 import { baseApisChannel, baseMetadataChannel, createDiagnosticContext, notifyCreation } from "../../diagnostic.js";
-import { MultipleErrors, NetworkError, UnsupportedApiError, UserError } from "../../errors.js";
+import { findErrorBy, MultipleErrors, NetworkError, UnsupportedApiError, UserError } from "../../errors.js";
 import { TypedEventEmitter } from "../../events.js";
 import { ConnectionPool } from "../../network/connection-pool.js";
 import { parseBroker } from "../../network/utils.js";
@@ -47,6 +48,7 @@ export class Base extends TypedEventEmitter {
     [kPrometheus];
     #metadata;
     #inflightDeduplications;
+    #apiVersionsVersions;
     constructor(options) {
         super();
         this.setMaxListeners(0);
@@ -54,8 +56,14 @@ export class Base extends TypedEventEmitter {
         this[kInstance] = currentInstance++;
         this[kApis] = [];
         this[kContext] = options.context;
+        this.#apiVersionsVersions = new WeakMap();
         // Validate options
-        this[kOptions] = Object.assign({}, defaultBaseOptions, options);
+        this[kOptions] = Object.assign({}, defaultBaseOptions);
+        for (const [key, value] of Object.entries(options)) {
+            if (value !== undefined) {
+                this[kOptions][key] = value;
+            }
+        }
         this[kValidateOptions](this[kOptions], baseOptionsValidator, '/options');
         this[kClientId] = options.clientId;
         if (typeof this[kOptions].retries === 'boolean') {
@@ -201,15 +209,36 @@ export class Base extends TypedEventEmitter {
     }
     [kListApis](callback) {
         this[kPerformDeduplicated]('listApis', deduplicateCallback => {
-            this[kPerformWithRetry]('listApis', retryCallback => {
+            this[kPerformWithRetry]('listApis', (retryCallback, attempt) => {
                 this[kGetBootstrapConnection]((error, connection) => {
                     if (error) {
                         retryCallback(error);
                         return;
                     }
-                    // We use V3 to be able to get APIS from Kafka 2.4.0+
-                    apiVersionsV3(connection, clientSoftwareName, clientSoftwareVersion, retryCallback);
-                });
+                    const apiVersionsVersion = this.#apiVersionsVersions.get(connection);
+                    const listApisV1 = () => {
+                        apiVersionsV1(connection, clientSoftwareName, clientSoftwareVersion, (error, response) => {
+                            retryCallback(error, response);
+                        });
+                    };
+                    if (apiVersionsVersion === 1) {
+                        listApisV1();
+                        return;
+                    }
+                    apiVersionsV3(connection, clientSoftwareName, clientSoftwareVersion, (error, response) => {
+                        if (!error) {
+                            this.#apiVersionsVersions.set(connection, 3);
+                            retryCallback(null, response);
+                            return;
+                        }
+                        if (findErrorBy(error, 'apiId', 'UNSUPPORTED_VERSION')) {
+                            this.#apiVersionsVersions.set(connection, 1);
+                            listApisV1();
+                            return;
+                        }
+                        retryCallback(error);
+                    });
+                }, attempt);
             }, (error, metadata) => {
                 if (error) {
                     deduplicateCallback(error);
@@ -224,7 +253,12 @@ export class Base extends TypedEventEmitter {
     }
     [kCheckNotClosed](callback) {
         if (this[kClosed]) {
-            const error = new NetworkError('Client is closed.', { closed: true, instance: this[kInstance] });
+            // A closed client never reopens, so this must not be retried.
+            const error = new NetworkError('Client is closed.', {
+                canRetry: false,
+                closed: true,
+                instance: this[kInstance]
+            });
             callback(error, undefined);
             return true;
         }
@@ -238,9 +272,8 @@ export class Base extends TypedEventEmitter {
         this.emitWithDebug('client', 'performWithRetry', operationId, attempt, retries);
         operation((error, result) => {
             if (error) {
-                const genericError = error;
                 // Only retry if all the errors in the chain are retriable
-                const retriable = !genericError.findBy?.('canRetry', false);
+                const retriable = !findErrorBy(error, 'canRetry', false);
                 errors.push(error);
                 if (attempt < retries && retriable && !shouldSkipRetry?.(error)) {
                     function onClose() {
@@ -255,7 +288,13 @@ export class Base extends TypedEventEmitter {
                     this.emitWithDebug('client', 'performWithRetry:retry', operationId, attempt, retries, delay);
                     const timeout = setTimeout(() => {
                         this.removeListener('client:close', onClose);
-                        this[kPerformWithRetry](operationId, operation, callback, attempt + 1, errors, shouldSkipRetry);
+                        try {
+                            this[kPerformWithRetry](operationId, operation, callback, attempt + 1, errors, shouldSkipRetry);
+                        }
+                        catch (error) {
+                            errors.push(error);
+                            callback(new MultipleErrors(`${operationId} failed ${attempt + 1} times.`, errors));
+                        }
                     }, delay);
                     this.once('client:close', onClose);
                 }
@@ -274,7 +313,7 @@ export class Base extends TypedEventEmitter {
                 errors.splice(0, errors.length);
             }
             callback(null, result);
-        });
+        }, attempt);
         return callback[kCallbackPromise];
     }
     [kPerformDeduplicated](operationId, operation, callback) {
@@ -329,13 +368,20 @@ export class Base extends TypedEventEmitter {
     [kGetConnection](broker, callback) {
         this[kConnections].get(broker, callback);
     }
-    [kGetBootstrapConnection](callback) {
+    [kGetBootstrapConnection](callback, attempt = 0) {
+        let brokers;
         if (!this.#metadata) {
-            this[kConnections].getFirstAvailable(this[kBootstrapBrokers], callback);
-            return;
+            brokers = this[kBootstrapBrokers];
+        }
+        else {
+            const discovered = Array.from(this.#metadata.brokers.values());
+            brokers = [...this[kBootstrapBrokers], ...discovered];
         }
-        const discovered = Array.from(this.#metadata.brokers.values());
-        this[kConnections].getFirstAvailable([...this[kBootstrapBrokers], ...discovered], callback);
+        if (attempt > 0 && brokers.length > 1) {
+            const offset = attempt % brokers.length;
+            brokers = [...brokers.slice(offset), ...brokers.slice(0, offset)];
+        }
+        this[kConnections].getFirstAvailable(brokers, callback);
     }
     [kValidateOptions](target, validator, targetName, throwOnErrors = true) {
         if (!this[kOptions].strict) {
@@ -390,7 +436,7 @@ export class Base extends TypedEventEmitter {
         this[kPerformDeduplicated](
         // Unique key to avoid mixing callbacks
         `metadata-${topics.sort().join(',')}-${autocreateTopics}-${options.forceUpdate}`, deduplicateCallback => {
-            this[kPerformWithRetry]('metadata', retryCallback => {
+            this[kPerformWithRetry]('metadata', (retryCallback, attempt) => {
                 this[kGetBootstrapConnection]((error, connection) => {
                     if (error) {
                         retryCallback(error);
@@ -401,12 +447,27 @@ export class Base extends TypedEventEmitter {
                             retryCallback(error);
                             return;
                         }
-                        api(connection, topicsToFetch, autocreateTopics, true, retryCallback);
+                        // includeTopicAuthorizedOperations: false. The response's
+                        // topic_authorized_operations bitfield is parsed but never read
+                        // by the producer/consumer, and requesting it makes IAM brokers
+                        // (e.g. MSK) authorize every topic operation and emit a CloudTrail
+                        // AccessDenied per denied op (CreateTopic/DeleteTopic/...) — pure
+                        // audit-log noise for permissions the client never exercises.
+                        api(connection, topicsToFetch, autocreateTopics, false, (error, response) => {
+                            // Metadata v0-v8 do not carry topic UUIDs. Topic names are stable
+                            // surrogates for the consumer paths that use internal topic IDs.
+                            if (!error && api.version <= 8) {
+                                for (const topic of response.topics) {
+                                    topic.topicId = topic.name;
+                                }
+                            }
+                            retryCallback(error, response);
+                        });
                     });
-                });
+                }, attempt);
             }, (error, metadata) => {
                 if (error) {
-                    const unknownTopicError = error.findBy('apiCode', 3);
+                    const unknownTopicError = findErrorBy(error, 'apiCode', 3);
                     if (unknownTopicError) {
                         const topicIndexMatch = unknownTopicError.path?.match(/\/topics\/(\d+)/);
                         /* c8 ignore next 3 - Hard to test  */
@@ -415,7 +476,7 @@ export class Base extends TypedEventEmitter {
                         deduplicateCallback(new UserError(`Unknown topic ${topicName}.`));
                         return;
                     }
-                    const hasStaleMetadata = error.findBy('hasStaleMetadata', true);
+                    const hasStaleMetadata = findErrorBy(error, 'hasStaleMetadata', true);
                     // Stale metadata, we need to fetch everything again
                     /* c8 ignore next 4 - Hard to test */
                     if (hasStaleMetadata) {
@@ -437,6 +498,7 @@ export class Base extends TypedEventEmitter {
                 }
                 else {
                     this.#metadata.lastUpdate = lastUpdate;
+                    this.#metadata.controllerId = metadata.controllerId;
                 }
                 const brokers = new Map();
                 // This should never change, but we act defensively here

package/dist/clients/base/options.d.ts

@@ -25,6 +25,9 @@ export declare const baseOptionsSchema: {
             type: string;
             pattern: string;
         };
+        clientRack: {
+            type: string;
+        };
         bootstrapBrokers: {
             oneOf: ({
                 type: string;

package/dist/clients/base/options.js

@@ -14,6 +14,7 @@ export const baseOptionsSchema = {
     type: 'object',
     properties: {
         clientId: idProperty,
+        clientRack: { type: 'string' },
         bootstrapBrokers: {
             oneOf: [
                 { type: 'array', items: { type: 'string' } },

package/dist/clients/base/types.d.ts

@@ -32,6 +32,7 @@ export interface ClusterMetadata {
 export type RetryDelayGetter<Owner = object> = (client: Owner, operationId: string, attempt: number, retries: number, error: Error) => number;
 export interface BaseOptions extends ConnectionOptions {
     clientId: string;
+    clientRack?: string;
     bootstrapBrokers: Broker[] | string[];
     context?: unknown;
     timeout?: number;

package/dist/clients/consumer/consumer.d.ts

@@ -1,10 +1,12 @@
 import { type CallbackWithPromise } from '../../apis/callbacks.ts';
 import { type FetchResponse } from '../../apis/consumer/fetch-v17.ts';
 import { type ConnectionPool } from '../../network/connection-pool.ts';
+import { kGetFetchNode } from '../../symbols.ts';
 import { Base, type BaseEvents, kCreateConnectionPool } from '../base/base.ts';
+import { type ClusterMetadata } from '../base/types.ts';
 import { MessagesStream } from './messages-stream.ts';
 import { TopicsMap } from './topics-map.ts';
-import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from './types.ts';
+import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerHeartbeatStalledPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from './types.ts';
 export interface ConsumerEvents extends BaseEvents {
     'consumer:group:join': (payload: ConsumerGroupJoinPayload) => void;
     'consumer:group:leave': (payload: ConsumerGroupLeavePayload) => void;
@@ -14,6 +16,7 @@ export interface ConsumerEvents extends BaseEvents {
     'consumer:heartbeat:cancel': (payload: ConsumerHeartbeatPayload) => void;
     'consumer:heartbeat:end': (payload?: ConsumerHeartbeatPayload) => void;
     'consumer:heartbeat:error': (payload: ConsumerHeartbeatErrorPayload) => void;
+    'consumer:heartbeat:stalled': (payload: ConsumerHeartbeatStalledPayload) => void;
     'consumer:lag': (lag: Offsets) => void;
     'consumer:lag:error': (error: Error) => void;
 }
@@ -56,4 +59,5 @@ export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
     joinGroup(options?: GroupOptions): Promise<string>;
     leaveGroup(force?: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
     leaveGroup(force?: boolean): Promise<void>;
+    [kGetFetchNode](metadata: ClusterMetadata, topic: string, partition: number, now: number): number;
 }