@platformatic/kafka 2.10.0 → 2.12.0

package/dist/clients/base/base.js

@@ -1,5 +1,6 @@
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
 import * as apis from "../../apis/index.js";
+import { api as apiVersionsV1 } from "../../apis/metadata/api-versions-v1.js";
 import { api as apiVersionsV3 } from "../../apis/metadata/api-versions-v3.js";
 import { baseApisChannel, baseMetadataChannel, createDiagnosticContext, notifyCreation } from "../../diagnostic.js";
 import { findErrorBy, MultipleErrors, NetworkError, UnsupportedApiError, UserError } from "../../errors.js";
@@ -47,6 +48,7 @@ export class Base extends TypedEventEmitter {
     [kPrometheus];
     #metadata;
     #inflightDeduplications;
+    #apiVersionsVersions;
     constructor(options) {
         super();
         this.setMaxListeners(0);
@@ -54,6 +56,7 @@ export class Base extends TypedEventEmitter {
         this[kInstance] = currentInstance++;
         this[kApis] = [];
         this[kContext] = options.context;
+        this.#apiVersionsVersions = new WeakMap();
         // Validate options
         this[kOptions] = Object.assign({}, defaultBaseOptions);
         for (const [key, value] of Object.entries(options)) {
@@ -212,8 +215,29 @@ export class Base extends TypedEventEmitter {
                         retryCallback(error);
                         return;
                     }
-                    // We use V3 to be able to get APIS from Kafka 2.4.0+
-                    apiVersionsV3(connection, clientSoftwareName, clientSoftwareVersion, retryCallback);
+                    const apiVersionsVersion = this.#apiVersionsVersions.get(connection);
+                    const listApisV1 = () => {
+                        apiVersionsV1(connection, clientSoftwareName, clientSoftwareVersion, (error, response) => {
+                            retryCallback(error, response);
+                        });
+                    };
+                    if (apiVersionsVersion === 1) {
+                        listApisV1();
+                        return;
+                    }
+                    apiVersionsV3(connection, clientSoftwareName, clientSoftwareVersion, (error, response) => {
+                        if (!error) {
+                            this.#apiVersionsVersions.set(connection, 3);
+                            retryCallback(null, response);
+                            return;
+                        }
+                        if (findErrorBy(error, 'apiId', 'UNSUPPORTED_VERSION')) {
+                            this.#apiVersionsVersions.set(connection, 1);
+                            listApisV1();
+                            return;
+                        }
+                        retryCallback(error);
+                    });
                 }, attempt);
             }, (error, metadata) => {
                 if (error) {
@@ -243,12 +267,12 @@ export class Base extends TypedEventEmitter {
     clearMetadata() {
         this.#metadata = undefined;
     }
-    [kPerformWithRetry](operationId, operation, callback, attempt = 0, errors = [], shouldSkipRetry) {
-        const retries = this[kOptions].retries;
+    [kPerformWithRetry](operationId, operation, callback, attempt = 0, errors = [], shouldSkipRetry, retryLimit) {
+        const retries = retryLimit ?? this[kOptions].retries;
         this.emitWithDebug('client', 'performWithRetry', operationId, attempt, retries);
         operation((error, result) => {
             if (error) {
-                // Only retry if all the errors in the chain are retriable
+                // Only retry if all the errors in the chain are retriable.
                 const retriable = !findErrorBy(error, 'canRetry', false);
                 errors.push(error);
                 if (attempt < retries && retriable && !shouldSkipRetry?.(error)) {
@@ -265,7 +289,7 @@ export class Base extends TypedEventEmitter {
                     const timeout = setTimeout(() => {
                         this.removeListener('client:close', onClose);
                         try {
-                            this[kPerformWithRetry](operationId, operation, callback, attempt + 1, errors, shouldSkipRetry);
+                            this[kPerformWithRetry](operationId, operation, callback, attempt + 1, errors, shouldSkipRetry, retries);
                         }
                         catch (error) {
                             errors.push(error);
@@ -411,7 +435,7 @@ export class Base extends TypedEventEmitter {
         const autocreateTopics = options.autocreateTopics ?? this[kOptions].autocreateTopics;
         this[kPerformDeduplicated](
         // Unique key to avoid mixing callbacks
-        `metadata-${topics.sort().join(',')}-${autocreateTopics}-${options.forceUpdate}`, deduplicateCallback => {
+        `metadata-${topics.sort().join(',')}-${autocreateTopics}-${options.forceUpdate}-${options.retries ?? 'default'}`, deduplicateCallback => {
             this[kPerformWithRetry]('metadata', (retryCallback, attempt) => {
                 this[kGetBootstrapConnection]((error, connection) => {
                     if (error) {
@@ -429,7 +453,16 @@ export class Base extends TypedEventEmitter {
                         // (e.g. MSK) authorize every topic operation and emit a CloudTrail
                         // AccessDenied per denied op (CreateTopic/DeleteTopic/...) — pure
                         // audit-log noise for permissions the client never exercises.
-                        api(connection, topicsToFetch, autocreateTopics, false, retryCallback);
+                        api(connection, topicsToFetch, autocreateTopics, false, (error, response) => {
+                            // Metadata v0-v8 do not carry topic UUIDs. Topic names are stable
+                            // surrogates for the consumer paths that use internal topic IDs.
+                            if (!error && api.version <= 8) {
+                                for (const topic of response.topics) {
+                                    topic.topicId = topic.name;
+                                }
+                            }
+                            retryCallback(error, response);
+                        });
                     });
                 }, attempt);
             }, (error, metadata) => {
@@ -499,7 +532,7 @@ export class Base extends TypedEventEmitter {
                 };
                 this.emitWithDebug('client', 'metadata', updatedMetadata);
                 deduplicateCallback(null, updatedMetadata);
-            }, 0);
+            }, 0, [], undefined, options.retries);
         }, callback);
     }
     #forwardEvents(source, events) {

package/dist/clients/base/options.d.ts

@@ -199,6 +199,10 @@ export declare const metadataOptionsSchema: {
             type: string;
             minimum: number;
         };
+        retries: {
+            type: string;
+            minimum: number;
+        };
     };
     additionalProperties: boolean;
 };

package/dist/clients/base/options.js

@@ -67,7 +67,8 @@ export const metadataOptionsSchema = {
         topics: { type: 'array', items: idProperty },
         autocreateTopics: { type: 'boolean' },
         forceUpdate: { type: 'boolean' },
-        metadataMaxAge: { type: 'number', minimum: 0 }
+        metadataMaxAge: { type: 'number', minimum: 0 },
+        retries: { type: 'number', minimum: 0 }
     },
     additionalProperties: false
 };

package/dist/clients/base/types.d.ts

@@ -48,4 +48,5 @@ export interface MetadataOptions {
     autocreateTopics?: boolean;
     forceUpdate?: boolean;
     metadataMaxAge?: number;
+    retries?: number;
 }

package/dist/clients/consumer/consumer-protocol.d.ts

@@ -0,0 +1,32 @@
+import { type GroupAssignment } from './types.ts';
+export declare const CONSUMER_PROTOCOL_LOWEST_VERSION = 0;
+export declare const CONSUMER_PROTOCOL_HIGHEST_VERSION = 3;
+export interface ConsumerProtocolSubscriptionData {
+    version: number;
+    topics: string[];
+    userData: Buffer;
+    ownedPartitions: GroupAssignment[];
+    generationId: number;
+    rackId: string | null;
+}
+export interface ConsumerProtocolAssignmentData {
+    version: number;
+    assignedPartitions: GroupAssignment[];
+    userData: Buffer;
+}
+export declare function encodeConsumerProtocolSubscription(data: {
+    version: number;
+    topics: string[];
+    userData?: Buffer | string;
+    ownedPartitions?: GroupAssignment[];
+    generationId?: number;
+    rackId?: string | null;
+}): Buffer;
+export declare function decodeConsumerProtocolSubscription(buffer: Buffer): ConsumerProtocolSubscriptionData;
+export declare function decodeCooperativeStickyGeneration(userData: Buffer): number;
+export declare function encodeConsumerProtocolAssignment(data: {
+    version: number;
+    assignedPartitions: GroupAssignment[];
+    userData?: Buffer | string | null;
+}): Buffer;
+export declare function decodeConsumerProtocolAssignment(buffer: Buffer): ConsumerProtocolAssignmentData;

package/dist/clients/consumer/consumer-protocol.js

@@ -0,0 +1,114 @@
+import { Reader } from "../../protocol/reader.js";
+import { Writer } from "../../protocol/writer.js";
+export const CONSUMER_PROTOCOL_LOWEST_VERSION = 0;
+export const CONSUMER_PROTOCOL_HIGHEST_VERSION = 3;
+function supportedVersion(version) {
+    if (version < CONSUMER_PROTOCOL_LOWEST_VERSION) {
+        return CONSUMER_PROTOCOL_LOWEST_VERSION;
+    }
+    if (version > CONSUMER_PROTOCOL_HIGHEST_VERSION) {
+        return CONSUMER_PROTOCOL_HIGHEST_VERSION;
+    }
+    return version;
+}
+function sortedAssignments(assignments) {
+    return assignments
+        .filter(({ partitions }) => partitions.length > 0)
+        .map(({ topic, partitions }) => ({ topic, partitions: [...partitions].sort((a, b) => a - b) }))
+        .sort((a, b) => a.topic.localeCompare(b.topic));
+}
+function appendTopicPartitions(writer, assignments) {
+    return writer.appendArray(sortedAssignments(assignments), (w, { topic, partitions }) => {
+        w.appendString(topic, false).appendArray(partitions, (w, partition) => w.appendInt32(partition), false, false);
+    }, false, false);
+}
+function readTopicPartitions(reader) {
+    return reader.readArray(r => ({
+        topic: r.readString(false),
+        partitions: r.readArray(r => r.readInt32(), false, false)
+    }), false, false);
+}
+export function encodeConsumerProtocolSubscription(data) {
+    const version = supportedVersion(data.version);
+    const userData = typeof data.userData === 'string' ? Buffer.from(data.userData) : data.userData;
+    const writer = Writer.create()
+        .appendInt16(version)
+        .appendArray([...data.topics].sort(), (w, topic) => w.appendString(topic, false), false, false)
+        .appendBytes(userData, false);
+    if (version >= 1) {
+        appendTopicPartitions(writer, data.ownedPartitions ?? []);
+    }
+    if (version >= 2) {
+        writer.appendInt32(data.generationId ?? -1);
+    }
+    if (version >= 3) {
+        writer.appendString(data.rackId || null, false);
+    }
+    return writer.buffer;
+}
+export function decodeConsumerProtocolSubscription(buffer) {
+    const reader = Reader.from(buffer);
+    const encodedVersion = reader.readInt16();
+    const version = supportedVersion(encodedVersion);
+    const subscription = {
+        version: encodedVersion,
+        topics: reader.readArray(r => r.readString(false), false, false),
+        userData: reader.readBytes(false),
+        ownedPartitions: [],
+        generationId: -1,
+        rackId: null
+    };
+    if (version >= 1) {
+        subscription.ownedPartitions = readTopicPartitions(reader);
+    }
+    if (version >= 2) {
+        subscription.generationId = reader.readInt32();
+    }
+    if (version >= 3) {
+        subscription.rackId = reader.readNullableString(false);
+    }
+    return subscription;
+}
+export function decodeCooperativeStickyGeneration(userData) {
+    if (userData.length < 4) {
+        return -1;
+    }
+    try {
+        return Reader.from(userData).readInt32();
+    }
+    catch {
+        return -1;
+    }
+}
+export function encodeConsumerProtocolAssignment(data) {
+    const userData = typeof data.userData === 'string' ? Buffer.from(data.userData) : data.userData;
+    const writer = Writer.create().appendInt16(supportedVersion(data.version));
+    appendTopicPartitions(writer, data.assignedPartitions);
+    writer.appendBytes(userData ?? null, false);
+    return writer.buffer;
+}
+export function decodeConsumerProtocolAssignment(buffer) {
+    if (buffer.length === 0) {
+        return {
+            version: 0,
+            assignedPartitions: [],
+            userData: Buffer.alloc(0)
+        };
+    }
+    const reader = Reader.from(buffer);
+    const encodedVersion = reader.readInt16();
+    if (reader.remaining === 0) {
+        return {
+            version: encodedVersion,
+            assignedPartitions: [],
+            userData: Buffer.alloc(0)
+        };
+    }
+    const assignedPartitions = readTopicPartitions(reader);
+    const userData = reader.remaining > 0 ? reader.readBytes(false) : Buffer.alloc(0);
+    return {
+        version: encodedVersion,
+        assignedPartitions,
+        userData
+    };
+}

package/dist/clients/consumer/consumer.d.ts

@@ -6,12 +6,13 @@ import { Base, type BaseEvents, kCreateConnectionPool } from '../base/base.ts';
 import { type ClusterMetadata } from '../base/types.ts';
 import { MessagesStream } from './messages-stream.ts';
 import { TopicsMap } from './topics-map.ts';
-import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerHeartbeatStalledPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from './types.ts';
+import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupAutocommitErrorPayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerHeartbeatStalledPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from './types.ts';
 export interface ConsumerEvents extends BaseEvents {
     'consumer:group:join': (payload: ConsumerGroupJoinPayload) => void;
     'consumer:group:leave': (payload: ConsumerGroupLeavePayload) => void;
     'consumer:group:rejoin': () => void;
     'consumer:group:rebalance': (payload: ConsumerGroupRebalancePayload) => void;
+    'consumer:group:autocommit:error': (payload: ConsumerGroupAutocommitErrorPayload) => void;
     'consumer:heartbeat:start': (payload?: ConsumerHeartbeatPayload) => void;
     'consumer:heartbeat:cancel': (payload: ConsumerHeartbeatPayload) => void;
     'consumer:heartbeat:end': (payload?: ConsumerHeartbeatPayload) => void;
@@ -33,8 +34,8 @@ export declare class Consumer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
     get lastHeartbeat(): Date | null;
     get coordinatorId(): number | null;
     get streamContext(): unknown;
-    close(force: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
     close(force?: boolean): Promise<void>;
+    close(force: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
     isActive(): boolean;
     consume(options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<MessagesStream<Key, Value, HeaderKey, HeaderValue>>): void;
     consume(options: ConsumeOptions<Key, Value, HeaderKey, HeaderValue>): Promise<MessagesStream<Key, Value, HeaderKey, HeaderValue>>;