@platformatic/kafka 2.1.0 → 2.2.3

package/dist/clients/base/base.d.ts

@@ -84,11 +84,11 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions, EventsT
     [kMetadata](options: MetadataOptions, callback: CallbackWithPromise<ClusterMetadata>): void;
     [kCheckNotClosed](callback: CallbackWithPromise<any>): boolean;
     clearMetadata(): void;
-    [kPerformWithRetry]<ReturnType>(operationId: string, operation: (callback: Callback<ReturnType>) => void, callback: CallbackWithPromise<ReturnType>, attempt?: number, errors?: Error[], shouldSkipRetry?: (e: Error) => boolean): void | Promise<ReturnType>;
+    [kPerformWithRetry]<ReturnType>(operationId: string, operation: (callback: Callback<ReturnType>, attempt: number) => void, callback: CallbackWithPromise<ReturnType>, attempt?: number, errors?: Error[], shouldSkipRetry?: (e: Error) => boolean): void | Promise<ReturnType>;
     [kPerformDeduplicated]<ReturnType>(operationId: string, operation: (callback: CallbackWithPromise<ReturnType>) => void, callback: CallbackWithPromise<ReturnType>): void | Promise<ReturnType>;
     [kGetApi]<RequestArguments extends Array<unknown>, ResponseType>(name: string, callback: Callback<API<RequestArguments, ResponseType>>): void;
     [kGetConnection](broker: Broker, callback: Callback<Connection>): void;
-    [kGetBootstrapConnection](callback: Callback<Connection>): void;
+    [kGetBootstrapConnection](callback: Callback<Connection>, attempt?: number): void;
     [kValidateOptions](target: unknown, validator: ValidateFunction<unknown>, targetName: string, throwOnErrors?: boolean): Error | null;
     [kInspect](...args: unknown[]): void;
     [kFormatValidationErrors](validator: ValidateFunction<unknown>, targetName: string): string;

package/dist/clients/base/base.js

@@ -206,7 +206,7 @@ export class Base extends TypedEventEmitter {
     }
     [kListApis](callback) {
         this[kPerformDeduplicated]('listApis', deduplicateCallback => {
-            this[kPerformWithRetry]('listApis', retryCallback => {
+            this[kPerformWithRetry]('listApis', (retryCallback, attempt) => {
                 this[kGetBootstrapConnection]((error, connection) => {
                     if (error) {
                         retryCallback(error);
@@ -214,7 +214,7 @@ export class Base extends TypedEventEmitter {
                     }
                     // We use V3 to be able to get APIS from Kafka 2.4.0+
                     apiVersionsV3(connection, clientSoftwareName, clientSoftwareVersion, retryCallback);
-                });
+                }, attempt);
             }, (error, metadata) => {
                 if (error) {
                     deduplicateCallback(error);
@@ -260,7 +260,13 @@ export class Base extends TypedEventEmitter {
                     this.emitWithDebug('client', 'performWithRetry:retry', operationId, attempt, retries, delay);
                     const timeout = setTimeout(() => {
                         this.removeListener('client:close', onClose);
-                        this[kPerformWithRetry](operationId, operation, callback, attempt + 1, errors, shouldSkipRetry);
+                        try {
+                            this[kPerformWithRetry](operationId, operation, callback, attempt + 1, errors, shouldSkipRetry);
+                        }
+                        catch (error) {
+                            errors.push(error);
+                            callback(new MultipleErrors(`${operationId} failed ${attempt + 1} times.`, errors));
+                        }
                     }, delay);
                     this.once('client:close', onClose);
                 }
@@ -279,7 +285,7 @@ export class Base extends TypedEventEmitter {
                 errors.splice(0, errors.length);
             }
             callback(null, result);
-        });
+        }, attempt);
         return callback[kCallbackPromise];
     }
     [kPerformDeduplicated](operationId, operation, callback) {
@@ -334,13 +340,20 @@ export class Base extends TypedEventEmitter {
     [kGetConnection](broker, callback) {
         this[kConnections].get(broker, callback);
     }
-    [kGetBootstrapConnection](callback) {
+    [kGetBootstrapConnection](callback, attempt = 0) {
+        let brokers;
         if (!this.#metadata) {
-            this[kConnections].getFirstAvailable(this[kBootstrapBrokers], callback);
-            return;
+            brokers = this[kBootstrapBrokers];
         }
-        const discovered = Array.from(this.#metadata.brokers.values());
-        this[kConnections].getFirstAvailable([...this[kBootstrapBrokers], ...discovered], callback);
+        else {
+            const discovered = Array.from(this.#metadata.brokers.values());
+            brokers = [...this[kBootstrapBrokers], ...discovered];
+        }
+        if (attempt > 0 && brokers.length > 1) {
+            const offset = attempt % brokers.length;
+            brokers = [...brokers.slice(offset), ...brokers.slice(0, offset)];
+        }
+        this[kConnections].getFirstAvailable(brokers, callback);
     }
     [kValidateOptions](target, validator, targetName, throwOnErrors = true) {
         if (!this[kOptions].strict) {
@@ -395,7 +408,7 @@ export class Base extends TypedEventEmitter {
         this[kPerformDeduplicated](
         // Unique key to avoid mixing callbacks
         `metadata-${topics.sort().join(',')}-${autocreateTopics}-${options.forceUpdate}`, deduplicateCallback => {
-            this[kPerformWithRetry]('metadata', retryCallback => {
+            this[kPerformWithRetry]('metadata', (retryCallback, attempt) => {
                 this[kGetBootstrapConnection]((error, connection) => {
                     if (error) {
                         retryCallback(error);
@@ -408,7 +421,7 @@ export class Base extends TypedEventEmitter {
                         }
                         api(connection, topicsToFetch, autocreateTopics, true, retryCallback);
                     });
-                });
+                }, attempt);
             }, (error, metadata) => {
                 if (error) {
                     const unknownTopicError = error.findBy('apiCode', 3);

package/dist/clients/consumer/messages-stream.js

@@ -302,7 +302,11 @@ export class MessagesStream extends Readable {
         return super[Symbol.asyncIterator]();
     }
     _construct(callback) {
-        this.#refreshOffsets(callback);
+        this.#refreshOffsetsInflight = true;
+        this.#refreshOffsets(error => {
+            this.#refreshOffsetsInflight = false;
+            callback(error ?? undefined);
+        });
     }
     _destroy(error, callback) {
         if (this.#autocommitInterval) {
@@ -409,10 +413,22 @@ export class MessagesStream extends Readable {
                             this.push(null);
                             return;
                         }
-                        if (this.#fallbackMode !== MessagesStreamFallbackModes.FAIL &&
-                            this.#handleOffsetOutOfRange(error, topicIds)) {
-                            process.nextTick(() => {
-                                this.#fetch();
+                        if (this.#fallbackMode !== MessagesStreamFallbackModes.FAIL) {
+                            this.#handleOffsetOutOfRange(error, topicIds, (recoveryError, recovered) => {
+                                if (this.#closed || this.closed || this.destroyed) {
+                                    return;
+                                }
+                                if (recoveryError) {
+                                    this.destroy(recoveryError);
+                                    return;
+                                }
+                                if (recovered) {
+                                    process.nextTick(() => {
+                                        this.#fetch();
+                                    });
+                                    return;
+                                }
+                                this.destroy(error);
                             });
                             return;
                         }
@@ -432,34 +448,81 @@ export class MessagesStream extends Readable {
             }
         });
     }
-    #handleOffsetOutOfRange(error, topicIds) {
+    #handleOffsetOutOfRange(error, topicIds, callback) {
         if (!error.findBy?.('apiId', 'OFFSET_OUT_OF_RANGE')) {
-            return false;
+            callback(null, false);
+            return;
         }
         const response = error.response;
         if (!response || response.errorCode !== 0) {
-            return false;
+            callback(null, false);
+            return;
         }
         const recoveredOffsets = [];
+        const partitionsToRefresh = new Map();
         for (const topicResponse of response.responses) {
             const topic = topicIds.get(topicResponse.topicId);
             if (!topic) {
-                return false;
+                callback(null, false);
+                return;
             }
             for (const partitionResponse of topicResponse.partitions) {
                 if (partitionResponse.errorCode === 0) {
                     continue;
                 }
                 if (partitionResponse.errorCode !== protocolErrors.OFFSET_OUT_OF_RANGE.code) {
-                    return false;
+                    callback(null, false);
+                    return;
                 }
-                const key = `${topic}:${partitionResponse.partitionIndex}`;
                 const offset = this.#fallbackMode === MessagesStreamFallbackModes.EARLIEST
                     ? partitionResponse.logStartOffset
                     : partitionResponse.highWatermark;
-                recoveredOffsets.push([key, offset]);
+                if (offset >= 0n) {
+                    recoveredOffsets.push([partitionKey(topic, partitionResponse.partitionIndex), offset]);
+                    continue;
+                }
+                let partitions = partitionsToRefresh.get(topic);
+                if (!partitions) {
+                    partitions = [];
+                    partitionsToRefresh.set(topic, partitions);
+                }
+                partitions.push(partitionResponse.partitionIndex);
             }
         }
+        if (partitionsToRefresh.size > 0) {
+            this.#refreshOutOfRangeOffsets(partitionsToRefresh, recoveredOffsets, callback);
+            return;
+        }
+        callback(null, this.#applyRecoveredOffsets(recoveredOffsets));
+    }
+    #refreshOutOfRangeOffsets(partitionsToRefresh, recoveredOffsets, callback) {
+        const partitions = Object.fromEntries(partitionsToRefresh);
+        this.#consumer.listOffsets({
+            topics: Array.from(partitionsToRefresh.keys()),
+            partitions,
+            timestamp: this.#fallbackMode === MessagesStreamFallbackModes.EARLIEST
+                ? ListOffsetTimestamps.EARLIEST
+                : ListOffsetTimestamps.LATEST
+        }, (error, offsets) => {
+            if (error) {
+                callback(error);
+                return;
+            }
+            for (const [topic, refreshedPartitions] of partitionsToRefresh) {
+                const topicOffsets = offsets.get(topic);
+                for (const partition of refreshedPartitions) {
+                    const offset = topicOffsets?.[partition];
+                    if (typeof offset !== 'bigint' || offset < 0n) {
+                        callback(new UserError(`Cannot recover offset out of range for topic ${topic} partition ${partition}.`));
+                        return;
+                    }
+                    recoveredOffsets.push([partitionKey(topic, partition), offset]);
+                }
+            }
+            callback(null, this.#applyRecoveredOffsets(recoveredOffsets));
+        });
+    }
+    #applyRecoveredOffsets(recoveredOffsets) {
         for (const [key, offset] of recoveredOffsets) {
             this.#offsetsToFetch.set(key, offset);
             this.#offsetsCommitted.set(key, offset);

package/dist/network/connection.js

@@ -129,7 +129,7 @@ export class Connection extends TypedEventEmitter {
             }
             /* c8 ignore next 13 - Hard to test */
             const connectingSocketTimeoutHandler = () => {
-                const error = new TimeoutError(`Connection to ${host}:${port} timed out.`);
+                const error = new TimeoutError(`Connection to ${host}:${port} timed out.`, { canRetry: true });
                 diagnosticContext.error = error;
                 this.#socket.destroy();
                 this.#status = ConnectionStatuses.ERROR;
@@ -219,7 +219,7 @@ export class Connection extends TypedEventEmitter {
             this.#socket?.destroy();
             callback(new TimeoutError(this.#host
                 ? `Connection to ${this.#host}:${this.#port} timed out.`
-                : `Connection ready timed out after ${this.#options.connectTimeout}ms.`));
+                : `Connection ready timed out after ${this.#options.connectTimeout}ms.`, { canRetry: true }));
         }, this.#options.connectTimeout);
         this.once('connect', onConnect);
         this.once('error', onError);
@@ -258,7 +258,9 @@ export class Connection extends TypedEventEmitter {
         return callback[kCallbackPromise];
     }
     send(apiKey, apiVersion, createPayload, responseParser, hasRequestHeaderTaggedFields, hasResponseHeaderTaggedFields, callback) {
-        const correlationId = ++this.#correlationId;
+        // Correlation ID is a 32-bit integer in the protocol, so we need to wrap around after 2^31 - 1
+        const correlationId = (this.#correlationId + 1) & 0x7FFFFFFF;
+        this.#correlationId = correlationId;
         const diagnostic = createDiagnosticContext({
             connection: this,
             operation: 'send',

package/dist/typescript-4/dist/clients/base/base.d.ts

@@ -84,11 +84,11 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions, EventsT
   [kMetadata](options: MetadataOptions, callback: CallbackWithPromise<ClusterMetadata>): void;
   [kCheckNotClosed](callback: CallbackWithPromise<any>): boolean;
   clearMetadata(): void;
-  [kPerformWithRetry]<ReturnType>(operationId: string, operation: (callback: Callback<ReturnType>) => void, callback: CallbackWithPromise<ReturnType>, attempt?: number, errors?: Error[], shouldSkipRetry?: (e: Error) => boolean): void | Promise<ReturnType>;
+  [kPerformWithRetry]<ReturnType>(operationId: string, operation: (callback: Callback<ReturnType>, attempt: number) => void, callback: CallbackWithPromise<ReturnType>, attempt?: number, errors?: Error[], shouldSkipRetry?: (e: Error) => boolean): void | Promise<ReturnType>;
   [kPerformDeduplicated]<ReturnType>(operationId: string, operation: (callback: CallbackWithPromise<ReturnType>) => void, callback: CallbackWithPromise<ReturnType>): void | Promise<ReturnType>;
   [kGetApi]<RequestArguments extends Array<unknown>, ResponseType>(name: string, callback: Callback<API<RequestArguments, ResponseType>>): void;
   [kGetConnection](broker: Broker, callback: Callback<Connection>): void;
-  [kGetBootstrapConnection](callback: Callback<Connection>): void;
+  [kGetBootstrapConnection](callback: Callback<Connection>, attempt?: number): void;
   [kValidateOptions](target: unknown, validator: ValidateFunction<unknown>, targetName: string, throwOnErrors?: boolean): Error | null;
   [kInspect](...args: unknown[]): void;
   [kFormatValidationErrors](validator: ValidateFunction<unknown>, targetName: string): string;

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "2.1.0";
+export const version = "2.2.3";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "2.1.0",
+  "version": "2.2.3",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",