@platformatic/kafka 2.10.0-alpha.1 → 2.10.0

package/dist/clients/consumer/consumer.js

@@ -32,6 +32,10 @@ export class Consumer extends Base {
     #heartbeatInterval;
     #lastHeartbeatIntervalMs;
     #lastHeartbeat;
+    #heartbeatStallTimer;
+    #heartbeatStallStart;
+    #heartbeatStalled;
+    #heartbeatStallError;
     #useConsumerGroupProtocol;
     #memberEpoch;
     #groupRemoteAssignor;
@@ -77,6 +81,10 @@ export class Consumer extends Base {
         this.#heartbeatInterval = null;
         this.#lastHeartbeatIntervalMs = 0;
         this.#lastHeartbeat = null;
+        this.#heartbeatStallTimer = null;
+        this.#heartbeatStallStart = null;
+        this.#heartbeatStalled = false;
+        this.#heartbeatStallError = undefined;
         this.#streams = new Set();
         this.#lagMonitoring = null;
         this.#memberEpoch = 0;
@@ -121,6 +129,7 @@ export class Consumer extends Base {
         }
         this[kClosed] = true;
         clearTimeout(this.#lagMonitoring);
+        this.#resetHeartbeatStall();
         let closer;
         if (this.#useConsumerGroupProtocol) {
             closer = this.#leaveGroupConsumerProtocol.bind(this);
@@ -424,6 +433,7 @@ export class Consumer extends Base {
             return callback[kCallbackPromise];
         }
         this.#membershipActive = false;
+        this.#resetHeartbeatStall();
         this.#leaveGroupClassicProtocol(force, error => {
             if (error) {
                 this.#membershipActive = true;
@@ -486,12 +496,15 @@ export class Consumer extends Base {
     }
     #fetch(options, callback) {
         const isolationLevel = options.isolationLevel ?? this[kOptions].isolationLevel;
+        let refreshMetadata = false;
         this[kPerformWithRetry]('fetch', retryCallback => {
-            this[kMetadata]({ topics: this.topics.current }, (error, metadata) => {
+            this[kMetadata]({ topics: this.topics.current, forceUpdate: refreshMetadata }, (error, metadata) => {
                 if (error) {
+                    refreshMetadata = NetworkError.isRetryable(error);
                     retryCallback(error);
                     return;
                 }
+                refreshMetadata = false;
                 const topicIds = this.#topicIdsById(metadata);
                 const node = this.#fetchNodeForRequest(metadata, options.node, options.topics, topicIds, Date.now());
                 const broker = metadata.brokers.get(node);
@@ -502,6 +515,7 @@ export class Consumer extends Base {
                 const pool = options.connectionPool ?? this[kConnections];
                 pool.get(broker, (error, connection) => {
                     if (error) {
+                        refreshMetadata = NetworkError.isRetryable(error);
                         this.#clearPreferredReadReplicas(options.topics, topicIds);
                         this.#fetchSessions.delete(node);
                         // When a connection was not available (either interrupted or not available) we
@@ -517,6 +531,7 @@ export class Consumer extends Base {
                     }
                     this[kGetApi]('Fetch', (error, api) => {
                         if (error) {
+                            refreshMetadata = NetworkError.isRetryable(error);
                             retryCallback(error);
                             return;
                         }
@@ -577,37 +592,9 @@ export class Consumer extends Base {
                                 }
                             }
                         }
-                        // Fetch v12 identifies topics by name, while later versions use UUIDs.
-                        const useTopicNames = api.version <= 12;
-                        let topicIdsByName;
-                        if (useTopicNames) {
-                            // Built here rather than in the response callback so that the fetch loop allocates
-                            // it once per request instead of once per request and once per response.
-                            topicIdsByName = new Map();
-                            for (const [id, name] of topicIds) {
-                                topicIdsByName.set(name, id);
-                            }
-                            requestTopics = requestTopics.map(topic => ({
-                                ...topic,
-                                topicId: topicIds.get(topic.topicId) ?? topic.topicId
-                            }));
-                            for (const forgotten of forgottenTopicsData) {
-                                const topicId = forgotten.topicId ?? forgotten.topic;
-                                forgotten.topicId = topicIds.get(topicId) ?? topicId;
-                                delete forgotten.topic;
-                            }
-                        }
                         api(connection, options.maxWaitTime ?? this[kOptions].maxWaitTime, options.minBytes ?? this[kOptions].minBytes, options.maxBytes ?? this[kOptions].maxBytes, isolationLevel, sessionId, sessionEpoch, requestTopics, forgottenTopicsData, this.#clientRack, (error, result) => {
-                            const response = result ?? error?.response;
-                            if (response && !response.nodeEndpoints) {
-                                response.nodeEndpoints = [];
-                            }
-                            if (useTopicNames && response) {
-                                for (const topic of response.responses) {
-                                    topic.topicId = topicIdsByName.get(topic.topicId) ?? topic.topicId;
-                                }
-                            }
                             if (error) {
+                                refreshMetadata = NetworkError.isRetryable(error);
                                 this.#clearPreferredReadReplicas(options.topics, topicIds);
                                 if (findErrorBy(error, 'apiId', 'FETCH_SESSION_ID_NOT_FOUND') ||
                                     findErrorBy(error, 'apiId', 'INVALID_FETCH_SESSION_EPOCH') ||
@@ -697,12 +684,12 @@ export class Consumer extends Base {
             callback(error);
         });
     }
-    #listOffsets(withTimestamps, options, callback) {
+    #listOffsets(withTimestamps, options, callback, forceUpdateMetadata = false) {
         let topics = options.topics;
         if (!topics || topics.length === 0) {
             topics = this.topics.current;
         }
-        this[kMetadata]({ topics }, (error, metadata) => {
+        this[kMetadata]({ topics, forceUpdate: forceUpdateMetadata }, (error, metadata) => {
             if (error) {
                 callback(error);
                 return;
@@ -757,6 +744,11 @@ export class Consumer extends Base {
                 }, concurrentCallback, 0);
             }, (error, responses) => {
                 if (error) {
+                    if (!forceUpdateMetadata && NetworkError.isRetryable(error)) {
+                        this.clearMetadata();
+                        this.#listOffsets(withTimestamps, options, callback, true);
+                        return;
+                    }
                     callback(this.#handleError(error));
                     return;
                 }
@@ -813,17 +805,7 @@ export class Consumer extends Base {
                         memberEpoch: this.#useConsumerGroupProtocol ? this.#memberEpoch : -1,
                         topics
                     }
-                ], false, (error, response) => {
-                    if (error || api.version >= 8) {
-                        groupCallback(error, response);
-                        return;
-                    }
-                    const legacyResponse = response;
-                    groupCallback(null, {
-                        throttleTimeMs: legacyResponse.throttleTimeMs,
-                        groups: [{ groupId: this.groupId, topics: legacyResponse.topics, errorCode: legacyResponse.errorCode }]
-                    });
-                });
+                ], false, groupCallback);
             });
         }, (error, response) => {
             if (error) {
@@ -881,6 +863,7 @@ export class Consumer extends Base {
                 return;
             }
             this.emitWithDebug('consumer:heartbeat', 'start', eventPayload);
+            this.#trackHeartbeatStall(options);
             this[kGetApi]('Heartbeat', (error, api) => {
                 if (error) {
                     groupCallback(error);
@@ -897,21 +880,43 @@ export class Consumer extends Base {
             if (error) {
                 this.#cancelHeartbeat();
                 if (this.#getRejoinError(error)) {
+                    // A rejoin can legitimately block up to rebalanceTimeout:
+                    // - extend the stall window by that bound rather than pausing it
+                    // - a rejoin that never completes still surfaces as a stall
+                    this.#resetHeartbeatStall();
+                    this.#trackHeartbeatStall(options, error, options.rebalanceTimeout + this.#heartbeatStallTimeoutFor(options));
                     this[kPerformWithRetry]('rejoinGroup', retryCallback => {
                         this.#joinGroup(options, retryCallback);
                     }, error => {
                         if (error) {
                             this.emitWithDebug(null, 'error', error);
+                            if (!this.#heartbeatStalled) {
+                                // No more heartbeats will be scheduled: restart the stall window without
+                                // the rejoin grace, keeping the triggering heartbeat error as lastError.
+                                const heartbeatError = this.#heartbeatStallError;
+                                this.#resetHeartbeatStall();
+                                this.#trackHeartbeatStall(options, heartbeatError);
+                            }
+                        }
+                        else {
+                            // Rejoin succeeded: drop the extended window; the next heartbeat opens the normal one.
+                            this.#resetHeartbeatStall();
                         }
                         this.emitWithDebug('consumer', 'rejoin');
                     }, 0);
                     return;
                 }
+                // The error was not group related, so we schedule another heartbeat.
+                // Do this before notifying listeners so they can still cancel it.
+                this.#heartbeatInterval = setTimeout(() => {
+                    this.#heartbeat(options);
+                }, options.heartbeatInterval);
                 this.emitWithDebug('consumer:heartbeat', 'error', { ...eventPayload, error });
-                // Note that here we purposely do not return, since it was not a group related problem we schedule another heartbeat
+                this.#trackHeartbeatStall(options, error);
             }
             else {
                 this.#lastHeartbeat = new Date();
+                this.#resetHeartbeatStall();
                 this.emitWithDebug('consumer:heartbeat', 'end', eventPayload);
             }
             this.#heartbeatInterval?.refresh();
@@ -921,6 +926,47 @@ export class Consumer extends Base {
         clearTimeout(this.#heartbeatInterval);
         this.#heartbeatInterval = null;
     }
+    #heartbeatStallTimeoutFor(options) {
+        return (options.heartbeatStallTimeout ??
+            this[kOptions].heartbeatStallTimeout ??
+            options.rebalanceTimeout ??
+            this[kOptions].rebalanceTimeout ??
+            defaultConsumerOptions.rebalanceTimeout);
+    }
+    // Opens the stall window on the first heartbeat attempt after the last success.
+    // The window survives retries and only #resetHeartbeatStall closes it.
+    #trackHeartbeatStall(options, error, timeout) {
+        if (error) {
+            this.#heartbeatStallError = error;
+        }
+        if (this.#heartbeatStalled || this.#heartbeatStallTimer) {
+            return;
+        }
+        timeout ??= this.#heartbeatStallTimeoutFor(options);
+        this.#heartbeatStallStart = Date.now();
+        this.#heartbeatStallTimer = setTimeout(() => {
+            this.#heartbeatStallTimer = null;
+            // Emit once per stall episode. The next successful heartbeat re-arms the tracking.
+            this.#heartbeatStalled = true;
+            if (this[kClosed] || !this.#membershipActive) {
+                return;
+            }
+            this.emitWithDebug('consumer:heartbeat', 'stalled', {
+                lastError: this.#heartbeatStallError,
+                lastHeartbeat: this.#lastHeartbeat,
+                stalledFor: Date.now() - this.#heartbeatStallStart
+            });
+        }, timeout);
+    }
+    #resetHeartbeatStall() {
+        if (this.#heartbeatStallTimer) {
+            clearTimeout(this.#heartbeatStallTimer);
+            this.#heartbeatStallTimer = null;
+        }
+        this.#heartbeatStallStart = null;
+        this.#heartbeatStalled = false;
+        this.#heartbeatStallError = undefined;
+    }
     #consumerGroupHeartbeat(options, callback) {
         options.rebalanceTimeout ??= this[kOptions].rebalanceTimeout;
         consumerHeartbeatChannel.traceCallback(this.#performConsumerGroupHeartbeat, 1, createDiagnosticContext({ client: this, operation: 'consumerGroupHeartbeat' }), this, options, callback);
@@ -928,6 +974,7 @@ export class Consumer extends Base {
     #performConsumerGroupHeartbeat(options, callback) {
         this.#performGroupOperation('consumerGroupHeartbeat', (connection, groupCallback) => {
             this.emitWithDebug('consumer:heartbeat', 'start');
+            this.#trackHeartbeatStall(options);
             this[kGetApi]('ConsumerGroupHeartbeat', (error, api) => {
                 if (error) {
                     groupCallback(error);
@@ -938,6 +985,7 @@ export class Consumer extends Base {
             });
         }, (error, response) => {
             if (this[kClosed]) {
+                this.#resetHeartbeatStall();
                 this.emitWithDebug('consumer:heartbeat', 'end');
                 callback(null);
                 return;
@@ -945,6 +993,7 @@ export class Consumer extends Base {
             if (error) {
                 this.#cancelHeartbeat();
                 this.emitWithDebug('consumer:heartbeat', 'error', { error });
+                this.#trackHeartbeatStall(options, error);
                 const fenced = error.response?.errorCode === protocolErrors.FENCED_MEMBER_EPOCH.code;
                 if (fenced) {
                     this.#assignments = [];
@@ -962,6 +1011,7 @@ export class Consumer extends Base {
                 return;
             }
             this.#lastHeartbeat = new Date();
+            this.#resetHeartbeatStall();
             this.#memberEpoch = response.memberEpoch;
             if (response.memberId) {
                 const changed = this.memberId !== response.memberId;
@@ -1011,6 +1061,7 @@ export class Consumer extends Base {
         if (toRevoke.length === 0) {
             return;
         }
+        this.emitWithDebug('consumer', 'group:rebalance', { groupId: this.groupId });
         for (const stream of this.#streams) {
             stream.pause();
             stream[kAutocommit]();
@@ -1200,12 +1251,7 @@ export class Consumer extends Base {
                             retryCallback(error);
                             return;
                         }
-                        api(connection, FindCoordinatorKeyTypes.GROUP, [this.groupId], (error, response) => {
-                            if (response && api.version < 4) {
-                                response.coordinators[0].key = this.groupId;
-                            }
-                            retryCallback(error, response);
-                        });
+                        api(connection, FindCoordinatorKeyTypes.GROUP, [this.groupId], retryCallback);
                     });
                 });
             }, (error, response) => {
@@ -1586,13 +1632,12 @@ export class Consumer extends Base {
                 if (!partition.records) {
                     continue;
                 }
-                const abortedTransactions = partition.abortedTransactions ?? [];
-                if (abortedTransactions.length) {
+                if (partition.abortedTransactions.length) {
                     const abortedRanges = new Map();
                     // Find first offsets
                     // For last offsets we set -1n as special value. It allows us to detect open ranges, whichs means
                     // that Kafka has not sent the control marker. In that case we ignore the range for safety.
-                    for (const aborted of abortedTransactions) {
+                    for (const aborted of partition.abortedTransactions) {
                         abortedRanges.set(aborted.producerId, [aborted.firstOffset, -1n]);
                     }
                     // Now find last offsets

package/dist/clients/consumer/messages-stream.js

@@ -5,7 +5,7 @@ import { consumerReceivesChannel, createDiagnosticContext, notifyCreation } from
 import { findErrorBy, protocolErrors, UserError } from "../../errors.js";
 import { IS_CONTROL } from "../../protocol/records.js";
 import { runAsyncSeries } from "../../registries/abstract.js";
-import { kAutocommit, kGetFetchNode, kInstance, kRefreshOffsetsAndFetch } from "../../symbols.js";
+import { kAutocommit, kDeserializationError, kGetFetchNode, kInstance, kRefreshOffsetsAndFetch } from "../../symbols.js";
 import { kConnections, kCreateConnectionPool, kInspect, kPrometheus } from "../base/base.js";
 import { ensureMetric } from "../metrics.js";
 import { defaultConsumerOptions } from "./options.js";
@@ -24,11 +24,12 @@ function messageToJSON() {
     return {
         key: this.key,
         value: this.value,
-        headers: Array.from(this.headers.entries()),
+        headers: this.headerEntries.map(([key, value]) => [key, value]),
         topic: this.topic,
         partition: this.partition,
         timestamp: this.timestamp.toString(),
         offset: this.offset.toString(),
+        leaderEpoch: this.leaderEpoch,
         metadata: this.metadata
     };
 }
@@ -309,7 +310,12 @@ export class MessagesStream extends Readable {
     _construct(callback) {
         this.#refreshOffsetsInflight = true;
         this.#refreshOffsets(error => {
+            const wasPending = this.#refreshOffsetsPending;
             this.#refreshOffsetsInflight = false;
+            this.#refreshOffsetsPending = false;
+            if (!error && wasPending) {
+                this.#scheduleRefreshOffsetsAndFetch();
+            }
             callback(error ?? undefined);
         });
     }
@@ -610,11 +616,20 @@ export class MessagesStream extends Readable {
                             : this.#commit.bind(this, topic, partition, offset + 1n, leaderEpoch);
                         try {
                             const headers = new Map();
+                            const headerEntries = [];
                             let payloadType = 'headerKey';
                             let deserializedKey;
                             let deserializedValue;
                             let deserializationError;
+                            let hookFailed = false;
                             try {
+                                // A before-deserialization hook already failed for this message, report it as is
+                                const hookError = messageToConsume[kDeserializationError];
+                                if (hookError) {
+                                    hookFailed = true;
+                                    payloadType = hookError.payloadType;
+                                    throw hookError.error;
+                                }
                                 for (const [headerKey, headerValue] of record.headers) {
                                     payloadType = 'headerKey';
                                     /* c8 ignore next - Hard to test */
@@ -623,6 +638,7 @@ export class MessagesStream extends Readable {
                                     /* c8 ignore next - Hard to test */
                                     const deserializedHeaderValue = headerValueDeserializer(headerValue ?? undefined, messageToConsume);
                                     headers.set(deserializedHeaderKey, deserializedHeaderValue);
+                                    headerEntries.push([deserializedHeaderKey, deserializedHeaderValue]);
                                 }
                                 payloadType = 'key';
                                 /* c8 ignore next - Hard to test */
@@ -659,11 +675,14 @@ export class MessagesStream extends Readable {
                                     */
                                     diagnosticContext.error = error;
                                     consumerReceivesChannel.error.publish(diagnosticContext);
-                                    const headersFailed = payloadType === 'headerKey' || payloadType === 'headerValue';
+                                    // A hook failure happens before anything is deserialized, so nothing can be kept
+                                    const headersFailed = hookFailed || payloadType === 'headerKey' || payloadType === 'headerValue';
                                     if (headersFailed) {
                                         headers.clear();
+                                        headerEntries.length = 0;
                                         for (const [headerKey, headerValue] of record.headers) {
                                             headers.set(headerKey, headerValue);
+                                            headerEntries.push([headerKey, headerValue]);
                                         }
                                     }
                                     if (headersFailed || payloadType === 'key') {
@@ -678,7 +697,7 @@ export class MessagesStream extends Readable {
                                 else {
                                     diagnosticContext.error = error;
                                     consumerReceivesChannel.error.publish(diagnosticContext);
-                                    this.destroy(new UserError('Failed to deserialize a message.', { cause: error }));
+                                    this.destroy(hookFailed ? error : new UserError('Failed to deserialize a message.', { cause: error }));
                                     return;
                                 }
                             }
@@ -687,10 +706,14 @@ export class MessagesStream extends Readable {
                                 key: deserializedKey,
                                 value: deserializedValue,
                                 headers,
+                                headerEntries,
                                 topic,
                                 partition,
                                 timestamp: firstTimestamp + record.timestampDelta,
                                 offset,
+                                // The batch header epoch is the epoch of the record itself, which is what
+                                // OffsetCommit expects as committedLeaderEpoch (KIP-320).
+                                leaderEpoch: batch.partitionLeaderEpoch,
                                 commit,
                                 // Deserializers and hooks can enrich the metadata of the message they processed
                                 metadata: deserializationError
@@ -831,7 +854,10 @@ export class MessagesStream extends Readable {
                 topics.push(assignment);
             }
             if (!topics.length) {
-                this.#assignOffsets(offsets, new Map(), callback);
+                // No assignment yet: do not seed fallback offsets for the whole topic.
+                // consumer:group:join will refresh once partitions are assigned.
+                this.emit('offsets');
+                callback(null);
                 return;
             }
             this.#consumer.listCommittedOffsets({ topics }, (error, commits) => {
@@ -858,6 +884,7 @@ export class MessagesStream extends Readable {
         this.#refreshOffsetsInflight = true;
         this.#refreshOffsets(error => {
             const shouldDestroyOnError = this.#refreshOffsetsDestroyOnError;
+            const wasPending = this.#refreshOffsetsPending;
             this.#refreshOffsetsInflight = false;
             this.#refreshOffsetsDestroyOnError = false;
             this.#refreshOffsetsPending = false;
@@ -869,8 +896,7 @@ export class MessagesStream extends Readable {
                 return;
             }
             // A new one was scheduled while the previous one was inflight, we need to run it immediately
-            /* c8 ignore next 4 - Hard to test */
-            if (this.#refreshOffsetsPending) {
+            if (wasPending) {
                 this.#scheduleRefreshOffsetsAndFetch(shouldDestroyOnError);
                 return;
             }
@@ -972,9 +998,23 @@ export class MessagesStream extends Readable {
         }
         runAsyncSeries((request, cb) => {
             const [data, type, message] = request;
-            const result = hook(data, type, message, cb);
+            /*
+              Without an error handler a failing hook destroys the stream, as it always did. With one,
+              the failure is recorded on the message it belongs to and reported to the handler in
+              #pushRecords, together with everything else which can fail while deserializing, so that
+              the caller can skip or degrade that single message instead of losing the stream.
+            */
+            const onHookSettled = error => {
+                if (error && this.#deserializationErrorHandler) {
+                    message[kDeserializationError] ??= { error, payloadType: type };
+                    cb(null);
+                    return;
+                }
+                cb(error);
+            };
+            const result = hook(data, type, message, onHookSettled);
             if (typeof result?.then === 'function') {
-                result.then(() => cb(null), cb);
+                result.then(() => onHookSettled(null), onHookSettled);
             }
         }, requests, 0, error => {
             if (error) {

package/dist/clients/consumer/options.d.ts

@@ -15,6 +15,10 @@ export declare const groupOptionsProperties: {
         type: string;
         minimum: number;
     };
+    heartbeatStallTimeout: {
+        type: string;
+        minimum: number;
+    };
     groupProtocol: {
         type: string;
         enum: ("consumer" | "classic")[];
@@ -160,6 +164,10 @@ export declare const groupOptionsSchema: {
             type: string;
             minimum: number;
         };
+        heartbeatStallTimeout: {
+            type: string;
+            minimum: number;
+        };
         groupProtocol: {
             type: string;
             enum: ("consumer" | "classic")[];
@@ -283,6 +291,10 @@ export declare const consumeOptionsSchema: {
             type: string;
             minimum: number;
         };
+        heartbeatStallTimeout: {
+            type: string;
+            minimum: number;
+        };
         groupProtocol: {
             type: string;
             enum: ("consumer" | "classic")[];
@@ -455,6 +467,10 @@ export declare const consumerOptionsSchema: {
             type: string;
             minimum: number;
         };
+        heartbeatStallTimeout: {
+            type: string;
+            minimum: number;
+        };
         groupProtocol: {
             type: string;
             enum: ("consumer" | "classic")[];
@@ -585,6 +601,10 @@ export declare const fetchOptionsSchema: {
             type: string;
             minimum: number;
         };
+        heartbeatStallTimeout: {
+            type: string;
+            minimum: number;
+        };
         groupProtocol: {
             type: string;
             enum: ("consumer" | "classic")[];

package/dist/clients/consumer/options.js

@@ -8,6 +8,7 @@ export const groupOptionsProperties = {
     sessionTimeout: { type: 'number', minimum: 0 },
     rebalanceTimeout: { type: 'number', minimum: 0 },
     heartbeatInterval: { type: 'number', minimum: 0 },
+    heartbeatStallTimeout: { type: 'number', minimum: 0 },
     groupProtocol: { type: 'string', enum: allowedGroupProtocols },
     groupRemoteAssignor: { type: 'string' },
     protocols: {

package/dist/clients/consumer/types.d.ts

@@ -82,6 +82,7 @@ export interface GroupOptions {
     sessionTimeout?: number;
     rebalanceTimeout?: number;
     heartbeatInterval?: number;
+    heartbeatStallTimeout?: number;
     protocols?: GroupProtocolSubscription[];
     protocolsMetadata?: GroupProtocolsMetadataCallback;
     partitionAssigner?: GroupPartitionsAssigner;
@@ -92,6 +93,7 @@ export interface ConsumerGroupOptions {
     groupInstanceId?: string;
     groupRemoteAssignor?: string;
     rebalanceTimeout?: number;
+    heartbeatStallTimeout?: number;
     assignmentUserData?: Buffer;
 }
 export interface ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> {
@@ -169,3 +171,8 @@ export interface ConsumerHeartbeatPayload {
 export interface ConsumerHeartbeatErrorPayload extends ConsumerHeartbeatPayload {
     error: Error;
 }
+export interface ConsumerHeartbeatStalledPayload {
+    lastError?: Error;
+    lastHeartbeat: Date | null;
+    stalledFor: number;
+}