@platformatic/kafka 2.1.0 → 2.2.0

package/dist/clients/consumer/messages-stream.js

@@ -2,14 +2,14 @@ import { Readable } from 'node:stream';
 import { createPromisifiedCallback, kCallbackPromise, noopCallback } from "../../apis/callbacks.js";
 import { ListOffsetTimestamps } from "../../apis/enumerations.js";
 import { consumerReceivesChannel, createDiagnosticContext, notifyCreation } from "../../diagnostic.js";
-import { protocolErrors, UserError } from "../../errors.js";
+import { findErrorBy, protocolErrors, UserError } from "../../errors.js";
 import { IS_CONTROL } from "../../protocol/records.js";
 import { runAsyncSeries } from "../../registries/abstract.js";
-import { kAutocommit, kGetFetchNode, kInstance, kRefreshOffsetsAndFetch } from "../../symbols.js";
+import { kAutocommit, kDeserializationError, kGetFetchNode, kInstance, kRefreshOffsetsAndFetch } from "../../symbols.js";
 import { kConnections, kCreateConnectionPool, kInspect, kPrometheus } from "../base/base.js";
 import { ensureMetric } from "../metrics.js";
 import { defaultConsumerOptions } from "./options.js";
-import { MessagesStreamFallbackModes, MessagesStreamModes } from "./types.js";
+import { DeserializationErrorActions, MessagesStreamFallbackModes, MessagesStreamModes } from "./types.js";
 import { partitionKey } from "./utils.js";
 // Don't move this function as being in the same file will enable V8 to remove.
 // For futher info, ask Matteo.
@@ -24,11 +24,12 @@ function messageToJSON() {
     return {
         key: this.key,
         value: this.value,
-        headers: Array.from(this.headers.entries()),
+        headers: this.headerEntries.map(([key, value]) => [key, value]),
         topic: this.topic,
         partition: this.partition,
         timestamp: this.timestamp.toString(),
         offset: this.offset.toString(),
+        leaderEpoch: this.leaderEpoch,
         metadata: this.metadata
     };
 }
@@ -61,6 +62,7 @@ export class MessagesStream extends Readable {
     #closeCallbacks;
     #metricsConsumedMessages;
     #corruptedMessageHandler;
+    #deserializationErrorHandler;
     #context;
     #onConsumerGroupJoin;
     #onBrokerDisconnect;
@@ -83,9 +85,12 @@ export class MessagesStream extends Readable {
     */
     [kConnections];
     constructor(consumer, options) {
-        const { autocommit, mode, fallbackMode, maxFetches, offsets, context, deserializers, onCorruptedMessage, registry, beforeDeserialization, 
+        const { autocommit, mode, fallbackMode, maxFetches, offsets, context, deserializers, onCorruptedMessage, onDeserializationError, registry, beforeDeserialization, 
         // The options below are only destructured to avoid being part of structuredClone below
-        partitionAssigner: _partitionAssigner, ...otherOptions } = options;
+        partitionAssigner: _partitionAssigner, protocolsMetadata: _protocolsMetadata, ...otherOptions } = options;
+        if (onCorruptedMessage && onDeserializationError) {
+            throw new UserError('Cannot specify both onCorruptedMessage and onDeserializationError.');
+        }
         if (offsets && mode !== MessagesStreamModes.MANUAL) {
             throw new UserError('Cannot specify offsets when the stream mode is not MANUAL.');
         }
@@ -124,6 +129,7 @@ export class MessagesStream extends Readable {
         this.#closed = false;
         this.#closeCallbacks = [];
         this.#corruptedMessageHandler = onCorruptedMessage ?? defaultCorruptedMessageHandler;
+        this.#deserializationErrorHandler = onDeserializationError;
         this.#context = context;
         this.#onConsumerGroupJoin = () => {
             this.#offsetsCommitted.clear();
@@ -302,7 +308,16 @@ export class MessagesStream extends Readable {
         return super[Symbol.asyncIterator]();
     }
     _construct(callback) {
-        this.#refreshOffsets(callback);
+        this.#refreshOffsetsInflight = true;
+        this.#refreshOffsets(error => {
+            const wasPending = this.#refreshOffsetsPending;
+            this.#refreshOffsetsInflight = false;
+            this.#refreshOffsetsPending = false;
+            if (!error && wasPending) {
+                this.#scheduleRefreshOffsetsAndFetch();
+            }
+            callback(error ?? undefined);
+        });
     }
     _destroy(error, callback) {
         if (this.#autocommitInterval) {
@@ -409,10 +424,22 @@ export class MessagesStream extends Readable {
                             this.push(null);
                             return;
                         }
-                        if (this.#fallbackMode !== MessagesStreamFallbackModes.FAIL &&
-                            this.#handleOffsetOutOfRange(error, topicIds)) {
-                            process.nextTick(() => {
-                                this.#fetch();
+                        if (this.#fallbackMode !== MessagesStreamFallbackModes.FAIL) {
+                            this.#handleOffsetOutOfRange(error, topicIds, (recoveryError, recovered) => {
+                                if (this.#closed || this.closed || this.destroyed) {
+                                    return;
+                                }
+                                if (recoveryError) {
+                                    this.destroy(recoveryError);
+                                    return;
+                                }
+                                if (recovered) {
+                                    process.nextTick(() => {
+                                        this.#fetch();
+                                    });
+                                    return;
+                                }
+                                this.destroy(error);
                             });
                             return;
                         }
@@ -432,34 +459,81 @@ export class MessagesStream extends Readable {
             }
         });
     }
-    #handleOffsetOutOfRange(error, topicIds) {
-        if (!error.findBy?.('apiId', 'OFFSET_OUT_OF_RANGE')) {
-            return false;
+    #handleOffsetOutOfRange(error, topicIds, callback) {
+        if (!findErrorBy(error, 'apiId', 'OFFSET_OUT_OF_RANGE')) {
+            callback(null, false);
+            return;
         }
         const response = error.response;
         if (!response || response.errorCode !== 0) {
-            return false;
+            callback(null, false);
+            return;
         }
         const recoveredOffsets = [];
+        const partitionsToRefresh = new Map();
         for (const topicResponse of response.responses) {
             const topic = topicIds.get(topicResponse.topicId);
             if (!topic) {
-                return false;
+                callback(null, false);
+                return;
             }
             for (const partitionResponse of topicResponse.partitions) {
                 if (partitionResponse.errorCode === 0) {
                     continue;
                 }
                 if (partitionResponse.errorCode !== protocolErrors.OFFSET_OUT_OF_RANGE.code) {
-                    return false;
+                    callback(null, false);
+                    return;
                 }
-                const key = `${topic}:${partitionResponse.partitionIndex}`;
                 const offset = this.#fallbackMode === MessagesStreamFallbackModes.EARLIEST
                     ? partitionResponse.logStartOffset
                     : partitionResponse.highWatermark;
-                recoveredOffsets.push([key, offset]);
+                if (offset >= 0n) {
+                    recoveredOffsets.push([partitionKey(topic, partitionResponse.partitionIndex), offset]);
+                    continue;
+                }
+                let partitions = partitionsToRefresh.get(topic);
+                if (!partitions) {
+                    partitions = [];
+                    partitionsToRefresh.set(topic, partitions);
+                }
+                partitions.push(partitionResponse.partitionIndex);
             }
         }
+        if (partitionsToRefresh.size > 0) {
+            this.#refreshOutOfRangeOffsets(partitionsToRefresh, recoveredOffsets, callback);
+            return;
+        }
+        callback(null, this.#applyRecoveredOffsets(recoveredOffsets));
+    }
+    #refreshOutOfRangeOffsets(partitionsToRefresh, recoveredOffsets, callback) {
+        const partitions = Object.fromEntries(partitionsToRefresh);
+        this.#consumer.listOffsets({
+            topics: Array.from(partitionsToRefresh.keys()),
+            partitions,
+            timestamp: this.#fallbackMode === MessagesStreamFallbackModes.EARLIEST
+                ? ListOffsetTimestamps.EARLIEST
+                : ListOffsetTimestamps.LATEST
+        }, (error, offsets) => {
+            if (error) {
+                callback(error);
+                return;
+            }
+            for (const [topic, refreshedPartitions] of partitionsToRefresh) {
+                const topicOffsets = offsets.get(topic);
+                for (const partition of refreshedPartitions) {
+                    const offset = topicOffsets?.[partition];
+                    if (typeof offset !== 'bigint' || offset < 0n) {
+                        callback(new UserError(`Cannot recover offset out of range for topic ${topic} partition ${partition}.`));
+                        return;
+                    }
+                    recoveredOffsets.push([partitionKey(topic, partition), offset]);
+                }
+            }
+            callback(null, this.#applyRecoveredOffsets(recoveredOffsets));
+        });
+    }
+    #applyRecoveredOffsets(recoveredOffsets) {
         for (const [key, offset] of recoveredOffsets) {
             this.#offsetsToFetch.set(key, offset);
             this.#offsetsCommitted.set(key, offset);
@@ -490,6 +564,11 @@ export class MessagesStream extends Readable {
             const topic = topicIds.get(topicResponse.topicId);
             for (const { records: recordsBatches, partitionIndex: partition } of topicResponse.partitions) {
                 const key = partitionKey(topic, partition);
+                // The broker can return partitions which were not requested when they linger
+                // in an incremental fetch session - their data is stale and must not rewind offsets
+                if (!requestedOffsets.has(key)) {
+                    continue;
+                }
                 if (!recordsBatches) {
                     continue;
                 }
@@ -537,25 +616,111 @@ export class MessagesStream extends Readable {
                             : this.#commit.bind(this, topic, partition, offset + 1n, leaderEpoch);
                         try {
                             const headers = new Map();
-                            for (const [headerKey, headerValue] of record.headers) {
-                                headers.set(
-                                /* c8 ignore next 2 - Hard to test */
-                                headerKeyDeserializer(headerKey ?? undefined, messageToConsume), headerValueDeserializer(headerValue ?? undefined, messageToConsume));
+                            const headerEntries = [];
+                            let payloadType = 'headerKey';
+                            let deserializedKey;
+                            let deserializedValue;
+                            let deserializationError;
+                            let hookFailed = false;
+                            try {
+                                // A before-deserialization hook already failed for this message, report it as is
+                                const hookError = messageToConsume[kDeserializationError];
+                                if (hookError) {
+                                    hookFailed = true;
+                                    payloadType = hookError.payloadType;
+                                    throw hookError.error;
+                                }
+                                for (const [headerKey, headerValue] of record.headers) {
+                                    payloadType = 'headerKey';
+                                    /* c8 ignore next - Hard to test */
+                                    const deserializedHeaderKey = headerKeyDeserializer(headerKey ?? undefined, messageToConsume);
+                                    payloadType = 'headerValue';
+                                    /* c8 ignore next - Hard to test */
+                                    const deserializedHeaderValue = headerValueDeserializer(headerValue ?? undefined, messageToConsume);
+                                    headers.set(deserializedHeaderKey, deserializedHeaderValue);
+                                    headerEntries.push([deserializedHeaderKey, deserializedHeaderValue]);
+                                }
+                                payloadType = 'key';
+                                /* c8 ignore next - Hard to test */
+                                deserializedKey = keyDeserializer(record.key ?? undefined, headers, messageToConsume);
+                                payloadType = 'value';
+                                deserializedValue = valueDeserializer(record.value ?? undefined, headers, messageToConsume);
+                            }
+                            catch (error) {
+                                const context = {
+                                    error,
+                                    payloadType,
+                                    record,
+                                    topic,
+                                    partition,
+                                    offset,
+                                    timestamp: firstTimestamp + record.timestampDelta,
+                                    commit: commit
+                                };
+                                const action = this.#deserializationErrorHandler
+                                    ? this.#deserializationErrorHandler(context)
+                                    : this.#corruptedMessageHandler(record, topic, partition, firstTimestamp, firstOffset, commit, error)
+                                        ? DeserializationErrorActions.FAIL
+                                        : DeserializationErrorActions.SKIP;
+                                if (action === DeserializationErrorActions.CONTINUE) {
+                                    /*
+                                      Deliver the message in a degraded form, so that nothing is lost: whatever was
+                                      already deserialized is kept, while the payload which failed and the ones which
+                                      were never attempted, since deserialization goes headers, key and then value,
+                                      are delivered as the original bytes.
+                  
+                                      The error is reported on the message metadata and published on the diagnostic
+                                      channel, which is where callers should react to it: logging here would be a log
+                                      storm waiting to happen.
+                                    */
+                                    diagnosticContext.error = error;
+                                    consumerReceivesChannel.error.publish(diagnosticContext);
+                                    // A hook failure happens before anything is deserialized, so nothing can be kept
+                                    const headersFailed = hookFailed || payloadType === 'headerKey' || payloadType === 'headerValue';
+                                    if (headersFailed) {
+                                        headers.clear();
+                                        headerEntries.length = 0;
+                                        for (const [headerKey, headerValue] of record.headers) {
+                                            headers.set(headerKey, headerValue);
+                                            headerEntries.push([headerKey, headerValue]);
+                                        }
+                                    }
+                                    if (headersFailed || payloadType === 'key') {
+                                        deserializedKey = (record.key ?? undefined);
+                                    }
+                                    deserializedValue = (record.value ?? undefined);
+                                    deserializationError = { error, payloadType };
+                                }
+                                else if (action === DeserializationErrorActions.SKIP) {
+                                    continue;
+                                }
+                                else {
+                                    diagnosticContext.error = error;
+                                    consumerReceivesChannel.error.publish(diagnosticContext);
+                                    this.destroy(hookFailed ? error : new UserError('Failed to deserialize a message.', { cause: error }));
+                                    return;
+                                }
                             }
-                            /* c8 ignore next 2 - Hard to test */
-                            const key = keyDeserializer(record.key ?? undefined, headers, messageToConsume);
-                            const value = valueDeserializer(record.value ?? undefined, headers, messageToConsume);
                             this.#metricsConsumedMessages?.inc();
                             const message = {
-                                key,
-                                value,
+                                key: deserializedKey,
+                                value: deserializedValue,
                                 headers,
+                                headerEntries,
                                 topic,
                                 partition,
                                 timestamp: firstTimestamp + record.timestampDelta,
                                 offset,
+                                // The batch header epoch is the epoch of the record itself, which is what
+                                // OffsetCommit expects as committedLeaderEpoch (KIP-320).
+                                leaderEpoch: batch.partitionLeaderEpoch,
                                 commit,
-                                metadata: messageMetadata,
+                                // Deserializers and hooks can enrich the metadata of the message they processed
+                                metadata: deserializationError
+                                    ? { ...messageMetadata, ...messageToConsume.metadata, deserializationError }
+                                    : messageToConsume.metadata
+                                        ? { ...messageMetadata, ...messageToConsume.metadata }
+                                        : messageMetadata,
                                 toJSON: messageToJSON
                             };
                             diagnosticContext.result = message;
@@ -564,13 +729,10 @@ export class MessagesStream extends Readable {
                             consumerReceivesChannel.asyncEnd.publish(diagnosticContext);
                         }
                         catch (error) {
-                            const shouldDestroy = this.#corruptedMessageHandler(record, topic, partition, firstTimestamp, firstOffset, commit);
-                            if (shouldDestroy) {
-                                diagnosticContext.error = error;
-                                consumerReceivesChannel.error.publish(diagnosticContext);
-                                this.destroy(new UserError('Failed to deserialize a message.', { cause: error }));
-                                return;
-                            }
+                            diagnosticContext.error = error;
+                            consumerReceivesChannel.error.publish(diagnosticContext);
+                            this.destroy(new UserError('Failed to process a message.', { cause: error }));
+                            return;
                         }
                         finally {
                             consumerReceivesChannel.end.publish(diagnosticContext);
@@ -644,7 +806,7 @@ export class MessagesStream extends Readable {
                 // Only destroy the stream when the broker requires a group rejoin
                 // (ILLEGAL_GENERATION, UNKNOWN_MEMBER_ID, REBALANCE_IN_PROGRESS).
                 // Transient coordinator errors must not tear down the consumption loop.
-                if (error.findBy?.('needsRejoin', true)) {
+                if (findErrorBy(error, 'needsRejoin', true)) {
                     this.destroy(error);
                 }
                 return;
@@ -692,7 +854,10 @@ export class MessagesStream extends Readable {
                 topics.push(assignment);
             }
             if (!topics.length) {
-                this.#assignOffsets(offsets, new Map(), callback);
+                // No assignment yet: do not seed fallback offsets for the whole topic.
+                // consumer:group:join will refresh once partitions are assigned.
+                this.emit('offsets');
+                callback(null);
                 return;
             }
             this.#consumer.listCommittedOffsets({ topics }, (error, commits) => {
@@ -719,6 +884,7 @@ export class MessagesStream extends Readable {
         this.#refreshOffsetsInflight = true;
         this.#refreshOffsets(error => {
             const shouldDestroyOnError = this.#refreshOffsetsDestroyOnError;
+            const wasPending = this.#refreshOffsetsPending;
             this.#refreshOffsetsInflight = false;
             this.#refreshOffsetsDestroyOnError = false;
             this.#refreshOffsetsPending = false;
@@ -730,8 +896,7 @@ export class MessagesStream extends Readable {
                 return;
             }
             // A new one was scheduled while the previous one was inflight, we need to run it immediately
-            /* c8 ignore next 4 - Hard to test */
-            if (this.#refreshOffsetsPending) {
+            if (wasPending) {
                 this.#scheduleRefreshOffsetsAndFetch(shouldDestroyOnError);
                 return;
             }
@@ -797,7 +962,13 @@ export class MessagesStream extends Readable {
         const requests = [];
         // Create the pre-deserialization requests
         for (const topicResponse of response.responses) {
+            const topic = topicIds.get(topicResponse.topicId);
             for (const { records: recordsBatches, partitionIndex: partition } of topicResponse.partitions) {
+                // The broker can return partitions which were not requested when they linger
+                // in an incremental fetch session - do not run hooks on their stale records
+                if (!requestedOffsets.has(partitionKey(topic, partition))) {
+                    continue;
+                }
                 /* c8 ignore next 3 - Hard to test */
                 if (!recordsBatches) {
                     continue;
@@ -809,7 +980,7 @@ export class MessagesStream extends Readable {
                         continue;
                     }
                     for (const message of batch.records) {
-                        message.topic = topicIds.get(topicResponse.topicId);
+                        message.topic = topic;
                         message.partition = partition;
                         requests.push([message.key, 'key', message]);
                         requests.push([message.value, 'value', message]);
@@ -827,9 +998,23 @@ export class MessagesStream extends Readable {
         }
         runAsyncSeries((request, cb) => {
             const [data, type, message] = request;
-            const result = hook(data, type, message, cb);
+            /*
+              Without an error handler a failing hook destroys the stream, as it always did. With one,
+              the failure is recorded on the message it belongs to and reported to the handler in
+              #pushRecords, together with everything else which can fail while deserializing, so that
+              the caller can skip or degrade that single message instead of losing the stream.
+            */
+            const onHookSettled = error => {
+                if (error && this.#deserializationErrorHandler) {
+                    message[kDeserializationError] ??= { error, payloadType: type };
+                    cb(null);
+                    return;
+                }
+                cb(error);
+            };
+            const result = hook(data, type, message, onHookSettled);
             if (typeof result?.then === 'function') {
-                result.then(() => cb(null), cb);
+                result.then(() => onHookSettled(null), onHookSettled);
             }
         }, requests, 0, error => {
             if (error) {

package/dist/clients/consumer/options.d.ts

@@ -15,6 +15,10 @@ export declare const groupOptionsProperties: {
         type: string;
         minimum: number;
     };
+    heartbeatStallTimeout: {
+        type: string;
+        minimum: number;
+    };
     groupProtocol: {
         type: string;
         enum: ("consumer" | "classic")[];
@@ -53,6 +57,9 @@ export declare const groupOptionsProperties: {
             };
         };
     };
+    protocolsMetadata: {
+        function: boolean;
+    };
     partitionAssigner: {
         function: boolean;
     };
@@ -157,6 +164,10 @@ export declare const groupOptionsSchema: {
             type: string;
             minimum: number;
         };
+        heartbeatStallTimeout: {
+            type: string;
+            minimum: number;
+        };
         groupProtocol: {
             type: string;
             enum: ("consumer" | "classic")[];
@@ -195,6 +206,9 @@ export declare const groupOptionsSchema: {
                 };
             };
         };
+        protocolsMetadata: {
+            function: boolean;
+        };
         partitionAssigner: {
             function: boolean;
         };
@@ -277,6 +291,10 @@ export declare const consumeOptionsSchema: {
             type: string;
             minimum: number;
         };
+        heartbeatStallTimeout: {
+            type: string;
+            minimum: number;
+        };
         groupProtocol: {
             type: string;
             enum: ("consumer" | "classic")[];
@@ -315,6 +333,9 @@ export declare const consumeOptionsSchema: {
                 };
             };
         };
+        protocolsMetadata: {
+            function: boolean;
+        };
         partitionAssigner: {
             function: boolean;
         };
@@ -363,6 +384,9 @@ export declare const consumeOptionsSchema: {
         onCorruptedMessage: {
             function: boolean;
         };
+        onDeserializationError: {
+            function: boolean;
+        };
     };
     required: string[];
     additionalProperties: boolean;
@@ -443,6 +467,10 @@ export declare const consumerOptionsSchema: {
             type: string;
             minimum: number;
         };
+        heartbeatStallTimeout: {
+            type: string;
+            minimum: number;
+        };
         groupProtocol: {
             type: string;
             enum: ("consumer" | "classic")[];
@@ -481,6 +509,9 @@ export declare const consumerOptionsSchema: {
                 };
             };
         };
+        protocolsMetadata: {
+            function: boolean;
+        };
         partitionAssigner: {
             function: boolean;
         };
@@ -570,6 +601,10 @@ export declare const fetchOptionsSchema: {
             type: string;
             minimum: number;
         };
+        heartbeatStallTimeout: {
+            type: string;
+            minimum: number;
+        };
         groupProtocol: {
             type: string;
             enum: ("consumer" | "classic")[];
@@ -608,6 +643,9 @@ export declare const fetchOptionsSchema: {
                 };
             };
         };
+        protocolsMetadata: {
+            function: boolean;
+        };
         partitionAssigner: {
             function: boolean;
         };

package/dist/clients/consumer/options.js

@@ -8,6 +8,7 @@ export const groupOptionsProperties = {
     sessionTimeout: { type: 'number', minimum: 0 },
     rebalanceTimeout: { type: 'number', minimum: 0 },
     heartbeatInterval: { type: 'number', minimum: 0 },
+    heartbeatStallTimeout: { type: 'number', minimum: 0 },
     groupProtocol: { type: 'string', enum: allowedGroupProtocols },
     groupRemoteAssignor: { type: 'string' },
     protocols: {
@@ -25,6 +26,7 @@ export const groupOptionsProperties = {
             }
         }
     },
+    protocolsMetadata: { function: true },
     partitionAssigner: { function: true }
 };
 export const groupOptionsAdditionalValidations = {
@@ -89,6 +91,7 @@ export const consumeOptionsSchema = {
             }
         },
         onCorruptedMessage: { function: true },
+        onDeserializationError: { function: true },
         ...groupOptionsProperties,
         ...consumeOptionsProperties
     },

package/dist/clients/consumer/types.d.ts

@@ -1,10 +1,11 @@
+import { type Callback } from '../../apis/definitions.ts';
 import { type FetchRequestTopic } from '../../apis/consumer/fetch-v17.ts';
 import { type GroupProtocols } from '../../apis/enumerations.ts';
 import { type ConnectionPool } from '../../network/connection-pool.ts';
 import { type KafkaRecord, type Message } from '../../protocol/records.ts';
 import { type SchemaRegistry } from '../../registries/abstract.ts';
 import { type BaseOptions, type ClusterMetadata, type TopicWithPartitionAndOffset } from '../base/types.ts';
-import { type BeforeDeserializationHook, type Deserializers } from '../serde.ts';
+import { type BeforeDeserializationHook, type BeforeHookPayloadType, type Deserializers } from '../serde.ts';
 export interface GroupProtocolSubscription {
     name: string;
     version: number;
@@ -31,8 +32,33 @@ export interface PreferredReadReplica {
     node: number;
     expiresAt: number;
 }
-export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit']) => boolean;
+export declare const DeserializationErrorActions: {
+    readonly FAIL: "fail";
+    readonly SKIP: "skip";
+    readonly CONTINUE: "continue";
+};
+export type DeserializationErrorAction = (typeof DeserializationErrorActions)[keyof typeof DeserializationErrorActions];
+export interface DeserializationErrorContext {
+    error: unknown;
+    payloadType: BeforeHookPayloadType;
+    record: KafkaRecord;
+    topic: string;
+    partition: number;
+    offset: bigint;
+    timestamp: bigint;
+    commit: Message['commit'];
+}
+export interface MessageDeserializationError {
+    error: unknown;
+    payloadType: BeforeHookPayloadType;
+}
+export type DeserializationErrorHandler = (context: DeserializationErrorContext) => DeserializationErrorAction;
+/**
+ * @deprecated Use `DeserializationErrorHandler` instead.
+ */
+export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit'], error: unknown) => boolean;
 export type GroupPartitionsAssigner = (current: string, members: Map<string, ExtendedGroupProtocolSubscription>, topics: Set<string>, metadata: ClusterMetadata) => GroupPartitionsAssignments[];
+export type GroupProtocolsMetadataCallback = (protocols: GroupProtocolSubscription[], topics: string[], metadata: ClusterMetadata, callback: Callback<Buffer>) => Buffer | Promise<Buffer> | undefined;
 export declare const MessagesStreamModes: {
     readonly LATEST: "latest";
     readonly EARLIEST: "earliest";
@@ -56,7 +82,9 @@ export interface GroupOptions {
     sessionTimeout?: number;
     rebalanceTimeout?: number;
     heartbeatInterval?: number;
+    heartbeatStallTimeout?: number;
     protocols?: GroupProtocolSubscription[];
+    protocolsMetadata?: GroupProtocolsMetadataCallback;
     partitionAssigner?: GroupPartitionsAssigner;
     assignmentUserData?: Buffer;
 }
@@ -65,6 +93,7 @@ export interface ConsumerGroupOptions {
     groupInstanceId?: string;
     groupRemoteAssignor?: string;
     rebalanceTimeout?: number;
+    heartbeatStallTimeout?: number;
     assignmentUserData?: Buffer;
 }
 export interface ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> {
@@ -86,6 +115,10 @@ export interface StreamOptions {
     fallbackMode?: MessagesStreamFallbackModeValue;
     maxFetches?: number;
     offsets?: TopicWithPartitionAndOffset[];
+    onDeserializationError?: DeserializationErrorHandler;
+    /**
+     * @deprecated Use `onDeserializationError` instead.
+     */
     onCorruptedMessage?: CorruptedMessageHandler;
 }
 export type ConsumeOptions<Key, Value, HeaderKey, HeaderValue> = StreamOptions & ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> & GroupOptions;
@@ -138,3 +171,8 @@ export interface ConsumerHeartbeatPayload {
 export interface ConsumerHeartbeatErrorPayload extends ConsumerHeartbeatPayload {
     error: Error;
 }
+export interface ConsumerHeartbeatStalledPayload {
+    lastError?: Error;
+    lastHeartbeat: Date | null;
+    stalledFor: number;
+}

package/dist/clients/consumer/types.js

@@ -1,3 +1,8 @@
+export const DeserializationErrorActions = {
+    FAIL: 'fail',
+    SKIP: 'skip',
+    CONTINUE: 'continue'
+};
 export const MessagesStreamModes = {
     LATEST: 'latest',
     EARLIEST: 'earliest',

package/dist/clients/consumer/utils.d.ts

@@ -1 +1,2 @@
 export declare function partitionKey(topic: string, partition: number): string;
+export declare function nextFetchEpoch(epoch: number): number;