@platformatic/kafka 2.0.1 → 2.2.0

package/dist/clients/consumer/messages-stream.js

@@ -2,14 +2,15 @@ import { Readable } from 'node:stream';
 import { createPromisifiedCallback, kCallbackPromise, noopCallback } from "../../apis/callbacks.js";
 import { ListOffsetTimestamps } from "../../apis/enumerations.js";
 import { consumerReceivesChannel, createDiagnosticContext, notifyCreation } from "../../diagnostic.js";
-import { protocolErrors, UserError } from "../../errors.js";
+import { findErrorBy, protocolErrors, UserError } from "../../errors.js";
 import { IS_CONTROL } from "../../protocol/records.js";
 import { runAsyncSeries } from "../../registries/abstract.js";
-import { kAutocommit, kInstance, kRefreshOffsetsAndFetch } from "../../symbols.js";
+import { kAutocommit, kDeserializationError, kGetFetchNode, kInstance, kRefreshOffsetsAndFetch } from "../../symbols.js";
 import { kConnections, kCreateConnectionPool, kInspect, kPrometheus } from "../base/base.js";
 import { ensureMetric } from "../metrics.js";
 import { defaultConsumerOptions } from "./options.js";
-import { MessagesStreamFallbackModes, MessagesStreamModes } from "./types.js";
+import { DeserializationErrorActions, MessagesStreamFallbackModes, MessagesStreamModes } from "./types.js";
+import { partitionKey } from "./utils.js";
 // Don't move this function as being in the same file will enable V8 to remove.
 // For futher info, ask Matteo.
 /* c8 ignore next 3 - Fallback deserializer, nothing to really test */
@@ -23,11 +24,12 @@ function messageToJSON() {
     return {
         key: this.key,
         value: this.value,
-        headers: Array.from(this.headers.entries()),
+        headers: this.headerEntries.map(([key, value]) => [key, value]),
         topic: this.topic,
         partition: this.partition,
         timestamp: this.timestamp.toString(),
         offset: this.offset.toString(),
+        leaderEpoch: this.leaderEpoch,
         metadata: this.metadata
     };
 }
@@ -60,6 +62,7 @@ export class MessagesStream extends Readable {
     #closeCallbacks;
     #metricsConsumedMessages;
     #corruptedMessageHandler;
+    #deserializationErrorHandler;
     #context;
     #onConsumerGroupJoin;
     #onBrokerDisconnect;
@@ -82,9 +85,12 @@ export class MessagesStream extends Readable {
     */
     [kConnections];
     constructor(consumer, options) {
-        const { autocommit, mode, fallbackMode, maxFetches, offsets, context, deserializers, onCorruptedMessage, registry, beforeDeserialization, 
+        const { autocommit, mode, fallbackMode, maxFetches, offsets, context, deserializers, onCorruptedMessage, onDeserializationError, registry, beforeDeserialization, 
         // The options below are only destructured to avoid being part of structuredClone below
-        partitionAssigner: _partitionAssigner, ...otherOptions } = options;
+        partitionAssigner: _partitionAssigner, protocolsMetadata: _protocolsMetadata, ...otherOptions } = options;
+        if (onCorruptedMessage && onDeserializationError) {
+            throw new UserError('Cannot specify both onCorruptedMessage and onDeserializationError.');
+        }
         if (offsets && mode !== MessagesStreamModes.MANUAL) {
             throw new UserError('Cannot specify offsets when the stream mode is not MANUAL.');
         }
@@ -123,6 +129,7 @@ export class MessagesStream extends Readable {
         this.#closed = false;
         this.#closeCallbacks = [];
         this.#corruptedMessageHandler = onCorruptedMessage ?? defaultCorruptedMessageHandler;
+        this.#deserializationErrorHandler = onDeserializationError;
         this.#context = context;
         this.#onConsumerGroupJoin = () => {
             this.#offsetsCommitted.clear();
@@ -145,7 +152,7 @@ export class MessagesStream extends Readable {
         this.#offsetsToFetch = new Map();
         if (offsets) {
             for (const { topic, partition, offset } of offsets) {
-                this.#offsetsToFetch.set(`${topic}:${partition}`, offset);
+                this.#offsetsToFetch.set(partitionKey(topic, partition), offset);
             }
         }
         // Clone the rest of the options so the user can never mutate them
@@ -301,7 +308,16 @@ export class MessagesStream extends Readable {
         return super[Symbol.asyncIterator]();
     }
     _construct(callback) {
-        this.#refreshOffsets(callback);
+        this.#refreshOffsetsInflight = true;
+        this.#refreshOffsets(error => {
+            const wasPending = this.#refreshOffsetsPending;
+            this.#refreshOffsetsInflight = false;
+            this.#refreshOffsetsPending = false;
+            if (!error && wasPending) {
+                this.#scheduleRefreshOffsetsAndFetch();
+            }
+            callback(error ?? undefined);
+        });
     }
     _destroy(error, callback) {
         if (this.#autocommitInterval) {
@@ -364,21 +380,22 @@ export class MessagesStream extends Readable {
                 }
                 const partitions = assignment.partitions;
                 for (const partition of partitions) {
-                    const leader = metadata.topics.get(topic).partitions[partition].leader;
-                    if (this.#inflightNodes.has(leader)) {
+                    const targetNode = this.#consumer[kGetFetchNode](metadata, topic, partition, now);
+                    const key = partitionKey(topic, partition);
+                    if (this.#inflightNodes.has(targetNode)) {
                         continue;
                     }
-                    let leaderRequests = requests.get(leader);
-                    if (!leaderRequests) {
-                        leaderRequests = [];
-                        requests.set(leader, leaderRequests);
+                    let nodeRequests = requests.get(targetNode);
+                    if (!nodeRequests) {
+                        nodeRequests = [];
+                        requests.set(targetNode, nodeRequests);
                     }
                     const topicId = metadata.topics.get(topic).id;
                     topicIds.set(topicId, topic);
-                    const fetchOffset = this.#offsetsToFetch.get(`${topic}:${partition}`);
-                    requestedOffsets.set(`${topic}:${partition}`, fetchOffset);
-                    const leaderEpoch = this.#partitionsEpochs.get(`${topic}:${partition}`) ?? -1;
-                    leaderRequests.push({
+                    const fetchOffset = this.#offsetsToFetch.get(key);
+                    requestedOffsets.set(key, fetchOffset);
+                    const leaderEpoch = this.#partitionsEpochs.get(key) ?? -1;
+                    nodeRequests.push({
                         topicId,
                         partitions: [
                             {
@@ -395,10 +412,10 @@ export class MessagesStream extends Readable {
             if (requests.size === 0) {
                 return;
             }
-            for (const [leader, leaderRequests] of requests) {
-                this.#inflightNodes.set(leader, Date.now());
-                this.#consumer.fetch({ ...this.#options, node: leader, topics: leaderRequests, connectionPool: this[kConnections] }, (error, response) => {
-                    this.#inflightNodes.delete(leader);
+            for (const [node, nodeRequests] of requests) {
+                this.#inflightNodes.set(node, Date.now());
+                this.#consumer.fetch({ ...this.#options, node, topics: nodeRequests, connectionPool: this[kConnections] }, (error, response) => {
+                    this.#inflightNodes.delete(node);
                     this.emit('fetch');
                     if (error) {
                         // The stream has been closed, ignore the error
@@ -407,10 +424,22 @@ export class MessagesStream extends Readable {
                             this.push(null);
                             return;
                         }
-                        if (this.#fallbackMode !== MessagesStreamFallbackModes.FAIL &&
-                            this.#handleOffsetOutOfRange(error, topicIds)) {
-                            process.nextTick(() => {
-                                this.#fetch();
+                        if (this.#fallbackMode !== MessagesStreamFallbackModes.FAIL) {
+                            this.#handleOffsetOutOfRange(error, topicIds, (recoveryError, recovered) => {
+                                if (this.#closed || this.closed || this.destroyed) {
+                                    return;
+                                }
+                                if (recoveryError) {
+                                    this.destroy(recoveryError);
+                                    return;
+                                }
+                                if (recovered) {
+                                    process.nextTick(() => {
+                                        this.#fetch();
+                                    });
+                                    return;
+                                }
+                                this.destroy(error);
                             });
                             return;
                         }
@@ -430,34 +459,81 @@ export class MessagesStream extends Readable {
             }
         });
     }
-    #handleOffsetOutOfRange(error, topicIds) {
-        if (!error.findBy?.('apiId', 'OFFSET_OUT_OF_RANGE')) {
-            return false;
+    #handleOffsetOutOfRange(error, topicIds, callback) {
+        if (!findErrorBy(error, 'apiId', 'OFFSET_OUT_OF_RANGE')) {
+            callback(null, false);
+            return;
         }
         const response = error.response;
         if (!response || response.errorCode !== 0) {
-            return false;
+            callback(null, false);
+            return;
         }
         const recoveredOffsets = [];
+        const partitionsToRefresh = new Map();
         for (const topicResponse of response.responses) {
             const topic = topicIds.get(topicResponse.topicId);
             if (!topic) {
-                return false;
+                callback(null, false);
+                return;
             }
             for (const partitionResponse of topicResponse.partitions) {
                 if (partitionResponse.errorCode === 0) {
                     continue;
                 }
                 if (partitionResponse.errorCode !== protocolErrors.OFFSET_OUT_OF_RANGE.code) {
-                    return false;
+                    callback(null, false);
+                    return;
                 }
-                const key = `${topic}:${partitionResponse.partitionIndex}`;
                 const offset = this.#fallbackMode === MessagesStreamFallbackModes.EARLIEST
                     ? partitionResponse.logStartOffset
                     : partitionResponse.highWatermark;
-                recoveredOffsets.push([key, offset]);
+                if (offset >= 0n) {
+                    recoveredOffsets.push([partitionKey(topic, partitionResponse.partitionIndex), offset]);
+                    continue;
+                }
+                let partitions = partitionsToRefresh.get(topic);
+                if (!partitions) {
+                    partitions = [];
+                    partitionsToRefresh.set(topic, partitions);
+                }
+                partitions.push(partitionResponse.partitionIndex);
             }
         }
+        if (partitionsToRefresh.size > 0) {
+            this.#refreshOutOfRangeOffsets(partitionsToRefresh, recoveredOffsets, callback);
+            return;
+        }
+        callback(null, this.#applyRecoveredOffsets(recoveredOffsets));
+    }
+    #refreshOutOfRangeOffsets(partitionsToRefresh, recoveredOffsets, callback) {
+        const partitions = Object.fromEntries(partitionsToRefresh);
+        this.#consumer.listOffsets({
+            topics: Array.from(partitionsToRefresh.keys()),
+            partitions,
+            timestamp: this.#fallbackMode === MessagesStreamFallbackModes.EARLIEST
+                ? ListOffsetTimestamps.EARLIEST
+                : ListOffsetTimestamps.LATEST
+        }, (error, offsets) => {
+            if (error) {
+                callback(error);
+                return;
+            }
+            for (const [topic, refreshedPartitions] of partitionsToRefresh) {
+                const topicOffsets = offsets.get(topic);
+                for (const partition of refreshedPartitions) {
+                    const offset = topicOffsets?.[partition];
+                    if (typeof offset !== 'bigint' || offset < 0n) {
+                        callback(new UserError(`Cannot recover offset out of range for topic ${topic} partition ${partition}.`));
+                        return;
+                    }
+                    recoveredOffsets.push([partitionKey(topic, partition), offset]);
+                }
+            }
+            callback(null, this.#applyRecoveredOffsets(recoveredOffsets));
+        });
+    }
+    #applyRecoveredOffsets(recoveredOffsets) {
         for (const [key, offset] of recoveredOffsets) {
             this.#offsetsToFetch.set(key, offset);
             this.#offsetsCommitted.set(key, offset);
@@ -487,6 +563,12 @@ export class MessagesStream extends Readable {
         for (const topicResponse of response.responses) {
             const topic = topicIds.get(topicResponse.topicId);
             for (const { records: recordsBatches, partitionIndex: partition } of topicResponse.partitions) {
+                const key = partitionKey(topic, partition);
+                // The broker can return partitions which were not requested when they linger
+                // in an incremental fetch session - their data is stale and must not rewind offsets
+                if (!requestedOffsets.has(key)) {
+                    continue;
+                }
                 if (!recordsBatches) {
                     continue;
                 }
@@ -494,15 +576,15 @@ export class MessagesStream extends Readable {
                     const firstTimestamp = batch.firstTimestamp;
                     const firstOffset = batch.firstOffset;
                     const leaderEpoch = metadata.topics.get(topic).partitions[partition].leaderEpoch;
-                    this.#partitionsEpochs.set(`${topic}:${partition}`, leaderEpoch);
+                    this.#partitionsEpochs.set(key, leaderEpoch);
                     // Track offsets
                     if (batch === recordsBatches[recordsBatches.length - 1]) {
                         // Track the last read offset
                         const lastOffset = batch.firstOffset + BigInt(batch.lastOffsetDelta);
-                        this.#offsetsToFetch.set(`${topic}:${partition}`, lastOffset + 1n);
+                        this.#offsetsToFetch.set(key, lastOffset + 1n);
                         // Autocommit if needed
                         if (autocommit) {
-                            this.#offsetsToCommit.set(`${topic}:${partition}`, {
+                            this.#offsetsToCommit.set(key, {
                                 topic,
                                 partition,
                                 offset: lastOffset + 1n,
@@ -518,7 +600,7 @@ export class MessagesStream extends Readable {
                     for (const record of batch.records) {
                         const messageToConsume = { ...record, topic, partition };
                         const offset = batch.firstOffset + BigInt(record.offsetDelta);
-                        if (offset < requestedOffsets.get(`${topic}:${partition}`)) {
+                        if (offset < requestedOffsets.get(key)) {
                             // Thi is a duplicate message, ignore it
                             continue;
                         }
@@ -534,25 +616,111 @@ export class MessagesStream extends Readable {
                             : this.#commit.bind(this, topic, partition, offset + 1n, leaderEpoch);
                         try {
                             const headers = new Map();
-                            for (const [headerKey, headerValue] of record.headers) {
-                                headers.set(
-                                /* c8 ignore next 2 - Hard to test */
-                                headerKeyDeserializer(headerKey ?? undefined, messageToConsume), headerValueDeserializer(headerValue ?? undefined, messageToConsume));
+                            const headerEntries = [];
+                            let payloadType = 'headerKey';
+                            let deserializedKey;
+                            let deserializedValue;
+                            let deserializationError;
+                            let hookFailed = false;
+                            try {
+                                // A before-deserialization hook already failed for this message, report it as is
+                                const hookError = messageToConsume[kDeserializationError];
+                                if (hookError) {
+                                    hookFailed = true;
+                                    payloadType = hookError.payloadType;
+                                    throw hookError.error;
+                                }
+                                for (const [headerKey, headerValue] of record.headers) {
+                                    payloadType = 'headerKey';
+                                    /* c8 ignore next - Hard to test */
+                                    const deserializedHeaderKey = headerKeyDeserializer(headerKey ?? undefined, messageToConsume);
+                                    payloadType = 'headerValue';
+                                    /* c8 ignore next - Hard to test */
+                                    const deserializedHeaderValue = headerValueDeserializer(headerValue ?? undefined, messageToConsume);
+                                    headers.set(deserializedHeaderKey, deserializedHeaderValue);
+                                    headerEntries.push([deserializedHeaderKey, deserializedHeaderValue]);
+                                }
+                                payloadType = 'key';
+                                /* c8 ignore next - Hard to test */
+                                deserializedKey = keyDeserializer(record.key ?? undefined, headers, messageToConsume);
+                                payloadType = 'value';
+                                deserializedValue = valueDeserializer(record.value ?? undefined, headers, messageToConsume);
+                            }
+                            catch (error) {
+                                const context = {
+                                    error,
+                                    payloadType,
+                                    record,
+                                    topic,
+                                    partition,
+                                    offset,
+                                    timestamp: firstTimestamp + record.timestampDelta,
+                                    commit: commit
+                                };
+                                const action = this.#deserializationErrorHandler
+                                    ? this.#deserializationErrorHandler(context)
+                                    : this.#corruptedMessageHandler(record, topic, partition, firstTimestamp, firstOffset, commit, error)
+                                        ? DeserializationErrorActions.FAIL
+                                        : DeserializationErrorActions.SKIP;
+                                if (action === DeserializationErrorActions.CONTINUE) {
+                                    /*
+                                      Deliver the message in a degraded form, so that nothing is lost: whatever was
+                                      already deserialized is kept, while the payload which failed and the ones which
+                                      were never attempted, since deserialization goes headers, key and then value,
+                                      are delivered as the original bytes.
+                  
+                                      The error is reported on the message metadata and published on the diagnostic
+                                      channel, which is where callers should react to it: logging here would be a log
+                                      storm waiting to happen.
+                                    */
+                                    diagnosticContext.error = error;
+                                    consumerReceivesChannel.error.publish(diagnosticContext);
+                                    // A hook failure happens before anything is deserialized, so nothing can be kept
+                                    const headersFailed = hookFailed || payloadType === 'headerKey' || payloadType === 'headerValue';
+                                    if (headersFailed) {
+                                        headers.clear();
+                                        headerEntries.length = 0;
+                                        for (const [headerKey, headerValue] of record.headers) {
+                                            headers.set(headerKey, headerValue);
+                                            headerEntries.push([headerKey, headerValue]);
+                                        }
+                                    }
+                                    if (headersFailed || payloadType === 'key') {
+                                        deserializedKey = (record.key ?? undefined);
+                                    }
+                                    deserializedValue = (record.value ?? undefined);
+                                    deserializationError = { error, payloadType };
+                                }
+                                else if (action === DeserializationErrorActions.SKIP) {
+                                    continue;
+                                }
+                                else {
+                                    diagnosticContext.error = error;
+                                    consumerReceivesChannel.error.publish(diagnosticContext);
+                                    this.destroy(hookFailed ? error : new UserError('Failed to deserialize a message.', { cause: error }));
+                                    return;
+                                }
                             }
-                            /* c8 ignore next 2 - Hard to test */
-                            const key = keyDeserializer(record.key ?? undefined, headers, messageToConsume);
-                            const value = valueDeserializer(record.value ?? undefined, headers, messageToConsume);
                             this.#metricsConsumedMessages?.inc();
                             const message = {
-                                key,
-                                value,
+                                key: deserializedKey,
+                                value: deserializedValue,
                                 headers,
+                                headerEntries,
                                 topic,
                                 partition,
                                 timestamp: firstTimestamp + record.timestampDelta,
                                 offset,
+                                // The batch header epoch is the epoch of the record itself, which is what
+                                // OffsetCommit expects as committedLeaderEpoch (KIP-320).
+                                leaderEpoch: batch.partitionLeaderEpoch,
                                 commit,
-                                metadata: messageMetadata,
+                                // Deserializers and hooks can enrich the metadata of the message they processed
+                                metadata: deserializationError
+                                    ? { ...messageMetadata, ...messageToConsume.metadata, deserializationError }
+                                    : messageToConsume.metadata
+                                        ? { ...messageMetadata, ...messageToConsume.metadata }
+                                        : messageMetadata,
                                 toJSON: messageToJSON
                             };
                             diagnosticContext.result = message;
@@ -561,13 +729,10 @@ export class MessagesStream extends Readable {
                             consumerReceivesChannel.asyncEnd.publish(diagnosticContext);
                         }
                         catch (error) {
-                            const shouldDestroy = this.#corruptedMessageHandler(record, topic, partition, firstTimestamp, firstOffset, commit);
-                            if (shouldDestroy) {
-                                diagnosticContext.error = error;
-                                consumerReceivesChannel.error.publish(diagnosticContext);
-                                this.destroy(new UserError('Failed to deserialize a message.', { cause: error }));
-                                return;
-                            }
+                            diagnosticContext.error = error;
+                            consumerReceivesChannel.error.publish(diagnosticContext);
+                            this.destroy(new UserError('Failed to process a message.', { cause: error }));
+                            return;
                         }
                         finally {
                             consumerReceivesChannel.end.publish(diagnosticContext);
@@ -605,7 +770,7 @@ export class MessagesStream extends Readable {
         }
     }
     #updateCommittedOffset(topic, partition, offset) {
-        const key = `${topic}:${partition}`;
+        const key = partitionKey(topic, partition);
         const previous = this.#offsetsCommitted.get(key);
         if (typeof previous === 'undefined' || previous < offset) {
             this.#offsetsCommitted.set(key, offset);
@@ -641,7 +806,7 @@ export class MessagesStream extends Readable {
                 // Only destroy the stream when the broker requires a group rejoin
                 // (ILLEGAL_GENERATION, UNKNOWN_MEMBER_ID, REBALANCE_IN_PROGRESS).
                 // Transient coordinator errors must not tear down the consumption loop.
-                if (error.findBy?.('needsRejoin', true)) {
+                if (findErrorBy(error, 'needsRejoin', true)) {
                     this.destroy(error);
                 }
                 return;
@@ -689,7 +854,10 @@ export class MessagesStream extends Readable {
                 topics.push(assignment);
             }
             if (!topics.length) {
-                this.#assignOffsets(offsets, new Map(), callback);
+                // No assignment yet: do not seed fallback offsets for the whole topic.
+                // consumer:group:join will refresh once partitions are assigned.
+                this.emit('offsets');
+                callback(null);
                 return;
             }
             this.#consumer.listCommittedOffsets({ topics }, (error, commits) => {
@@ -716,6 +884,7 @@ export class MessagesStream extends Readable {
         this.#refreshOffsetsInflight = true;
         this.#refreshOffsets(error => {
             const shouldDestroyOnError = this.#refreshOffsetsDestroyOnError;
+            const wasPending = this.#refreshOffsetsPending;
             this.#refreshOffsetsInflight = false;
             this.#refreshOffsetsDestroyOnError = false;
             this.#refreshOffsetsPending = false;
@@ -727,8 +896,7 @@ export class MessagesStream extends Readable {
                 return;
             }
             // A new one was scheduled while the previous one was inflight, we need to run it immediately
-            /* c8 ignore next 4 - Hard to test */
-            if (this.#refreshOffsetsPending) {
+            if (wasPending) {
                 this.#scheduleRefreshOffsetsAndFetch(shouldDestroyOnError);
                 return;
             }
@@ -767,8 +935,9 @@ export class MessagesStream extends Readable {
             }
             const partitions = assignment.partitions;
             for (const partition of partitions) {
-                const committed = this.#offsetsToFetch.get(`${topic}:${partition}`);
-                this.#offsetsCommitted.set(`${topic}:${partition}`, committed);
+                const key = partitionKey(topic, partition);
+                const committed = this.#offsetsToFetch.get(key);
+                this.#offsetsCommitted.set(key, committed);
             }
         }
         this.emit('offsets');
@@ -793,7 +962,13 @@ export class MessagesStream extends Readable {
         const requests = [];
         // Create the pre-deserialization requests
         for (const topicResponse of response.responses) {
+            const topic = topicIds.get(topicResponse.topicId);
             for (const { records: recordsBatches, partitionIndex: partition } of topicResponse.partitions) {
+                // The broker can return partitions which were not requested when they linger
+                // in an incremental fetch session - do not run hooks on their stale records
+                if (!requestedOffsets.has(partitionKey(topic, partition))) {
+                    continue;
+                }
                 /* c8 ignore next 3 - Hard to test */
                 if (!recordsBatches) {
                     continue;
@@ -805,7 +980,7 @@ export class MessagesStream extends Readable {
                         continue;
                     }
                     for (const message of batch.records) {
-                        message.topic = topicIds.get(topicResponse.topicId);
+                        message.topic = topic;
                         message.partition = partition;
                         requests.push([message.key, 'key', message]);
                         requests.push([message.value, 'value', message]);
@@ -823,9 +998,23 @@ export class MessagesStream extends Readable {
         }
         runAsyncSeries((request, cb) => {
             const [data, type, message] = request;
-            const result = hook(data, type, message, cb);
+            /*
+              Without an error handler a failing hook destroys the stream, as it always did. With one,
+              the failure is recorded on the message it belongs to and reported to the handler in
+              #pushRecords, together with everything else which can fail while deserializing, so that
+              the caller can skip or degrade that single message instead of losing the stream.
+            */
+            const onHookSettled = error => {
+                if (error && this.#deserializationErrorHandler) {
+                    message[kDeserializationError] ??= { error, payloadType: type };
+                    cb(null);
+                    return;
+                }
+                cb(error);
+            };
+            const result = hook(data, type, message, onHookSettled);
             if (typeof result?.then === 'function') {
-                result.then(() => cb(null), cb);
+                result.then(() => onHookSettled(null), onHookSettled);
             }
         }, requests, 0, error => {
             if (error) {

package/dist/clients/consumer/options.d.ts

@@ -15,6 +15,10 @@ export declare const groupOptionsProperties: {
         type: string;
         minimum: number;
     };
+    heartbeatStallTimeout: {
+        type: string;
+        minimum: number;
+    };
     groupProtocol: {
         type: string;
         enum: ("consumer" | "classic")[];
@@ -53,6 +57,9 @@ export declare const groupOptionsProperties: {
             };
         };
     };
+    protocolsMetadata: {
+        function: boolean;
+    };
     partitionAssigner: {
         function: boolean;
     };
@@ -157,6 +164,10 @@ export declare const groupOptionsSchema: {
             type: string;
             minimum: number;
         };
+        heartbeatStallTimeout: {
+            type: string;
+            minimum: number;
+        };
         groupProtocol: {
             type: string;
             enum: ("consumer" | "classic")[];
@@ -195,6 +206,9 @@ export declare const groupOptionsSchema: {
                 };
             };
         };
+        protocolsMetadata: {
+            function: boolean;
+        };
         partitionAssigner: {
             function: boolean;
         };
@@ -277,6 +291,10 @@ export declare const consumeOptionsSchema: {
             type: string;
             minimum: number;
         };
+        heartbeatStallTimeout: {
+            type: string;
+            minimum: number;
+        };
         groupProtocol: {
             type: string;
             enum: ("consumer" | "classic")[];
@@ -315,6 +333,9 @@ export declare const consumeOptionsSchema: {
                 };
             };
         };
+        protocolsMetadata: {
+            function: boolean;
+        };
         partitionAssigner: {
             function: boolean;
         };
@@ -363,6 +384,9 @@ export declare const consumeOptionsSchema: {
         onCorruptedMessage: {
             function: boolean;
         };
+        onDeserializationError: {
+            function: boolean;
+        };
     };
     required: string[];
     additionalProperties: boolean;
@@ -443,6 +467,10 @@ export declare const consumerOptionsSchema: {
             type: string;
             minimum: number;
         };
+        heartbeatStallTimeout: {
+            type: string;
+            minimum: number;
+        };
         groupProtocol: {
             type: string;
             enum: ("consumer" | "classic")[];
@@ -481,6 +509,9 @@ export declare const consumerOptionsSchema: {
                 };
             };
         };
+        protocolsMetadata: {
+            function: boolean;
+        };
         partitionAssigner: {
             function: boolean;
         };
@@ -570,6 +601,10 @@ export declare const fetchOptionsSchema: {
             type: string;
             minimum: number;
         };
+        heartbeatStallTimeout: {
+            type: string;
+            minimum: number;
+        };
         groupProtocol: {
             type: string;
             enum: ("consumer" | "classic")[];
@@ -608,6 +643,9 @@ export declare const fetchOptionsSchema: {
                 };
             };
         };
+        protocolsMetadata: {
+            function: boolean;
+        };
         partitionAssigner: {
             function: boolean;
         };