@platformatic/kafka 2.9.0 → 2.10.0

package/dist/clients/admin/admin.js

@@ -1,7 +1,7 @@
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
 import { ConfigResourceTypes, FetchIsolationLevels, FindCoordinatorKeyTypes } from "../../apis/enumerations.js";
 import { adminAclsChannel, adminClientQuotasChannel, adminConfigsChannel, adminConsumerGroupOffsetsChannel, adminGroupsChannel, adminLogDirsChannel, adminOffsetsChannel, adminTopicsChannel, createDiagnosticContext } from "../../diagnostic.js";
-import { findErrorBy, MultipleErrors, UserError } from "../../errors.js";
+import { findErrorBy, MultipleErrors, NetworkError, UserError } from "../../errors.js";
 import { Reader } from "../../protocol/reader.js";
 import { Base, kAfterCreate, kCheckNotClosed, kConnections, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from "../base/base.js";
 import { adminListOffsetsOptionsValidator, alterClientQuotasOptionsValidator, alterConfigsOptionsValidator, alterConsumerGroupOffsetsOptionsValidator, createAclsOptionsValidator, createPartitionsOptionsValidator, createTopicsOptionsValidator, deleteAclsOptionsValidator, deleteConsumerGroupOffsetsOptionsValidator, deleteGroupsOptionsValidator, deleteRecordsOptionsValidator, deleteTopicsOptionsValidator, describeAclsOptionsValidator, describeClientQuotasOptionsValidator, describeConfigsOptionsValidator, describeGroupsOptionsValidator, describeLogDirsOptionsValidator, findCoordinatorOptionsValidator, incrementalAlterConfigsOptionsValidator, listConsumerGroupOffsetsOptionsValidator, listGroupsOptionsValidator, listTopicsOptionsValidator, removeMembersFromConsumerGroupOptionsValidator } from "./options.js";
@@ -366,7 +366,12 @@ export class Admin extends Base {
     }
     #getControllerConnection(callback) {
         if (this.#controller) {
-            this[kConnections].get(this.#controller, callback);
+            this[kConnections].get(this.#controller, (error, connection) => {
+                if (NetworkError.isRetryable(error)) {
+                    this.#controller = null;
+                }
+                callback(error, connection);
+            });
         }
         else {
             this[kGetBootstrapConnection](callback);
@@ -422,26 +427,34 @@ export class Admin extends Base {
             }, 0);
         }, callback);
     }
+    #mapTopicAssignments(assignments) {
+        const mapped = [];
+        for (const { partition, brokers } of assignments ?? []) {
+            mapped.push({ partitionIndex: partition, brokerIds: brokers });
+        }
+        return mapped;
+    }
     #createTopics(options, callback) {
         // -1 is required if manual assignments are used. If no manual assignments are used, -1 will default to broker settings.
         const numPartitions = options.partitions ?? -1;
         const replicationFactor = options.replicas ?? -1;
-        const assignments = [];
+        const assignments = this.#mapTopicAssignments(options.assignments);
         const configs = options.configs ?? [];
-        for (const { partition, brokers } of options.assignments ?? []) {
-            assignments.push({ partitionIndex: partition, brokerIds: brokers });
-        }
         const requests = [];
         for (const topic of options.topics) {
+            if (typeof topic === 'string') {
+                requests.push({ name: topic, numPartitions, replicationFactor, assignments, configs });
+                continue;
+            }
             requests.push({
-                name: topic,
-                numPartitions,
-                replicationFactor,
-                assignments,
+                name: topic.topic,
+                numPartitions: topic.partitions ?? numPartitions,
+                replicationFactor: topic.replicas ?? replicationFactor,
+                assignments: topic.assignments ? this.#mapTopicAssignments(topic.assignments) : assignments,
                 configs
             });
         }
-        this[kPerformDeduplicated](`createTopics-${options.topics.join(',')}`, deduplicateCallback => {
+        this[kPerformDeduplicated](`createTopics-${options.topics.map(topic => (typeof topic === 'string' ? topic : topic.topic)).join(',')}`, deduplicateCallback => {
             this[kPerformWithRetry]('createTopics', retryCallback => {
                 this.#getControllerConnection((error, connection) => {
                     if (error) {

package/dist/clients/admin/options.d.ts

@@ -14,8 +14,48 @@ export declare const createTopicOptionsSchema: {
         topics: {
             type: string;
             items: {
-                type: string;
-                pattern: string;
+                oneOf: ({
+                    type: string;
+                    pattern: string;
+                } | {
+                    type: string;
+                    properties: {
+                        topic: {
+                            type: string;
+                            pattern: string;
+                        };
+                        partitions: {
+                            type: string;
+                        };
+                        replicas: {
+                            type: string;
+                        };
+                        assignments: {
+                            type: string;
+                            items: {
+                                type: string;
+                                properties: {
+                                    partition: {
+                                        type: string;
+                                        minimum: number;
+                                    };
+                                    brokers: {
+                                        type: string;
+                                        items: {
+                                            type: string;
+                                        };
+                                        minItems: number;
+                                    };
+                                };
+                                required: string[];
+                                additionalProperties: boolean;
+                            };
+                            minItems: number;
+                        };
+                    };
+                    required: string[];
+                    additionalProperties: boolean;
+                })[];
             };
         };
         partitions: {
@@ -46,6 +86,22 @@ export declare const createTopicOptionsSchema: {
             };
             minItems: number;
         };
+        configs: {
+            type: string;
+            items: {
+                type: string;
+                properties: {
+                    name: {
+                        type: string;
+                    };
+                    value: {
+                        type: string[];
+                    };
+                };
+                required: string[];
+                additionalProperties: boolean;
+            };
+        };
     };
     required: string[];
     additionalProperties: boolean;

package/dist/clients/admin/options.js

@@ -8,24 +8,55 @@ export const groupsProperties = {
         minItems: 1
     }
 };
+const createTopicAssignmentsSchema = {
+    type: 'array',
+    items: {
+        type: 'object',
+        properties: {
+            partition: { type: 'number', minimum: 0 },
+            brokers: { type: 'array', items: { type: 'number' }, minItems: 1 }
+        },
+        required: ['partition', 'brokers'],
+        additionalProperties: false
+    },
+    minItems: 1
+};
 export const createTopicOptionsSchema = {
     type: 'object',
     properties: {
-        topics: { type: 'array', items: idProperty },
+        topics: {
+            type: 'array',
+            items: {
+                oneOf: [
+                    idProperty,
+                    {
+                        type: 'object',
+                        properties: {
+                            topic: idProperty,
+                            partitions: { type: 'number' },
+                            replicas: { type: 'number' },
+                            assignments: createTopicAssignmentsSchema
+                        },
+                        required: ['topic'],
+                        additionalProperties: false
+                    }
+                ]
+            }
+        },
         partitions: { type: 'number' },
         replicas: { type: 'number' },
-        assignments: {
+        assignments: createTopicAssignmentsSchema,
+        configs: {
             type: 'array',
             items: {
                 type: 'object',
                 properties: {
-                    partition: { type: 'number', minimum: 0 },
-                    brokers: { type: 'array', items: { type: 'number' }, minItems: 1 }
+                    name: { type: 'string' },
+                    value: { type: ['string', 'null'] }
                 },
-                required: ['partition', 'brokers'],
+                required: ['name'],
                 additionalProperties: false
-            },
-            minItems: 1
+            }
         }
     },
     required: ['topics'],

package/dist/clients/admin/types.d.ts

@@ -43,8 +43,14 @@ export interface Group extends Omit<GroupBase, 'groupType'> {
 }
 export interface AdminOptions extends BaseOptions {
 }
+export interface CreateTopicsTopicOptions {
+    topic: string;
+    partitions?: number;
+    replicas?: number;
+    assignments?: BrokerAssignment[];
+}
 export interface CreateTopicsOptions {
-    topics: string[];
+    topics: (string | CreateTopicsTopicOptions)[];
     partitions?: number;
     replicas?: number;
     assignments?: BrokerAssignment[];

package/dist/clients/consumer/consumer.d.ts

@@ -6,7 +6,7 @@ import { Base, type BaseEvents, kCreateConnectionPool } from '../base/base.ts';
 import { type ClusterMetadata } from '../base/types.ts';
 import { MessagesStream } from './messages-stream.ts';
 import { TopicsMap } from './topics-map.ts';
-import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from './types.ts';
+import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerHeartbeatStalledPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from './types.ts';
 export interface ConsumerEvents extends BaseEvents {
     'consumer:group:join': (payload: ConsumerGroupJoinPayload) => void;
     'consumer:group:leave': (payload: ConsumerGroupLeavePayload) => void;
@@ -16,6 +16,7 @@ export interface ConsumerEvents extends BaseEvents {
     'consumer:heartbeat:cancel': (payload: ConsumerHeartbeatPayload) => void;
     'consumer:heartbeat:end': (payload?: ConsumerHeartbeatPayload) => void;
     'consumer:heartbeat:error': (payload: ConsumerHeartbeatErrorPayload) => void;
+    'consumer:heartbeat:stalled': (payload: ConsumerHeartbeatStalledPayload) => void;
     'consumer:lag': (lag: Offsets) => void;
     'consumer:lag:error': (error: Error) => void;
 }

package/dist/clients/consumer/consumer.js

@@ -32,6 +32,10 @@ export class Consumer extends Base {
     #heartbeatInterval;
     #lastHeartbeatIntervalMs;
     #lastHeartbeat;
+    #heartbeatStallTimer;
+    #heartbeatStallStart;
+    #heartbeatStalled;
+    #heartbeatStallError;
     #useConsumerGroupProtocol;
     #memberEpoch;
     #groupRemoteAssignor;
@@ -77,6 +81,10 @@ export class Consumer extends Base {
         this.#heartbeatInterval = null;
         this.#lastHeartbeatIntervalMs = 0;
         this.#lastHeartbeat = null;
+        this.#heartbeatStallTimer = null;
+        this.#heartbeatStallStart = null;
+        this.#heartbeatStalled = false;
+        this.#heartbeatStallError = undefined;
         this.#streams = new Set();
         this.#lagMonitoring = null;
         this.#memberEpoch = 0;
@@ -121,6 +129,7 @@ export class Consumer extends Base {
         }
         this[kClosed] = true;
         clearTimeout(this.#lagMonitoring);
+        this.#resetHeartbeatStall();
         let closer;
         if (this.#useConsumerGroupProtocol) {
             closer = this.#leaveGroupConsumerProtocol.bind(this);
@@ -424,6 +433,7 @@ export class Consumer extends Base {
             return callback[kCallbackPromise];
         }
         this.#membershipActive = false;
+        this.#resetHeartbeatStall();
         this.#leaveGroupClassicProtocol(force, error => {
             if (error) {
                 this.#membershipActive = true;
@@ -486,12 +496,15 @@ export class Consumer extends Base {
     }
     #fetch(options, callback) {
         const isolationLevel = options.isolationLevel ?? this[kOptions].isolationLevel;
+        let refreshMetadata = false;
         this[kPerformWithRetry]('fetch', retryCallback => {
-            this[kMetadata]({ topics: this.topics.current }, (error, metadata) => {
+            this[kMetadata]({ topics: this.topics.current, forceUpdate: refreshMetadata }, (error, metadata) => {
                 if (error) {
+                    refreshMetadata = NetworkError.isRetryable(error);
                     retryCallback(error);
                     return;
                 }
+                refreshMetadata = false;
                 const topicIds = this.#topicIdsById(metadata);
                 const node = this.#fetchNodeForRequest(metadata, options.node, options.topics, topicIds, Date.now());
                 const broker = metadata.brokers.get(node);
@@ -502,6 +515,7 @@ export class Consumer extends Base {
                 const pool = options.connectionPool ?? this[kConnections];
                 pool.get(broker, (error, connection) => {
                     if (error) {
+                        refreshMetadata = NetworkError.isRetryable(error);
                         this.#clearPreferredReadReplicas(options.topics, topicIds);
                         this.#fetchSessions.delete(node);
                         // When a connection was not available (either interrupted or not available) we
@@ -517,6 +531,7 @@ export class Consumer extends Base {
                     }
                     this[kGetApi]('Fetch', (error, api) => {
                         if (error) {
+                            refreshMetadata = NetworkError.isRetryable(error);
                             retryCallback(error);
                             return;
                         }
@@ -579,6 +594,7 @@ export class Consumer extends Base {
                         }
                         api(connection, options.maxWaitTime ?? this[kOptions].maxWaitTime, options.minBytes ?? this[kOptions].minBytes, options.maxBytes ?? this[kOptions].maxBytes, isolationLevel, sessionId, sessionEpoch, requestTopics, forgottenTopicsData, this.#clientRack, (error, result) => {
                             if (error) {
+                                refreshMetadata = NetworkError.isRetryable(error);
                                 this.#clearPreferredReadReplicas(options.topics, topicIds);
                                 if (findErrorBy(error, 'apiId', 'FETCH_SESSION_ID_NOT_FOUND') ||
                                     findErrorBy(error, 'apiId', 'INVALID_FETCH_SESSION_EPOCH') ||
@@ -668,12 +684,12 @@ export class Consumer extends Base {
             callback(error);
         });
     }
-    #listOffsets(withTimestamps, options, callback) {
+    #listOffsets(withTimestamps, options, callback, forceUpdateMetadata = false) {
         let topics = options.topics;
         if (!topics || topics.length === 0) {
             topics = this.topics.current;
         }
-        this[kMetadata]({ topics }, (error, metadata) => {
+        this[kMetadata]({ topics, forceUpdate: forceUpdateMetadata }, (error, metadata) => {
             if (error) {
                 callback(error);
                 return;
@@ -728,6 +744,11 @@ export class Consumer extends Base {
                 }, concurrentCallback, 0);
             }, (error, responses) => {
                 if (error) {
+                    if (!forceUpdateMetadata && NetworkError.isRetryable(error)) {
+                        this.clearMetadata();
+                        this.#listOffsets(withTimestamps, options, callback, true);
+                        return;
+                    }
                     callback(this.#handleError(error));
                     return;
                 }
@@ -842,6 +863,7 @@ export class Consumer extends Base {
                 return;
             }
             this.emitWithDebug('consumer:heartbeat', 'start', eventPayload);
+            this.#trackHeartbeatStall(options);
             this[kGetApi]('Heartbeat', (error, api) => {
                 if (error) {
                     groupCallback(error);
@@ -858,21 +880,43 @@ export class Consumer extends Base {
             if (error) {
                 this.#cancelHeartbeat();
                 if (this.#getRejoinError(error)) {
+                    // A rejoin can legitimately block up to rebalanceTimeout:
+                    // - extend the stall window by that bound rather than pausing it
+                    // - a rejoin that never completes still surfaces as a stall
+                    this.#resetHeartbeatStall();
+                    this.#trackHeartbeatStall(options, error, options.rebalanceTimeout + this.#heartbeatStallTimeoutFor(options));
                     this[kPerformWithRetry]('rejoinGroup', retryCallback => {
                         this.#joinGroup(options, retryCallback);
                     }, error => {
                         if (error) {
                             this.emitWithDebug(null, 'error', error);
+                            if (!this.#heartbeatStalled) {
+                                // No more heartbeats will be scheduled: restart the stall window without
+                                // the rejoin grace, keeping the triggering heartbeat error as lastError.
+                                const heartbeatError = this.#heartbeatStallError;
+                                this.#resetHeartbeatStall();
+                                this.#trackHeartbeatStall(options, heartbeatError);
+                            }
+                        }
+                        else {
+                            // Rejoin succeeded: drop the extended window; the next heartbeat opens the normal one.
+                            this.#resetHeartbeatStall();
                         }
                         this.emitWithDebug('consumer', 'rejoin');
                     }, 0);
                     return;
                 }
+                // The error was not group related, so we schedule another heartbeat.
+                // Do this before notifying listeners so they can still cancel it.
+                this.#heartbeatInterval = setTimeout(() => {
+                    this.#heartbeat(options);
+                }, options.heartbeatInterval);
                 this.emitWithDebug('consumer:heartbeat', 'error', { ...eventPayload, error });
-                // Note that here we purposely do not return, since it was not a group related problem we schedule another heartbeat
+                this.#trackHeartbeatStall(options, error);
             }
             else {
                 this.#lastHeartbeat = new Date();
+                this.#resetHeartbeatStall();
                 this.emitWithDebug('consumer:heartbeat', 'end', eventPayload);
             }
             this.#heartbeatInterval?.refresh();
@@ -882,6 +926,47 @@ export class Consumer extends Base {
         clearTimeout(this.#heartbeatInterval);
         this.#heartbeatInterval = null;
     }
+    #heartbeatStallTimeoutFor(options) {
+        return (options.heartbeatStallTimeout ??
+            this[kOptions].heartbeatStallTimeout ??
+            options.rebalanceTimeout ??
+            this[kOptions].rebalanceTimeout ??
+            defaultConsumerOptions.rebalanceTimeout);
+    }
+    // Opens the stall window on the first heartbeat attempt after the last success.
+    // The window survives retries and only #resetHeartbeatStall closes it.
+    #trackHeartbeatStall(options, error, timeout) {
+        if (error) {
+            this.#heartbeatStallError = error;
+        }
+        if (this.#heartbeatStalled || this.#heartbeatStallTimer) {
+            return;
+        }
+        timeout ??= this.#heartbeatStallTimeoutFor(options);
+        this.#heartbeatStallStart = Date.now();
+        this.#heartbeatStallTimer = setTimeout(() => {
+            this.#heartbeatStallTimer = null;
+            // Emit once per stall episode. The next successful heartbeat re-arms the tracking.
+            this.#heartbeatStalled = true;
+            if (this[kClosed] || !this.#membershipActive) {
+                return;
+            }
+            this.emitWithDebug('consumer:heartbeat', 'stalled', {
+                lastError: this.#heartbeatStallError,
+                lastHeartbeat: this.#lastHeartbeat,
+                stalledFor: Date.now() - this.#heartbeatStallStart
+            });
+        }, timeout);
+    }
+    #resetHeartbeatStall() {
+        if (this.#heartbeatStallTimer) {
+            clearTimeout(this.#heartbeatStallTimer);
+            this.#heartbeatStallTimer = null;
+        }
+        this.#heartbeatStallStart = null;
+        this.#heartbeatStalled = false;
+        this.#heartbeatStallError = undefined;
+    }
     #consumerGroupHeartbeat(options, callback) {
         options.rebalanceTimeout ??= this[kOptions].rebalanceTimeout;
         consumerHeartbeatChannel.traceCallback(this.#performConsumerGroupHeartbeat, 1, createDiagnosticContext({ client: this, operation: 'consumerGroupHeartbeat' }), this, options, callback);
@@ -889,6 +974,7 @@ export class Consumer extends Base {
     #performConsumerGroupHeartbeat(options, callback) {
         this.#performGroupOperation('consumerGroupHeartbeat', (connection, groupCallback) => {
             this.emitWithDebug('consumer:heartbeat', 'start');
+            this.#trackHeartbeatStall(options);
             this[kGetApi]('ConsumerGroupHeartbeat', (error, api) => {
                 if (error) {
                     groupCallback(error);
@@ -899,6 +985,7 @@ export class Consumer extends Base {
             });
         }, (error, response) => {
             if (this[kClosed]) {
+                this.#resetHeartbeatStall();
                 this.emitWithDebug('consumer:heartbeat', 'end');
                 callback(null);
                 return;
@@ -906,6 +993,7 @@ export class Consumer extends Base {
             if (error) {
                 this.#cancelHeartbeat();
                 this.emitWithDebug('consumer:heartbeat', 'error', { error });
+                this.#trackHeartbeatStall(options, error);
                 const fenced = error.response?.errorCode === protocolErrors.FENCED_MEMBER_EPOCH.code;
                 if (fenced) {
                     this.#assignments = [];
@@ -923,6 +1011,7 @@ export class Consumer extends Base {
                 return;
             }
             this.#lastHeartbeat = new Date();
+            this.#resetHeartbeatStall();
             this.#memberEpoch = response.memberEpoch;
             if (response.memberId) {
                 const changed = this.memberId !== response.memberId;
@@ -972,6 +1061,7 @@ export class Consumer extends Base {
         if (toRevoke.length === 0) {
             return;
         }
+        this.emitWithDebug('consumer', 'group:rebalance', { groupId: this.groupId });
         for (const stream of this.#streams) {
             stream.pause();
             stream[kAutocommit]();

package/dist/clients/consumer/messages-stream.js

@@ -5,7 +5,7 @@ import { consumerReceivesChannel, createDiagnosticContext, notifyCreation } from
 import { findErrorBy, protocolErrors, UserError } from "../../errors.js";
 import { IS_CONTROL } from "../../protocol/records.js";
 import { runAsyncSeries } from "../../registries/abstract.js";
-import { kAutocommit, kGetFetchNode, kInstance, kRefreshOffsetsAndFetch } from "../../symbols.js";
+import { kAutocommit, kDeserializationError, kGetFetchNode, kInstance, kRefreshOffsetsAndFetch } from "../../symbols.js";
 import { kConnections, kCreateConnectionPool, kInspect, kPrometheus } from "../base/base.js";
 import { ensureMetric } from "../metrics.js";
 import { defaultConsumerOptions } from "./options.js";
@@ -24,11 +24,12 @@ function messageToJSON() {
     return {
         key: this.key,
         value: this.value,
-        headers: Array.from(this.headers.entries()),
+        headers: this.headerEntries.map(([key, value]) => [key, value]),
         topic: this.topic,
         partition: this.partition,
         timestamp: this.timestamp.toString(),
         offset: this.offset.toString(),
+        leaderEpoch: this.leaderEpoch,
         metadata: this.metadata
     };
 }
@@ -309,7 +310,12 @@ export class MessagesStream extends Readable {
     _construct(callback) {
         this.#refreshOffsetsInflight = true;
         this.#refreshOffsets(error => {
+            const wasPending = this.#refreshOffsetsPending;
             this.#refreshOffsetsInflight = false;
+            this.#refreshOffsetsPending = false;
+            if (!error && wasPending) {
+                this.#scheduleRefreshOffsetsAndFetch();
+            }
             callback(error ?? undefined);
         });
     }
@@ -610,11 +616,20 @@ export class MessagesStream extends Readable {
                             : this.#commit.bind(this, topic, partition, offset + 1n, leaderEpoch);
                         try {
                             const headers = new Map();
+                            const headerEntries = [];
                             let payloadType = 'headerKey';
                             let deserializedKey;
                             let deserializedValue;
                             let deserializationError;
+                            let hookFailed = false;
                             try {
+                                // A before-deserialization hook already failed for this message, report it as is
+                                const hookError = messageToConsume[kDeserializationError];
+                                if (hookError) {
+                                    hookFailed = true;
+                                    payloadType = hookError.payloadType;
+                                    throw hookError.error;
+                                }
                                 for (const [headerKey, headerValue] of record.headers) {
                                     payloadType = 'headerKey';
                                     /* c8 ignore next - Hard to test */
@@ -623,6 +638,7 @@ export class MessagesStream extends Readable {
                                     /* c8 ignore next - Hard to test */
                                     const deserializedHeaderValue = headerValueDeserializer(headerValue ?? undefined, messageToConsume);
                                     headers.set(deserializedHeaderKey, deserializedHeaderValue);
+                                    headerEntries.push([deserializedHeaderKey, deserializedHeaderValue]);
                                 }
                                 payloadType = 'key';
                                 /* c8 ignore next - Hard to test */
@@ -659,11 +675,14 @@ export class MessagesStream extends Readable {
                                     */
                                     diagnosticContext.error = error;
                                     consumerReceivesChannel.error.publish(diagnosticContext);
-                                    const headersFailed = payloadType === 'headerKey' || payloadType === 'headerValue';
+                                    // A hook failure happens before anything is deserialized, so nothing can be kept
+                                    const headersFailed = hookFailed || payloadType === 'headerKey' || payloadType === 'headerValue';
                                     if (headersFailed) {
                                         headers.clear();
+                                        headerEntries.length = 0;
                                         for (const [headerKey, headerValue] of record.headers) {
                                             headers.set(headerKey, headerValue);
+                                            headerEntries.push([headerKey, headerValue]);
                                         }
                                     }
                                     if (headersFailed || payloadType === 'key') {
@@ -678,7 +697,7 @@ export class MessagesStream extends Readable {
                                 else {
                                     diagnosticContext.error = error;
                                     consumerReceivesChannel.error.publish(diagnosticContext);
-                                    this.destroy(new UserError('Failed to deserialize a message.', { cause: error }));
+                                    this.destroy(hookFailed ? error : new UserError('Failed to deserialize a message.', { cause: error }));
                                     return;
                                 }
                             }
@@ -687,10 +706,14 @@ export class MessagesStream extends Readable {
                                 key: deserializedKey,
                                 value: deserializedValue,
                                 headers,
+                                headerEntries,
                                 topic,
                                 partition,
                                 timestamp: firstTimestamp + record.timestampDelta,
                                 offset,
+                                // The batch header epoch is the epoch of the record itself, which is what
+                                // OffsetCommit expects as committedLeaderEpoch (KIP-320).
+                                leaderEpoch: batch.partitionLeaderEpoch,
                                 commit,
                                 // Deserializers and hooks can enrich the metadata of the message they processed
                                 metadata: deserializationError
@@ -831,7 +854,10 @@ export class MessagesStream extends Readable {
                 topics.push(assignment);
             }
             if (!topics.length) {
-                this.#assignOffsets(offsets, new Map(), callback);
+                // No assignment yet: do not seed fallback offsets for the whole topic.
+                // consumer:group:join will refresh once partitions are assigned.
+                this.emit('offsets');
+                callback(null);
                 return;
             }
             this.#consumer.listCommittedOffsets({ topics }, (error, commits) => {
@@ -858,6 +884,7 @@ export class MessagesStream extends Readable {
         this.#refreshOffsetsInflight = true;
         this.#refreshOffsets(error => {
             const shouldDestroyOnError = this.#refreshOffsetsDestroyOnError;
+            const wasPending = this.#refreshOffsetsPending;
             this.#refreshOffsetsInflight = false;
             this.#refreshOffsetsDestroyOnError = false;
             this.#refreshOffsetsPending = false;
@@ -869,8 +896,7 @@ export class MessagesStream extends Readable {
                 return;
             }
             // A new one was scheduled while the previous one was inflight, we need to run it immediately
-            /* c8 ignore next 4 - Hard to test */
-            if (this.#refreshOffsetsPending) {
+            if (wasPending) {
                 this.#scheduleRefreshOffsetsAndFetch(shouldDestroyOnError);
                 return;
             }
@@ -972,9 +998,23 @@ export class MessagesStream extends Readable {
         }
         runAsyncSeries((request, cb) => {
             const [data, type, message] = request;
-            const result = hook(data, type, message, cb);
+            /*
+              Without an error handler a failing hook destroys the stream, as it always did. With one,
+              the failure is recorded on the message it belongs to and reported to the handler in
+              #pushRecords, together with everything else which can fail while deserializing, so that
+              the caller can skip or degrade that single message instead of losing the stream.
+            */
+            const onHookSettled = error => {
+                if (error && this.#deserializationErrorHandler) {
+                    message[kDeserializationError] ??= { error, payloadType: type };
+                    cb(null);
+                    return;
+                }
+                cb(error);
+            };
+            const result = hook(data, type, message, onHookSettled);
             if (typeof result?.then === 'function') {
-                result.then(() => cb(null), cb);
+                result.then(() => onHookSettled(null), onHookSettled);
             }
         }, requests, 0, error => {
             if (error) {

package/dist/clients/consumer/options.d.ts

@@ -15,6 +15,10 @@ export declare const groupOptionsProperties: {
         type: string;
         minimum: number;
     };
+    heartbeatStallTimeout: {
+        type: string;
+        minimum: number;
+    };
     groupProtocol: {
         type: string;
         enum: ("consumer" | "classic")[];
@@ -160,6 +164,10 @@ export declare const groupOptionsSchema: {
             type: string;
             minimum: number;
         };
+        heartbeatStallTimeout: {
+            type: string;
+            minimum: number;
+        };
         groupProtocol: {
             type: string;
             enum: ("consumer" | "classic")[];
@@ -283,6 +291,10 @@ export declare const consumeOptionsSchema: {
             type: string;
             minimum: number;
         };
+        heartbeatStallTimeout: {
+            type: string;
+            minimum: number;
+        };
         groupProtocol: {
             type: string;
             enum: ("consumer" | "classic")[];
@@ -455,6 +467,10 @@ export declare const consumerOptionsSchema: {
             type: string;
             minimum: number;
         };
+        heartbeatStallTimeout: {
+            type: string;
+            minimum: number;
+        };
         groupProtocol: {
             type: string;
             enum: ("consumer" | "classic")[];
@@ -585,6 +601,10 @@ export declare const fetchOptionsSchema: {
             type: string;
             minimum: number;
         };
+        heartbeatStallTimeout: {
+            type: string;
+            minimum: number;
+        };
         groupProtocol: {
             type: string;
             enum: ("consumer" | "classic")[];

package/dist/clients/consumer/options.js

@@ -8,6 +8,7 @@ export const groupOptionsProperties = {
     sessionTimeout: { type: 'number', minimum: 0 },
     rebalanceTimeout: { type: 'number', minimum: 0 },
     heartbeatInterval: { type: 'number', minimum: 0 },
+    heartbeatStallTimeout: { type: 'number', minimum: 0 },
     groupProtocol: { type: 'string', enum: allowedGroupProtocols },
     groupRemoteAssignor: { type: 'string' },
     protocols: {

package/dist/clients/consumer/types.d.ts

@@ -82,6 +82,7 @@ export interface GroupOptions {
     sessionTimeout?: number;
     rebalanceTimeout?: number;
     heartbeatInterval?: number;
+    heartbeatStallTimeout?: number;
     protocols?: GroupProtocolSubscription[];
     protocolsMetadata?: GroupProtocolsMetadataCallback;
     partitionAssigner?: GroupPartitionsAssigner;
@@ -92,6 +93,7 @@ export interface ConsumerGroupOptions {
     groupInstanceId?: string;
     groupRemoteAssignor?: string;
     rebalanceTimeout?: number;
+    heartbeatStallTimeout?: number;
     assignmentUserData?: Buffer;
 }
 export interface ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> {
@@ -169,3 +171,8 @@ export interface ConsumerHeartbeatPayload {
 export interface ConsumerHeartbeatErrorPayload extends ConsumerHeartbeatPayload {
     error: Error;
 }
+export interface ConsumerHeartbeatStalledPayload {
+    lastError?: Error;
+    lastHeartbeat: Date | null;
+    stalledFor: number;
+}

package/dist/clients/producer/producer.js

@@ -2,7 +2,7 @@ import { randomUUID } from 'crypto';
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
 import { FindCoordinatorKeyTypes, ProduceAcks } from "../../apis/enumerations.js";
 import { createDiagnosticContext, producerInitIdempotentChannel, producerSendsChannel, producerTransactionsChannel } from "../../diagnostic.js";
-import { findErrorBy, UserError } from "../../errors.js";
+import { findErrorBy, NetworkError, UserError } from "../../errors.js";
 import { runAsyncSeries } from "../../registries/abstract.js";
 import { kInstance, kTransaction, kTransactionAddOffsets, kTransactionAddPartitions, kTransactionCancel, kTransactionCommitOffset, kTransactionEnd, kTransactionFindCoordinator, kTransactionPrepare } from "../../symbols.js";
 import { emitExperimentalApiWarning, NumericMap } from "../../utils.js";
@@ -744,65 +744,71 @@ export class Producer extends Base {
         });
     }
     #performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, repeatOnStaleMetadata, produceOptions, recoverOnProducerStateLoss, callback) {
-        // Get the metadata with the topic/partitions informations
-        this[kMetadata]({ topics, autocreateTopics }, (error, metadata) => {
-            if (error) {
-                callback(error);
-                return;
-            }
-            const { topic, partition } = messages[0];
-            const leader = metadata.topics.get(topic).partitions[partition].leader;
-            this[kPerformWithRetry]('produce', retryCallback => {
+        const { topic, partition } = messages[0];
+        let refreshMetadata = false;
+        this[kPerformWithRetry]('produce', retryCallback => {
+            this[kMetadata]({ topics, autocreateTopics, forceUpdate: refreshMetadata }, (error, metadata) => {
+                if (error) {
+                    refreshMetadata = NetworkError.isRetryable(error);
+                    retryCallback(error);
+                    return;
+                }
+                refreshMetadata = false;
+                const leader = metadata.topics.get(topic).partitions[partition].leader;
                 this[kGetConnection](metadata.brokers.get(leader), (error, connection) => {
                     if (error) {
+                        refreshMetadata = NetworkError.isRetryable(error);
                         retryCallback(error);
                         return;
                     }
                     this[kGetApi]('Produce', (error, api) => {
                         if (error) {
+                            refreshMetadata = NetworkError.isRetryable(error);
                             retryCallback(error);
                             return;
                         }
-                        api(connection, acks, timeout, messages, produceOptions, retryCallback);
+                        api(connection, acks, timeout, messages, produceOptions, (error, response) => {
+                            refreshMetadata = NetworkError.isRetryable(error);
+                            retryCallback(error, response);
+                        });
                     });
                 });
-            }, (error, results) => {
-                if (error) {
-                    // If the last error was due to stale metadata, we retry the operation with this set of messages
-                    // since the partition is already set, it should attempt on the new destination
-                    /* c8 ignore next - Hard to test */
-                    const hasStaleMetadata = findErrorBy(error, 'hasStaleMetadata', true);
-                    if (hasStaleMetadata && repeatOnStaleMetadata) {
-                        this.clearMetadata();
-                        this.#performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, false, produceOptions, recoverOnProducerStateLoss, callback);
-                        return;
-                    }
-                    if (recoverOnProducerStateLoss && !this.#transaction && this.#isProducerStateLost(error)) {
-                        this.#recoverIdempotentProducer(messages, produceOptions, (error, recoveredProduceOptions) => {
-                            if (error) {
-                                callback(error);
-                                return;
-                            }
-                            this.#performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, repeatOnStaleMetadata, recoveredProduceOptions, false, callback);
-                        });
-                        return;
-                    }
-                    callback(error, results);
+            });
+        }, (error, results) => {
+            if (error) {
+                // If the last error was due to stale metadata, we retry the operation with this set of messages
+                // since the partition is already set, it should attempt on the new destination
+                /* c8 ignore next - Hard to test */
+                const hasStaleMetadata = findErrorBy(error, 'hasStaleMetadata', true);
+                if (hasStaleMetadata && repeatOnStaleMetadata) {
+                    this.clearMetadata();
+                    this.#performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, false, produceOptions, recoverOnProducerStateLoss, callback);
                     return;
                 }
-                callback(error, results);
-            }, 0, [], error => {
-                /* c8 ignore next 3 - Hard to test */
-                if (!repeatOnStaleMetadata) {
-                    return false;
+                if (recoverOnProducerStateLoss && !this.#transaction && this.#isProducerStateLost(error)) {
+                    this.#recoverIdempotentProducer(messages, produceOptions, (error, recoveredProduceOptions) => {
+                        if (error) {
+                            callback(error);
+                            return;
+                        }
+                        this.#performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, repeatOnStaleMetadata, recoveredProduceOptions, false, callback);
+                    });
+                    return;
                 }
-                return !!findErrorBy(error, 'hasStaleMetadata', true);
-            });
+                callback(error, results);
+                return;
+            }
+            callback(error, results);
+        }, 0, [], error => {
+            /* c8 ignore next 3 - Hard to test */
+            if (!repeatOnStaleMetadata) {
+                return false;
+            }
+            return !!findErrorBy(error, 'hasStaleMetadata', true);
         });
     }
     #isProducerStateLost(error) {
-        return !!(findErrorBy(error, 'apiId', 'UNKNOWN_PRODUCER_ID') ||
-            findErrorBy(error, 'apiId', 'OUT_OF_ORDER_SEQUENCE_NUMBER'));
+        return !!(findErrorBy(error, 'apiId', 'UNKNOWN_PRODUCER_ID') || findErrorBy(error, 'apiId', 'OUT_OF_ORDER_SEQUENCE_NUMBER'));
     }
     #recoverIdempotentProducer(messages, produceOptions, callback) {
         this.#producerInfo = undefined;

package/dist/errors.d.ts

@@ -36,6 +36,7 @@ export declare class AuthenticationError extends GenericError {
 }
 export declare class NetworkError extends GenericError {
     static code: ErrorCode;
+    static isRetryable(error: Error | null | undefined): boolean;
     constructor(message: string, properties?: ErrorProperties);
 }
 export declare class ProtocolError extends GenericError {

package/dist/errors.js

@@ -119,6 +119,10 @@ export class AuthenticationError extends GenericError {
 }
 export class NetworkError extends GenericError {
     static code = 'PLT_KFK_NETWORK';
+    static isRetryable(error) {
+        const networkError = findErrorBy(error, 'code', NetworkError.code);
+        return networkError?.canRetry === true;
+    }
     constructor(message, properties = {}) {
         super(NetworkError.code, message, { canRetry: true, ...properties });
     }

package/dist/network/utils.js

@@ -1,12 +1,30 @@
+const BRACKETED_BROKER_RE = /^\[([^\]]+)](?::(\d*))?$/;
 export function parseBroker(broker, defaultPort = 9092) {
-    if (typeof broker === 'string') {
-        if (broker.includes(':')) {
-            const [host, port] = broker.split(':');
-            return { host, port: Number(port) };
-        }
-        else {
-            return { host: broker, port: defaultPort };
+    if (typeof broker !== 'string') {
+        const host = stripBrackets(broker.host);
+        if (host === broker.host) {
+            return broker;
         }
+        return { ...broker, host };
+    }
+    const bracketed = BRACKETED_BROKER_RE.exec(broker);
+    if (bracketed) {
+        return { host: bracketed[1], port: parsePort(bracketed[2], defaultPort) };
+    }
+    const separator = broker.lastIndexOf(':');
+    // Bare IPv6 addresses contain multiple colons and no port. The host:port form
+    // is unambiguous only with a single colon, or with brackets around the host.
+    if (separator === -1 || broker.indexOf(':') !== separator) {
+        return { host: broker, port: defaultPort };
+    }
+    return { host: broker.slice(0, separator), port: parsePort(broker.slice(separator + 1), defaultPort) };
+}
+function parsePort(port, defaultPort) {
+    return port === undefined || port.length === 0 ? defaultPort : Number(port);
+}
+function stripBrackets(host) {
+    if (host.startsWith('[') && host.endsWith(']')) {
+        return host.slice(1, -1);
     }
-    return broker;
+    return host;
 }

package/dist/protocol/records.d.ts

@@ -33,11 +33,14 @@ export interface MessageJSON<Key = unknown, Value = unknown, HeaderKey = unknown
     partition: number;
     timestamp: string;
     offset: string;
+    leaderEpoch: number;
     metadata: Record<string, unknown>;
 }
 export interface Message<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends Required<MessageBase<Key, Value>> {
     headers: Map<HeaderKey, HeaderValue>;
+    headerEntries: Array<[HeaderKey, HeaderValue]>;
     offset: bigint;
+    leaderEpoch: number;
     metadata: Record<string, unknown>;
     commit(callback?: (error?: Error) => void): void | Promise<void>;
     toJSON(): MessageJSON<Key, Value, HeaderKey, HeaderValue>;

package/dist/registries/abstract.js

@@ -1,19 +1,38 @@
 export function runAsyncSeries(operation, collection, index, callback) {
-    if (collection.length === 0 || index >= collection.length) {
+    const length = collection.length;
+    if (length === 0 || index >= length) {
         callback(null);
         return;
     }
-    operation(collection[index], error => {
-        if (error) {
-            callback(error);
+    while (index < length) {
+        let invocationReturned = false;
+        let cbCalled = false;
+        let cbError;
+        operation(collection[index], error => {
+            if (invocationReturned) {
+                // The callback was invoked asynchronously, so continuing recursively does not grow the current stack.
+                if (error) {
+                    callback(error);
+                    return;
+                }
+                runAsyncSeries(operation, collection, index + 1, callback);
+                return;
+            }
+            cbCalled = true;
+            cbError = error;
+        });
+        invocationReturned = true;
+        if (!cbCalled) {
+            // Wait for the async callback to continue the series.
             return;
         }
-        else if (index === collection.length - 1) {
-            callback(null);
+        if (cbError) {
+            callback(cbError);
             return;
         }
-        runAsyncSeries(operation, collection, index + 1, callback);
-    });
+        index++;
+    }
+    callback(null);
 }
 /* c8 ignore start */
 export class AbstractSchemaRegistry {

package/dist/registries/confluent-schema-registry.js

@@ -1,3 +1,5 @@
+import * as AjvDraft04Module from 'ajv-draft-04';
+import draft06MetaSchema from 'ajv/dist/refs/json-schema-draft-06.json' with { type: 'json' };
 import { Ajv } from 'ajv';
 import { Ajv2020 } from 'ajv/dist/2020.js';
 import avro from 'avsc';
@@ -8,8 +10,7 @@ import { EMPTY_BUFFER, MultipleErrors, NetworkError, TimeoutError, UnsupportedFo
 import { getCredential } from "../protocol/sasl/utils.js";
 import { AbstractSchemaRegistry } from "./abstract.js";
 const require = createRequire(import.meta.url);
-const AjvDraft04 = require('ajv-draft-04');
-const draft06MetaSchema = require('ajv/dist/refs/json-schema-draft-06.json');
+const AjvDraft04 = AjvDraft04Module.default;
 const undiciGlobalDispatcherSymbol = /^undici\.globalDispatcher\.(\d+)$/;
 export function createUndiciAgent(options) {
     /*

package/dist/symbols.d.ts

@@ -2,6 +2,7 @@ export declare const kInstance: unique symbol;
 export declare const kRefreshOffsetsAndFetch: unique symbol;
 export declare const kAutocommit: unique symbol;
 export declare const kGetFetchNode: unique symbol;
+export declare const kDeserializationError: unique symbol;
 export declare const kTransaction: unique symbol;
 export declare const kTransactionPrepare: unique symbol;
 export declare const kTransactionAddPartitions: unique symbol;

package/dist/symbols.js

@@ -3,6 +3,7 @@ export const kInstance = Symbol('plt.kafka.base.instance');
 export const kRefreshOffsetsAndFetch = Symbol('plt.kafka.messagesStream.refreshOffsetsAndFetch');
 export const kAutocommit = Symbol('plt.kafka.messagesStream.autocommit');
 export const kGetFetchNode = Symbol('plt.kafka.consumer.getFetchNode');
+export const kDeserializationError = Symbol('plt.kafka.messagesStream.deserializationError');
 export const kTransaction = Symbol('plt.kafka.producer.transactions');
 export const kTransactionPrepare = Symbol('plt.kafka.producer.transactions.prepare');
 export const kTransactionAddPartitions = Symbol('plt.kafka.producer.transactions.addPartitions');

package/dist/typescript-4/dist/clients/admin/options.d.ts

@@ -14,8 +14,48 @@ export declare const createTopicOptionsSchema: {
     topics: {
       type: string;
       items: {
-        type: string;
-        pattern: string;
+        oneOf: ({
+          type: string;
+          pattern: string;
+        } | {
+          type: string;
+          properties: {
+            topic: {
+              type: string;
+              pattern: string;
+            };
+            partitions: {
+              type: string;
+            };
+            replicas: {
+              type: string;
+            };
+            assignments: {
+              type: string;
+              items: {
+                type: string;
+                properties: {
+                  partition: {
+                    type: string;
+                    minimum: number;
+                  };
+                  brokers: {
+                    type: string;
+                    items: {
+                      type: string;
+                    };
+                    minItems: number;
+                  };
+                };
+                required: string[];
+                additionalProperties: boolean;
+              };
+              minItems: number;
+            };
+          };
+          required: string[];
+          additionalProperties: boolean;
+        })[];
       };
     };
     partitions: {
@@ -46,6 +86,22 @@ export declare const createTopicOptionsSchema: {
       };
       minItems: number;
     };
+    configs: {
+      type: string;
+      items: {
+        type: string;
+        properties: {
+          name: {
+            type: string;
+          };
+          value: {
+            type: string[];
+          };
+        };
+        required: string[];
+        additionalProperties: boolean;
+      };
+    };
   };
   required: string[];
   additionalProperties: boolean;

package/dist/typescript-4/dist/clients/admin/types.d.ts

@@ -42,8 +42,14 @@ export interface Group extends Omit<GroupBase, 'groupType'> {
   authorizedOperations: number;
 }
 export interface AdminOptions extends BaseOptions {}
+export interface CreateTopicsTopicOptions {
+  topic: string;
+  partitions?: number;
+  replicas?: number;
+  assignments?: BrokerAssignment[];
+}
 export interface CreateTopicsOptions {
-  topics: string[];
+  topics: (string | CreateTopicsTopicOptions)[];
   partitions?: number;
   replicas?: number;
   assignments?: BrokerAssignment[];

package/dist/typescript-4/dist/clients/consumer/consumer.d.ts

@@ -6,7 +6,7 @@ import { Base, type BaseEvents, kCreateConnectionPool } from "../base/base";
 import { type ClusterMetadata } from "../base/types";
 import { MessagesStream } from "./messages-stream";
 import { TopicsMap } from "./topics-map";
-import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from "./types";
+import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerHeartbeatStalledPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from "./types";
 export interface ConsumerEvents extends BaseEvents {
   'consumer:group:join': (payload: ConsumerGroupJoinPayload) => void;
   'consumer:group:leave': (payload: ConsumerGroupLeavePayload) => void;
@@ -16,6 +16,7 @@ export interface ConsumerEvents extends BaseEvents {
   'consumer:heartbeat:cancel': (payload: ConsumerHeartbeatPayload) => void;
   'consumer:heartbeat:end': (payload?: ConsumerHeartbeatPayload) => void;
   'consumer:heartbeat:error': (payload: ConsumerHeartbeatErrorPayload) => void;
+  'consumer:heartbeat:stalled': (payload: ConsumerHeartbeatStalledPayload) => void;
   'consumer:lag': (lag: Offsets) => void;
   'consumer:lag:error': (error: Error) => void;
 }

package/dist/typescript-4/dist/clients/consumer/options.d.ts

@@ -15,6 +15,10 @@ export declare const groupOptionsProperties: {
     type: string;
     minimum: number;
   };
+  heartbeatStallTimeout: {
+    type: string;
+    minimum: number;
+  };
   groupProtocol: {
     type: string;
     enum: ("consumer" | "classic")[];
@@ -160,6 +164,10 @@ export declare const groupOptionsSchema: {
       type: string;
       minimum: number;
     };
+    heartbeatStallTimeout: {
+      type: string;
+      minimum: number;
+    };
     groupProtocol: {
       type: string;
       enum: ("consumer" | "classic")[];
@@ -283,6 +291,10 @@ export declare const consumeOptionsSchema: {
       type: string;
       minimum: number;
     };
+    heartbeatStallTimeout: {
+      type: string;
+      minimum: number;
+    };
     groupProtocol: {
       type: string;
       enum: ("consumer" | "classic")[];
@@ -455,6 +467,10 @@ export declare const consumerOptionsSchema: {
       type: string;
       minimum: number;
     };
+    heartbeatStallTimeout: {
+      type: string;
+      minimum: number;
+    };
     groupProtocol: {
       type: string;
       enum: ("consumer" | "classic")[];
@@ -585,6 +601,10 @@ export declare const fetchOptionsSchema: {
       type: string;
       minimum: number;
     };
+    heartbeatStallTimeout: {
+      type: string;
+      minimum: number;
+    };
     groupProtocol: {
       type: string;
       enum: ("consumer" | "classic")[];

package/dist/typescript-4/dist/clients/consumer/types.d.ts

@@ -82,6 +82,7 @@ export interface GroupOptions {
   sessionTimeout?: number;
   rebalanceTimeout?: number;
   heartbeatInterval?: number;
+  heartbeatStallTimeout?: number;
   protocols?: GroupProtocolSubscription[];
   protocolsMetadata?: GroupProtocolsMetadataCallback;
   partitionAssigner?: GroupPartitionsAssigner;
@@ -92,6 +93,7 @@ export interface ConsumerGroupOptions {
   groupInstanceId?: string;
   groupRemoteAssignor?: string;
   rebalanceTimeout?: number;
+  heartbeatStallTimeout?: number;
   assignmentUserData?: Buffer;
 }
 export interface ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> {
@@ -168,4 +170,9 @@ export interface ConsumerHeartbeatPayload {
 }
 export interface ConsumerHeartbeatErrorPayload extends ConsumerHeartbeatPayload {
   error: Error;
+}
+export interface ConsumerHeartbeatStalledPayload {
+  lastError?: Error;
+  lastHeartbeat: Date | null;
+  stalledFor: number;
 }

package/dist/typescript-4/dist/errors.d.ts

@@ -42,6 +42,7 @@ export declare class AuthenticationError extends GenericError {
 }
 export declare class NetworkError extends GenericError {
   static code: ErrorCode;
+  static isRetryable(error: Error | null | undefined): boolean;
   constructor(message: string, properties?: ErrorProperties);
 }
 export declare class ProtocolError extends GenericError {

package/dist/typescript-4/dist/protocol/records.d.ts

@@ -33,11 +33,14 @@ export interface MessageJSON<Key = unknown, Value = unknown, HeaderKey = unknown
   partition: number;
   timestamp: string;
   offset: string;
+  leaderEpoch: number;
   metadata: Record<string, unknown>;
 }
 export interface Message<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends Required<MessageBase<Key, Value>> {
   headers: Map<HeaderKey, HeaderValue>;
+  headerEntries: Array<[HeaderKey, HeaderValue]>;
   offset: bigint;
+  leaderEpoch: number;
   metadata: Record<string, unknown>;
   commit(callback?: (error?: Error) => void): void | Promise<void>;
   toJSON(): MessageJSON<Key, Value, HeaderKey, HeaderValue>;

package/dist/typescript-4/dist/symbols.d.ts

@@ -2,6 +2,7 @@ export declare const kInstance: unique symbol;
 export declare const kRefreshOffsetsAndFetch: unique symbol;
 export declare const kAutocommit: unique symbol;
 export declare const kGetFetchNode: unique symbol;
+export declare const kDeserializationError: unique symbol;
 export declare const kTransaction: unique symbol;
 export declare const kTransactionPrepare: unique symbol;
 export declare const kTransactionAddPartitions: unique symbol;

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "2.9.0";
+export const version = "2.10.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "2.9.0",
+  "version": "2.10.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",