@platformatic/kafka 2.2.3 → 2.3.1

package/dist/clients/base/base.js

@@ -419,7 +419,13 @@ export class Base extends TypedEventEmitter {
                             retryCallback(error);
                             return;
                         }
-                        api(connection, topicsToFetch, autocreateTopics, true, retryCallback);
+                        // includeTopicAuthorizedOperations: false. The response's
+                        // topic_authorized_operations bitfield is parsed but never read
+                        // by the producer/consumer, and requesting it makes IAM brokers
+                        // (e.g. MSK) authorize every topic operation and emit a CloudTrail
+                        // AccessDenied per denied op (CreateTopic/DeleteTopic/...) — pure
+                        // audit-log noise for permissions the client never exercises.
+                        api(connection, topicsToFetch, autocreateTopics, false, retryCallback);
                     });
                 }, attempt);
             }, (error, metadata) => {
@@ -455,6 +461,7 @@ export class Base extends TypedEventEmitter {
                 }
                 else {
                     this.#metadata.lastUpdate = lastUpdate;
+                    this.#metadata.controllerId = metadata.controllerId;
                 }
                 const brokers = new Map();
                 // This should never change, but we act defensively here

package/dist/clients/consumer/consumer.js

@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto';
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
 import { FetchIsolationLevels, FindCoordinatorKeyTypes } from "../../apis/enumerations.js";
 import { consumerCommitsChannel, consumerConsumesChannel, consumerFetchesChannel, consumerGroupChannel, consumerHeartbeatChannel, consumerLagChannel, consumerOffsetsChannel, createDiagnosticContext } from "../../diagnostic.js";
-import { protocolErrors, UserError } from "../../errors.js";
+import { NetworkError, protocolErrors, UserError } from "../../errors.js";
 import { INT32_SIZE } from "../../protocol/definitions.js";
 import { Reader } from "../../protocol/reader.js";
 import { IS_CONTROL } from "../../protocol/records.js";
@@ -15,7 +15,7 @@ import { MessagesStream } from "./messages-stream.js";
 import { commitOptionsValidator, consumeOptionsValidator, consumerOptionsValidator, defaultConsumerOptions, fetchOptionsValidator, getLagOptionsValidator, groupIdAndOptionsValidator, groupOptionsValidator, listCommitsOptionsValidator, listOffsetsOptionsValidator } from "./options.js";
 import { roundRobinAssigner } from "./partitions-assigners.js";
 import { TopicsMap } from "./topics-map.js";
-import { partitionKey } from "./utils.js";
+import { nextFetchEpoch, partitionKey } from "./utils.js";
 export class Consumer extends Base {
     groupId;
     groupInstanceId;
@@ -36,6 +36,7 @@ export class Consumer extends Base {
     #memberEpoch;
     #groupRemoteAssignor;
     #clientRack;
+    #fetchSessions;
     #preferredReadReplicas;
     #streams;
     #lagMonitoring;
@@ -82,6 +83,7 @@ export class Consumer extends Base {
         this.#useConsumerGroupProtocol = this[kOptions].groupProtocol === 'consumer';
         this.#groupRemoteAssignor = this[kOptions].groupRemoteAssignor ?? null;
         this.#clientRack = this[kOptions].clientRack ?? '';
+        this.#fetchSessions = new Map();
         this.#preferredReadReplicas = new Map();
         this.#streamContext = options.streamContext ?? options.context;
         this.#validateGroupOptions(this[kOptions], groupIdAndOptionsValidator);
@@ -144,6 +146,7 @@ export class Consumer extends Base {
                     return;
                 }
                 this.topics.clear();
+                this.#fetchSessions.clear();
                 if (this[kPrometheus]) {
                     ensureMetric(this[kPrometheus], 'Gauge', 'kafka_consumers', 'Number of active Kafka consumers').dec();
                 }
@@ -492,6 +495,7 @@ export class Consumer extends Base {
                 pool.get(broker, (error, connection) => {
                     if (error) {
                         this.#clearPreferredReadReplicas(options.topics, topicIds);
+                        this.#fetchSessions.delete(node);
                         // When a connection was not available (either interrupted or not available) we
                         // reset the leader epoch in the options so that when connection is re-established again we can continue
                         for (const topic of options.topics) {
@@ -508,11 +512,74 @@ export class Consumer extends Base {
                             retryCallback(error);
                             return;
                         }
-                        api(connection, options.maxWaitTime ?? this[kOptions].maxWaitTime, options.minBytes ?? this[kOptions].minBytes, options.maxBytes ?? this[kOptions].maxBytes, isolationLevel, 0, 0, options.topics, [], this.#clientRack, (error, result) => {
+                        let session = this.#fetchSessions.get(node);
+                        if (session && session.pool !== pool) {
+                            session = undefined;
+                        }
+                        const sessionId = session?.id ?? 0;
+                        const sessionEpoch = session?.epoch ?? 0;
+                        // The desired state of the session: options.topics always lists all the wanted partitions
+                        const desired = new Map();
+                        for (const topic of options.topics) {
+                            let partitions = desired.get(topic.topicId);
+                            if (!partitions) {
+                                partitions = new Map();
+                                desired.set(topic.topicId, partitions);
+                            }
+                            for (const partition of topic.partitions) {
+                                // Copy the partition since the data is mutated in the error paths
+                                partitions.set(partition.partition, { ...partition });
+                            }
+                        }
+                        let requestTopics = options.topics;
+                        const forgottenTopicsData = [];
+                        if (session) {
+                            // Only send the partitions which are new or whose state changed: the broker
+                            // keeps serving the unchanged ones from the session (KIP-227)
+                            requestTopics = [];
+                            for (const [topicId, partitions] of desired) {
+                                const cached = session.partitions.get(topicId);
+                                const changed = [];
+                                for (const partition of partitions.values()) {
+                                    const previous = cached?.get(partition.partition);
+                                    if (!previous ||
+                                        previous.fetchOffset !== partition.fetchOffset ||
+                                        previous.currentLeaderEpoch !== partition.currentLeaderEpoch ||
+                                        previous.lastFetchedEpoch !== partition.lastFetchedEpoch ||
+                                        previous.partitionMaxBytes !== partition.partitionMaxBytes) {
+                                        changed.push(partition);
+                                    }
+                                }
+                                if (changed.length > 0) {
+                                    requestTopics.push({ topicId, partitions: changed });
+                                }
+                            }
+                            // Ask the broker to remove from the session the partitions which are no
+                            // longer wanted, otherwise it keeps serving them forever
+                            for (const [topicId, partitions] of session.partitions) {
+                                const stillDesired = desired.get(topicId);
+                                const forgotten = [];
+                                for (const partition of partitions.keys()) {
+                                    if (!stillDesired?.has(partition)) {
+                                        forgotten.push(partition);
+                                    }
+                                }
+                                if (forgotten.length > 0) {
+                                    forgottenTopicsData.push({ topic: topicId, partitions: forgotten });
+                                }
+                            }
+                        }
+                        api(connection, options.maxWaitTime ?? this[kOptions].maxWaitTime, options.minBytes ?? this[kOptions].minBytes, options.maxBytes ?? this[kOptions].maxBytes, isolationLevel, sessionId, sessionEpoch, requestTopics, forgottenTopicsData, this.#clientRack, (error, result) => {
                             if (error) {
                                 this.#clearPreferredReadReplicas(options.topics, topicIds);
                                 const genericError = error;
+                                if (genericError.findBy?.('apiId', 'FETCH_SESSION_ID_NOT_FOUND') ||
+                                    genericError.findBy?.('apiId', 'INVALID_FETCH_SESSION_EPOCH') ||
+                                    genericError.findBy?.('apiId', 'FETCH_SESSION_TOPIC_ID_ERROR')) {
+                                    this.#fetchSessions.delete(node);
+                                }
                                 if (genericError.findBy?.('apiId', 'FENCED_LEADER_EPOCH')) {
+                                    this.#fetchSessions.delete(node);
                                     this.clearMetadata();
                                     for (const topic of options.topics) {
                                         for (const partition of topic.partitions) {
@@ -524,6 +591,7 @@ export class Consumer extends Base {
                             }
                             else {
                                 this.#updatePreferredReadReplicas(metadata, topicIds, result);
+                                this.#updateFetchSession(node, pool, sessionEpoch, result.sessionId, desired);
                             }
                             retryCallback(error, result);
                         });
@@ -542,6 +610,20 @@ export class Consumer extends Base {
             callback(error, response);
         }, 0);
     }
+    #updateFetchSession(node, pool, sentEpoch, responseSessionId, desired) {
+        if (!responseSessionId) {
+            this.#fetchSessions.delete(node);
+            return;
+        }
+        // After a successful fetch the broker session contains exactly the desired partitions:
+        // unchanged ones already matched it, changed ones were sent and removed ones were forgotten
+        this.#fetchSessions.set(node, {
+            id: responseSessionId,
+            epoch: nextFetchEpoch(sentEpoch),
+            pool,
+            partitions: desired
+        });
+    }
     #commit(options, callback, rejoinAttempts = 0) {
         this.#performGroupOperation('commit', (connection, groupCallback) => {
             const topics = new Map();
@@ -1209,11 +1291,17 @@ export class Consumer extends Base {
                         topicSubscriptions.push(subscription);
                     }
                 }
-                this[kMetadata]({ topics: Array.from(topicsSubscriptions.keys()) }, (error, metadata) => {
+                const subscribedTopics = Array.from(topicsSubscriptions.keys());
+                this[kMetadata]({ topics: subscribedTopics }, (error, metadata) => {
                     if (error) {
                         callback(this.#handleError(error));
                         return;
                     }
+                    const missingTopic = subscribedTopics.find(topic => !metadata.topics.has(topic));
+                    if (missingTopic) {
+                        callback(this.#handleError(new NetworkError(`Cannot compute group assignments: metadata for subscribed topic "${missingTopic}" is not available in the current cluster metadata.`, { canRetry: true, hasStaleMetadata: true })));
+                        return;
+                    }
                     this.#performSyncGroup(partitionsAssigner, this.#createAssignments(partitionsAssigner, metadata), callback);
                 });
                 return;

package/dist/clients/consumer/messages-stream.js

@@ -553,6 +553,11 @@ export class MessagesStream extends Readable {
             const topic = topicIds.get(topicResponse.topicId);
             for (const { records: recordsBatches, partitionIndex: partition } of topicResponse.partitions) {
                 const key = partitionKey(topic, partition);
+                // The broker can return partitions which were not requested when they linger
+                // in an incremental fetch session - their data is stale and must not rewind offsets
+                if (!requestedOffsets.has(key)) {
+                    continue;
+                }
                 if (!recordsBatches) {
                     continue;
                 }
@@ -860,7 +865,13 @@ export class MessagesStream extends Readable {
         const requests = [];
         // Create the pre-deserialization requests
         for (const topicResponse of response.responses) {
+            const topic = topicIds.get(topicResponse.topicId);
             for (const { records: recordsBatches, partitionIndex: partition } of topicResponse.partitions) {
+                // The broker can return partitions which were not requested when they linger
+                // in an incremental fetch session - do not run hooks on their stale records
+                if (!requestedOffsets.has(partitionKey(topic, partition))) {
+                    continue;
+                }
                 /* c8 ignore next 3 - Hard to test */
                 if (!recordsBatches) {
                     continue;
@@ -872,7 +883,7 @@ export class MessagesStream extends Readable {
                         continue;
                     }
                     for (const message of batch.records) {
-                        message.topic = topicIds.get(topicResponse.topicId);
+                        message.topic = topic;
                         message.partition = partition;
                         requests.push([message.key, 'key', message]);
                         requests.push([message.value, 'value', message]);

package/dist/clients/consumer/utils.d.ts

@@ -1 +1,2 @@
 export declare function partitionKey(topic: string, partition: number): string;
+export declare function nextFetchEpoch(epoch: number): number;

package/dist/clients/consumer/utils.js

@@ -1,3 +1,9 @@
 export function partitionKey(topic, partition) {
     return `${topic}:${partition}`;
 }
+export function nextFetchEpoch(epoch) {
+    if (epoch < 0) {
+        return 0;
+    }
+    return epoch >= 0x7fffffff ? 1 : epoch + 1;
+}

package/dist/clients/producer/producer.js

@@ -342,7 +342,7 @@ export class Producer extends Base {
         }
         const transactionalId = this.#transaction.id;
         this[kPerformDeduplicated]('findCoordinator', deduplicateCallback => {
-            this[kPerformWithRetry]('findCoordinator', retryCallback => {
+            this[kPerformWithRetry]('findCoordinator', (retryCallback, attempt) => {
                 this[kGetBootstrapConnection]((error, connection) => {
                     if (error) {
                         retryCallback(error);
@@ -355,7 +355,7 @@ export class Producer extends Base {
                         }
                         api(connection, FindCoordinatorKeyTypes.TRANSACTION, [transactionalId], retryCallback);
                     });
-                });
+                }, attempt);
             }, (error, response) => {
                 if (error) {
                     deduplicateCallback(error);
@@ -531,11 +531,8 @@ export class Producer extends Base {
     #initIdempotentProducer(options, callback) {
         const transactionalId = this.#transaction?.id;
         this[kPerformDeduplicated]('initProducerId', deduplicateCallback => {
-            this[kPerformWithRetry]('initProducerId', retryCallback => {
-                const connector = transactionalId
-                    ? this.#getCoordinatorConnection.bind(this)
-                    : this[kGetBootstrapConnection].bind(this);
-                connector((error, connection) => {
+            this[kPerformWithRetry]('initProducerId', (retryCallback, attempt) => {
+                const onConnection = (error, connection) => {
                     if (error) {
                         retryCallback(error);
                         return;
@@ -549,7 +546,16 @@ export class Producer extends Base {
                             retryCallback(this.#handleError(error), response);
                         });
                     });
-                });
+                };
+                // The transactional path fails over inside #getCoordinatorConnection (via
+                // #findCoordinator -> kGetBootstrapConnection). The idempotent path must thread
+                // `attempt` so kGetBootstrapConnection rotates past a reachable-but-hung broker.
+                if (transactionalId) {
+                    this.#getCoordinatorConnection(onConnection);
+                }
+                else {
+                    this[kGetBootstrapConnection](onConnection, attempt);
+                }
             }, (error, response) => {
                 if (error) {
                     this.#handleFencingError(error);
@@ -685,9 +691,16 @@ export class Producer extends Base {
             }
             // Track nodes so that we can get their ID for delayed write reporting
             const nodes = [];
+            const sequenceSnapshot = produceOptions.sequences ? new NumericMap() : undefined;
+            if (sequenceSnapshot) {
+                for (const [partitionKey, messagesCount] of messagesPerPartition) {
+                    sequenceSnapshot.set(partitionKey, this.#sequences.postIncrement(partitionKey, messagesCount, 0));
+                }
+                produceOptions = { ...produceOptions, sequences: sequenceSnapshot };
+            }
             runConcurrentCallbacks('Producing messages failed.', messagesByDestination, ([destination, destinationMessages], concurrentCallback) => {
                 nodes.push(destination);
-                this.#performSingleDestinationSend(topics, destinationMessages, this[kOptions].timeout, sendOptions.acks, sendOptions.autocreateTopics, sendOptions.repeatOnStaleMetadata, produceOptions, concurrentCallback);
+                this.#performSingleDestinationSend(topics, destinationMessages, this[kOptions].timeout, sendOptions.acks, sendOptions.autocreateTopics, sendOptions.repeatOnStaleMetadata, produceOptions, true, concurrentCallback);
             }, (error, apiResults) => {
                 this.#metricsProducedMessages?.inc(messages.length);
                 const results = {};
@@ -710,14 +723,16 @@ export class Producer extends Base {
                                     partition: partitionResponse.index,
                                     offset: partitionResponse.baseOffset
                                 });
-                                const partitionKey = `${name}:${partitionResponse.index}`;
-                                this.#sequences.postIncrement(partitionKey, messagesPerPartition.get(partitionKey), 0);
                             }
                         }
                         results.offsets = topics;
                     }
                 }
                 if (error) {
+                    if (sendOptions.idempotent && !this.#transaction) {
+                        this.#producerInfo = undefined;
+                        this.#sequences.clear();
+                    }
                     ;
                     error.produced = results;
                     callback(error);
@@ -728,7 +743,7 @@ export class Producer extends Base {
             });
         });
     }
-    #performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, repeatOnStaleMetadata, produceOptions, callback) {
+    #performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, repeatOnStaleMetadata, produceOptions, recoverOnProducerStateLoss, callback) {
         // Get the metadata with the topic/partitions informations
         this[kMetadata]({ topics, autocreateTopics }, (error, metadata) => {
             if (error) {
@@ -762,7 +777,17 @@ export class Producer extends Base {
                     const hasStaleMetadata = kafkaError?.findBy('hasStaleMetadata', true);
                     if (hasStaleMetadata && repeatOnStaleMetadata) {
                         this.clearMetadata();
-                        this.#performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, false, produceOptions, callback);
+                        this.#performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, false, produceOptions, recoverOnProducerStateLoss, callback);
+                        return;
+                    }
+                    if (recoverOnProducerStateLoss && !this.#transaction && this.#isProducerStateLost(error)) {
+                        this.#recoverIdempotentProducer(messages, produceOptions, (error, recoveredProduceOptions) => {
+                            if (error) {
+                                callback(error);
+                                return;
+                            }
+                            this.#performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, repeatOnStaleMetadata, recoveredProduceOptions, false, callback);
+                        });
                         return;
                     }
                     callback(error, results);
@@ -778,6 +803,35 @@ export class Producer extends Base {
             });
         });
     }
+    #isProducerStateLost(error) {
+        const kafkaError = error;
+        return !!(GenericError.isGenericError(error) &&
+            (kafkaError.findBy('apiId', 'UNKNOWN_PRODUCER_ID') || kafkaError.findBy('apiId', 'OUT_OF_ORDER_SEQUENCE_NUMBER')));
+    }
+    #recoverIdempotentProducer(messages, produceOptions, callback) {
+        this.#producerInfo = undefined;
+        this.#sequences.clear();
+        this.initIdempotentProducer({}, (error, producerInfo) => {
+            if (error) {
+                callback(error);
+                return;
+            }
+            const sequences = new NumericMap();
+            const messagesPerPartition = new NumericMap();
+            for (const { topic, partition } of messages) {
+                messagesPerPartition.postIncrement(`${topic}:${partition}`, 1, 0);
+            }
+            for (const [partitionKey, messagesCount] of messagesPerPartition) {
+                sequences.set(partitionKey, this.#sequences.postIncrement(partitionKey, messagesCount, 0));
+            }
+            callback(null, {
+                ...produceOptions,
+                producerId: producerInfo.producerId,
+                producerEpoch: producerInfo.producerEpoch,
+                sequences
+            });
+        });
+    }
     #getCoordinatorConnection(callback) {
         if (this.#coordinatorId === undefined) {
             this[kTransactionFindCoordinator](error => {

package/dist/clients/serde.d.ts

@@ -17,7 +17,7 @@ export interface Deserializers<Key, Value, HeaderKey, HeaderValue> {
     headerValue: Deserializer<HeaderValue>;
 }
 export type BeforeHookPayloadType = 'key' | 'value' | 'headerKey' | 'headerValue';
-export type BeforeDeserializationHook = (payload: Buffer, type: BeforeHookPayloadType, message: MessageToConsume, callback: Callback<void>) => void | Promise<void>;
+export type BeforeDeserializationHook = (payload: Buffer | null, type: BeforeHookPayloadType, message: MessageToConsume, callback: Callback<void>) => void | Promise<void>;
 export type BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue> = (payload: unknown, type: BeforeHookPayloadType, message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, callback: Callback<void>) => void | Promise<void>;
 export declare function stringSerializer(data?: string): Buffer | undefined;
 export declare function stringDeserializer(data?: string | Buffer): string | undefined;

package/dist/errors.d.ts

@@ -14,6 +14,7 @@ export declare class GenericError extends Error {
     static isGenericError(error: Error): error is GenericError | MultipleErrors;
     constructor(code: ErrorCode, message: string, { cause, ...rest }?: ErrorProperties);
     findBy<ErrorType extends GenericError = GenericError>(property: string, value: unknown): ErrorType | null;
+    toString(): string;
 }
 export declare class MultipleErrors extends AggregateError {
     code: string;
@@ -25,6 +26,7 @@ export declare class MultipleErrors extends AggregateError {
     static isMultipleErrors(error: Error): error is MultipleErrors;
     constructor(message: string, errors: Error[], { cause, ...rest }?: ErrorProperties);
     findBy<ErrorType extends GenericError | MultipleErrors = MultipleErrors>(property: string, value: unknown): ErrorType | null;
+    toString(): string;
 }
 export * from './protocol/errors.ts';
 export declare class AuthenticationError extends GenericError {

package/dist/errors.js

@@ -45,6 +45,9 @@ export class GenericError extends Error {
         }
         return null;
     }
+    toString() {
+        return `Error ${this.code}: ${this.message}`;
+    }
 }
 export class MultipleErrors extends AggregateError {
     code;
@@ -91,6 +94,9 @@ export class MultipleErrors extends AggregateError {
         }
         return null;
     }
+    toString() {
+        return `Error ${this.code}: ${this.message} (caused by: ${this.errors.map(error => error.toString()).join(', ')})`;
+    }
 }
 export * from "./protocol/errors.js";
 export class AuthenticationError extends GenericError {

package/dist/network/connection-pool.js

@@ -1,6 +1,6 @@
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../apis/callbacks.js";
 import { connectionsPoolGetsChannel, createDiagnosticContext, notifyCreation } from "../diagnostic.js";
-import { MultipleErrors } from "../errors.js";
+import { MultipleErrors, NetworkError } from "../errors.js";
 import { TypedEventEmitter } from "../events.js";
 import { Connection, ConnectionStatuses } from "./connection.js";
 let currentInstance = 0;
@@ -95,6 +95,14 @@ export class ConnectionPool extends TypedEventEmitter {
             callback(new Error('Connection pool is closed.'));
             return;
         }
+        if (broker == null || broker.host == null || broker.port == null) {
+            callback(new NetworkError('Cannot get a connection: the broker is not available in the current cluster metadata.', {
+                canRetry: true,
+                hasStaleMetadata: true,
+                broker
+            }));
+            return;
+        }
         const key = `${broker.host}:${broker.port}`;
         const existing = this.#connections.get(key);
         if (existing) {

package/dist/protocol/reader.js

@@ -302,9 +302,10 @@ export class Reader {
     }
     // TODO: Tagged fields are not supported yet
     readTaggedFields() {
-        const length = this.readVarInt();
-        if (length > 0) {
-            this.skip(length);
+        const length = this.readUnsignedVarInt();
+        for (let i = 0; i < length; i++) {
+            this.readUnsignedVarInt();
+            this.skip(this.readUnsignedVarInt());
         }
     }
 }

package/dist/registries/abstract.d.ts

@@ -4,7 +4,7 @@ import { type MessageToProduce } from '../protocol/records.ts';
 export interface SchemaRegistry<Id = unknown, Schema = unknown, Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> {
     get(id: Id): Schema | undefined;
     fetch(id: Id, callback?: (error?: Error) => void): void | Promise<void>;
-    getSchemaId(payload: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): Id;
+    getSchemaId(payload: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): Id;
     getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
     getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
     getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;
@@ -14,7 +14,7 @@ export declare function runAsyncSeries<V>(operation: (item: V, cb: Callback<void
 export declare class AbstractSchemaRegistry<Id = unknown, Schema = unknown, Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> implements SchemaRegistry<Id, Schema, Key, Value, HeaderKey, HeaderValue> {
     get(_: Id): Schema | undefined;
     fetch(_i: unknown, _c?: (error?: Error) => void): void | Promise<void>;
-    getSchemaId(_p: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, _t?: BeforeHookPayloadType): Id;
+    getSchemaId(_p: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, _t?: BeforeHookPayloadType): Id;
     getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
     getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
     getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;

package/dist/registries/confluent-schema-registry.d.ts

@@ -30,7 +30,7 @@ export declare function defaultProtobufTypeMapper(_: number, type: BeforeHookPay
 export declare class ConfluentSchemaRegistry<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends AbstractSchemaRegistry<number | undefined, Schema, Key, Value, HeaderKey, HeaderValue> {
     #private;
     constructor(options: ConfluentSchemaRegistryOptions);
-    getSchemaId(message: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): number | undefined;
+    getSchemaId(message: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): number | undefined;
     get(id: number): Schema | undefined;
     fetchSchema(id: number, callback: Callback<void>): Promise<void>;
     getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;

package/dist/registries/confluent-schema-registry.js

@@ -27,6 +27,9 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
         this.#auth = options.auth;
     }
     getSchemaId(message, type) {
+        if (message === null) {
+            return undefined;
+        }
         if (Buffer.isBuffer(message)) {
             if (type !== 'value') {
                 return undefined;

package/dist/typescript-4/dist/apis/callbacks.d.ts

@@ -6,6 +6,4 @@ export type CallbackWithPromise<ReturnType> = Callback<ReturnType> & {
   [kCallbackPromise]?: Promise<ReturnType>;
 };
 export declare function createPromisifiedCallback<ReturnType>(): CallbackWithPromise<ReturnType>;
-export declare function runConcurrentCallbacks<ReturnType, K, V>(errorMessage: string, collection: Map<K, V>, operation: (item: [K, V], cb: Callback<ReturnType>) => void, callback: Callback<ReturnType[]>): void;
-export declare function runConcurrentCallbacks<ReturnType, V>(errorMessage: string, collection: Set<V>, operation: (item: V, cb: Callback<ReturnType>) => void, callback: Callback<ReturnType[]>): void;
-export declare function runConcurrentCallbacks<ReturnType, V>(errorMessage: string, collection: V[], operation: (item: V, cb: Callback<ReturnType>) => void, callback: Callback<ReturnType[]>): void;
+export declare function runConcurrentCallbacks<R, V>(errorMessage: string, collection: Iterable<V>, operation: (item: V, cb: Callback<R>) => void, callback: Callback<R[]>): void;

package/dist/typescript-4/dist/clients/consumer/utils.d.ts

@@ -1 +1,2 @@
-export declare function partitionKey(topic: string, partition: number): string;
+export declare function partitionKey(topic: string, partition: number): string;
+export declare function nextFetchEpoch(epoch: number): number;

package/dist/typescript-4/dist/clients/serde.d.ts

@@ -17,7 +17,7 @@ export interface Deserializers<Key, Value, HeaderKey, HeaderValue> {
   headerValue: Deserializer<HeaderValue>;
 }
 export type BeforeHookPayloadType = 'key' | 'value' | 'headerKey' | 'headerValue';
-export type BeforeDeserializationHook = (payload: Buffer, type: BeforeHookPayloadType, message: MessageToConsume, callback: Callback<void>) => void | Promise<void>;
+export type BeforeDeserializationHook = (payload: Buffer | null, type: BeforeHookPayloadType, message: MessageToConsume, callback: Callback<void>) => void | Promise<void>;
 export type BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue> = (payload: unknown, type: BeforeHookPayloadType, message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, callback: Callback<void>) => void | Promise<void>;
 export declare function stringSerializer(data?: string): Buffer | undefined;
 export declare function stringDeserializer(data?: string | Buffer): string | undefined;

package/dist/typescript-4/dist/errors.d.ts

@@ -17,6 +17,7 @@ export declare class GenericError extends Error {
     ...rest
   }?: ErrorProperties);
   findBy<ErrorType extends GenericError = GenericError>(property: string, value: unknown): ErrorType | null;
+  toString(): string;
 }
 export declare class MultipleErrors extends AggregateError {
   code: string;
@@ -31,6 +32,7 @@ export declare class MultipleErrors extends AggregateError {
     ...rest
   }?: ErrorProperties);
   findBy<ErrorType extends GenericError | MultipleErrors = MultipleErrors>(property: string, value: unknown): ErrorType | null;
+  toString(): string;
 }
 export * from "./protocol/errors";
 export declare class AuthenticationError extends GenericError {

package/dist/typescript-4/dist/registries/abstract.d.ts

@@ -4,7 +4,7 @@ import { type MessageToProduce } from "../protocol/records";
 export interface SchemaRegistry<Id = unknown, Schema = unknown, Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> {
   get(id: Id): Schema | undefined;
   fetch(id: Id, callback?: (error?: Error) => void): void | Promise<void>;
-  getSchemaId(payload: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): Id;
+  getSchemaId(payload: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): Id;
   getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
   getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
   getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;
@@ -14,7 +14,7 @@ export declare function runAsyncSeries<V>(operation: (item: V, cb: Callback<void
 export declare class AbstractSchemaRegistry<Id = unknown, Schema = unknown, Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> implements SchemaRegistry<Id, Schema, Key, Value, HeaderKey, HeaderValue> {
   get(_: Id): Schema | undefined;
   fetch(_i: unknown, _c?: (error?: Error) => void): void | Promise<void>;
-  getSchemaId(_p: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, _t?: BeforeHookPayloadType): Id;
+  getSchemaId(_p: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, _t?: BeforeHookPayloadType): Id;
   getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
   getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
   getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;

package/dist/typescript-4/dist/registries/confluent-schema-registry.d.ts

@@ -30,7 +30,7 @@ export declare function defaultProtobufTypeMapper(_: number, type: BeforeHookPay
 export declare class ConfluentSchemaRegistry<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends AbstractSchemaRegistry<number | undefined, Schema, Key, Value, HeaderKey, HeaderValue> {
   #private;
   constructor(options: ConfluentSchemaRegistryOptions);
-  getSchemaId(message: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): number | undefined;
+  getSchemaId(message: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): number | undefined;
   get(id: number): Schema | undefined;
   fetchSchema(id: number, callback: Callback<void>): Promise<void>;
   getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "2.2.3";
+export const version = "2.3.1";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "2.2.3",
+  "version": "2.3.1",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
@@ -91,7 +91,7 @@
     "typecheck": "tsc -p . --noEmit",
     "format": "prettier -w benchmarks playground regression src test",
     "test": "c8 -c test/config/c8-local.json ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
-    "test:ci": "c8 -c test/config/c8-ci.json ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
+    "test:ci": "./scripts/node --test --experimental-test-coverage --test-coverage-include='src/**' --test-coverage-lines=90 --test-coverage-branches=90 --test-coverage-functions=90 --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
     "test:integrity": "./scripts/node --test --test-reporter=cleaner-spec-reporter 'regression/integrity/*.test.ts'",
     "test:performance": "./scripts/node --test --test-reporter=cleaner-spec-reporter 'regression/performance/*.test.ts'",
     "test:regression": "npm run test:integrity",