@platformatic/kafka 2.8.0 → 2.9.0

package/dist/clients/admin/admin.js

@@ -1,7 +1,7 @@
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
 import { ConfigResourceTypes, FetchIsolationLevels, FindCoordinatorKeyTypes } from "../../apis/enumerations.js";
 import { adminAclsChannel, adminClientQuotasChannel, adminConfigsChannel, adminConsumerGroupOffsetsChannel, adminGroupsChannel, adminLogDirsChannel, adminOffsetsChannel, adminTopicsChannel, createDiagnosticContext } from "../../diagnostic.js";
-import { MultipleErrors, UserError } from "../../errors.js";
+import { findErrorBy, MultipleErrors, UserError } from "../../errors.js";
 import { Reader } from "../../protocol/reader.js";
 import { Base, kAfterCreate, kCheckNotClosed, kConnections, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from "../base/base.js";
 import { adminListOffsetsOptionsValidator, alterClientQuotasOptionsValidator, alterConfigsOptionsValidator, alterConsumerGroupOffsetsOptionsValidator, createAclsOptionsValidator, createPartitionsOptionsValidator, createTopicsOptionsValidator, deleteAclsOptionsValidator, deleteConsumerGroupOffsetsOptionsValidator, deleteGroupsOptionsValidator, deleteRecordsOptionsValidator, deleteTopicsOptionsValidator, describeAclsOptionsValidator, describeClientQuotasOptionsValidator, describeConfigsOptionsValidator, describeGroupsOptionsValidator, describeLogDirsOptionsValidator, findCoordinatorOptionsValidator, incrementalAlterConfigsOptionsValidator, listConsumerGroupOffsetsOptionsValidator, listGroupsOptionsValidator, listTopicsOptionsValidator, removeMembersFromConsumerGroupOptionsValidator } from "./options.js";
@@ -373,7 +373,7 @@ export class Admin extends Base {
         }
     }
     #handleNotControllerError(error, value, callback) {
-        if (error && error?.findBy?.('apiCode', 41)) {
+        if (findErrorBy(error, 'apiCode', 41)) {
             this[kMetadata]({ topics: [], forceUpdate: true }, (metadataError, metadata) => {
                 /* c8 ignore next 4 - Hard to test */
                 if (metadataError) {

package/dist/clients/base/base.js

@@ -2,7 +2,7 @@ import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } f
 import * as apis from "../../apis/index.js";
 import { api as apiVersionsV3 } from "../../apis/metadata/api-versions-v3.js";
 import { baseApisChannel, baseMetadataChannel, createDiagnosticContext, notifyCreation } from "../../diagnostic.js";
-import { MultipleErrors, NetworkError, UnsupportedApiError, UserError } from "../../errors.js";
+import { findErrorBy, MultipleErrors, NetworkError, UnsupportedApiError, UserError } from "../../errors.js";
 import { TypedEventEmitter } from "../../events.js";
 import { ConnectionPool } from "../../network/connection-pool.js";
 import { parseBroker } from "../../network/utils.js";
@@ -229,7 +229,12 @@ export class Base extends TypedEventEmitter {
     }
     [kCheckNotClosed](callback) {
         if (this[kClosed]) {
-            const error = new NetworkError('Client is closed.', { closed: true, instance: this[kInstance] });
+            // A closed client never reopens, so this must not be retried.
+            const error = new NetworkError('Client is closed.', {
+                canRetry: false,
+                closed: true,
+                instance: this[kInstance]
+            });
             callback(error, undefined);
             return true;
         }
@@ -243,9 +248,8 @@ export class Base extends TypedEventEmitter {
         this.emitWithDebug('client', 'performWithRetry', operationId, attempt, retries);
         operation((error, result) => {
             if (error) {
-                const genericError = error;
                 // Only retry if all the errors in the chain are retriable
-                const retriable = !genericError.findBy?.('canRetry', false);
+                const retriable = !findErrorBy(error, 'canRetry', false);
                 errors.push(error);
                 if (attempt < retries && retriable && !shouldSkipRetry?.(error)) {
                     function onClose() {
@@ -430,7 +434,7 @@ export class Base extends TypedEventEmitter {
                 }, attempt);
             }, (error, metadata) => {
                 if (error) {
-                    const unknownTopicError = error.findBy('apiCode', 3);
+                    const unknownTopicError = findErrorBy(error, 'apiCode', 3);
                     if (unknownTopicError) {
                         const topicIndexMatch = unknownTopicError.path?.match(/\/topics\/(\d+)/);
                         /* c8 ignore next 3 - Hard to test  */
@@ -439,7 +443,7 @@ export class Base extends TypedEventEmitter {
                         deduplicateCallback(new UserError(`Unknown topic ${topicName}.`));
                         return;
                     }
-                    const hasStaleMetadata = error.findBy('hasStaleMetadata', true);
+                    const hasStaleMetadata = findErrorBy(error, 'hasStaleMetadata', true);
                     // Stale metadata, we need to fetch everything again
                     /* c8 ignore next 4 - Hard to test */
                     if (hasStaleMetadata) {

package/dist/clients/consumer/consumer.js

@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto';
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
 import { FetchIsolationLevels, FindCoordinatorKeyTypes } from "../../apis/enumerations.js";
 import { consumerCommitsChannel, consumerConsumesChannel, consumerFetchesChannel, consumerGroupChannel, consumerHeartbeatChannel, consumerLagChannel, consumerOffsetsChannel, createDiagnosticContext } from "../../diagnostic.js";
-import { NetworkError, protocolErrors, UserError } from "../../errors.js";
+import { findErrorBy, NetworkError, protocolErrors, UserError } from "../../errors.js";
 import { INT32_SIZE } from "../../protocol/definitions.js";
 import { Reader } from "../../protocol/reader.js";
 import { IS_CONTROL } from "../../protocol/records.js";
@@ -580,13 +580,12 @@ export class Consumer extends Base {
                         api(connection, options.maxWaitTime ?? this[kOptions].maxWaitTime, options.minBytes ?? this[kOptions].minBytes, options.maxBytes ?? this[kOptions].maxBytes, isolationLevel, sessionId, sessionEpoch, requestTopics, forgottenTopicsData, this.#clientRack, (error, result) => {
                             if (error) {
                                 this.#clearPreferredReadReplicas(options.topics, topicIds);
-                                const genericError = error;
-                                if (genericError.findBy?.('apiId', 'FETCH_SESSION_ID_NOT_FOUND') ||
-                                    genericError.findBy?.('apiId', 'INVALID_FETCH_SESSION_EPOCH') ||
-                                    genericError.findBy?.('apiId', 'FETCH_SESSION_TOPIC_ID_ERROR')) {
+                                if (findErrorBy(error, 'apiId', 'FETCH_SESSION_ID_NOT_FOUND') ||
+                                    findErrorBy(error, 'apiId', 'INVALID_FETCH_SESSION_EPOCH') ||
+                                    findErrorBy(error, 'apiId', 'FETCH_SESSION_TOPIC_ID_ERROR')) {
                                     this.#fetchSessions.delete(node);
                                 }
-                                if (genericError.findBy?.('apiId', 'FENCED_LEADER_EPOCH')) {
+                                if (findErrorBy(error, 'apiId', 'FENCED_LEADER_EPOCH')) {
                                     this.#fetchSessions.delete(node);
                                     this.clearMetadata();
                                     for (const topic of options.topics) {
@@ -791,7 +790,7 @@ export class Consumer extends Base {
             if (error) {
                 if (this.#useConsumerGroupProtocol &&
                     staleMemberEpochRetries < this[kOptions].retries &&
-                    error.findBy?.('apiId', 'STALE_MEMBER_EPOCH')) {
+                    findErrorBy(error, 'apiId', 'STALE_MEMBER_EPOCH')) {
                     this.#consumerGroupHeartbeat(this[kOptions], heartbeatError => {
                         if (heartbeatError) {
                             callback(this.#handleError(heartbeatError));
@@ -1041,7 +1040,35 @@ export class Consumer extends Base {
             callback(err);
         });
     }
-    #leaveGroupConsumerProtocol(_, callback) {
+    #leaveGroupConsumerProtocol(force, callback) {
+        // Remove streams that might have been exited in the meanwhile
+        for (const stream of this.#streams) {
+            if (stream.closed || stream.destroyed) {
+                this.#streams.delete(stream);
+            }
+        }
+        // Active streams must be closed before leaving the group, exactly like in the classic protocol.
+        // Their fetch loop keeps running until they are closed and, since Consumer.close marks the client
+        // as closed before leaving, any fetch scheduled in the meanwhile would fail with "Client is closed."
+        // and destroy the stream with an error the user never asked for.
+        if (this.#streams.size) {
+            if (!force) {
+                callback(new UserError('Cannot leave group while consuming messages.'));
+                return;
+            }
+            runConcurrentCallbacks('Closing streams failed.', this.#streams, (stream, concurrentCallback) => {
+                stream.close(concurrentCallback);
+            }, error => {
+                /* c8 ignore next 4 - Hard to test */
+                if (error) {
+                    callback(error);
+                    return;
+                }
+                // All streams are closed, try the operation again without force
+                this.#leaveGroupConsumerProtocol(false, callback);
+            });
+            return;
+        }
         // Leave by sending a heartbeat with memberEpoch = -1
         this.#cancelHeartbeat();
         this.#performDeduplicateGroupOperaton('leaveGroupConsumerProtocol', (connection, groupCallback) => {
@@ -1260,7 +1287,7 @@ export class Consumer extends Base {
             });
         }, error => {
             if (error) {
-                const unknownMemberError = error.findBy?.('unknownMemberId', true);
+                const unknownMemberError = findErrorBy(error, 'unknownMemberId', true);
                 // This is to avoid throwing an error if a group join was cancelled.
                 if (!unknownMemberError) {
                     callback(error);
@@ -1541,7 +1568,7 @@ export class Consumer extends Base {
         }
     }
     #getRejoinError(error) {
-        const protocolError = error.findBy?.('needsRejoin', true);
+        const protocolError = findErrorBy(error, 'needsRejoin', true);
         if (!protocolError) {
             return null;
         }
@@ -1627,17 +1654,14 @@ export class Consumer extends Base {
         }
     }
     #handleError(error) {
-        const kafkaError = error;
-        if (kafkaError) {
-            if (kafkaError.findBy('hasStaleMetadata', true)) {
-                this.clearMetadata();
-            }
-            else if (kafkaError.findBy('code', 'PLT_KFK_NETWORK') ||
-                kafkaError.findBy('apiId', 'COORDINATOR_LOAD_IN_PROGRESS') ||
-                kafkaError.findBy('apiId', 'COORDINATOR_NOT_AVAILABLE') ||
-                kafkaError.findBy('apiId', 'NOT_COORDINATOR')) {
-                this.#coordinatorId = null;
-            }
+        if (findErrorBy(error, 'hasStaleMetadata', true)) {
+            this.clearMetadata();
+        }
+        else if (findErrorBy(error, 'code', 'PLT_KFK_NETWORK') ||
+            findErrorBy(error, 'apiId', 'COORDINATOR_LOAD_IN_PROGRESS') ||
+            findErrorBy(error, 'apiId', 'COORDINATOR_NOT_AVAILABLE') ||
+            findErrorBy(error, 'apiId', 'NOT_COORDINATOR')) {
+            this.#coordinatorId = null;
         }
         return error;
     }

package/dist/clients/consumer/messages-stream.js

@@ -2,7 +2,7 @@ import { Readable } from 'node:stream';
 import { createPromisifiedCallback, kCallbackPromise, noopCallback } from "../../apis/callbacks.js";
 import { ListOffsetTimestamps } from "../../apis/enumerations.js";
 import { consumerReceivesChannel, createDiagnosticContext, notifyCreation } from "../../diagnostic.js";
-import { protocolErrors, UserError } from "../../errors.js";
+import { findErrorBy, protocolErrors, UserError } from "../../errors.js";
 import { IS_CONTROL } from "../../protocol/records.js";
 import { runAsyncSeries } from "../../registries/abstract.js";
 import { kAutocommit, kGetFetchNode, kInstance, kRefreshOffsetsAndFetch } from "../../symbols.js";
@@ -454,7 +454,7 @@ export class MessagesStream extends Readable {
         });
     }
     #handleOffsetOutOfRange(error, topicIds, callback) {
-        if (!error.findBy?.('apiId', 'OFFSET_OUT_OF_RANGE')) {
+        if (!findErrorBy(error, 'apiId', 'OFFSET_OUT_OF_RANGE')) {
             callback(null, false);
             return;
         }
@@ -613,6 +613,7 @@ export class MessagesStream extends Readable {
                             let payloadType = 'headerKey';
                             let deserializedKey;
                             let deserializedValue;
+                            let deserializationError;
                             try {
                                 for (const [headerKey, headerValue] of record.headers) {
                                     payloadType = 'headerKey';
@@ -640,16 +641,46 @@ export class MessagesStream extends Readable {
                                     timestamp: firstTimestamp + record.timestampDelta,
                                     commit: commit
                                 };
-                                const shouldDestroy = this.#deserializationErrorHandler
-                                    ? this.#deserializationErrorHandler(context) !== DeserializationErrorActions.SKIP
-                                    : this.#corruptedMessageHandler(record, topic, partition, firstTimestamp, firstOffset, commit, error);
-                                if (shouldDestroy) {
+                                const action = this.#deserializationErrorHandler
+                                    ? this.#deserializationErrorHandler(context)
+                                    : this.#corruptedMessageHandler(record, topic, partition, firstTimestamp, firstOffset, commit, error)
+                                        ? DeserializationErrorActions.FAIL
+                                        : DeserializationErrorActions.SKIP;
+                                if (action === DeserializationErrorActions.CONTINUE) {
+                                    /*
+                                      Deliver the message in a degraded form, so that nothing is lost: whatever was
+                                      already deserialized is kept, while the payload which failed and the ones which
+                                      were never attempted, since deserialization goes headers, key and then value,
+                                      are delivered as the original bytes.
+                  
+                                      The error is reported on the message metadata and published on the diagnostic
+                                      channel, which is where callers should react to it: logging here would be a log
+                                      storm waiting to happen.
+                                    */
+                                    diagnosticContext.error = error;
+                                    consumerReceivesChannel.error.publish(diagnosticContext);
+                                    const headersFailed = payloadType === 'headerKey' || payloadType === 'headerValue';
+                                    if (headersFailed) {
+                                        headers.clear();
+                                        for (const [headerKey, headerValue] of record.headers) {
+                                            headers.set(headerKey, headerValue);
+                                        }
+                                    }
+                                    if (headersFailed || payloadType === 'key') {
+                                        deserializedKey = (record.key ?? undefined);
+                                    }
+                                    deserializedValue = (record.value ?? undefined);
+                                    deserializationError = { error, payloadType };
+                                }
+                                else if (action === DeserializationErrorActions.SKIP) {
+                                    continue;
+                                }
+                                else {
                                     diagnosticContext.error = error;
                                     consumerReceivesChannel.error.publish(diagnosticContext);
                                     this.destroy(new UserError('Failed to deserialize a message.', { cause: error }));
                                     return;
                                 }
-                                continue;
                             }
                             this.#metricsConsumedMessages?.inc();
                             const message = {
@@ -661,7 +692,12 @@ export class MessagesStream extends Readable {
                                 timestamp: firstTimestamp + record.timestampDelta,
                                 offset,
                                 commit,
-                                metadata: messageMetadata,
+                                // Deserializers and hooks can enrich the metadata of the message they processed
+                                metadata: deserializationError
+                                    ? { ...messageMetadata, ...messageToConsume.metadata, deserializationError }
+                                    : messageToConsume.metadata
+                                        ? { ...messageMetadata, ...messageToConsume.metadata }
+                                        : messageMetadata,
                                 toJSON: messageToJSON
                             };
                             diagnosticContext.result = message;
@@ -747,7 +783,7 @@ export class MessagesStream extends Readable {
                 // Only destroy the stream when the broker requires a group rejoin
                 // (ILLEGAL_GENERATION, UNKNOWN_MEMBER_ID, REBALANCE_IN_PROGRESS).
                 // Transient coordinator errors must not tear down the consumption loop.
-                if (error.findBy?.('needsRejoin', true)) {
+                if (findErrorBy(error, 'needsRejoin', true)) {
                     this.destroy(error);
                 }
                 return;

package/dist/clients/consumer/types.d.ts

@@ -35,6 +35,7 @@ export interface PreferredReadReplica {
 export declare const DeserializationErrorActions: {
     readonly FAIL: "fail";
     readonly SKIP: "skip";
+    readonly CONTINUE: "continue";
 };
 export type DeserializationErrorAction = (typeof DeserializationErrorActions)[keyof typeof DeserializationErrorActions];
 export interface DeserializationErrorContext {
@@ -47,6 +48,10 @@ export interface DeserializationErrorContext {
     timestamp: bigint;
     commit: Message['commit'];
 }
+export interface MessageDeserializationError {
+    error: unknown;
+    payloadType: BeforeHookPayloadType;
+}
 export type DeserializationErrorHandler = (context: DeserializationErrorContext) => DeserializationErrorAction;
 /**
  * @deprecated Use `DeserializationErrorHandler` instead.

package/dist/clients/consumer/types.js

@@ -1,6 +1,7 @@
 export const DeserializationErrorActions = {
     FAIL: 'fail',
-    SKIP: 'skip'
+    SKIP: 'skip',
+    CONTINUE: 'continue'
 };
 export const MessagesStreamModes = {
     LATEST: 'latest',

package/dist/clients/producer/producer.js

@@ -2,7 +2,7 @@ import { randomUUID } from 'crypto';
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../../apis/callbacks.js";
 import { FindCoordinatorKeyTypes, ProduceAcks } from "../../apis/enumerations.js";
 import { createDiagnosticContext, producerInitIdempotentChannel, producerSendsChannel, producerTransactionsChannel } from "../../diagnostic.js";
-import { GenericError, UserError } from "../../errors.js";
+import { findErrorBy, UserError } from "../../errors.js";
 import { runAsyncSeries } from "../../registries/abstract.js";
 import { kInstance, kTransaction, kTransactionAddOffsets, kTransactionAddPartitions, kTransactionCancel, kTransactionCommitOffset, kTransactionEnd, kTransactionFindCoordinator, kTransactionPrepare } from "../../symbols.js";
 import { emitExperimentalApiWarning, NumericMap } from "../../utils.js";
@@ -770,11 +770,8 @@ export class Producer extends Base {
                 if (error) {
                     // If the last error was due to stale metadata, we retry the operation with this set of messages
                     // since the partition is already set, it should attempt on the new destination
-                    /* c8 ignore next 3 - Hard to test */
-                    const kafkaError = GenericError.isGenericError(error)
-                        ? error
-                        : null;
-                    const hasStaleMetadata = kafkaError?.findBy('hasStaleMetadata', true);
+                    /* c8 ignore next - Hard to test */
+                    const hasStaleMetadata = findErrorBy(error, 'hasStaleMetadata', true);
                     if (hasStaleMetadata && repeatOnStaleMetadata) {
                         this.clearMetadata();
                         this.#performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, false, produceOptions, recoverOnProducerStateLoss, callback);
@@ -796,17 +793,16 @@ export class Producer extends Base {
                 callback(error, results);
             }, 0, [], error => {
                 /* c8 ignore next 3 - Hard to test */
-                if (!repeatOnStaleMetadata || !GenericError.isGenericError(error)) {
+                if (!repeatOnStaleMetadata) {
                     return false;
                 }
-                return !!error.findBy('hasStaleMetadata', true);
+                return !!findErrorBy(error, 'hasStaleMetadata', true);
             });
         });
     }
     #isProducerStateLost(error) {
-        const kafkaError = error;
-        return !!(GenericError.isGenericError(error) &&
-            (kafkaError.findBy('apiId', 'UNKNOWN_PRODUCER_ID') || kafkaError.findBy('apiId', 'OUT_OF_ORDER_SEQUENCE_NUMBER')));
+        return !!(findErrorBy(error, 'apiId', 'UNKNOWN_PRODUCER_ID') ||
+            findErrorBy(error, 'apiId', 'OUT_OF_ORDER_SEQUENCE_NUMBER'));
     }
     #recoverIdempotentProducer(messages, produceOptions, callback) {
         this.#producerInfo = undefined;
@@ -859,20 +855,17 @@ export class Producer extends Base {
         });
     }
     #handleFencingError(error) {
-        if (error.findBy?.('producerFenced', true)) {
+        if (findErrorBy(error, 'producerFenced', true)) {
             this.#producerInfo = undefined;
             this.#transaction = undefined;
         }
     }
     #handleError(error) {
-        const kafkaError = error;
-        if (kafkaError) {
-            if (kafkaError.findBy('code', 'PLT_KFK_NETWORK') ||
-                kafkaError.findBy('apiId', 'COORDINATOR_LOAD_IN_PROGRESS') ||
-                kafkaError.findBy('apiId', 'COORDINATOR_NOT_AVAILABLE') ||
-                kafkaError.findBy('apiId', 'NOT_COORDINATOR')) {
-                this.#coordinatorId = undefined;
-            }
+        if (findErrorBy(error, 'code', 'PLT_KFK_NETWORK') ||
+            findErrorBy(error, 'apiId', 'COORDINATOR_LOAD_IN_PROGRESS') ||
+            findErrorBy(error, 'apiId', 'COORDINATOR_NOT_AVAILABLE') ||
+            findErrorBy(error, 'apiId', 'NOT_COORDINATOR')) {
+            this.#coordinatorId = undefined;
         }
         return error;
     }

package/dist/errors.d.ts

@@ -28,6 +28,7 @@ export declare class MultipleErrors extends AggregateError {
     findBy<ErrorType extends GenericError | MultipleErrors = MultipleErrors>(property: string, value: unknown): ErrorType | null;
     toString(): string;
 }
+export declare function findErrorBy<ErrorType extends GenericError = GenericError>(error: Error | null | undefined, property: string, value: unknown): ErrorType | null;
 export * from './protocol/errors.ts';
 export declare class AuthenticationError extends GenericError {
     static code: ErrorCode;

package/dist/errors.js

@@ -98,6 +98,18 @@ export class MultipleErrors extends AggregateError {
         return `Error ${this.code}: ${this.message} (caused by: ${this.errors.map(error => error.toString()).join(', ')})`;
     }
 }
+// findBy only exists on the library's own error classes, but errors also come from user code, Node
+// internals and third parties. This brand checks first, so classification is total: an error that
+// cannot be classified reports no match instead of throwing. Use it for any error whose origin is
+// not statically known.
+export function findErrorBy(error, property, value) {
+    if (!error || !GenericError.isGenericError(error)) {
+        return null;
+    }
+    // GenericError and MultipleErrors declare findBy with different generic defaults, so the union is
+    // not callable. The brand check above guarantees the call is safe either way.
+    return error.findBy(property, value);
+}
 export * from "./protocol/errors.js";
 export class AuthenticationError extends GenericError {
     static code = 'PLT_KFK_AUTHENTICATION';

package/dist/network/connection-pool.js

@@ -92,7 +92,9 @@ export class ConnectionPool extends TypedEventEmitter {
     }
     #get(broker, callback) {
         if (this.#closed) {
-            callback(new Error('Connection pool is closed.'));
+            // A closed pool never reopens, so this must not be retried. It also has to be a library error:
+            // clients classify errors via findBy, which only exists on GenericError subclasses.
+            callback(new NetworkError('Connection pool is closed.', { canRetry: false, closed: true, broker }));
             return;
         }
         if (broker == null || broker.host == null || broker.port == null) {

package/dist/protocol/records.d.ts

@@ -71,6 +71,7 @@ export interface KafkaRecord {
 export interface MessageToConsume extends KafkaRecord {
     topic: string;
     partition: number;
+    metadata?: Record<string, unknown>;
 }
 export interface RecordsBatch {
     firstOffset: bigint;

package/dist/registries/confluent-schema-registry.d.ts

@@ -1,5 +1,6 @@
 import { type ErrorObject, type Options, type ValidateFunction } from 'ajv';
 import { type Type } from 'avsc';
+import { type ConnectionOptions as TLSConnectionOptions } from 'node:tls';
 import { type Root } from 'protobufjs';
 import { type Callback } from '../apis/definitions.ts';
 import { type BeforeDeserializationHook, type BeforeHookPayloadType, type BeforeSerializationHook, type Deserializers, type Serializers } from '../clients/serde.ts';
@@ -7,10 +8,25 @@ import { type CredentialProvider, UserError } from '../index.ts';
 import { type MessageToConsume, type MessageToProduce } from '../protocol/records.ts';
 import { AbstractSchemaRegistry } from './abstract.ts';
 type ConfluentSchemaRegistryMessageToProduce = MessageToProduce<unknown, unknown, unknown, unknown>;
+export type UndiciDispatcher = NonNullable<RequestInit['dispatcher']>;
+export interface UndiciAgentOptions {
+    connect: TLSConnectionOptions;
+}
+export type UndiciAgentConstructor = new (options: UndiciAgentOptions) => UndiciDispatcher;
+export declare function createUndiciAgent(options: UndiciAgentOptions): UndiciDispatcher;
 export interface ConfluentSchemaRegistryMetadata {
-    schemas?: Record<BeforeHookPayloadType, number>;
+    schemas?: Partial<Record<BeforeHookPayloadType, number>>;
+}
+export interface ConfluentSchemaRegistryConsumedMetadata {
+    lengths?: {
+        key?: number;
+        value?: number;
+    };
 }
 export type ConfluentSchemaRegistryProtobufTypeMapper = (id: number, type: BeforeHookPayloadType, context: ConfluentSchemaRegistryMessageToProduce | MessageToConsume) => string;
+export type ConfluentSchemaRegistryHeaders = Record<string, string>;
+export type ConfluentSchemaRegistryHeadersProvider = () => ConfluentSchemaRegistryHeaders | Promise<ConfluentSchemaRegistryHeaders>;
+export type ConfluentSchemaRegistryRetryDelayGetter = (attempt: number, retries: number, error: Error) => number;
 export interface ConfluentSchemaRegistryOptions {
     url: string;
     auth?: {
@@ -18,10 +34,22 @@ export interface ConfluentSchemaRegistryOptions {
         password?: string | CredentialProvider;
         token?: string | CredentialProvider;
     };
+    headers?: ConfluentSchemaRegistryHeaders | ConfluentSchemaRegistryHeadersProvider;
+    timeout?: number;
+    retries?: number;
+    retryDelay?: number | ConfluentSchemaRegistryRetryDelayGetter;
+    tls?: TLSConnectionOptions;
+    ssl?: TLSConnectionOptions;
     protobufTypeMapper?: ConfluentSchemaRegistryProtobufTypeMapper;
     jsonValidateSend?: boolean;
     jsonAjvOptions?: Options;
 }
+export declare const defaultConfluentSchemaRegistryOptions: {
+    timeout: number;
+    retries: number;
+    retryDelay: number;
+};
+export declare const retriableSchemaRegistryStatuses: number[];
 export type SchemaValidationPhase = 'serialization' | 'deserialization';
 export interface SchemaValidationErrorProperties {
     schemaId: number;
@@ -49,6 +77,8 @@ export interface Schema {
     type: 'avro' | 'protobuf' | 'json';
     schema: Type | Root | ValidateFunction;
 }
+export declare function trackConsumedSchemaId(message: MessageToConsume | undefined, type: BeforeHookPayloadType, schemaId: number): void;
+export declare function trackConsumedPayloadLength(message: MessageToConsume | undefined, type: 'key' | 'value', data: Buffer): void;
 export declare function defaultProtobufTypeMapper(_: number, type: BeforeHookPayloadType, context: ConfluentSchemaRegistryMessageToProduce | MessageToConsume): string;
 export declare class ConfluentSchemaRegistry<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends AbstractSchemaRegistry<number | undefined, Schema, Key, Value, HeaderKey, HeaderValue> {
     #private;

package/dist/registries/confluent-schema-registry.js

@@ -2,19 +2,74 @@ import { Ajv } from 'ajv';
 import { Ajv2020 } from 'ajv/dist/2020.js';
 import avro from 'avsc';
 import { createRequire } from 'node:module';
+import { setTimeout as sleep } from 'node:timers/promises';
 import { jsonDeserializer, jsonSerializer, stringDeserializer, stringSerializer } from "../clients/serde.js";
-import { EMPTY_BUFFER, UnsupportedFormatError, UserError } from "../index.js";
+import { EMPTY_BUFFER, MultipleErrors, NetworkError, TimeoutError, UnsupportedFormatError, UserError } from "../index.js";
 import { getCredential } from "../protocol/sasl/utils.js";
 import { AbstractSchemaRegistry } from "./abstract.js";
 const require = createRequire(import.meta.url);
 const AjvDraft04 = require('ajv-draft-04');
 const draft06MetaSchema = require('ajv/dist/refs/json-schema-draft-06.json');
+const undiciGlobalDispatcherSymbol = /^undici\.globalDispatcher\.(\d+)$/;
+export function createUndiciAgent(options) {
+    /*
+      The undici globals are installed the first time undici is loaded, which creating a Request is
+      enough to do and which performs no I/O.
+    */
+    new Request('http://localhost'); // eslint-disable-line no-new
+    /*
+      The symbol is versioned and more than one version is registered at the same time, pointing at
+      different objects: on Node 26 undici.globalDispatcher.1 is a backwards compatibility wrapper and
+      only undici.globalDispatcher.2 is the Agent. The highest version available therefore wins.
+    */
+    const globals = globalThis;
+    let version = -1;
+    let Agent;
+    for (const symbol of Object.getOwnPropertySymbols(globalThis)) {
+        const symbolVersion = symbol.description?.match(undiciGlobalDispatcherSymbol)?.[1];
+        if (typeof symbolVersion === 'undefined' || Number(symbolVersion) <= version) {
+            continue;
+        }
+        version = Number(symbolVersion);
+        Agent = globals[symbol]?.constructor;
+    }
+    /* c8 ignore next 6 - Only reachable on a Node.js build which does not bundle undici */
+    if (typeof Agent !== 'function') {
+        throw new UserError('Cannot access the undici Agent bundled with Node.js, which is required to configure TLS for the Schema Registry.');
+    }
+    return new Agent(options);
+}
+export const defaultConfluentSchemaRegistryOptions = {
+    timeout: 5000,
+    retries: 3,
+    retryDelay: 1000
+};
+// Statuses which are worth retrying: the registry is overloaded, restarting or behind a proxy which is
+export const retriableSchemaRegistryStatuses = [408, 425, 429, 500, 502, 503, 504];
 export class SchemaValidationError extends UserError {
     constructor(message, properties) {
         super(message, properties);
         this.name = 'SchemaValidationError';
     }
 }
+export function trackConsumedSchemaId(message, type, schemaId) {
+    /* c8 ignore next 3 - The message is always provided for keys and values */
+    if (!message) {
+        return;
+    }
+    const metadata = (message.metadata ??= {});
+    metadata.schemas ??= {};
+    metadata.schemas[type] = schemaId;
+}
+export function trackConsumedPayloadLength(message, type, data) {
+    /* c8 ignore next 3 - The message is always provided for keys and values */
+    if (!message) {
+        return;
+    }
+    const metadata = (message.metadata ??= {});
+    metadata.lengths ??= {};
+    metadata.lengths[type] = data.length;
+}
 /* c8 ignore next 8 */
 export function defaultProtobufTypeMapper(_, type, context) {
     // Confluent Schema Registry convention
@@ -31,6 +86,12 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
     #jsonAjvDraft04;
     #pendingFetches;
     #auth;
+    #headers;
+    #timeout;
+    #retries;
+    #retryDelay;
+    #tls;
+    #tlsDispatcher;
     constructor(options) {
         super();
         this.#url = options.url;
@@ -43,7 +104,15 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
         this.#jsonAjvDraft7.addMetaSchema(draft06MetaSchema);
         this.#jsonAjvDraft04 = new AjvDraft04(jsonAjvOptions);
         this.#auth = options.auth;
+        this.#headers = options.headers;
+        this.#timeout = options.timeout ?? defaultConfluentSchemaRegistryOptions.timeout;
+        this.#retries = options.retries ?? defaultConfluentSchemaRegistryOptions.retries;
+        this.#retryDelay = options.retryDelay ?? defaultConfluentSchemaRegistryOptions.retryDelay;
+        this.#tls = options.tls ?? options.ssl;
         this.#pendingFetches = new Map();
+        if (this.#tls && new URL(this.#url).protocol !== 'https:') {
+            throw new UserError('TLS options can only be used with a HTTPS Schema Registry URL.', { url: this.#url });
+        }
     }
     getSchemaId(message, type) {
         if (message === null) {
@@ -112,7 +181,7 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
     }
     getBeforeDeserializationHook() {
         const registry = this;
-        return function beforeDeserialization(payload, type, _message, callback) {
+        return function beforeDeserialization(payload, type, message, callback) {
             // Extract the schema ID from the message metadata
             const schemaId = registry.getSchemaId(payload, type);
             // When no schema ID is found, nothing to do
@@ -120,6 +189,11 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
                 callback(null);
                 return;
             }
+            /*
+              Track the schema ID before fetching, so that it is exposed on the message metadata even when
+              the schema cannot be fetched or the payload cannot be decoded.
+            */
+            trackConsumedSchemaId(message, type, schemaId);
             // The schema is already fetch
             if (registry.get(schemaId)) {
                 callback(null);
@@ -128,28 +202,86 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
             registry.fetchSchema(schemaId, callback);
         };
     }
-    async #fetchSchema(id) {
-        const requestInit = {};
+    async #requestHeaders() {
+        let headers;
+        if (this.#headers) {
+            headers = { ...(typeof this.#headers === 'function' ? await this.#headers() : this.#headers) };
+        }
+        // Authentication headers are applied last so that auth options always win over custom headers
         if (this.#auth) {
+            headers ??= {};
             if (this.#auth.token) {
                 const token = await getCredential('token', this.#auth.token);
-                requestInit.headers = {
-                    Authorization: `Bearer ${token}`
-                };
+                headers.Authorization = `Bearer ${token}`;
             }
             else {
                 const username = await getCredential('username', this.#auth.username);
                 const password = await getCredential('password', this.#auth.password);
-                requestInit.headers = {
-                    Authorization: `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`
-                };
+                headers.Authorization = `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`;
+            }
+        }
+        return headers;
+    }
+    async #requestSchema(id) {
+        const requestInit = {};
+        const headers = await this.#requestHeaders();
+        if (headers) {
+            requestInit.headers = headers;
+        }
+        if (this.#tls) {
+            // Created once per registry, so that connections are pooled across schema fetches
+            this.#tlsDispatcher ??= createUndiciAgent({ connect: this.#tls });
+            requestInit.dispatcher = this.#tlsDispatcher;
+        }
+        if (this.#timeout > 0) {
+            requestInit.signal = AbortSignal.timeout(this.#timeout);
+        }
+        let response;
+        try {
+            response = await fetch(`${this.#url}/schemas/ids/${id}`, requestInit);
+        }
+        catch (error) {
+            if (error.name === 'TimeoutError') {
+                throw new TimeoutError(`Fetching a schema timed out after ${this.#timeout} ms.`, {
+                    canRetry: true,
+                    cause: error
+                });
             }
+            throw new NetworkError('Failed to fetch a schema.', { cause: error });
         }
-        const response = await fetch(`${this.#url}/schemas/ids/${id}`, requestInit);
         if (!response.ok) {
-            throw new UserError(`Failed to fetch a schema: [HTTP ${response.status}]`, { response: await response.text() });
+            throw new UserError(`Failed to fetch a schema: [HTTP ${response.status}]`, {
+                response: await response.text(),
+                canRetry: retriableSchemaRegistryStatuses.includes(response.status)
+            });
         }
-        const responseBody = (await response.json());
+        return (await response.json());
+    }
+    async #requestSchemaWithRetries(id) {
+        const errors = [];
+        for (let attempt = 0;; attempt++) {
+            try {
+                return await this.#requestSchema(id);
+            }
+            catch (error) {
+                errors.push(error);
+                if (attempt >= this.#retries || error.canRetry !== true) {
+                    if (errors.length === 1) {
+                        throw errors[0];
+                    }
+                    throw new MultipleErrors(`Failed to fetch a schema after ${errors.length} attempts.`, errors);
+                }
+                const delay = typeof this.#retryDelay === 'function'
+                    ? this.#retryDelay(attempt + 1, this.#retries, error)
+                    : this.#retryDelay;
+                if (delay > 0) {
+                    await sleep(delay);
+                }
+            }
+        }
+    }
+    async #fetchSchema(id) {
+        const responseBody = await this.#requestSchemaWithRetries(id);
         const { schema } = responseBody;
         const schemaType = responseBody.schemaType ?? 'AVRO';
         switch (schemaType) {
@@ -232,17 +364,32 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
         return Buffer.concat([header, encodedMessage]);
     }
     #schemaDeserializer(type, fallbackDeserializer, data, headers, message) {
+        /*
+          Deserializing replaces the original payload, so record its size beforehand: it is the only
+          chance callers have to account for the actual ingress size of the message.
+        */
+        if ((type === 'key' || type === 'value') && Buffer.isBuffer(data)) {
+            trackConsumedPayloadLength(message, type, data);
+        }
         /* c8 ignore next 3 - Hard to test */
         if (typeof data === 'undefined' || data.length === 0) {
             return EMPTY_BUFFER;
         }
-        if (type === 'headerKey' || type === 'headerValue') {
+        const isHeader = type === 'headerKey' || type === 'headerValue';
+        if (isHeader) {
             message = headers;
         }
         const schemaId = this.getSchemaId(data, type);
         if (!schemaId) {
             return fallbackDeserializer(data);
         }
+        /*
+          Also track the schema ID here, so that it is exposed on the message metadata even when the
+          deserializers are used without the before-deserialization hook of this registry.
+        */
+        if (!isHeader) {
+            trackConsumedSchemaId(message, type, schemaId);
+        }
         const schema = this.#schemas.get(schemaId);
         if (!schema) {
             throw new UserError(`Schema with ID ${schemaId} not found.`, { missingSchema: schemaId });

package/dist/typescript-4/dist/clients/consumer/types.d.ts

@@ -35,6 +35,7 @@ export interface PreferredReadReplica {
 export declare const DeserializationErrorActions: {
   readonly FAIL: "fail";
   readonly SKIP: "skip";
+  readonly CONTINUE: "continue";
 };
 export type DeserializationErrorAction = (typeof DeserializationErrorActions)[keyof typeof DeserializationErrorActions];
 export interface DeserializationErrorContext {
@@ -47,6 +48,10 @@ export interface DeserializationErrorContext {
   timestamp: bigint;
   commit: Message['commit'];
 }
+export interface MessageDeserializationError {
+  error: unknown;
+  payloadType: BeforeHookPayloadType;
+}
 export type DeserializationErrorHandler = (context: DeserializationErrorContext) => DeserializationErrorAction;
 /**
  * @deprecated Use `DeserializationErrorHandler` instead.

package/dist/typescript-4/dist/errors.d.ts

@@ -34,6 +34,7 @@ export declare class MultipleErrors extends AggregateError {
   findBy<ErrorType extends GenericError | MultipleErrors = MultipleErrors>(property: string, value: unknown): ErrorType | null;
   toString(): string;
 }
+export declare function findErrorBy<ErrorType extends GenericError = GenericError>(error: Error | null | undefined, property: string, value: unknown): ErrorType | null;
 export * from "./protocol/errors";
 export declare class AuthenticationError extends GenericError {
   static code: ErrorCode;

package/dist/typescript-4/dist/protocol/records.d.ts

@@ -71,6 +71,7 @@ export interface KafkaRecord {
 export interface MessageToConsume extends KafkaRecord {
   topic: string;
   partition: number;
+  metadata?: Record<string, unknown>;
 }
 export interface RecordsBatch {
   firstOffset: bigint;

package/dist/typescript-4/dist/registries/confluent-schema-registry.d.ts

@@ -1,5 +1,6 @@
 import { type ErrorObject, type Options, type ValidateFunction } from 'ajv';
 import { type Type } from 'avsc';
+import { type ConnectionOptions as TLSConnectionOptions } from 'node:tls';
 import { type Root } from 'protobufjs';
 import { type Callback } from "../apis/definitions";
 import { type BeforeDeserializationHook, type BeforeHookPayloadType, type BeforeSerializationHook, type Deserializers, type Serializers } from "../clients/serde";
@@ -7,10 +8,25 @@ import { type CredentialProvider, UserError } from "../index";
 import { type MessageToConsume, type MessageToProduce } from "../protocol/records";
 import { AbstractSchemaRegistry } from "./abstract";
 type ConfluentSchemaRegistryMessageToProduce = MessageToProduce<unknown, unknown, unknown, unknown>;
+export type UndiciDispatcher = NonNullable<RequestInit['dispatcher']>;
+export interface UndiciAgentOptions {
+  connect: TLSConnectionOptions;
+}
+export type UndiciAgentConstructor = new (options: UndiciAgentOptions) => UndiciDispatcher;
+export declare function createUndiciAgent(options: UndiciAgentOptions): UndiciDispatcher;
 export interface ConfluentSchemaRegistryMetadata {
-  schemas?: Record<BeforeHookPayloadType, number>;
+  schemas?: Partial<Record<BeforeHookPayloadType, number>>;
+}
+export interface ConfluentSchemaRegistryConsumedMetadata {
+  lengths?: {
+    key?: number;
+    value?: number;
+  };
 }
 export type ConfluentSchemaRegistryProtobufTypeMapper = (id: number, type: BeforeHookPayloadType, context: ConfluentSchemaRegistryMessageToProduce | MessageToConsume) => string;
+export type ConfluentSchemaRegistryHeaders = Record<string, string>;
+export type ConfluentSchemaRegistryHeadersProvider = () => ConfluentSchemaRegistryHeaders | Promise<ConfluentSchemaRegistryHeaders>;
+export type ConfluentSchemaRegistryRetryDelayGetter = (attempt: number, retries: number, error: Error) => number;
 export interface ConfluentSchemaRegistryOptions {
   url: string;
   auth?: {
@@ -18,10 +34,22 @@ export interface ConfluentSchemaRegistryOptions {
     password?: string | CredentialProvider;
     token?: string | CredentialProvider;
   };
+  headers?: ConfluentSchemaRegistryHeaders | ConfluentSchemaRegistryHeadersProvider;
+  timeout?: number;
+  retries?: number;
+  retryDelay?: number | ConfluentSchemaRegistryRetryDelayGetter;
+  tls?: TLSConnectionOptions;
+  ssl?: TLSConnectionOptions;
   protobufTypeMapper?: ConfluentSchemaRegistryProtobufTypeMapper;
   jsonValidateSend?: boolean;
   jsonAjvOptions?: Options;
 }
+export declare const defaultConfluentSchemaRegistryOptions: {
+  timeout: number;
+  retries: number;
+  retryDelay: number;
+};
+export declare const retriableSchemaRegistryStatuses: number[];
 export type SchemaValidationPhase = 'serialization' | 'deserialization';
 export interface SchemaValidationErrorProperties {
   schemaId: number;
@@ -49,6 +77,8 @@ export interface Schema {
   type: 'avro' | 'protobuf' | 'json';
   schema: Type | Root | ValidateFunction;
 }
+export declare function trackConsumedSchemaId(message: MessageToConsume | undefined, type: BeforeHookPayloadType, schemaId: number): void;
+export declare function trackConsumedPayloadLength(message: MessageToConsume | undefined, type: 'key' | 'value', data: Buffer): void;
 export declare function defaultProtobufTypeMapper(_: number, type: BeforeHookPayloadType, context: ConfluentSchemaRegistryMessageToProduce | MessageToConsume): string;
 export declare class ConfluentSchemaRegistry<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends AbstractSchemaRegistry<number | undefined, Schema, Key, Value, HeaderKey, HeaderValue> {
   #private;

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "2.8.0";
+export const version = "2.9.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "2.8.0",
+  "version": "2.9.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",