@platformatic/kafka 2.0.1 → 2.2.0

package/dist/network/connection.js

@@ -4,7 +4,8 @@ import { createConnection } from 'node:net';
 import { connect as createTLSConnection } from 'node:tls';
 import { createPromisifiedCallback, kCallbackPromise } from "../apis/callbacks.js";
 import { allowedSASLMechanisms, SASLMechanisms } from "../apis/enumerations.js";
-import { saslAuthenticateV2, saslHandshakeV1 } from "../apis/index.js";
+import { clientSoftwareName, clientSoftwareVersion } from "../clients/base/options.js";
+import { apiVersionsV1, saslAuthenticateV0, saslAuthenticateV1, saslAuthenticateV2, saslHandshakeV1 } from "../apis/index.js";
 import { connectionsApiChannel, connectionsConnectsChannel, createDiagnosticContext, notifyCreation } from "../diagnostic.js";
 import { AuthenticationError, NetworkError, TimeoutError, UnexpectedCorrelationIdError, UserError } from "../errors.js";
 import { TypedEventEmitter } from "../events.js";
@@ -50,12 +51,20 @@ export class Connection extends TypedEventEmitter {
     #responseReader;
     #socket;
     #socketMustBeDrained;
+    #detectMissingTLS;
     #reauthenticationTimeout;
+    // Negotiated once per connection: reauthentication runs the same path again.
+    #saslAuthenticateApi;
     constructor(clientId, options = {}) {
         super();
         this.setMaxListeners(0);
         this.#instanceId = currentInstance++;
-        this.#options = Object.assign({}, defaultOptions, options);
+        this.#options = Object.assign({}, defaultOptions);
+        for (const [key, value] of Object.entries(options)) {
+            if (value !== undefined) {
+                this.#options[key] = value;
+            }
+        }
         this.#options.tls ??= this.#options.ssl;
         this.#status = ConnectionStatuses.NONE;
         this.#clientId = clientId;
@@ -70,6 +79,7 @@ export class Connection extends TypedEventEmitter {
         this.#responseBuffer = new DynamicBuffer();
         this.#responseReader = new Reader(this.#responseBuffer);
         this.#socketMustBeDrained = false;
+        this.#detectMissingTLS = !this.#options.tls;
         notifyCreation('connection', this);
     }
     /* c8 ignore next 3 - Simple getter */
@@ -115,6 +125,7 @@ export class Connection extends TypedEventEmitter {
                 return callback[kCallbackPromise];
             }
             this.#status = ConnectionStatuses.CONNECTING;
+            this.#detectMissingTLS = !this.#options.tls;
             const connectionOptions = {
                 timeout: this.#options.connectTimeout
             };
@@ -124,7 +135,7 @@ export class Connection extends TypedEventEmitter {
             }
             /* c8 ignore next 13 - Hard to test */
             const connectingSocketTimeoutHandler = () => {
-                const error = new TimeoutError(`Connection to ${host}:${port} timed out.`);
+                const error = new TimeoutError(`Connection to ${host}:${port} timed out.`, { canRetry: true });
                 diagnosticContext.error = error;
                 this.#socket.destroy();
                 this.#status = ConnectionStatuses.ERROR;
@@ -214,7 +225,7 @@ export class Connection extends TypedEventEmitter {
             this.#socket?.destroy();
             callback(new TimeoutError(this.#host
                 ? `Connection to ${this.#host}:${this.#port} timed out.`
-                : `Connection ready timed out after ${this.#options.connectTimeout}ms.`));
+                : `Connection ready timed out after ${this.#options.connectTimeout}ms.`, { canRetry: true }));
         }, this.#options.connectTimeout);
         this.once('connect', onConnect);
         this.once('error', onError);
@@ -253,7 +264,9 @@ export class Connection extends TypedEventEmitter {
         return callback[kCallbackPromise];
     }
     send(apiKey, apiVersion, createPayload, responseParser, hasRequestHeaderTaggedFields, hasResponseHeaderTaggedFields, callback) {
-        const correlationId = ++this.#correlationId;
+        // Correlation ID is a 32-bit integer in the protocol, so we need to wrap around after 2^31 - 1
+        const correlationId = (this.#correlationId + 1) & 0x7fffffff;
+        this.#correlationId = correlationId;
         const diagnostic = createDiagnosticContext({
             connection: this,
             operation: 'send',
@@ -330,28 +343,71 @@ export class Connection extends TypedEventEmitter {
             this.#onConnectionError(host, port, diagnosticContext, new UserError(`SASL mechanism ${mechanism} not supported.`));
             return;
         }
-        saslHandshakeV1.api(this, mechanism, (error, response) => {
+        this.#negotiateSaslAuthenticate(saslAuthenticate => {
+            saslHandshakeV1.api(this, mechanism, (error, response) => {
+                if (error) {
+                    this.#onConnectionError(host, port, diagnosticContext, new AuthenticationError('Cannot find a suitable SASL mechanism.', { cause: error }));
+                    return;
+                }
+                this.emit('sasl:handshake', response.mechanisms);
+                const callback = this.#onSaslAuthenticate.bind(this, host, port, diagnosticContext);
+                if (authenticate) {
+                    authenticate(mechanism, this, saslAuthenticate, username, password, token, callback);
+                }
+                else if (mechanism === SASLMechanisms.PLAIN) {
+                    saslPlain.authenticate(saslAuthenticate, this, username, password, callback);
+                }
+                else if (mechanism === SASLMechanisms.OAUTHBEARER) {
+                    saslOAuthBearer.authenticate(saslAuthenticate, this, token, oauthBearerExtensions, callback);
+                }
+                else if (mechanism === SASLMechanisms.GSSAPI) {
+                    callback(new UserError('No custom SASL/GSSAPI authenticator provided.'));
+                }
+                else {
+                    saslScramSha.authenticate(saslAuthenticate, this, mechanism.substring(6), username, password, defaultCrypto, callback);
+                }
+            });
+        });
+    }
+    /**
+     * Chooses the SaslAuthenticate version to authenticate with.
+     *
+     * The version cannot be assumed: v1 arrived in Apache Kafka 2.0 and v2 in 2.4, so pinning the
+     * newest makes authentication fail outright on older brokers, which answer an unparseable request
+     * with InvalidRequestException and drop the connection.
+     *
+     * Brokers accept ApiVersions before authentication precisely so that clients can ask, and it has
+     * to happen before SaslHandshake: once the handshake is sent the connection only accepts
+     * SaslAuthenticate. The result is cached because reauthentication runs this path again.
+     *
+     * SaslHandshake itself stays at v1. v0 signals the pre-1.0 flow, where the SASL tokens are written
+     * raw on the wire rather than wrapped in SaslAuthenticate requests, which this package does not
+     * implement.
+     */
+    #negotiateSaslAuthenticate(callback) {
+        if (this.#saslAuthenticateApi) {
+            callback(this.#saslAuthenticateApi);
+            return;
+        }
+        const candidates = [
+            [2, saslAuthenticateV2.api],
+            [1, saslAuthenticateV1.api],
+            [0, saslAuthenticateV0.api]
+        ];
+        apiVersionsV1.api(this, clientSoftwareName, clientSoftwareVersion, (error, response) => {
+            let selected;
+            /* c8 ignore next 3 - Only reachable against a broker which refuses ApiVersions before authentication */
             if (error) {
-                this.#onConnectionError(host, port, diagnosticContext, new AuthenticationError('Cannot find a suitable SASL mechanism.', { cause: error }));
-                return;
-            }
-            this.emit('sasl:handshake', response.mechanisms);
-            const callback = this.#onSaslAuthenticate.bind(this, host, port, diagnosticContext);
-            if (authenticate) {
-                authenticate(mechanism, this, saslAuthenticateV2.api, username, password, token, callback);
-            }
-            else if (mechanism === SASLMechanisms.PLAIN) {
-                saslPlain.authenticate(saslAuthenticateV2.api, this, username, password, callback);
-            }
-            else if (mechanism === SASLMechanisms.OAUTHBEARER) {
-                saslOAuthBearer.authenticate(saslAuthenticateV2.api, this, token, oauthBearerExtensions, callback);
-            }
-            else if (mechanism === SASLMechanisms.GSSAPI) {
-                callback(new UserError('No custom SASL/GSSAPI authenticator provided.'));
+                selected = undefined;
             }
             else {
-                saslScramSha.authenticate(saslAuthenticateV2.api, this, mechanism.substring(6), username, password, defaultCrypto, callback);
+                const advertised = response.apiKeys.find(api => api.apiKey === saslAuthenticateV2.api.key);
+                selected = candidates.find(([version]) => advertised && version >= advertised.minVersion && version <= advertised.maxVersion)?.[1];
             }
+            // Falling back to the newest version keeps the previous behaviour for brokers which do not
+            // answer the probe, rather than failing a connection which would otherwise have worked.
+            this.#saslAuthenticateApi = selected ?? saslAuthenticateV2.api;
+            callback(this.#saslAuthenticateApi);
         });
     }
     /*
@@ -460,6 +516,27 @@ export class Connection extends TypedEventEmitter {
         correlation_id => INT32
     */
     #onData(chunk) {
+        if (this.#detectMissingTLS) {
+            this.#detectMissingTLS = false;
+            if (this.#isTLSRecord(chunk)) {
+                const error = new UserError('Broker requires TLS.');
+                this.#status = ConnectionStatuses.ERROR;
+                for (const request of this.#afterDrainRequests) {
+                    if (!request.noResponse) {
+                        clearTimeout(request.timeoutHandle);
+                        request.callback(error, undefined);
+                    }
+                }
+                this.#afterDrainRequests = [];
+                for (const request of this.#inflightRequests.values()) {
+                    clearTimeout(request.timeoutHandle);
+                    request.callback(error, undefined);
+                }
+                this.#inflightRequests.clear();
+                this.#socket.destroy();
+                return;
+            }
+        }
         this.#responseBuffer.append(chunk);
         // There is at least one message size to add
         // Note that here the initial position is always 0
@@ -540,6 +617,12 @@ export class Connection extends TypedEventEmitter {
         this.#socketMustBeDrained = false;
         this.emit('drain');
     }
+    #isTLSRecord(chunk) {
+        return (chunk.length >= 3 &&
+            (chunk[0] === 0x14 || chunk[0] === 0x15 || chunk[0] === 0x16 || chunk[0] === 0x17) &&
+            chunk[1] === 0x03 &&
+            chunk[2] <= 0x04);
+    }
     #onClose() {
         this.#status = ConnectionStatuses.CLOSED;
         this.emit('close');

package/dist/network/utils.js

@@ -1,12 +1,30 @@
+const BRACKETED_BROKER_RE = /^\[([^\]]+)](?::(\d*))?$/;
 export function parseBroker(broker, defaultPort = 9092) {
-    if (typeof broker === 'string') {
-        if (broker.includes(':')) {
-            const [host, port] = broker.split(':');
-            return { host, port: Number(port) };
-        }
-        else {
-            return { host: broker, port: defaultPort };
+    if (typeof broker !== 'string') {
+        const host = stripBrackets(broker.host);
+        if (host === broker.host) {
+            return broker;
         }
+        return { ...broker, host };
+    }
+    const bracketed = BRACKETED_BROKER_RE.exec(broker);
+    if (bracketed) {
+        return { host: bracketed[1], port: parsePort(bracketed[2], defaultPort) };
+    }
+    const separator = broker.lastIndexOf(':');
+    // Bare IPv6 addresses contain multiple colons and no port. The host:port form
+    // is unambiguous only with a single colon, or with brackets around the host.
+    if (separator === -1 || broker.indexOf(':') !== separator) {
+        return { host: broker, port: defaultPort };
+    }
+    return { host: broker.slice(0, separator), port: parsePort(broker.slice(separator + 1), defaultPort) };
+}
+function parsePort(port, defaultPort) {
+    return port === undefined || port.length === 0 ? defaultPort : Number(port);
+}
+function stripBrackets(host) {
+    if (host.startsWith('[') && host.endsWith(']')) {
+        return host.slice(1, -1);
     }
-    return broker;
+    return host;
 }

package/dist/protocol/definitions.js

@@ -7,5 +7,5 @@ export const EMPTY_BUFFER = Buffer.alloc(0);
 export const EMPTY_UUID = Buffer.alloc(UUID_SIZE);
 // Since it is serialized at either 0 (for nullable) or 1 (since length is stored as length + 1), it always uses a single byte
 export const EMPTY_OR_SINGLE_COMPACT_LENGTH_SIZE = INT8_SIZE;
-// TODO: Tagged fields are not supported yet
+// Flexible-version messages use this zero-length tagged-field section.
 export const EMPTY_TAGGED_FIELDS_BUFFER = Buffer.from([0]);

package/dist/protocol/reader.js

@@ -187,9 +187,13 @@ export class Reader {
         return this.readNullableString(compact, encoding) || '';
     }
     readUUID() {
-        const value = this.peekUUID();
+        const hex = this.peekUUID();
         this.position += UUID_SIZE;
-        return value.replace(/(.{8})(.{4})(.{4})(.{4})(.{12})/, '$1-$2-$3-$4-$5');
+        // Hyphenated by slicing rather than by a capture group regex. This runs once per topic per
+        // Fetch response from v13 up, and the regex form measured about fourteen times the cost, which
+        // made the newest Fetch versions measurably slower to decode than the ones identifying topics
+        // by name.
+        return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
     }
     readNullableBytes(compact = true) {
         let length;
@@ -300,11 +304,12 @@ export class Reader {
         }
         return reader();
     }
-    // TODO: Tagged fields are not supported yet
+    // Skip unknown tagged fields because codecs do not expose them to callers.
     readTaggedFields() {
-        const length = this.readVarInt();
-        if (length > 0) {
-            this.skip(length);
+        const length = this.readUnsignedVarInt();
+        for (let i = 0; i < length; i++) {
+            this.readUnsignedVarInt();
+            this.skip(this.readUnsignedVarInt());
         }
     }
 }

package/dist/protocol/records.d.ts

@@ -33,11 +33,14 @@ export interface MessageJSON<Key = unknown, Value = unknown, HeaderKey = unknown
     partition: number;
     timestamp: string;
     offset: string;
+    leaderEpoch: number;
     metadata: Record<string, unknown>;
 }
 export interface Message<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends Required<MessageBase<Key, Value>> {
     headers: Map<HeaderKey, HeaderValue>;
+    headerEntries: Array<[HeaderKey, HeaderValue]>;
     offset: bigint;
+    leaderEpoch: number;
     metadata: Record<string, unknown>;
     commit(callback?: (error?: Error) => void): void | Promise<void>;
     toJSON(): MessageJSON<Key, Value, HeaderKey, HeaderValue>;
@@ -71,6 +74,7 @@ export interface KafkaRecord {
 export interface MessageToConsume extends KafkaRecord {
     topic: string;
     partition: number;
+    metadata?: Record<string, unknown>;
 }
 export interface RecordsBatch {
     firstOffset: bigint;

package/dist/protocol/writer.js

@@ -1,6 +1,6 @@
 import { DynamicBuffer } from '@platformatic/dynamic-buffer';
 import { humanize } from "../utils.js";
-import { EMPTY_TAGGED_FIELDS_BUFFER, EMPTY_UUID } from "./definitions.js";
+import { EMPTY_TAGGED_FIELDS_BUFFER, EMPTY_UUID, UUID_SIZE } from "./definitions.js";
 const instanceIdentifier = Symbol('plt.kafka.writer.instanceIdentifier');
 export class Writer {
     context;
@@ -125,6 +125,12 @@ export class Writer {
             return this.append(EMPTY_UUID);
         }
         const buffer = Buffer.from(value.replaceAll('-', ''), 'hex');
+        // UUIDs are fixed width and carry no length prefix, so anything that is not exactly 16 bytes
+        // would silently desync the rest of the request. Buffer.from() stops at the first non hex
+        // character, so a topic name used as a topic id would append nothing at all here.
+        if (buffer.length !== UUID_SIZE) {
+            throw new RangeError(`Expected a UUID, got ${JSON.stringify(value)}.`);
+        }
         this.#buffer.append(buffer);
         return this;
     }
@@ -138,7 +144,9 @@ export class Writer {
         else {
             this.appendInt32(value.length);
         }
-        this.#buffer.append(value);
+        if (value.length > 0) {
+            this.#buffer.append(value);
+        }
         return this;
     }
     // Note that this does not follow the wire protocol specification and thus the length is not +1ed
@@ -147,7 +155,12 @@ export class Writer {
             return this.appendVarInt(-1);
         }
         this.appendVarInt(value.length);
-        this.#buffer.append(value);
+        // Appending an empty buffer corrupts DynamicBuffer's positional reads, so it must be skipped:
+        // see https://github.com/platformatic/dynamic-buffer/issues/12. Record keys and values are
+        // allowed to be empty without being null, so this is reachable.
+        if (value.length > 0) {
+            this.#buffer.append(value);
+        }
         return this;
     }
     appendArray(value, entryWriter, compact = true, appendTrailingTaggedFields = true) {
@@ -210,7 +223,7 @@ export class Writer {
         }
         return this;
     }
-    // TODO: Tagged fields are not supported yet
+    // This client emits only empty tagged-field sections; it does not expose tagged fields to callers.
     appendTaggedFields(_ = []) {
         return this.append(EMPTY_TAGGED_FIELDS_BUFFER);
     }

package/dist/registries/abstract.d.ts

@@ -4,7 +4,7 @@ import { type MessageToProduce } from '../protocol/records.ts';
 export interface SchemaRegistry<Id = unknown, Schema = unknown, Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> {
     get(id: Id): Schema | undefined;
     fetch(id: Id, callback?: (error?: Error) => void): void | Promise<void>;
-    getSchemaId(payload: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): Id;
+    getSchemaId(payload: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): Id;
     getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
     getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
     getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;
@@ -14,7 +14,7 @@ export declare function runAsyncSeries<V>(operation: (item: V, cb: Callback<void
 export declare class AbstractSchemaRegistry<Id = unknown, Schema = unknown, Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> implements SchemaRegistry<Id, Schema, Key, Value, HeaderKey, HeaderValue> {
     get(_: Id): Schema | undefined;
     fetch(_i: unknown, _c?: (error?: Error) => void): void | Promise<void>;
-    getSchemaId(_p: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, _t?: BeforeHookPayloadType): Id;
+    getSchemaId(_p: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, _t?: BeforeHookPayloadType): Id;
     getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
     getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
     getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;

package/dist/registries/abstract.js

@@ -1,19 +1,38 @@
 export function runAsyncSeries(operation, collection, index, callback) {
-    if (collection.length === 0 || index >= collection.length) {
+    const length = collection.length;
+    if (length === 0 || index >= length) {
         callback(null);
         return;
     }
-    operation(collection[index], error => {
-        if (error) {
-            callback(error);
+    while (index < length) {
+        let invocationReturned = false;
+        let cbCalled = false;
+        let cbError;
+        operation(collection[index], error => {
+            if (invocationReturned) {
+                // The callback was invoked asynchronously, so continuing recursively does not grow the current stack.
+                if (error) {
+                    callback(error);
+                    return;
+                }
+                runAsyncSeries(operation, collection, index + 1, callback);
+                return;
+            }
+            cbCalled = true;
+            cbError = error;
+        });
+        invocationReturned = true;
+        if (!cbCalled) {
+            // Wait for the async callback to continue the series.
             return;
         }
-        else if (index === collection.length - 1) {
-            callback(null);
+        if (cbError) {
+            callback(cbError);
             return;
         }
-        runAsyncSeries(operation, collection, index + 1, callback);
-    });
+        index++;
+    }
+    callback(null);
 }
 /* c8 ignore start */
 export class AbstractSchemaRegistry {

package/dist/registries/confluent-schema-registry.d.ts

@@ -1,16 +1,32 @@
-import { type ValidateFunction } from 'ajv';
+import { type ErrorObject, type Options, type ValidateFunction } from 'ajv';
 import { type Type } from 'avsc';
+import { type ConnectionOptions as TLSConnectionOptions } from 'node:tls';
 import { type Root } from 'protobufjs';
 import { type Callback } from '../apis/definitions.ts';
 import { type BeforeDeserializationHook, type BeforeHookPayloadType, type BeforeSerializationHook, type Deserializers, type Serializers } from '../clients/serde.ts';
-import { type CredentialProvider } from '../index.ts';
+import { type CredentialProvider, UserError } from '../index.ts';
 import { type MessageToConsume, type MessageToProduce } from '../protocol/records.ts';
 import { AbstractSchemaRegistry } from './abstract.ts';
 type ConfluentSchemaRegistryMessageToProduce = MessageToProduce<unknown, unknown, unknown, unknown>;
+export type UndiciDispatcher = NonNullable<RequestInit['dispatcher']>;
+export interface UndiciAgentOptions {
+    connect: TLSConnectionOptions;
+}
+export type UndiciAgentConstructor = new (options: UndiciAgentOptions) => UndiciDispatcher;
+export declare function createUndiciAgent(options: UndiciAgentOptions): UndiciDispatcher;
 export interface ConfluentSchemaRegistryMetadata {
-    schemas?: Record<BeforeHookPayloadType, number>;
+    schemas?: Partial<Record<BeforeHookPayloadType, number>>;
+}
+export interface ConfluentSchemaRegistryConsumedMetadata {
+    lengths?: {
+        key?: number;
+        value?: number;
+    };
 }
 export type ConfluentSchemaRegistryProtobufTypeMapper = (id: number, type: BeforeHookPayloadType, context: ConfluentSchemaRegistryMessageToProduce | MessageToConsume) => string;
+export type ConfluentSchemaRegistryHeaders = Record<string, string>;
+export type ConfluentSchemaRegistryHeadersProvider = () => ConfluentSchemaRegistryHeaders | Promise<ConfluentSchemaRegistryHeaders>;
+export type ConfluentSchemaRegistryRetryDelayGetter = (attempt: number, retries: number, error: Error) => number;
 export interface ConfluentSchemaRegistryOptions {
     url: string;
     auth?: {
@@ -18,19 +34,56 @@ export interface ConfluentSchemaRegistryOptions {
         password?: string | CredentialProvider;
         token?: string | CredentialProvider;
     };
+    headers?: ConfluentSchemaRegistryHeaders | ConfluentSchemaRegistryHeadersProvider;
+    timeout?: number;
+    retries?: number;
+    retryDelay?: number | ConfluentSchemaRegistryRetryDelayGetter;
+    tls?: TLSConnectionOptions;
+    ssl?: TLSConnectionOptions;
     protobufTypeMapper?: ConfluentSchemaRegistryProtobufTypeMapper;
     jsonValidateSend?: boolean;
+    jsonAjvOptions?: Options;
+}
+export declare const defaultConfluentSchemaRegistryOptions: {
+    timeout: number;
+    retries: number;
+    retryDelay: number;
+};
+export declare const retriableSchemaRegistryStatuses: number[];
+export type SchemaValidationPhase = 'serialization' | 'deserialization';
+export interface SchemaValidationErrorProperties {
+    schemaId: number;
+    schemaType: 'json';
+    phase: SchemaValidationPhase;
+    payloadType: BeforeHookPayloadType;
+    type: BeforeHookPayloadType;
+    data: unknown;
+    headers?: ReadonlyMap<unknown, unknown>;
+    validationErrors: ErrorObject[] | null | undefined;
+}
+export declare class SchemaValidationError extends UserError {
+    readonly schemaId: number;
+    readonly schemaType: 'json';
+    readonly phase: SchemaValidationPhase;
+    readonly payloadType: BeforeHookPayloadType;
+    readonly type: BeforeHookPayloadType;
+    readonly data: unknown;
+    readonly headers?: ReadonlyMap<unknown, unknown>;
+    readonly validationErrors: ErrorObject[] | null | undefined;
+    constructor(message: string, properties: SchemaValidationErrorProperties);
 }
 export interface Schema {
     id: number;
     type: 'avro' | 'protobuf' | 'json';
     schema: Type | Root | ValidateFunction;
 }
+export declare function trackConsumedSchemaId(message: MessageToConsume | undefined, type: BeforeHookPayloadType, schemaId: number): void;
+export declare function trackConsumedPayloadLength(message: MessageToConsume | undefined, type: 'key' | 'value', data: Buffer): void;
 export declare function defaultProtobufTypeMapper(_: number, type: BeforeHookPayloadType, context: ConfluentSchemaRegistryMessageToProduce | MessageToConsume): string;
 export declare class ConfluentSchemaRegistry<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends AbstractSchemaRegistry<number | undefined, Schema, Key, Value, HeaderKey, HeaderValue> {
     #private;
     constructor(options: ConfluentSchemaRegistryOptions);
-    getSchemaId(message: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): number | undefined;
+    getSchemaId(message: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): number | undefined;
     get(id: number): Schema | undefined;
     fetchSchema(id: number, callback: Callback<void>): Promise<void>;
     getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;