@platformatic/kafka 2.1.0 → 2.2.0

package/dist/protocol/records.d.ts

@@ -33,11 +33,14 @@ export interface MessageJSON<Key = unknown, Value = unknown, HeaderKey = unknown
     partition: number;
     timestamp: string;
     offset: string;
+    leaderEpoch: number;
     metadata: Record<string, unknown>;
 }
 export interface Message<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends Required<MessageBase<Key, Value>> {
     headers: Map<HeaderKey, HeaderValue>;
+    headerEntries: Array<[HeaderKey, HeaderValue]>;
     offset: bigint;
+    leaderEpoch: number;
     metadata: Record<string, unknown>;
     commit(callback?: (error?: Error) => void): void | Promise<void>;
     toJSON(): MessageJSON<Key, Value, HeaderKey, HeaderValue>;
@@ -71,6 +74,7 @@ export interface KafkaRecord {
 export interface MessageToConsume extends KafkaRecord {
     topic: string;
     partition: number;
+    metadata?: Record<string, unknown>;
 }
 export interface RecordsBatch {
     firstOffset: bigint;

package/dist/protocol/writer.js

@@ -1,6 +1,6 @@
 import { DynamicBuffer } from '@platformatic/dynamic-buffer';
 import { humanize } from "../utils.js";
-import { EMPTY_TAGGED_FIELDS_BUFFER, EMPTY_UUID } from "./definitions.js";
+import { EMPTY_TAGGED_FIELDS_BUFFER, EMPTY_UUID, UUID_SIZE } from "./definitions.js";
 const instanceIdentifier = Symbol('plt.kafka.writer.instanceIdentifier');
 export class Writer {
     context;
@@ -125,6 +125,12 @@ export class Writer {
             return this.append(EMPTY_UUID);
         }
         const buffer = Buffer.from(value.replaceAll('-', ''), 'hex');
+        // UUIDs are fixed width and carry no length prefix, so anything that is not exactly 16 bytes
+        // would silently desync the rest of the request. Buffer.from() stops at the first non hex
+        // character, so a topic name used as a topic id would append nothing at all here.
+        if (buffer.length !== UUID_SIZE) {
+            throw new RangeError(`Expected a UUID, got ${JSON.stringify(value)}.`);
+        }
         this.#buffer.append(buffer);
         return this;
     }
@@ -138,7 +144,9 @@ export class Writer {
         else {
             this.appendInt32(value.length);
         }
-        this.#buffer.append(value);
+        if (value.length > 0) {
+            this.#buffer.append(value);
+        }
         return this;
     }
     // Note that this does not follow the wire protocol specification and thus the length is not +1ed
@@ -147,7 +155,12 @@ export class Writer {
             return this.appendVarInt(-1);
         }
         this.appendVarInt(value.length);
-        this.#buffer.append(value);
+        // Appending an empty buffer corrupts DynamicBuffer's positional reads, so it must be skipped:
+        // see https://github.com/platformatic/dynamic-buffer/issues/12. Record keys and values are
+        // allowed to be empty without being null, so this is reachable.
+        if (value.length > 0) {
+            this.#buffer.append(value);
+        }
         return this;
     }
     appendArray(value, entryWriter, compact = true, appendTrailingTaggedFields = true) {
@@ -210,7 +223,7 @@ export class Writer {
         }
         return this;
     }
-    // TODO: Tagged fields are not supported yet
+    // This client emits only empty tagged-field sections; it does not expose tagged fields to callers.
     appendTaggedFields(_ = []) {
         return this.append(EMPTY_TAGGED_FIELDS_BUFFER);
     }

package/dist/registries/abstract.d.ts

@@ -4,7 +4,7 @@ import { type MessageToProduce } from '../protocol/records.ts';
 export interface SchemaRegistry<Id = unknown, Schema = unknown, Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> {
     get(id: Id): Schema | undefined;
     fetch(id: Id, callback?: (error?: Error) => void): void | Promise<void>;
-    getSchemaId(payload: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): Id;
+    getSchemaId(payload: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): Id;
     getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
     getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
     getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;
@@ -14,7 +14,7 @@ export declare function runAsyncSeries<V>(operation: (item: V, cb: Callback<void
 export declare class AbstractSchemaRegistry<Id = unknown, Schema = unknown, Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> implements SchemaRegistry<Id, Schema, Key, Value, HeaderKey, HeaderValue> {
     get(_: Id): Schema | undefined;
     fetch(_i: unknown, _c?: (error?: Error) => void): void | Promise<void>;
-    getSchemaId(_p: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, _t?: BeforeHookPayloadType): Id;
+    getSchemaId(_p: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, _t?: BeforeHookPayloadType): Id;
     getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
     getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
     getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;

package/dist/registries/abstract.js

@@ -1,19 +1,38 @@
 export function runAsyncSeries(operation, collection, index, callback) {
-    if (collection.length === 0 || index >= collection.length) {
+    const length = collection.length;
+    if (length === 0 || index >= length) {
         callback(null);
         return;
     }
-    operation(collection[index], error => {
-        if (error) {
-            callback(error);
+    while (index < length) {
+        let invocationReturned = false;
+        let cbCalled = false;
+        let cbError;
+        operation(collection[index], error => {
+            if (invocationReturned) {
+                // The callback was invoked asynchronously, so continuing recursively does not grow the current stack.
+                if (error) {
+                    callback(error);
+                    return;
+                }
+                runAsyncSeries(operation, collection, index + 1, callback);
+                return;
+            }
+            cbCalled = true;
+            cbError = error;
+        });
+        invocationReturned = true;
+        if (!cbCalled) {
+            // Wait for the async callback to continue the series.
             return;
         }
-        else if (index === collection.length - 1) {
-            callback(null);
+        if (cbError) {
+            callback(cbError);
             return;
         }
-        runAsyncSeries(operation, collection, index + 1, callback);
-    });
+        index++;
+    }
+    callback(null);
 }
 /* c8 ignore start */
 export class AbstractSchemaRegistry {

package/dist/registries/confluent-schema-registry.d.ts

@@ -1,16 +1,32 @@
-import { type ValidateFunction } from 'ajv';
+import { type ErrorObject, type Options, type ValidateFunction } from 'ajv';
 import { type Type } from 'avsc';
+import { type ConnectionOptions as TLSConnectionOptions } from 'node:tls';
 import { type Root } from 'protobufjs';
 import { type Callback } from '../apis/definitions.ts';
 import { type BeforeDeserializationHook, type BeforeHookPayloadType, type BeforeSerializationHook, type Deserializers, type Serializers } from '../clients/serde.ts';
-import { type CredentialProvider } from '../index.ts';
+import { type CredentialProvider, UserError } from '../index.ts';
 import { type MessageToConsume, type MessageToProduce } from '../protocol/records.ts';
 import { AbstractSchemaRegistry } from './abstract.ts';
 type ConfluentSchemaRegistryMessageToProduce = MessageToProduce<unknown, unknown, unknown, unknown>;
+export type UndiciDispatcher = NonNullable<RequestInit['dispatcher']>;
+export interface UndiciAgentOptions {
+    connect: TLSConnectionOptions;
+}
+export type UndiciAgentConstructor = new (options: UndiciAgentOptions) => UndiciDispatcher;
+export declare function createUndiciAgent(options: UndiciAgentOptions): UndiciDispatcher;
 export interface ConfluentSchemaRegistryMetadata {
-    schemas?: Record<BeforeHookPayloadType, number>;
+    schemas?: Partial<Record<BeforeHookPayloadType, number>>;
+}
+export interface ConfluentSchemaRegistryConsumedMetadata {
+    lengths?: {
+        key?: number;
+        value?: number;
+    };
 }
 export type ConfluentSchemaRegistryProtobufTypeMapper = (id: number, type: BeforeHookPayloadType, context: ConfluentSchemaRegistryMessageToProduce | MessageToConsume) => string;
+export type ConfluentSchemaRegistryHeaders = Record<string, string>;
+export type ConfluentSchemaRegistryHeadersProvider = () => ConfluentSchemaRegistryHeaders | Promise<ConfluentSchemaRegistryHeaders>;
+export type ConfluentSchemaRegistryRetryDelayGetter = (attempt: number, retries: number, error: Error) => number;
 export interface ConfluentSchemaRegistryOptions {
     url: string;
     auth?: {
@@ -18,19 +34,56 @@ export interface ConfluentSchemaRegistryOptions {
         password?: string | CredentialProvider;
         token?: string | CredentialProvider;
     };
+    headers?: ConfluentSchemaRegistryHeaders | ConfluentSchemaRegistryHeadersProvider;
+    timeout?: number;
+    retries?: number;
+    retryDelay?: number | ConfluentSchemaRegistryRetryDelayGetter;
+    tls?: TLSConnectionOptions;
+    ssl?: TLSConnectionOptions;
     protobufTypeMapper?: ConfluentSchemaRegistryProtobufTypeMapper;
     jsonValidateSend?: boolean;
+    jsonAjvOptions?: Options;
+}
+export declare const defaultConfluentSchemaRegistryOptions: {
+    timeout: number;
+    retries: number;
+    retryDelay: number;
+};
+export declare const retriableSchemaRegistryStatuses: number[];
+export type SchemaValidationPhase = 'serialization' | 'deserialization';
+export interface SchemaValidationErrorProperties {
+    schemaId: number;
+    schemaType: 'json';
+    phase: SchemaValidationPhase;
+    payloadType: BeforeHookPayloadType;
+    type: BeforeHookPayloadType;
+    data: unknown;
+    headers?: ReadonlyMap<unknown, unknown>;
+    validationErrors: ErrorObject[] | null | undefined;
+}
+export declare class SchemaValidationError extends UserError {
+    readonly schemaId: number;
+    readonly schemaType: 'json';
+    readonly phase: SchemaValidationPhase;
+    readonly payloadType: BeforeHookPayloadType;
+    readonly type: BeforeHookPayloadType;
+    readonly data: unknown;
+    readonly headers?: ReadonlyMap<unknown, unknown>;
+    readonly validationErrors: ErrorObject[] | null | undefined;
+    constructor(message: string, properties: SchemaValidationErrorProperties);
 }
 export interface Schema {
     id: number;
     type: 'avro' | 'protobuf' | 'json';
     schema: Type | Root | ValidateFunction;
 }
+export declare function trackConsumedSchemaId(message: MessageToConsume | undefined, type: BeforeHookPayloadType, schemaId: number): void;
+export declare function trackConsumedPayloadLength(message: MessageToConsume | undefined, type: 'key' | 'value', data: Buffer): void;
 export declare function defaultProtobufTypeMapper(_: number, type: BeforeHookPayloadType, context: ConfluentSchemaRegistryMessageToProduce | MessageToConsume): string;
 export declare class ConfluentSchemaRegistry<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends AbstractSchemaRegistry<number | undefined, Schema, Key, Value, HeaderKey, HeaderValue> {
     #private;
     constructor(options: ConfluentSchemaRegistryOptions);
-    getSchemaId(message: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): number | undefined;
+    getSchemaId(message: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): number | undefined;
     get(id: number): Schema | undefined;
     fetchSchema(id: number, callback: Callback<void>): Promise<void>;
     getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;

package/dist/registries/confluent-schema-registry.js

@@ -1,32 +1,124 @@
+import * as AjvDraft04Module from 'ajv-draft-04';
+import draft06MetaSchema from 'ajv/dist/refs/json-schema-draft-06.json' with { type: 'json' };
+import { Ajv } from 'ajv';
+import { Ajv2020 } from 'ajv/dist/2020.js';
 import avro from 'avsc';
 import { createRequire } from 'node:module';
+import { setTimeout as sleep } from 'node:timers/promises';
 import { jsonDeserializer, jsonSerializer, stringDeserializer, stringSerializer } from "../clients/serde.js";
-import { ajv, EMPTY_BUFFER, UnsupportedFormatError, UserError } from "../index.js";
+import { EMPTY_BUFFER, MultipleErrors, NetworkError, TimeoutError, UnsupportedFormatError, UserError } from "../index.js";
 import { getCredential } from "../protocol/sasl/utils.js";
 import { AbstractSchemaRegistry } from "./abstract.js";
 const require = createRequire(import.meta.url);
+const AjvDraft04 = AjvDraft04Module.default;
+const undiciGlobalDispatcherSymbol = /^undici\.globalDispatcher\.(\d+)$/;
+export function createUndiciAgent(options) {
+    /*
+      The undici globals are installed the first time undici is loaded, which creating a Request is
+      enough to do and which performs no I/O.
+    */
+    new Request('http://localhost'); // eslint-disable-line no-new
+    /*
+      The symbol is versioned and more than one version is registered at the same time, pointing at
+      different objects: on Node 26 undici.globalDispatcher.1 is a backwards compatibility wrapper and
+      only undici.globalDispatcher.2 is the Agent. The highest version available therefore wins.
+    */
+    const globals = globalThis;
+    let version = -1;
+    let Agent;
+    for (const symbol of Object.getOwnPropertySymbols(globalThis)) {
+        const symbolVersion = symbol.description?.match(undiciGlobalDispatcherSymbol)?.[1];
+        if (typeof symbolVersion === 'undefined' || Number(symbolVersion) <= version) {
+            continue;
+        }
+        version = Number(symbolVersion);
+        Agent = globals[symbol]?.constructor;
+    }
+    /* c8 ignore next 6 - Only reachable on a Node.js build which does not bundle undici */
+    if (typeof Agent !== 'function') {
+        throw new UserError('Cannot access the undici Agent bundled with Node.js, which is required to configure TLS for the Schema Registry.');
+    }
+    return new Agent(options);
+}
+export const defaultConfluentSchemaRegistryOptions = {
+    timeout: 5000,
+    retries: 3,
+    retryDelay: 1000
+};
+// Statuses which are worth retrying: the registry is overloaded, restarting or behind a proxy which is
+export const retriableSchemaRegistryStatuses = [408, 425, 429, 500, 502, 503, 504];
+export class SchemaValidationError extends UserError {
+    constructor(message, properties) {
+        super(message, properties);
+        this.name = 'SchemaValidationError';
+    }
+}
+export function trackConsumedSchemaId(message, type, schemaId) {
+    /* c8 ignore next 3 - The message is always provided for keys and values */
+    if (!message) {
+        return;
+    }
+    const metadata = (message.metadata ??= {});
+    metadata.schemas ??= {};
+    metadata.schemas[type] = schemaId;
+}
+export function trackConsumedPayloadLength(message, type, data) {
+    /* c8 ignore next 3 - The message is always provided for keys and values */
+    if (!message) {
+        return;
+    }
+    const metadata = (message.metadata ??= {});
+    metadata.lengths ??= {};
+    metadata.lengths[type] = data.length;
+}
 /* c8 ignore next 8 */
 export function defaultProtobufTypeMapper(_, type, context) {
     // Confluent Schema Registry convention
     return `${context.topic}-${type}`;
 }
-// TODO(ShogunPanda): Authentication support
 export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
     #url;
     #schemas;
     #protobufParse;
     #protobufTypeMapper;
     #jsonValidateSend;
+    #jsonAjv;
+    #jsonAjvDraft7;
+    #jsonAjvDraft04;
+    #pendingFetches;
     #auth;
+    #headers;
+    #timeout;
+    #retries;
+    #retryDelay;
+    #tls;
+    #tlsDispatcher;
     constructor(options) {
         super();
         this.#url = options.url;
         this.#schemas = new Map();
         this.#protobufTypeMapper = options.protobufTypeMapper ?? defaultProtobufTypeMapper;
         this.#jsonValidateSend = options.jsonValidateSend ?? false;
+        const jsonAjvOptions = { allErrors: true, coerceTypes: false, strict: true, ...options.jsonAjvOptions };
+        this.#jsonAjv = new Ajv2020(jsonAjvOptions);
+        this.#jsonAjvDraft7 = new Ajv(jsonAjvOptions);
+        this.#jsonAjvDraft7.addMetaSchema(draft06MetaSchema);
+        this.#jsonAjvDraft04 = new AjvDraft04(jsonAjvOptions);
         this.#auth = options.auth;
+        this.#headers = options.headers;
+        this.#timeout = options.timeout ?? defaultConfluentSchemaRegistryOptions.timeout;
+        this.#retries = options.retries ?? defaultConfluentSchemaRegistryOptions.retries;
+        this.#retryDelay = options.retryDelay ?? defaultConfluentSchemaRegistryOptions.retryDelay;
+        this.#tls = options.tls ?? options.ssl;
+        this.#pendingFetches = new Map();
+        if (this.#tls && new URL(this.#url).protocol !== 'https:') {
+            throw new UserError('TLS options can only be used with a HTTPS Schema Registry URL.', { url: this.#url });
+        }
     }
     getSchemaId(message, type) {
+        if (message === null) {
+            return undefined;
+        }
         if (Buffer.isBuffer(message)) {
             if (type !== 'value') {
                 return undefined;
@@ -39,42 +131,15 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
         return this.#schemas.get(id);
     }
     async fetchSchema(id, callback) {
+        let fetch = this.#pendingFetches.get(id);
+        if (!fetch) {
+            fetch = this.#fetchSchema(id).finally(() => {
+                this.#pendingFetches.delete(id);
+            });
+            this.#pendingFetches.set(id, fetch);
+        }
         try {
-            const requestInit = {};
-            if (this.#auth) {
-                if (this.#auth.token) {
-                    const token = await getCredential('token', this.#auth.token);
-                    requestInit.headers = {
-                        Authorization: `Bearer ${token}`
-                    };
-                }
-                else {
-                    const username = await getCredential('username', this.#auth.username);
-                    const password = await getCredential('password', this.#auth.password);
-                    requestInit.headers = {
-                        Authorization: `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`
-                    };
-                }
-            }
-            const response = await fetch(`${this.#url}/schemas/ids/${id}`, requestInit);
-            if (!response.ok) {
-                throw new UserError(`Failed to fetch a schema: [HTTP ${response.status}]`, { response: await response.text() });
-            }
-            const responseBody = (await response.json());
-            const { schema } = responseBody;
-            const schemaType = responseBody.schemaType ?? 'AVRO';
-            switch (schemaType) {
-                case 'AVRO':
-                    this.#schemas.set(id, { id, type: 'avro', schema: avro.Type.forSchema(JSON.parse(schema)) });
-                    break;
-                case 'PROTOBUF':
-                    this.#protobufParse ??= this.#loadProtobuf();
-                    this.#schemas.set(id, { id, type: 'protobuf', schema: this.#protobufParse(schema).root });
-                    break;
-                case 'JSON':
-                    this.#schemas.set(id, { id, type: 'json', schema: ajv.compile(JSON.parse(schema)) });
-                    break;
-            }
+            await fetch;
             process.nextTick(callback);
         }
         catch (err) {
@@ -117,7 +182,7 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
     }
     getBeforeDeserializationHook() {
         const registry = this;
-        return function beforeDeserialization(payload, type, _message, callback) {
+        return function beforeDeserialization(payload, type, message, callback) {
             // Extract the schema ID from the message metadata
             const schemaId = registry.getSchemaId(payload, type);
             // When no schema ID is found, nothing to do
@@ -125,6 +190,11 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
                 callback(null);
                 return;
             }
+            /*
+              Track the schema ID before fetching, so that it is exposed on the message metadata even when
+              the schema cannot be fetched or the payload cannot be decoded.
+            */
+            trackConsumedSchemaId(message, type, schemaId);
             // The schema is already fetch
             if (registry.get(schemaId)) {
                 callback(null);
@@ -133,6 +203,115 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
             registry.fetchSchema(schemaId, callback);
         };
     }
+    async #requestHeaders() {
+        let headers;
+        if (this.#headers) {
+            headers = { ...(typeof this.#headers === 'function' ? await this.#headers() : this.#headers) };
+        }
+        // Authentication headers are applied last so that auth options always win over custom headers
+        if (this.#auth) {
+            headers ??= {};
+            if (this.#auth.token) {
+                const token = await getCredential('token', this.#auth.token);
+                headers.Authorization = `Bearer ${token}`;
+            }
+            else {
+                const username = await getCredential('username', this.#auth.username);
+                const password = await getCredential('password', this.#auth.password);
+                headers.Authorization = `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`;
+            }
+        }
+        return headers;
+    }
+    async #requestSchema(id) {
+        const requestInit = {};
+        const headers = await this.#requestHeaders();
+        if (headers) {
+            requestInit.headers = headers;
+        }
+        if (this.#tls) {
+            // Created once per registry, so that connections are pooled across schema fetches
+            this.#tlsDispatcher ??= createUndiciAgent({ connect: this.#tls });
+            requestInit.dispatcher = this.#tlsDispatcher;
+        }
+        if (this.#timeout > 0) {
+            requestInit.signal = AbortSignal.timeout(this.#timeout);
+        }
+        let response;
+        try {
+            response = await fetch(`${this.#url}/schemas/ids/${id}`, requestInit);
+        }
+        catch (error) {
+            if (error.name === 'TimeoutError') {
+                throw new TimeoutError(`Fetching a schema timed out after ${this.#timeout} ms.`, {
+                    canRetry: true,
+                    cause: error
+                });
+            }
+            throw new NetworkError('Failed to fetch a schema.', { cause: error });
+        }
+        if (!response.ok) {
+            throw new UserError(`Failed to fetch a schema: [HTTP ${response.status}]`, {
+                response: await response.text(),
+                canRetry: retriableSchemaRegistryStatuses.includes(response.status)
+            });
+        }
+        return (await response.json());
+    }
+    async #requestSchemaWithRetries(id) {
+        const errors = [];
+        for (let attempt = 0;; attempt++) {
+            try {
+                return await this.#requestSchema(id);
+            }
+            catch (error) {
+                errors.push(error);
+                if (attempt >= this.#retries || error.canRetry !== true) {
+                    if (errors.length === 1) {
+                        throw errors[0];
+                    }
+                    throw new MultipleErrors(`Failed to fetch a schema after ${errors.length} attempts.`, errors);
+                }
+                const delay = typeof this.#retryDelay === 'function'
+                    ? this.#retryDelay(attempt + 1, this.#retries, error)
+                    : this.#retryDelay;
+                if (delay > 0) {
+                    await sleep(delay);
+                }
+            }
+        }
+    }
+    async #fetchSchema(id) {
+        const responseBody = await this.#requestSchemaWithRetries(id);
+        const { schema } = responseBody;
+        const schemaType = responseBody.schemaType ?? 'AVRO';
+        switch (schemaType) {
+            case 'AVRO':
+                this.#schemas.set(id, { id, type: 'avro', schema: avro.Type.forSchema(JSON.parse(schema)) });
+                break;
+            case 'PROTOBUF':
+                this.#protobufParse ??= this.#loadProtobuf();
+                this.#schemas.set(id, { id, type: 'protobuf', schema: this.#protobufParse(schema).root });
+                break;
+            case 'JSON': {
+                const jsonSchema = JSON.parse(schema);
+                this.#schemas.set(id, { id, type: 'json', schema: this.#getJsonAjv(jsonSchema).compile(jsonSchema) });
+                break;
+            }
+        }
+    }
+    #getJsonAjv(schema) {
+        if (typeof schema !== 'object' || schema === null || typeof schema.$schema !== 'string') {
+            return this.#jsonAjv;
+        }
+        if (schema.$schema.includes('draft-04')) {
+            return this.#jsonAjvDraft04;
+        }
+        if (schema.$schema.includes('draft-06') || schema.$schema.includes('draft-07')) {
+            return this.#jsonAjvDraft7;
+        }
+        return this.#jsonAjv;
+    }
     #schemaSerializer(type, fallbackSerializer, data, headers, message) {
         /* c8 ignore next 3 - Hard to test */
         if (typeof data === 'undefined') {
@@ -166,7 +345,16 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
                     const validate = schema.schema;
                     const valid = validate(data);
                     if (!valid) {
-                        throw new UserError(`JSON Schema validation failed before serialization: ${ajv.errorsText(validate.errors)}`, { type, data, headers, validationErrors: validate.errors });
+                        throw new SchemaValidationError(`JSON Schema validation failed before serialization: ${this.#jsonAjv.errorsText(validate.errors)}`, {
+                            schemaId,
+                            schemaType: 'json',
+                            phase: 'serialization',
+                            payloadType: type,
+                            type,
+                            data,
+                            headers,
+                            validationErrors: validate.errors
+                        });
                     }
                 }
                 encodedMessage = Buffer.from(JSON.stringify(data));
@@ -177,17 +365,32 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
         return Buffer.concat([header, encodedMessage]);
     }
     #schemaDeserializer(type, fallbackDeserializer, data, headers, message) {
+        /*
+          Deserializing replaces the original payload, so record its size beforehand: it is the only
+          chance callers have to account for the actual ingress size of the message.
+        */
+        if ((type === 'key' || type === 'value') && Buffer.isBuffer(data)) {
+            trackConsumedPayloadLength(message, type, data);
+        }
         /* c8 ignore next 3 - Hard to test */
         if (typeof data === 'undefined' || data.length === 0) {
             return EMPTY_BUFFER;
         }
-        if (type === 'headerKey' || type === 'headerValue') {
+        const isHeader = type === 'headerKey' || type === 'headerValue';
+        if (isHeader) {
             message = headers;
         }
         const schemaId = this.getSchemaId(data, type);
         if (!schemaId) {
             return fallbackDeserializer(data);
         }
+        /*
+          Also track the schema ID here, so that it is exposed on the message metadata even when the
+          deserializers are used without the before-deserialization hook of this registry.
+        */
+        if (!isHeader) {
+            trackConsumedSchemaId(message, type, schemaId);
+        }
         const schema = this.#schemas.get(schemaId);
         if (!schema) {
             throw new UserError(`Schema with ID ${schemaId} not found.`, { missingSchema: schemaId });
@@ -205,7 +408,16 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
                 const validate = schema.schema;
                 const valid = validate(parsed);
                 if (!valid) {
-                    throw new UserError(`JSON Schema validation failed before deserialization: ${ajv.errorsText(validate.errors)}`, { type, data: parsed, headers, validationErrors: validate.errors });
+                    throw new SchemaValidationError(`JSON Schema validation failed before deserialization: ${this.#jsonAjv.errorsText(validate.errors)}`, {
+                        schemaId,
+                        schemaType: 'json',
+                        phase: 'deserialization',
+                        payloadType: type,
+                        type,
+                        data: parsed,
+                        headers,
+                        validationErrors: validate.errors
+                    });
                 }
                 return parsed;
             }

package/dist/symbols.d.ts

@@ -2,6 +2,7 @@ export declare const kInstance: unique symbol;
 export declare const kRefreshOffsetsAndFetch: unique symbol;
 export declare const kAutocommit: unique symbol;
 export declare const kGetFetchNode: unique symbol;
+export declare const kDeserializationError: unique symbol;
 export declare const kTransaction: unique symbol;
 export declare const kTransactionPrepare: unique symbol;
 export declare const kTransactionAddPartitions: unique symbol;

package/dist/symbols.js

@@ -3,6 +3,7 @@ export const kInstance = Symbol('plt.kafka.base.instance');
 export const kRefreshOffsetsAndFetch = Symbol('plt.kafka.messagesStream.refreshOffsetsAndFetch');
 export const kAutocommit = Symbol('plt.kafka.messagesStream.autocommit');
 export const kGetFetchNode = Symbol('plt.kafka.consumer.getFetchNode');
+export const kDeserializationError = Symbol('plt.kafka.messagesStream.deserializationError');
 export const kTransaction = Symbol('plt.kafka.producer.transactions');
 export const kTransactionPrepare = Symbol('plt.kafka.producer.transactions.prepare');
 export const kTransactionAddPartitions = Symbol('plt.kafka.producer.transactions.addPartitions');