@platformatic/kafka 1.27.0-alpha.1 → 1.27.0-alpha.2

package/dist/protocol/compression.js

@@ -1,7 +1,8 @@
+import { createRequire } from 'node:module';
 import zlib from 'node:zlib';
 import { UnsupportedCompressionError } from "../errors.js";
 import { DynamicBuffer } from "./dynamic-buffer.js";
-import { lz4Compress, lz4Decompress, snappyCompress, snappyDecompress } from "./native.js";
+const require = createRequire(import.meta.url);
 const { zstdCompressSync, zstdDecompressSync, gzipSync, gunzipSync } = zlib;
 export const CompressionAlgorithms = {
     NONE: 'none',
@@ -12,12 +13,34 @@ export const CompressionAlgorithms = {
 };
 export const allowedCompressionsAlgorithms = Object.values(CompressionAlgorithms);
 function ensureBuffer(data) {
-    return DynamicBuffer.isDynamicBuffer(data) ? data.buffer : data;
+    return DynamicBuffer.isDynamicBuffer(data) ? data.slice() : data;
+}
+let snappyCompressSync;
+let snappyDecompressSync;
+let lz4CompressFrameSync;
+let lz4DecompressFrameSync;
+function loadSnappy() {
+    try {
+        const snappy = require('snappy');
+        snappyCompressSync = snappy.compressSync;
+        snappyDecompressSync = snappy.uncompressSync;
+        /* c8 ignore next 5 - In tests snappy is always available */
+    }
+    catch (e) {
+        throw new UnsupportedCompressionError('Cannot load snappy module, which is an optionalDependency. Please check your local installation.');
+    }
+}
+function loadLZ4() {
+    try {
+        const lz4 = require('lz4-napi');
+        lz4CompressFrameSync = lz4.compressFrameSync;
+        lz4DecompressFrameSync = lz4.decompressFrameSync;
+        /* c8 ignore next 5 - In tests lz4-napi is always available */
+    }
+    catch (e) {
+        throw new UnsupportedCompressionError('Cannot load lz4-napi module, which is an optionalDependency. Please check your local installation.');
+    }
 }
-const snappyCompressSync = snappyCompress;
-const snappyDecompressSync = snappyDecompress;
-const lz4CompressFrameSync = lz4Compress;
-const lz4DecompressFrameSync = lz4Decompress;
 export const compressionsAlgorithms = {
     /* c8 ignore next 8 - 'none' is actually never used but this is to please Typescript */
     none: {
@@ -42,9 +65,17 @@ export const compressionsAlgorithms = {
     },
     snappy: {
         compressSync(data) {
+            /* c8 ignore next 4 - In tests snappy is always available */
+            if (!snappyCompressSync) {
+                loadSnappy();
+            }
             return snappyCompressSync(ensureBuffer(data));
         },
         decompressSync(data) {
+            /* c8 ignore next 4 - In tests snappy is always available */
+            if (!snappyDecompressSync) {
+                loadSnappy();
+            }
             return snappyDecompressSync(ensureBuffer(data));
         },
         bitmask: 2,
@@ -52,9 +83,17 @@ export const compressionsAlgorithms = {
     },
     lz4: {
         compressSync(data) {
+            /* c8 ignore next 4 - In tests lz4-napi is always available */
+            if (!lz4CompressFrameSync) {
+                loadLZ4();
+            }
             return lz4CompressFrameSync(ensureBuffer(data));
         },
         decompressSync(data) {
+            /* c8 ignore next 4 - In tests lz4-napi is always available */
+            if (!lz4DecompressFrameSync) {
+                loadLZ4();
+            }
             return lz4DecompressFrameSync(ensureBuffer(data));
         },
         bitmask: 3,

package/dist/protocol/crc32c.js

@@ -1,7 +1,6 @@
 // Based on the work from: https://github.com/tulios/kafkajs/blob/master/src/protocol/recordBatch/crc32C/crc32C.js
 import { createRequire } from 'node:module';
 import { DynamicBuffer } from "./dynamic-buffer.js";
-import { crc32c as wasmCRC32C } from "./native.js";
 /* prettier-ignore */
 const CRC = [
     0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4,
@@ -86,7 +85,9 @@ export function loadNativeCRC32C() {
     }
 }
 export function jsCRC32C(data) {
-    const bytes = DynamicBuffer.isDynamicBuffer(data) ? data.buffer : data;
+    const bytes = DynamicBuffer.isDynamicBuffer(data)
+        ? data.buffer
+        : new Uint8Array(data);
     let crc = 0xffffffff;
     for (let i = 0, len = bytes.length; i < len; ++i) {
         crc = CRC[(crc ^ bytes[i]) & 0xff] ^ (crc >>> 8);
@@ -94,4 +95,4 @@ export function jsCRC32C(data) {
     return (crc ^ 0xffffffff) >>> 0;
 }
 /* c8 ignore next - Hard to test */
-export const crc32c = loadNativeCRC32C() ?? wasmCRC32C;
+export const crc32c = loadNativeCRC32C() ?? jsCRC32C;

package/dist/protocol/definitions.js

@@ -7,5 +7,5 @@ export const EMPTY_BUFFER = Buffer.alloc(0);
 export const EMPTY_UUID = Buffer.alloc(UUID_SIZE);
 // Since it is serialized at either 0 (for nullable) or 1 (since length is stored as length + 1), it always uses a single byte
 export const EMPTY_OR_SINGLE_COMPACT_LENGTH_SIZE = INT8_SIZE;
-// TODO(ShogunPanda): Tagged fields are not supported yet
+// TODO: Tagged fields are not supported yet
 export const EMPTY_TAGGED_FIELDS_BUFFER = Buffer.from([0]);

package/dist/protocol/reader.js

@@ -300,7 +300,7 @@ export class Reader {
         }
         return reader();
     }
-    // TODO(ShogunPanda): Tagged fields are not supported yet
+    // TODO: Tagged fields are not supported yet
     readTaggedFields() {
         const length = this.readVarInt();
         if (length > 0) {

package/dist/protocol/records.d.ts

@@ -17,6 +17,7 @@ export interface MessageBase<Key = Buffer, Value = Buffer> {
 }
 export interface MessageToProduce<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends MessageBase<Key, Value> {
     headers?: Map<HeaderKey, HeaderValue> | Record<string, HeaderValue>;
+    metadata?: unknown;
 }
 export interface MessageConsumerMetadata {
     coordinatorId: number;
@@ -67,6 +68,10 @@ export interface KafkaRecord {
     value: Buffer;
     headers: [Buffer, Buffer][];
 }
+export interface MessageToConsume extends KafkaRecord {
+    topic: string;
+    partition: number;
+}
 export interface RecordsBatch {
     firstOffset: bigint;
     length: number;
@@ -85,24 +90,8 @@ export interface RecordsBatch {
 export declare const messageSchema: {
     type: string;
     properties: {
-        key: {
-            oneOf: ({
-                type: string;
-                buffer?: undefined;
-            } | {
-                buffer: boolean;
-                type?: undefined;
-            })[];
-        };
-        value: {
-            oneOf: ({
-                type: string;
-                buffer?: undefined;
-            } | {
-                buffer: boolean;
-                type?: undefined;
-            })[];
-        };
+        key: boolean;
+        value: boolean;
         headers: {
             anyOf: ({
                 map: boolean;

package/dist/protocol/records.js

@@ -14,12 +14,8 @@ export const BATCH_HEAD = INT64_SIZE + INT32_SIZE; // FirstOffset + Length
 export const messageSchema = {
     type: 'object',
     properties: {
-        key: {
-            oneOf: [{ type: 'string' }, { buffer: true }]
-        },
-        value: {
-            oneOf: [{ type: 'string' }, { buffer: true }]
-        },
+        key: true,
+        value: true,
         headers: {
             // Note: we can't use oneOf here since a Map is also a 'object'. Thanks JS.
             anyOf: [

package/dist/protocol/sasl/oauth-bearer.d.ts

@@ -1,6 +1,6 @@
 import { type CallbackWithPromise } from '../../apis/callbacks.ts';
 import { type SASLAuthenticationAPI, type SaslAuthenticateResponse } from '../../apis/security/sasl-authenticate-v2.ts';
-import { type Connection, type SASLCredentialProvider } from '../../network/connection.ts';
+import { type Connection, type CredentialProvider } from '../../network/connection.ts';
 export declare function jwtValidateAuthenticationBytes(authBytes: Buffer, callback: CallbackWithPromise<Buffer>): void;
-export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, tokenOrProvider: string | SASLCredentialProvider, extensions: Record<string, string> | SASLCredentialProvider<Record<string, string>>, callback: CallbackWithPromise<SaslAuthenticateResponse>): void;
-export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, tokenOrProvider: string | SASLCredentialProvider, extensions: Record<string, string> | SASLCredentialProvider<Record<string, string>>): Promise<SaslAuthenticateResponse>;
+export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, tokenOrProvider: string | CredentialProvider, extensions: Record<string, string> | CredentialProvider<Record<string, string>>, callback: CallbackWithPromise<SaslAuthenticateResponse>): void;
+export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, tokenOrProvider: string | CredentialProvider, extensions: Record<string, string> | CredentialProvider<Record<string, string>>): Promise<SaslAuthenticateResponse>;

package/dist/protocol/sasl/plain.d.ts

@@ -1,5 +1,5 @@
 import { type CallbackWithPromise } from '../../apis/callbacks.ts';
 import { type SASLAuthenticationAPI, type SaslAuthenticateResponse } from '../../apis/security/sasl-authenticate-v2.ts';
-import { type Connection, type SASLCredentialProvider } from '../../network/connection.ts';
-export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, usernameProvider: string | SASLCredentialProvider, passwordProvider: string | SASLCredentialProvider, callback: CallbackWithPromise<SaslAuthenticateResponse>): void;
-export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, usernameProvider: string | SASLCredentialProvider, passwordProvider: string | SASLCredentialProvider): Promise<SaslAuthenticateResponse>;
+import { type Connection, type CredentialProvider } from '../../network/connection.ts';
+export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, usernameProvider: string | CredentialProvider, passwordProvider: string | CredentialProvider, callback: CallbackWithPromise<SaslAuthenticateResponse>): void;
+export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, usernameProvider: string | CredentialProvider, passwordProvider: string | CredentialProvider): Promise<SaslAuthenticateResponse>;

package/dist/protocol/sasl/scram-sha.d.ts

@@ -1,6 +1,6 @@
 import { type CallbackWithPromise } from '../../apis/callbacks.ts';
 import { type SASLAuthenticationAPI, type SaslAuthenticateResponse } from '../../apis/security/sasl-authenticate-v2.ts';
-import { type Connection, type SASLCredentialProvider } from '../../network/connection.ts';
+import { type Connection, type CredentialProvider } from '../../network/connection.ts';
 export interface ScramAlgorithmDefinition {
     keyLength: number;
     algorithm: string;
@@ -35,5 +35,5 @@ export declare function hi(definition: ScramAlgorithmDefinition, password: strin
 export declare function hmac(definition: ScramAlgorithmDefinition, key: Buffer, data: string | Buffer): Buffer;
 export declare function xor(a: Buffer, b: Buffer): Buffer;
 export declare const defaultCrypto: ScramCryptoModule;
-export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, algorithm: ScramAlgorithm, usernameProvider: string | SASLCredentialProvider, passwordProvider: string | SASLCredentialProvider, crypto: ScramCryptoModule, callback: CallbackWithPromise<SaslAuthenticateResponse>): void;
-export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, algorithm: ScramAlgorithm, usernameProvider: string | SASLCredentialProvider, passwordProvider: string | SASLCredentialProvider, crypto?: ScramCryptoModule): Promise<SaslAuthenticateResponse>;
+export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, algorithm: ScramAlgorithm, usernameProvider: string | CredentialProvider, passwordProvider: string | CredentialProvider, crypto: ScramCryptoModule, callback: CallbackWithPromise<SaslAuthenticateResponse>): void;
+export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, algorithm: ScramAlgorithm, usernameProvider: string | CredentialProvider, passwordProvider: string | CredentialProvider, crypto?: ScramCryptoModule): Promise<SaslAuthenticateResponse>;

package/dist/protocol/sasl/utils.d.ts

@@ -1,4 +1,4 @@
 import { type CallbackWithPromise } from '../../apis/index.ts';
-import { type SASLCredentialProvider } from '../../network/connection.ts';
-export declare function getCredential<T>(label: string, credentialOrProvider: T | SASLCredentialProvider<T>, callback: CallbackWithPromise<T>): void;
-export declare function getCredential<T>(label: string, credentialOrProvider: T | SASLCredentialProvider<T>): Promise<T>;
+import { type CredentialProvider } from '../../network/connection.ts';
+export declare function getCredential<T>(label: string, credentialOrProvider: T | CredentialProvider<T>, callback: CallbackWithPromise<T>): void;
+export declare function getCredential<T>(label: string, credentialOrProvider: T | CredentialProvider<T>): Promise<T>;

package/dist/protocol/writer.js

@@ -210,7 +210,7 @@ export class Writer {
         }
         return this;
     }
-    // TODO(ShogunPanda): Tagged fields are not supported yet
+    // TODO: Tagged fields are not supported yet
     appendTaggedFields(_ = []) {
         return this.append(EMPTY_TAGGED_FIELDS_BUFFER);
     }

package/dist/registries/abstract.d.ts

@@ -0,0 +1,22 @@
+import { type Callback } from '../apis/definitions.ts';
+import { type BeforeDeserializationHook, type BeforeHookPayloadType, type BeforeSerializationHook, type Deserializers, type Serializers } from '../clients/serde.ts';
+import { type MessageToProduce } from '../protocol/records.ts';
+export interface SchemaRegistry<Id = unknown, Schema = unknown, Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> {
+    get(id: Id): Schema | undefined;
+    fetch(id: Id, callback?: (error?: Error) => void): void | Promise<void>;
+    getSchemaId(payload: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): Id;
+    getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
+    getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
+    getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;
+    getBeforeDeserializationHook(): BeforeDeserializationHook;
+}
+export declare function runAsyncSeries<V>(operation: (item: V, cb: Callback<void>) => void | Promise<void>, collection: V[], index: number, callback: Callback<void>): void;
+export declare class AbstractSchemaRegistry<Id = unknown, Schema = unknown, Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> implements SchemaRegistry<Id, Schema, Key, Value, HeaderKey, HeaderValue> {
+    get(_: Id): Schema | undefined;
+    fetch(_i: unknown, _c?: (error?: Error) => void): void | Promise<void>;
+    getSchemaId(_p: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, _t?: BeforeHookPayloadType): Id;
+    getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
+    getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
+    getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;
+    getBeforeDeserializationHook(): BeforeDeserializationHook;
+}

package/dist/registries/abstract.js

@@ -0,0 +1,38 @@
+export function runAsyncSeries(operation, collection, index, callback) {
+    operation(collection[index], error => {
+        if (error) {
+            callback(error);
+            return;
+        }
+        else if (index === collection.length - 1) {
+            callback(null);
+            return;
+        }
+        runAsyncSeries(operation, collection, index + 1, callback);
+    });
+}
+/* c8 ignore start */
+export class AbstractSchemaRegistry {
+    get(_) {
+        throw new Error('AbstractSchemaRegistry.get() should be implemented in subclasses.');
+    }
+    fetch(_i, _c) {
+        throw new Error('AbstractSchemaRegistry.fetch() should be implemented in subclasses.');
+    }
+    getSchemaId(_p, _t) {
+        throw new Error('AbstractSchemaRegistry.getSchemaId() should be implemented in subclasses.');
+    }
+    getSerializers() {
+        throw new Error('AbstractSchemaRegistry.getSerializers() should be implemented in subclasses.');
+    }
+    getDeserializers() {
+        throw new Error('AbstractSchemaRegistry.getDeserializers() should be implemented in subclasses.');
+    }
+    getBeforeSerializationHook() {
+        throw new Error('AbstractSchemaRegistry.getBeforeSerializationHook() should be implemented in subclasses.');
+    }
+    getBeforeDeserializationHook() {
+        throw new Error('AbstractSchemaRegistry.getBeforeDeserializationHook() should be implemented in subclasses.');
+    }
+}
+/* c8 ignore end */

package/dist/registries/confluent-schema-registry.d.ts

@@ -0,0 +1,41 @@
+import { type ValidateFunction } from 'ajv';
+import { type Type } from 'avsc';
+import { type Root } from 'protobufjs';
+import { type Callback } from '../apis/definitions.ts';
+import { type BeforeDeserializationHook, type BeforeHookPayloadType, type BeforeSerializationHook, type Deserializers, type Serializers } from '../clients/serde.ts';
+import { type CredentialProvider } from '../index.ts';
+import { type MessageToConsume, type MessageToProduce } from '../protocol/records.ts';
+import { AbstractSchemaRegistry } from './abstract.ts';
+type ConfluentSchemaRegistryMessageToProduce = MessageToProduce<unknown, unknown, unknown, unknown>;
+export interface ConfluentSchemaRegistryMetadata {
+    schemas?: Record<BeforeHookPayloadType, number>;
+}
+export type ConfluentSchemaRegistryProtobufTypeMapper = (id: number, type: BeforeHookPayloadType, context: ConfluentSchemaRegistryMessageToProduce | MessageToConsume) => string;
+export interface ConfluentSchemaRegistryOptions {
+    url: string;
+    auth?: {
+        username?: string | CredentialProvider;
+        password?: string | CredentialProvider;
+        token?: string | CredentialProvider;
+    };
+    protobufTypeMapper?: ConfluentSchemaRegistryProtobufTypeMapper;
+    jsonValidateSend?: boolean;
+}
+export interface Schema {
+    id: number;
+    type: 'avro' | 'protobuf' | 'json';
+    schema: Type | Root | ValidateFunction;
+}
+export declare function defaultProtobufTypeMapper(_: number, type: BeforeHookPayloadType, context: ConfluentSchemaRegistryMessageToProduce | MessageToConsume): string;
+export declare class ConfluentSchemaRegistry<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends AbstractSchemaRegistry<number | undefined, Schema, Key, Value, HeaderKey, HeaderValue> {
+    #private;
+    constructor(options: ConfluentSchemaRegistryOptions);
+    getSchemaId(message: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): number | undefined;
+    get(id: number): Schema | undefined;
+    fetchSchema(id: number, callback: Callback<void>): Promise<void>;
+    getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
+    getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
+    getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;
+    getBeforeDeserializationHook(): BeforeDeserializationHook;
+}
+export {};

package/dist/registries/confluent-schema-registry.js

@@ -0,0 +1,222 @@
+import avro from 'avsc';
+import { createRequire } from 'node:module';
+import { jsonDeserializer, jsonSerializer, stringDeserializer, stringSerializer } from "../clients/serde.js";
+import { ajv, EMPTY_BUFFER, UnsupportedFormatError, UserError } from "../index.js";
+import { getCredential } from "../protocol/sasl/utils.js";
+import { AbstractSchemaRegistry } from "./abstract.js";
+const require = createRequire(import.meta.url);
+/* c8 ignore next 8 */
+export function defaultProtobufTypeMapper(_, type, context) {
+    // Confluent Schema Registry convention
+    return `${context.topic}-${type}`;
+}
+// TODO(ShogunPanda): Authentication support
+export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
+    #url;
+    #schemas;
+    #protobufParse;
+    #protobufTypeMapper;
+    #jsonValidateSend;
+    #auth;
+    constructor(options) {
+        super();
+        this.#url = options.url;
+        this.#schemas = new Map();
+        this.#protobufTypeMapper = options.protobufTypeMapper ?? defaultProtobufTypeMapper;
+        this.#jsonValidateSend = options.jsonValidateSend ?? false;
+        this.#auth = options.auth;
+    }
+    getSchemaId(message, type) {
+        if (Buffer.isBuffer(message)) {
+            if (type !== 'value') {
+                return undefined;
+            }
+            return message.readInt32BE(1);
+        }
+        return message.metadata?.schemas?.[type];
+    }
+    get(id) {
+        return this.#schemas.get(id);
+    }
+    async fetchSchema(id, callback) {
+        try {
+            const requestInit = {};
+            if (this.#auth) {
+                if (this.#auth.token) {
+                    const token = await getCredential('token', this.#auth.token);
+                    requestInit.headers = {
+                        Authorization: `Bearer ${token}`
+                    };
+                }
+                else {
+                    const username = await getCredential('username', this.#auth.username);
+                    const password = await getCredential('password', this.#auth.password);
+                    requestInit.headers = {
+                        Authorization: `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`
+                    };
+                }
+            }
+            const response = await fetch(`${this.#url}/schemas/ids/${id}`, requestInit);
+            if (!response.ok) {
+                throw new UserError(`Failed to fetch a schema: [HTTP ${response.status}]`, { response: await response.text() });
+            }
+            const responseBody = await response.json();
+            const { schema, schemaType } = responseBody;
+            switch (schemaType) {
+                case 'AVRO':
+                    this.#schemas.set(id, { id, type: 'avro', schema: avro.Type.forSchema(JSON.parse(schema)) });
+                    break;
+                case 'PROTOBUF':
+                    this.#protobufParse ??= this.#loadProtobuf();
+                    this.#schemas.set(id, { id, type: 'protobuf', schema: this.#protobufParse(schema).root });
+                    break;
+                case 'JSON':
+                    this.#schemas.set(id, { id, type: 'json', schema: ajv.compile(JSON.parse(schema)) });
+                    break;
+            }
+            process.nextTick(callback);
+        }
+        catch (err) {
+            process.nextTick(() => callback(err));
+        }
+    }
+    getSerializers() {
+        return {
+            key: this.#schemaSerializer.bind(this, 'key', stringSerializer),
+            value: this.#schemaSerializer.bind(this, 'value', jsonSerializer),
+            headerKey: this.#schemaSerializer.bind(this, 'headerKey', stringSerializer),
+            headerValue: this.#schemaSerializer.bind(this, 'headerValue', jsonSerializer)
+        };
+    }
+    getDeserializers() {
+        return {
+            key: this.#schemaDeserializer.bind(this, 'key', stringDeserializer),
+            value: this.#schemaDeserializer.bind(this, 'value', jsonDeserializer),
+            headerKey: this.#schemaDeserializer.bind(this, 'headerKey', stringDeserializer),
+            headerValue: this.#schemaDeserializer.bind(this, 'headerValue', jsonDeserializer)
+        };
+    }
+    getBeforeSerializationHook() {
+        const registry = this;
+        return function beforeSerialization(_, type, message, callback) {
+            // Extract the schema ID from the message metadata
+            const schemaId = registry.getSchemaId(message, type);
+            // When no schema ID is found, nothing to do
+            if (!schemaId) {
+                callback(null);
+                return;
+            }
+            // The schema is already fetch
+            if (registry.get(schemaId)) {
+                callback(null);
+                return;
+            }
+            registry.fetchSchema(schemaId, callback);
+        };
+    }
+    getBeforeDeserializationHook() {
+        const registry = this;
+        return function beforeDeserialization(payload, type, _message, callback) {
+            // Extract the schema ID from the message metadata
+            const schemaId = registry.getSchemaId(payload, type);
+            // When no schema ID is found, nothing to do
+            if (!schemaId) {
+                callback(null);
+                return;
+            }
+            // The schema is already fetch
+            if (registry.get(schemaId)) {
+                callback(null);
+                return;
+            }
+            registry.fetchSchema(schemaId, callback);
+        };
+    }
+    #schemaSerializer(type, fallbackSerializer, data, headers, message) {
+        /* c8 ignore next 3 - Hard to test */
+        if (typeof data === 'undefined') {
+            return EMPTY_BUFFER;
+        }
+        if (type === 'headerKey' || type === 'headerValue') {
+            message = headers;
+        }
+        const schemaId = message?.metadata?.schemas?.[type];
+        if (!schemaId) {
+            return fallbackSerializer(data);
+        }
+        const schema = this.#schemas.get(schemaId);
+        if (!schema) {
+            throw new UserError(`Schema with ID ${schemaId} not found.`, { missingSchema: schemaId });
+        }
+        let encodedMessage;
+        switch (schema.type) {
+            case 'avro':
+                encodedMessage = schema.schema.toBuffer(data);
+                break;
+            case 'protobuf':
+                {
+                    const typeName = this.#protobufTypeMapper(schemaId, type, message);
+                    const Type = schema.schema.lookupType(typeName);
+                    encodedMessage = Buffer.from(Type.encode(Type.create(data)).finish());
+                }
+                break;
+            case 'json':
+                if (this.#jsonValidateSend) {
+                    const validate = schema.schema;
+                    const valid = validate(data);
+                    if (!valid) {
+                        throw new UserError(`JSON Schema validation failed before serialization: ${ajv.errorsText(validate.errors)}`, { type, data, headers, validationErrors: validate.errors });
+                    }
+                }
+                encodedMessage = Buffer.from(JSON.stringify(data));
+                break;
+        }
+        const header = Buffer.alloc(5);
+        header.writeInt32BE(schemaId, 1);
+        return Buffer.concat([header, encodedMessage]);
+    }
+    #schemaDeserializer(type, fallbackDeserializer, data, headers, message) {
+        /* c8 ignore next 3 - Hard to test */
+        if (typeof data === 'undefined' || data.length === 0) {
+            return EMPTY_BUFFER;
+        }
+        if (type === 'headerKey' || type === 'headerValue') {
+            message = headers;
+        }
+        const schemaId = this.getSchemaId(data, type);
+        if (!schemaId) {
+            return fallbackDeserializer(data);
+        }
+        const schema = this.#schemas.get(schemaId);
+        if (!schema) {
+            throw new UserError(`Schema with ID ${schemaId} not found.`, { missingSchema: schemaId });
+        }
+        switch (schema.type) {
+            case 'avro':
+                return schema.schema.fromBuffer(data.subarray(5));
+            case 'protobuf': {
+                const typeName = this.#protobufTypeMapper(schemaId, type, message);
+                const Type = schema.schema.lookupType(typeName);
+                return Type.decode(data.subarray(5));
+            }
+            case 'json': {
+                const parsed = JSON.parse(data.subarray(5).toString('utf-8'));
+                const validate = schema.schema;
+                const valid = validate(parsed);
+                if (!valid) {
+                    throw new UserError(`JSON Schema validation failed before deserialization: ${ajv.errorsText(validate.errors)}`, { type, data: parsed, headers, validationErrors: validate.errors });
+                }
+                return parsed;
+            }
+        }
+    }
+    #loadProtobuf() {
+        try {
+            return require('protobufjs').parse;
+            /* c8 ignore next 5 - In tests protobufjs is always available */
+        }
+        catch (e) {
+            throw new UnsupportedFormatError('Cannot load protobufjs module, which is an optionalDependency. Please check your local installation.');
+        }
+    }
+}

package/dist/registries/index.d.ts

@@ -0,0 +1,2 @@
+export * from './abstract.ts';
+export * from './confluent-schema-registry.ts';

package/dist/registries/index.js

@@ -0,0 +1,2 @@
+export * from "./abstract.js";
+export * from "./confluent-schema-registry.js";

package/dist/typescript-4/dist/clients/consumer/options.d.ts

@@ -127,6 +127,12 @@ export declare const consumeOptionsProperties: {
     type: string;
     minimum: number;
   };
+  beforeDeserialization: {
+    function: boolean;
+  };
+  registry: {
+    type: string;
+  };
 };
 export declare const groupOptionsSchema: {
   type: string;
@@ -241,6 +247,12 @@ export declare const consumeOptionsSchema: {
       type: string;
       minimum: number;
     };
+    beforeDeserialization: {
+      function: boolean;
+    };
+    registry: {
+      type: string;
+    };
     groupInstanceId: {
       type: string;
       pattern: string;
@@ -396,6 +408,12 @@ export declare const consumerOptionsSchema: {
       type: string;
       minimum: number;
     };
+    beforeDeserialization: {
+      function: boolean;
+    };
+    registry: {
+      type: string;
+    };
     groupInstanceId: {
       type: string;
       pattern: string;
@@ -511,6 +529,12 @@ export declare const fetchOptionsSchema: {
       type: string;
       minimum: number;
     };
+    beforeDeserialization: {
+      function: boolean;
+    };
+    registry: {
+      type: string;
+    };
     groupInstanceId: {
       type: string;
       pattern: string;

package/dist/typescript-4/dist/clients/consumer/types.d.ts

@@ -2,8 +2,9 @@ import { type FetchRequestTopic } from "../../apis/consumer/fetch-v17";
 import { type GroupProtocols } from "../../apis/enumerations";
 import { type ConnectionPool } from "../../network/connection-pool";
 import { type KafkaRecord, type Message } from "../../protocol/records";
+import { type SchemaRegistry } from "../../registries/abstract";
 import { type BaseOptions, type ClusterMetadata, type TopicWithPartitionAndOffset } from "../base/types";
-import { type Deserializers } from "../serde";
+import { type BeforeDeserializationHook, type Deserializers } from "../serde";
 export interface GroupProtocolSubscription {
   name: string;
   version: number;
@@ -68,6 +69,8 @@ export interface ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> {
   isolationLevel?: number;
   deserializers?: Partial<Deserializers<Key, Value, HeaderKey, HeaderValue>>;
   highWaterMark?: number;
+  beforeDeserialization?: BeforeDeserializationHook;
+  registry?: SchemaRegistry<unknown, unknown, Key, Value, HeaderKey, HeaderValue>;
 }
 export interface StreamOptions {
   topics: string[];