@platformatic/kafka 1.26.0 → 1.27.0-alpha.2

package/dist/protocol/definitions.js

@@ -7,5 +7,5 @@ export const EMPTY_BUFFER = Buffer.alloc(0);
 export const EMPTY_UUID = Buffer.alloc(UUID_SIZE);
 // Since it is serialized at either 0 (for nullable) or 1 (since length is stored as length + 1), it always uses a single byte
 export const EMPTY_OR_SINGLE_COMPACT_LENGTH_SIZE = INT8_SIZE;
-// TODO(ShogunPanda): Tagged fields are not supported yet
+// TODO: Tagged fields are not supported yet
 export const EMPTY_TAGGED_FIELDS_BUFFER = Buffer.from([0]);

package/dist/protocol/reader.js

@@ -300,7 +300,7 @@ export class Reader {
         }
         return reader();
     }
-    // TODO(ShogunPanda): Tagged fields are not supported yet
+    // TODO: Tagged fields are not supported yet
     readTaggedFields() {
         const length = this.readVarInt();
         if (length > 0) {

package/dist/protocol/records.d.ts

@@ -17,6 +17,7 @@ export interface MessageBase<Key = Buffer, Value = Buffer> {
 }
 export interface MessageToProduce<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends MessageBase<Key, Value> {
     headers?: Map<HeaderKey, HeaderValue> | Record<string, HeaderValue>;
+    metadata?: unknown;
 }
 export interface MessageConsumerMetadata {
     coordinatorId: number;
@@ -67,6 +68,10 @@ export interface KafkaRecord {
     value: Buffer;
     headers: [Buffer, Buffer][];
 }
+export interface MessageToConsume extends KafkaRecord {
+    topic: string;
+    partition: number;
+}
 export interface RecordsBatch {
     firstOffset: bigint;
     length: number;
@@ -85,24 +90,8 @@ export interface RecordsBatch {
 export declare const messageSchema: {
     type: string;
     properties: {
-        key: {
-            oneOf: ({
-                type: string;
-                buffer?: undefined;
-            } | {
-                buffer: boolean;
-                type?: undefined;
-            })[];
-        };
-        value: {
-            oneOf: ({
-                type: string;
-                buffer?: undefined;
-            } | {
-                buffer: boolean;
-                type?: undefined;
-            })[];
-        };
+        key: boolean;
+        value: boolean;
         headers: {
             anyOf: ({
                 map: boolean;

package/dist/protocol/records.js

@@ -14,12 +14,8 @@ export const BATCH_HEAD = INT64_SIZE + INT32_SIZE; // FirstOffset + Length
 export const messageSchema = {
     type: 'object',
     properties: {
-        key: {
-            oneOf: [{ type: 'string' }, { buffer: true }]
-        },
-        value: {
-            oneOf: [{ type: 'string' }, { buffer: true }]
-        },
+        key: true,
+        value: true,
         headers: {
             // Note: we can't use oneOf here since a Map is also a 'object'. Thanks JS.
             anyOf: [

package/dist/protocol/sasl/oauth-bearer.d.ts

@@ -1,6 +1,6 @@
 import { type CallbackWithPromise } from '../../apis/callbacks.ts';
 import { type SASLAuthenticationAPI, type SaslAuthenticateResponse } from '../../apis/security/sasl-authenticate-v2.ts';
-import { type Connection, type SASLCredentialProvider } from '../../network/connection.ts';
+import { type Connection, type CredentialProvider } from '../../network/connection.ts';
 export declare function jwtValidateAuthenticationBytes(authBytes: Buffer, callback: CallbackWithPromise<Buffer>): void;
-export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, tokenOrProvider: string | SASLCredentialProvider, extensions: Record<string, string> | SASLCredentialProvider<Record<string, string>>, callback: CallbackWithPromise<SaslAuthenticateResponse>): void;
-export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, tokenOrProvider: string | SASLCredentialProvider, extensions: Record<string, string> | SASLCredentialProvider<Record<string, string>>): Promise<SaslAuthenticateResponse>;
+export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, tokenOrProvider: string | CredentialProvider, extensions: Record<string, string> | CredentialProvider<Record<string, string>>, callback: CallbackWithPromise<SaslAuthenticateResponse>): void;
+export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, tokenOrProvider: string | CredentialProvider, extensions: Record<string, string> | CredentialProvider<Record<string, string>>): Promise<SaslAuthenticateResponse>;

package/dist/protocol/sasl/plain.d.ts

@@ -1,5 +1,5 @@
 import { type CallbackWithPromise } from '../../apis/callbacks.ts';
 import { type SASLAuthenticationAPI, type SaslAuthenticateResponse } from '../../apis/security/sasl-authenticate-v2.ts';
-import { type Connection, type SASLCredentialProvider } from '../../network/connection.ts';
-export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, usernameProvider: string | SASLCredentialProvider, passwordProvider: string | SASLCredentialProvider, callback: CallbackWithPromise<SaslAuthenticateResponse>): void;
-export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, usernameProvider: string | SASLCredentialProvider, passwordProvider: string | SASLCredentialProvider): Promise<SaslAuthenticateResponse>;
+import { type Connection, type CredentialProvider } from '../../network/connection.ts';
+export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, usernameProvider: string | CredentialProvider, passwordProvider: string | CredentialProvider, callback: CallbackWithPromise<SaslAuthenticateResponse>): void;
+export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, usernameProvider: string | CredentialProvider, passwordProvider: string | CredentialProvider): Promise<SaslAuthenticateResponse>;

package/dist/protocol/sasl/scram-sha.d.ts

@@ -1,6 +1,6 @@
 import { type CallbackWithPromise } from '../../apis/callbacks.ts';
 import { type SASLAuthenticationAPI, type SaslAuthenticateResponse } from '../../apis/security/sasl-authenticate-v2.ts';
-import { type Connection, type SASLCredentialProvider } from '../../network/connection.ts';
+import { type Connection, type CredentialProvider } from '../../network/connection.ts';
 export interface ScramAlgorithmDefinition {
     keyLength: number;
     algorithm: string;
@@ -35,5 +35,5 @@ export declare function hi(definition: ScramAlgorithmDefinition, password: strin
 export declare function hmac(definition: ScramAlgorithmDefinition, key: Buffer, data: string | Buffer): Buffer;
 export declare function xor(a: Buffer, b: Buffer): Buffer;
 export declare const defaultCrypto: ScramCryptoModule;
-export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, algorithm: ScramAlgorithm, usernameProvider: string | SASLCredentialProvider, passwordProvider: string | SASLCredentialProvider, crypto: ScramCryptoModule, callback: CallbackWithPromise<SaslAuthenticateResponse>): void;
-export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, algorithm: ScramAlgorithm, usernameProvider: string | SASLCredentialProvider, passwordProvider: string | SASLCredentialProvider, crypto?: ScramCryptoModule): Promise<SaslAuthenticateResponse>;
+export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, algorithm: ScramAlgorithm, usernameProvider: string | CredentialProvider, passwordProvider: string | CredentialProvider, crypto: ScramCryptoModule, callback: CallbackWithPromise<SaslAuthenticateResponse>): void;
+export declare function authenticate(authenticateAPI: SASLAuthenticationAPI, connection: Connection, algorithm: ScramAlgorithm, usernameProvider: string | CredentialProvider, passwordProvider: string | CredentialProvider, crypto?: ScramCryptoModule): Promise<SaslAuthenticateResponse>;

package/dist/protocol/sasl/utils.d.ts

@@ -1,4 +1,4 @@
 import { type CallbackWithPromise } from '../../apis/index.ts';
-import { type SASLCredentialProvider } from '../../network/connection.ts';
-export declare function getCredential<T>(label: string, credentialOrProvider: T | SASLCredentialProvider<T>, callback: CallbackWithPromise<T>): void;
-export declare function getCredential<T>(label: string, credentialOrProvider: T | SASLCredentialProvider<T>): Promise<T>;
+import { type CredentialProvider } from '../../network/connection.ts';
+export declare function getCredential<T>(label: string, credentialOrProvider: T | CredentialProvider<T>, callback: CallbackWithPromise<T>): void;
+export declare function getCredential<T>(label: string, credentialOrProvider: T | CredentialProvider<T>): Promise<T>;

package/dist/protocol/writer.js

@@ -210,7 +210,7 @@ export class Writer {
         }
         return this;
     }
-    // TODO(ShogunPanda): Tagged fields are not supported yet
+    // TODO: Tagged fields are not supported yet
     appendTaggedFields(_ = []) {
         return this.append(EMPTY_TAGGED_FIELDS_BUFFER);
     }

package/dist/registries/abstract.d.ts

@@ -0,0 +1,22 @@
+import { type Callback } from '../apis/definitions.ts';
+import { type BeforeDeserializationHook, type BeforeHookPayloadType, type BeforeSerializationHook, type Deserializers, type Serializers } from '../clients/serde.ts';
+import { type MessageToProduce } from '../protocol/records.ts';
+export interface SchemaRegistry<Id = unknown, Schema = unknown, Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> {
+    get(id: Id): Schema | undefined;
+    fetch(id: Id, callback?: (error?: Error) => void): void | Promise<void>;
+    getSchemaId(payload: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): Id;
+    getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
+    getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
+    getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;
+    getBeforeDeserializationHook(): BeforeDeserializationHook;
+}
+export declare function runAsyncSeries<V>(operation: (item: V, cb: Callback<void>) => void | Promise<void>, collection: V[], index: number, callback: Callback<void>): void;
+export declare class AbstractSchemaRegistry<Id = unknown, Schema = unknown, Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> implements SchemaRegistry<Id, Schema, Key, Value, HeaderKey, HeaderValue> {
+    get(_: Id): Schema | undefined;
+    fetch(_i: unknown, _c?: (error?: Error) => void): void | Promise<void>;
+    getSchemaId(_p: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, _t?: BeforeHookPayloadType): Id;
+    getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
+    getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
+    getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;
+    getBeforeDeserializationHook(): BeforeDeserializationHook;
+}

package/dist/registries/abstract.js

@@ -0,0 +1,38 @@
+export function runAsyncSeries(operation, collection, index, callback) {
+    operation(collection[index], error => {
+        if (error) {
+            callback(error);
+            return;
+        }
+        else if (index === collection.length - 1) {
+            callback(null);
+            return;
+        }
+        runAsyncSeries(operation, collection, index + 1, callback);
+    });
+}
+/* c8 ignore start */
+export class AbstractSchemaRegistry {
+    get(_) {
+        throw new Error('AbstractSchemaRegistry.get() should be implemented in subclasses.');
+    }
+    fetch(_i, _c) {
+        throw new Error('AbstractSchemaRegistry.fetch() should be implemented in subclasses.');
+    }
+    getSchemaId(_p, _t) {
+        throw new Error('AbstractSchemaRegistry.getSchemaId() should be implemented in subclasses.');
+    }
+    getSerializers() {
+        throw new Error('AbstractSchemaRegistry.getSerializers() should be implemented in subclasses.');
+    }
+    getDeserializers() {
+        throw new Error('AbstractSchemaRegistry.getDeserializers() should be implemented in subclasses.');
+    }
+    getBeforeSerializationHook() {
+        throw new Error('AbstractSchemaRegistry.getBeforeSerializationHook() should be implemented in subclasses.');
+    }
+    getBeforeDeserializationHook() {
+        throw new Error('AbstractSchemaRegistry.getBeforeDeserializationHook() should be implemented in subclasses.');
+    }
+}
+/* c8 ignore end */

package/dist/registries/confluent-schema-registry.d.ts

@@ -0,0 +1,41 @@
+import { type ValidateFunction } from 'ajv';
+import { type Type } from 'avsc';
+import { type Root } from 'protobufjs';
+import { type Callback } from '../apis/definitions.ts';
+import { type BeforeDeserializationHook, type BeforeHookPayloadType, type BeforeSerializationHook, type Deserializers, type Serializers } from '../clients/serde.ts';
+import { type CredentialProvider } from '../index.ts';
+import { type MessageToConsume, type MessageToProduce } from '../protocol/records.ts';
+import { AbstractSchemaRegistry } from './abstract.ts';
+type ConfluentSchemaRegistryMessageToProduce = MessageToProduce<unknown, unknown, unknown, unknown>;
+export interface ConfluentSchemaRegistryMetadata {
+    schemas?: Record<BeforeHookPayloadType, number>;
+}
+export type ConfluentSchemaRegistryProtobufTypeMapper = (id: number, type: BeforeHookPayloadType, context: ConfluentSchemaRegistryMessageToProduce | MessageToConsume) => string;
+export interface ConfluentSchemaRegistryOptions {
+    url: string;
+    auth?: {
+        username?: string | CredentialProvider;
+        password?: string | CredentialProvider;
+        token?: string | CredentialProvider;
+    };
+    protobufTypeMapper?: ConfluentSchemaRegistryProtobufTypeMapper;
+    jsonValidateSend?: boolean;
+}
+export interface Schema {
+    id: number;
+    type: 'avro' | 'protobuf' | 'json';
+    schema: Type | Root | ValidateFunction;
+}
+export declare function defaultProtobufTypeMapper(_: number, type: BeforeHookPayloadType, context: ConfluentSchemaRegistryMessageToProduce | MessageToConsume): string;
+export declare class ConfluentSchemaRegistry<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends AbstractSchemaRegistry<number | undefined, Schema, Key, Value, HeaderKey, HeaderValue> {
+    #private;
+    constructor(options: ConfluentSchemaRegistryOptions);
+    getSchemaId(message: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): number | undefined;
+    get(id: number): Schema | undefined;
+    fetchSchema(id: number, callback: Callback<void>): Promise<void>;
+    getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
+    getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
+    getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;
+    getBeforeDeserializationHook(): BeforeDeserializationHook;
+}
+export {};

package/dist/registries/confluent-schema-registry.js

@@ -0,0 +1,222 @@
+import avro from 'avsc';
+import { createRequire } from 'node:module';
+import { jsonDeserializer, jsonSerializer, stringDeserializer, stringSerializer } from "../clients/serde.js";
+import { ajv, EMPTY_BUFFER, UnsupportedFormatError, UserError } from "../index.js";
+import { getCredential } from "../protocol/sasl/utils.js";
+import { AbstractSchemaRegistry } from "./abstract.js";
+const require = createRequire(import.meta.url);
+/* c8 ignore next 8 */
+export function defaultProtobufTypeMapper(_, type, context) {
+    // Confluent Schema Registry convention
+    return `${context.topic}-${type}`;
+}
+// TODO(ShogunPanda): Authentication support
+export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
+    #url;
+    #schemas;
+    #protobufParse;
+    #protobufTypeMapper;
+    #jsonValidateSend;
+    #auth;
+    constructor(options) {
+        super();
+        this.#url = options.url;
+        this.#schemas = new Map();
+        this.#protobufTypeMapper = options.protobufTypeMapper ?? defaultProtobufTypeMapper;
+        this.#jsonValidateSend = options.jsonValidateSend ?? false;
+        this.#auth = options.auth;
+    }
+    getSchemaId(message, type) {
+        if (Buffer.isBuffer(message)) {
+            if (type !== 'value') {
+                return undefined;
+            }
+            return message.readInt32BE(1);
+        }
+        return message.metadata?.schemas?.[type];
+    }
+    get(id) {
+        return this.#schemas.get(id);
+    }
+    async fetchSchema(id, callback) {
+        try {
+            const requestInit = {};
+            if (this.#auth) {
+                if (this.#auth.token) {
+                    const token = await getCredential('token', this.#auth.token);
+                    requestInit.headers = {
+                        Authorization: `Bearer ${token}`
+                    };
+                }
+                else {
+                    const username = await getCredential('username', this.#auth.username);
+                    const password = await getCredential('password', this.#auth.password);
+                    requestInit.headers = {
+                        Authorization: `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`
+                    };
+                }
+            }
+            const response = await fetch(`${this.#url}/schemas/ids/${id}`, requestInit);
+            if (!response.ok) {
+                throw new UserError(`Failed to fetch a schema: [HTTP ${response.status}]`, { response: await response.text() });
+            }
+            const responseBody = await response.json();
+            const { schema, schemaType } = responseBody;
+            switch (schemaType) {
+                case 'AVRO':
+                    this.#schemas.set(id, { id, type: 'avro', schema: avro.Type.forSchema(JSON.parse(schema)) });
+                    break;
+                case 'PROTOBUF':
+                    this.#protobufParse ??= this.#loadProtobuf();
+                    this.#schemas.set(id, { id, type: 'protobuf', schema: this.#protobufParse(schema).root });
+                    break;
+                case 'JSON':
+                    this.#schemas.set(id, { id, type: 'json', schema: ajv.compile(JSON.parse(schema)) });
+                    break;
+            }
+            process.nextTick(callback);
+        }
+        catch (err) {
+            process.nextTick(() => callback(err));
+        }
+    }
+    getSerializers() {
+        return {
+            key: this.#schemaSerializer.bind(this, 'key', stringSerializer),
+            value: this.#schemaSerializer.bind(this, 'value', jsonSerializer),
+            headerKey: this.#schemaSerializer.bind(this, 'headerKey', stringSerializer),
+            headerValue: this.#schemaSerializer.bind(this, 'headerValue', jsonSerializer)
+        };
+    }
+    getDeserializers() {
+        return {
+            key: this.#schemaDeserializer.bind(this, 'key', stringDeserializer),
+            value: this.#schemaDeserializer.bind(this, 'value', jsonDeserializer),
+            headerKey: this.#schemaDeserializer.bind(this, 'headerKey', stringDeserializer),
+            headerValue: this.#schemaDeserializer.bind(this, 'headerValue', jsonDeserializer)
+        };
+    }
+    getBeforeSerializationHook() {
+        const registry = this;
+        return function beforeSerialization(_, type, message, callback) {
+            // Extract the schema ID from the message metadata
+            const schemaId = registry.getSchemaId(message, type);
+            // When no schema ID is found, nothing to do
+            if (!schemaId) {
+                callback(null);
+                return;
+            }
+            // The schema is already fetch
+            if (registry.get(schemaId)) {
+                callback(null);
+                return;
+            }
+            registry.fetchSchema(schemaId, callback);
+        };
+    }
+    getBeforeDeserializationHook() {
+        const registry = this;
+        return function beforeDeserialization(payload, type, _message, callback) {
+            // Extract the schema ID from the message metadata
+            const schemaId = registry.getSchemaId(payload, type);
+            // When no schema ID is found, nothing to do
+            if (!schemaId) {
+                callback(null);
+                return;
+            }
+            // The schema is already fetch
+            if (registry.get(schemaId)) {
+                callback(null);
+                return;
+            }
+            registry.fetchSchema(schemaId, callback);
+        };
+    }
+    #schemaSerializer(type, fallbackSerializer, data, headers, message) {
+        /* c8 ignore next 3 - Hard to test */
+        if (typeof data === 'undefined') {
+            return EMPTY_BUFFER;
+        }
+        if (type === 'headerKey' || type === 'headerValue') {
+            message = headers;
+        }
+        const schemaId = message?.metadata?.schemas?.[type];
+        if (!schemaId) {
+            return fallbackSerializer(data);
+        }
+        const schema = this.#schemas.get(schemaId);
+        if (!schema) {
+            throw new UserError(`Schema with ID ${schemaId} not found.`, { missingSchema: schemaId });
+        }
+        let encodedMessage;
+        switch (schema.type) {
+            case 'avro':
+                encodedMessage = schema.schema.toBuffer(data);
+                break;
+            case 'protobuf':
+                {
+                    const typeName = this.#protobufTypeMapper(schemaId, type, message);
+                    const Type = schema.schema.lookupType(typeName);
+                    encodedMessage = Buffer.from(Type.encode(Type.create(data)).finish());
+                }
+                break;
+            case 'json':
+                if (this.#jsonValidateSend) {
+                    const validate = schema.schema;
+                    const valid = validate(data);
+                    if (!valid) {
+                        throw new UserError(`JSON Schema validation failed before serialization: ${ajv.errorsText(validate.errors)}`, { type, data, headers, validationErrors: validate.errors });
+                    }
+                }
+                encodedMessage = Buffer.from(JSON.stringify(data));
+                break;
+        }
+        const header = Buffer.alloc(5);
+        header.writeInt32BE(schemaId, 1);
+        return Buffer.concat([header, encodedMessage]);
+    }
+    #schemaDeserializer(type, fallbackDeserializer, data, headers, message) {
+        /* c8 ignore next 3 - Hard to test */
+        if (typeof data === 'undefined' || data.length === 0) {
+            return EMPTY_BUFFER;
+        }
+        if (type === 'headerKey' || type === 'headerValue') {
+            message = headers;
+        }
+        const schemaId = this.getSchemaId(data, type);
+        if (!schemaId) {
+            return fallbackDeserializer(data);
+        }
+        const schema = this.#schemas.get(schemaId);
+        if (!schema) {
+            throw new UserError(`Schema with ID ${schemaId} not found.`, { missingSchema: schemaId });
+        }
+        switch (schema.type) {
+            case 'avro':
+                return schema.schema.fromBuffer(data.subarray(5));
+            case 'protobuf': {
+                const typeName = this.#protobufTypeMapper(schemaId, type, message);
+                const Type = schema.schema.lookupType(typeName);
+                return Type.decode(data.subarray(5));
+            }
+            case 'json': {
+                const parsed = JSON.parse(data.subarray(5).toString('utf-8'));
+                const validate = schema.schema;
+                const valid = validate(parsed);
+                if (!valid) {
+                    throw new UserError(`JSON Schema validation failed before deserialization: ${ajv.errorsText(validate.errors)}`, { type, data: parsed, headers, validationErrors: validate.errors });
+                }
+                return parsed;
+            }
+        }
+    }
+    #loadProtobuf() {
+        try {
+            return require('protobufjs').parse;
+            /* c8 ignore next 5 - In tests protobufjs is always available */
+        }
+        catch (e) {
+            throw new UnsupportedFormatError('Cannot load protobufjs module, which is an optionalDependency. Please check your local installation.');
+        }
+    }
+}

package/dist/registries/index.d.ts

@@ -0,0 +1,2 @@
+export * from './abstract.ts';
+export * from './confluent-schema-registry.ts';

package/dist/registries/index.js

@@ -0,0 +1,2 @@
+export * from "./abstract.js";
+export * from "./confluent-schema-registry.js";

package/dist/typescript-4/dist/clients/consumer/options.d.ts

@@ -127,6 +127,12 @@ export declare const consumeOptionsProperties: {
     type: string;
     minimum: number;
   };
+  beforeDeserialization: {
+    function: boolean;
+  };
+  registry: {
+    type: string;
+  };
 };
 export declare const groupOptionsSchema: {
   type: string;
@@ -241,6 +247,12 @@ export declare const consumeOptionsSchema: {
       type: string;
       minimum: number;
     };
+    beforeDeserialization: {
+      function: boolean;
+    };
+    registry: {
+      type: string;
+    };
     groupInstanceId: {
       type: string;
       pattern: string;
@@ -396,6 +408,12 @@ export declare const consumerOptionsSchema: {
       type: string;
       minimum: number;
     };
+    beforeDeserialization: {
+      function: boolean;
+    };
+    registry: {
+      type: string;
+    };
     groupInstanceId: {
       type: string;
       pattern: string;
@@ -511,6 +529,12 @@ export declare const fetchOptionsSchema: {
       type: string;
       minimum: number;
     };
+    beforeDeserialization: {
+      function: boolean;
+    };
+    registry: {
+      type: string;
+    };
     groupInstanceId: {
       type: string;
       pattern: string;

package/dist/typescript-4/dist/clients/consumer/types.d.ts

@@ -2,8 +2,9 @@ import { type FetchRequestTopic } from "../../apis/consumer/fetch-v17";
 import { type GroupProtocols } from "../../apis/enumerations";
 import { type ConnectionPool } from "../../network/connection-pool";
 import { type KafkaRecord, type Message } from "../../protocol/records";
+import { type SchemaRegistry } from "../../registries/abstract";
 import { type BaseOptions, type ClusterMetadata, type TopicWithPartitionAndOffset } from "../base/types";
-import { type Deserializers } from "../serde";
+import { type BeforeDeserializationHook, type Deserializers } from "../serde";
 export interface GroupProtocolSubscription {
   name: string;
   version: number;
@@ -68,6 +69,8 @@ export interface ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> {
   isolationLevel?: number;
   deserializers?: Partial<Deserializers<Key, Value, HeaderKey, HeaderValue>>;
   highWaterMark?: number;
+  beforeDeserialization?: BeforeDeserializationHook;
+  registry?: SchemaRegistry<unknown, unknown, Key, Value, HeaderKey, HeaderValue>;
 }
 export interface StreamOptions {
   topics: string[];

package/dist/typescript-4/dist/clients/producer/options.d.ts

@@ -121,24 +121,8 @@ export declare const sendOptionsSchema: {
       items: {
         type: string;
         properties: {
-          key: {
-            oneOf: ({
-              type: string;
-              buffer?: undefined;
-            } | {
-              buffer: boolean;
-              type?: undefined;
-            })[];
-          };
-          value: {
-            oneOf: ({
-              type: string;
-              buffer?: undefined;
-            } | {
-              buffer: boolean;
-              type?: undefined;
-            })[];
-          };
+          key: boolean;
+          value: boolean;
           headers: {
             anyOf: ({
               map: boolean;

package/dist/typescript-4/dist/clients/producer/types.d.ts

@@ -1,7 +1,8 @@
 import { type CompressionAlgorithmValue } from "../../protocol/compression";
 import { type MessageToProduce } from "../../protocol/records";
+import { type SchemaRegistry } from "../../registries/abstract";
 import { type BaseOptions, type TopicWithPartitionAndOffset } from "../base/types";
-import { type Serializers } from "../serde";
+import { type BeforeSerializationHook, type Serializers } from "../serde";
 export interface ProducerInfo {
   producerId: bigint;
   producerEpoch: number;
@@ -25,6 +26,8 @@ export interface ProduceOptions<Key, Value, HeaderKey, HeaderValue> {
 export type ProducerOptions<Key, Value, HeaderKey, HeaderValue> = BaseOptions & ProduceOptions<Key, Value, HeaderKey, HeaderValue> & {
   transactionalId?: string;
   serializers?: Partial<Serializers<Key, Value, HeaderKey, HeaderValue>>;
+  beforeSerialization?: BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;
+  registry?: SchemaRegistry<unknown, unknown, Key, Value, HeaderKey, HeaderValue>;
 };
 export type SendOptions<Key, Value, HeaderKey, HeaderValue> = {
   messages: MessageToProduce<Key, Value, HeaderKey, HeaderValue>[];

package/dist/typescript-4/dist/clients/serde.d.ts

@@ -1,7 +1,9 @@
-export type Serializer<InputType = unknown> = (data?: InputType) => Buffer | undefined;
-export type Deserializer<OutputType = unknown> = (data?: Buffer) => OutputType | undefined;
-export type SerializerWithHeaders<InputType = unknown, HeaderKey = unknown, HeaderValue = unknown> = (data?: InputType, headers?: Map<HeaderKey, HeaderValue>) => Buffer | undefined;
-export type DeserializerWithHeaders<OutputType = unknown, HeaderKey = unknown, HeaderValue = unknown> = (data?: Buffer, headers?: Map<HeaderKey, HeaderValue>) => OutputType | undefined;
+import { type Callback } from "../apis/definitions";
+import { type MessageToConsume, type MessageToProduce } from "../protocol/records";
+export type Serializer<InputType = unknown> = (data?: InputType, metadata?: unknown) => Buffer | undefined;
+export type SerializerWithHeaders<InputType = unknown, HeaderKey = unknown, HeaderValue = unknown> = (data?: InputType, headers?: Map<HeaderKey, HeaderValue>, message?: MessageToProduce<unknown, unknown, unknown, unknown>) => Buffer | undefined;
+export type Deserializer<OutputType = unknown> = (data?: Buffer, message?: MessageToConsume) => OutputType | undefined;
+export type DeserializerWithHeaders<OutputType = unknown, HeaderKey = unknown, HeaderValue = unknown> = (data?: Buffer, headers?: Map<HeaderKey, HeaderValue>, message?: MessageToConsume) => OutputType | undefined;
 export interface Serializers<Key, Value, HeaderKey, HeaderValue> {
   key: SerializerWithHeaders<Key, HeaderKey, HeaderValue>;
   value: SerializerWithHeaders<Value, HeaderKey, HeaderValue>;
@@ -9,11 +11,14 @@ export interface Serializers<Key, Value, HeaderKey, HeaderValue> {
   headerValue: Serializer<HeaderValue>;
 }
 export interface Deserializers<Key, Value, HeaderKey, HeaderValue> {
-  key: DeserializerWithHeaders<Key>;
-  value: DeserializerWithHeaders<Value>;
+  key: DeserializerWithHeaders<Key, HeaderKey, HeaderValue>;
+  value: DeserializerWithHeaders<Value, HeaderKey, HeaderValue>;
   headerKey: Deserializer<HeaderKey>;
   headerValue: Deserializer<HeaderValue>;
 }
+export type BeforeHookPayloadType = 'key' | 'value' | 'headerKey' | 'headerValue';
+export type BeforeDeserializationHook = (payload: Buffer, type: BeforeHookPayloadType, message: MessageToConsume, callback: Callback<void>) => void | Promise<void>;
+export type BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue> = (payload: unknown, type: BeforeHookPayloadType, message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, callback: Callback<void>) => void | Promise<void>;
 export declare function stringSerializer(data?: string): Buffer | undefined;
 export declare function stringDeserializer(data?: string | Buffer): string | undefined;
 export declare function jsonSerializer<T = Record<string, any>>(data?: T): Buffer | undefined;

package/dist/typescript-4/dist/errors.d.ts

@@ -2,7 +2,7 @@ import { type NullableString } from "./protocol/definitions";
 declare const kGenericError: unique symbol;
 declare const kMultipleErrors: unique symbol;
 export declare const ERROR_PREFIX = "PLT_KFK_";
-export declare const errorCodes: readonly ["PLT_KFK_AUTHENTICATION", "PLT_KFK_MULTIPLE", "PLT_KFK_NETWORK", "PLT_KFK_OUT_OF_BOUNDS", "PLT_KFK_PROTOCOL", "PLT_KFK_RESPONSE", "PLT_KFK_TIMEOUT", "PLT_KFK_UNEXPECTED_CORRELATION_ID", "PLT_KFK_UNFINISHED_WRITE_BUFFER", "PLT_KFK_UNSUPPORTED_API", "PLT_KFK_UNSUPPORTED_COMPRESSION", "PLT_KFK_UNSUPPORTED", "PLT_KFK_USER"];
+export declare const errorCodes: readonly ["PLT_KFK_AUTHENTICATION", "PLT_KFK_MULTIPLE", "PLT_KFK_NETWORK", "PLT_KFK_OUT_OF_BOUNDS", "PLT_KFK_PROTOCOL", "PLT_KFK_RESPONSE", "PLT_KFK_TIMEOUT", "PLT_KFK_UNEXPECTED_CORRELATION_ID", "PLT_KFK_UNFINISHED_WRITE_BUFFER", "PLT_KFK_UNSUPPORTED_API", "PLT_KFK_UNSUPPORTED_COMPRESSION", "PLT_KFK_UNSUPPORTED_FORMAT", "PLT_KFK_UNSUPPORTED", "PLT_KFK_USER"];
 export type ErrorCode = (typeof errorCodes)[number];
 export type ErrorProperties = {
   cause?: Error;
@@ -73,6 +73,10 @@ export declare class UnsupportedCompressionError extends GenericError {
   static code: ErrorCode;
   constructor(message: string, properties?: ErrorProperties);
 }
+export declare class UnsupportedFormatError extends GenericError {
+  static code: ErrorCode;
+  constructor(message: string, properties?: ErrorProperties);
+}
 export declare class UnsupportedError extends GenericError {
   static code: ErrorCode;
   constructor(message: string, properties?: ErrorProperties);