@platformatic/kafka 1.27.0-alpha.1 → 1.27.0-alpha.2

package/README.md

@@ -10,6 +10,7 @@ A modern, high-performance, pure TypeScript/JavaScript type safe client for Apac
 - **Flexible API**: You can use promises or callbacks on all APIs.
 - **Streaming or Event-based Consumers**: Thanks to Node.js streams, you can choose your preferred consuming method.
 - **Flexible Serialisation**: Pluggable serialisers and deserialisers.
+- **Schema Registry Support**: Built-in Confluent Schema Registry integration with AVRO, Protobuf, and JSON Schema support.
 - **Connection Management**: Automatic connection pooling and recovery.
 - **Low Dependencies**: Minimal external dependencies.
 
@@ -141,6 +142,63 @@ await admin.deleteTopics({ topics: ['my-topic'] })
 await admin.close()
 ```
 
+### Schema Registry
+
+The library includes built-in support for Confluent Schema Registry with AVRO, Protocol Buffers, and JSON Schema:
+
+```typescript
+import { Producer, Consumer } from '@platformatic/kafka'
+import { ConfluentSchemaRegistry } from '@platformatic/kafka/registries'
+
+// Create a schema registry instance
+const registry = new ConfluentSchemaRegistry({
+  url: 'http://localhost:8081',
+  auth: {
+    username: 'user',
+    password: 'password'
+  }
+})
+
+// Producer with schema registry
+const producer = new Producer({
+  clientId: 'schema-producer',
+  bootstrapBrokers: ['localhost:9092'],
+  registry // Automatic serialization with schemas
+})
+
+// Send messages with schema IDs
+await producer.send({
+  messages: [{
+    topic: 'users',
+    value: { id: 1, name: 'Alice' },
+    metadata: {
+      schemas: {
+        value: 100 // Schema ID in the registry
+      }
+    }
+  }]
+})
+
+// Consumer with schema registry
+const consumer = new Consumer({
+  groupId: 'schema-consumers',
+  clientId: 'schema-consumer',
+  bootstrapBrokers: ['localhost:9092'],
+  registry // Automatic deserialization with schemas
+})
+
+const stream = await consumer.consume({
+  topics: ['users']
+})
+
+// Messages are automatically deserialized
+for await (const message of stream) {
+  console.log('User:', message.value) // Typed object
+}
+```
+
+For more details, see the [Confluent Schema Registry documentation](./docs/confluent-schema-registry.md).
+
 ## TLS and SASL
 
 See the relevant sections in the the [Base Client](./docs/base.md) page.

package/dist/clients/consumer/consumer.js

@@ -40,6 +40,15 @@ export class Consumer extends Base {
     constructor(options) {
         super({ ...defaultConsumerOptions, ...options });
         this[kValidateOptions](options, consumerOptionsValidator, '/options');
+        if (options.registry) {
+            if (options.beforeDeserialization) {
+                throw new UserError('/options/beforeDeserialization cannot be provided when /options/registry is provided.');
+            }
+            else if (options.deserializers) {
+                throw new UserError('/options/deserializers cannot be provided when /options/registry is provided.');
+            }
+            options.registry.getDeserializers();
+        }
         this.groupId = options.groupId;
         this.groupInstanceId = options.groupInstanceId ?? null;
         this.generationId = 0;
@@ -150,8 +159,22 @@ export class Consumer extends Base {
         }
         options.autocommit ??= this[kOptions].autocommit ?? true;
         options.maxBytes ??= this[kOptions].maxBytes;
-        options.deserializers = Object.assign({}, options.deserializers, this[kOptions].deserializers);
         options.highWaterMark ??= this[kOptions].highWaterMark;
+        options.registry ??= this[kOptions].registry;
+        options.beforeDeserialization ??= this[kOptions].beforeDeserialization;
+        if (options.registry) {
+            if (options.beforeDeserialization) {
+                throw new UserError('/options/beforeDeserialization cannot be provided when /options/registry is provided.');
+                /* c8 ignore next - Hard to test */
+            }
+            else if (options.deserializers || this[kOptions].deserializers) {
+                throw new UserError('/options/deserializers cannot be provided when /options/registry is provided.');
+            }
+            options.deserializers = options.registry.getDeserializers();
+        }
+        else {
+            options.deserializers = Object.assign({}, options.deserializers, this[kOptions].deserializers);
+        }
         this.#consume(options, callback);
         return callback[kCallbackPromise];
     }

package/dist/clients/consumer/messages-stream.js

@@ -4,6 +4,7 @@ import { ListOffsetTimestamps } from "../../apis/enumerations.js";
 import { consumerReceivesChannel, createDiagnosticContext, notifyCreation } from "../../diagnostic.js";
 import { UserError } from "../../errors.js";
 import { IS_CONTROL } from "../../protocol/records.js";
+import { runAsyncSeries } from "../../registries/abstract.js";
 import { kAutocommit, kInstance, kRefreshOffsetsAndFetch } from "../../symbols.js";
 import { kConnections, kCreateConnectionPool, kInspect, kPrometheus } from "../base/base.js";
 import { ensureMetric } from "../metrics.js";
@@ -56,6 +57,7 @@ export class MessagesStream extends Readable {
     #closeCallbacks;
     #metricsConsumedMessages;
     #corruptedMessageHandler;
+    #pushRecordsOperation;
     [kInstance];
     /*
       The following requests are blocking in Kafka:
@@ -74,7 +76,7 @@ export class MessagesStream extends Readable {
     */
     [kConnections];
     constructor(consumer, options) {
-        const { autocommit, mode, fallbackMode, maxFetches, offsets, deserializers, onCorruptedMessage, 
+        const { autocommit, mode, fallbackMode, maxFetches, offsets, deserializers, onCorruptedMessage, registry, beforeDeserialization, 
         // The options below are only destructured to avoid being part of structuredClone below
         partitionAssigner: _partitionAssigner, ...otherOptions } = options;
         if (offsets && mode !== MessagesStreamModes.MANUAL) {
@@ -101,8 +103,10 @@ export class MessagesStream extends Readable {
         this.#maxFetches = maxFetches ?? 0;
         this.#topics = structuredClone(options.topics);
         this.#inflightNodes = new Set();
-        this.#keyDeserializer = deserializers?.key ?? noopDeserializer;
-        this.#valueDeserializer = deserializers?.value ?? noopDeserializer;
+        this.#keyDeserializer =
+            deserializers?.key ?? noopDeserializer;
+        this.#valueDeserializer =
+            deserializers?.value ?? noopDeserializer;
         this.#headerKeyDeserializer = deserializers?.headerKey ?? noopDeserializer;
         this.#headerValueDeserializer = deserializers?.headerValue ?? noopDeserializer;
         this.#autocommitEnabled = !!options.autocommit;
@@ -110,6 +114,15 @@ export class MessagesStream extends Readable {
         this.#closed = false;
         this.#closeCallbacks = [];
         this.#corruptedMessageHandler = onCorruptedMessage ?? defaultCorruptedMessageHandler;
+        if (registry) {
+            this.#pushRecordsOperation = this.#beforeDeserialization.bind(this, registry.getBeforeDeserializationHook());
+        }
+        else if (beforeDeserialization) {
+            this.#pushRecordsOperation = this.#beforeDeserialization.bind(this, beforeDeserialization);
+        }
+        else {
+            this.#pushRecordsOperation = this.#pushRecords.bind(this);
+        }
         // Restore offsets
         this.#offsetsToFetch = new Map();
         if (offsets) {
@@ -362,10 +375,7 @@ export class MessagesStream extends Readable {
                         }
                         return;
                     }
-                    this.#pushRecords(metadata, topicIds, response, requestedOffsets);
-                    if (this.#maxFetches > 0 && ++this.#fetches >= this.#maxFetches) {
-                        this.push(null);
-                    }
+                    this.#pushRecordsOperation(metadata, topicIds, response, requestedOffsets);
                 });
             }
         });
@@ -419,6 +429,7 @@ export class MessagesStream extends Readable {
                     }
                     // Process messages
                     for (const record of batch.records) {
+                        const messageToConsume = { ...record, topic, partition };
                         const offset = batch.firstOffset + BigInt(record.offsetDelta);
                         if (offset < requestedOffsets.get(`${topic}:${partition}`)) {
                             // Thi is a duplicate message, ignore it
@@ -437,10 +448,10 @@ export class MessagesStream extends Readable {
                         try {
                             const headers = new Map();
                             for (const [headerKey, headerValue] of record.headers) {
-                                headers.set(headerKeyDeserializer(headerKey), headerValueDeserializer(headerValue));
+                                headers.set(headerKeyDeserializer(headerKey, messageToConsume), headerValueDeserializer(headerValue, messageToConsume));
                             }
-                            const key = keyDeserializer(record.key, headers);
-                            const value = valueDeserializer(record.value, headers);
+                            const key = keyDeserializer(record.key, headers, messageToConsume);
+                            const value = valueDeserializer(record.value, headers, messageToConsume);
                             this.#metricsConsumedMessages?.inc();
                             const message = {
                                 key,
@@ -483,6 +494,9 @@ export class MessagesStream extends Readable {
                 this.#fetch();
             });
         }
+        if (this.#maxFetches > 0 && ++this.#fetches >= this.#maxFetches) {
+            this.push(null);
+        }
     }
     #updateCommittedOffset(topic, partition, offset) {
         const key = `${topic}:${partition}`;
@@ -637,4 +651,46 @@ export class MessagesStream extends Readable {
     [kInspect](...args) {
         this.#consumer[kInspect](...args);
     }
+    #beforeDeserialization(hook, metadata, topicIds, response, requestedOffsets) {
+        const requests = [];
+        // Create the pre-deserialization requests
+        for (const topicResponse of response.responses) {
+            for (const { records: recordsBatches, partitionIndex: partition } of topicResponse.partitions) {
+                /* c8 ignore next 3 - Hard to test */
+                if (!recordsBatches) {
+                    continue;
+                }
+                for (const batch of recordsBatches) {
+                    // Filter control markers
+                    /* c8 ignore next 3 - Hard to test */
+                    if (batch.attributes & IS_CONTROL) {
+                        continue;
+                    }
+                    for (const message of batch.records) {
+                        message.topic = topicIds.get(topicResponse.topicId);
+                        message.partition = partition;
+                        requests.push([message.key, 'key', message]);
+                        requests.push([message.value, 'value', message]);
+                        for (const [headerKey, headerValue] of message.headers) {
+                            requests.push([headerKey, 'headerKey', message]);
+                            requests.push([headerValue, 'headerValue', message]);
+                        }
+                    }
+                }
+            }
+        }
+        runAsyncSeries((request, cb) => {
+            const [data, type, message] = request;
+            const result = hook(data, type, message, cb);
+            if (typeof result?.then === 'function') {
+                result.then(() => cb(null), cb);
+            }
+        }, requests, 0, error => {
+            if (error) {
+                this.destroy(error);
+                return;
+            }
+            this.#pushRecords(metadata, topicIds, response, requestedOffsets);
+        });
+    }
 }

package/dist/clients/consumer/options.d.ts

@@ -127,6 +127,12 @@ export declare const consumeOptionsProperties: {
         type: string;
         minimum: number;
     };
+    beforeDeserialization: {
+        function: boolean;
+    };
+    registry: {
+        type: string;
+    };
 };
 export declare const groupOptionsSchema: {
     type: string;
@@ -241,6 +247,12 @@ export declare const consumeOptionsSchema: {
             type: string;
             minimum: number;
         };
+        beforeDeserialization: {
+            function: boolean;
+        };
+        registry: {
+            type: string;
+        };
         groupInstanceId: {
             type: string;
             pattern: string;
@@ -396,6 +408,12 @@ export declare const consumerOptionsSchema: {
             type: string;
             minimum: number;
         };
+        beforeDeserialization: {
+            function: boolean;
+        };
+        registry: {
+            type: string;
+        };
         groupInstanceId: {
             type: string;
             pattern: string;
@@ -511,6 +529,12 @@ export declare const fetchOptionsSchema: {
             type: string;
             minimum: number;
         };
+        beforeDeserialization: {
+            function: boolean;
+        };
+        registry: {
+            type: string;
+        };
         groupInstanceId: {
             type: string;
             pattern: string;

package/dist/clients/consumer/options.js

@@ -61,7 +61,9 @@ export const consumeOptionsProperties = {
     maxWaitTime: { type: 'number', minimum: 0 },
     isolationLevel: { type: 'number', enum: allowedFetchIsolationLevels },
     deserializers: serdeProperties,
-    highWaterMark: { type: 'number', minimum: 1 }
+    highWaterMark: { type: 'number', minimum: 1 },
+    beforeDeserialization: { function: true },
+    registry: { type: 'object' }
 };
 export const groupOptionsSchema = {
     type: 'object',

package/dist/clients/consumer/types.d.ts

@@ -2,8 +2,9 @@ import { type FetchRequestTopic } from '../../apis/consumer/fetch-v17.ts';
 import { type GroupProtocols } from '../../apis/enumerations.ts';
 import { type ConnectionPool } from '../../network/connection-pool.ts';
 import { type KafkaRecord, type Message } from '../../protocol/records.ts';
+import { type SchemaRegistry } from '../../registries/abstract.ts';
 import { type BaseOptions, type ClusterMetadata, type TopicWithPartitionAndOffset } from '../base/types.ts';
-import { type Deserializers } from '../serde.ts';
+import { type BeforeDeserializationHook, type Deserializers } from '../serde.ts';
 export interface GroupProtocolSubscription {
     name: string;
     version: number;
@@ -68,6 +69,8 @@ export interface ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> {
     isolationLevel?: number;
     deserializers?: Partial<Deserializers<Key, Value, HeaderKey, HeaderValue>>;
     highWaterMark?: number;
+    beforeDeserialization?: BeforeDeserializationHook;
+    registry?: SchemaRegistry<unknown, unknown, Key, Value, HeaderKey, HeaderValue>;
 }
 export interface StreamOptions {
     topics: string[];

package/dist/clients/producer/options.d.ts

@@ -121,24 +121,8 @@ export declare const sendOptionsSchema: {
             items: {
                 type: string;
                 properties: {
-                    key: {
-                        oneOf: ({
-                            type: string;
-                            buffer?: undefined;
-                        } | {
-                            buffer: boolean;
-                            type?: undefined;
-                        })[];
-                    };
-                    value: {
-                        oneOf: ({
-                            type: string;
-                            buffer?: undefined;
-                        } | {
-                            buffer: boolean;
-                            type?: undefined;
-                        })[];
-                    };
+                    key: boolean;
+                    value: boolean;
                     headers: {
                         anyOf: ({
                             map: boolean;

package/dist/clients/producer/options.js

@@ -36,7 +36,9 @@ export const producerOptionsValidator = ajv.compile({
     type: 'object',
     properties: {
         ...produceOptionsProperties,
-        serializers: serdeProperties
+        serializers: serdeProperties,
+        beforeSerialization: { function: true },
+        registry: { type: 'object' }
     },
     additionalProperties: true
 });

package/dist/clients/producer/producer.js

@@ -4,6 +4,7 @@ import { FindCoordinatorKeyTypes, ProduceAcks } from "../../apis/enumerations.js
 import { createDiagnosticContext, producerInitIdempotentChannel, producerSendsChannel, producerTransactionsChannel } from "../../diagnostic.js";
 import { UserError } from "../../errors.js";
 import { murmur2 } from "../../protocol/murmur2.js";
+import { runAsyncSeries } from "../../registries/abstract.js";
 import { kInstance, kTransaction, kTransactionAddOffsets, kTransactionAddPartitions, kTransactionCancel, kTransactionCommitOffset, kTransactionEnd, kTransactionFindCoordinator, kTransactionPrepare } from "../../symbols.js";
 import { NumericMap } from "../../utils.js";
 import { Base, kAfterCreate, kCheckNotClosed, kClosed, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
@@ -28,6 +29,7 @@ export class Producer extends Base {
     #metricsProducedMessages;
     #coordinatorId;
     #transaction;
+    #sendOperation;
     constructor(options) {
         if (options.idempotent) {
             options.maxInflights = 1;
@@ -39,18 +41,37 @@ export class Producer extends Base {
         }
         options.repeatOnStaleMetadata ??= true;
         super(options);
+        this[kValidateOptions](options, producerOptionsValidator, '/options');
+        let serializers = options.serializers;
+        if (options.registry) {
+            if (options.beforeSerialization) {
+                throw new UserError('/options/beforeSerialization cannot be provided when /options/registry is provided.');
+            }
+            else if (options.serializers) {
+                throw new UserError('/options/serializers cannot be provided when /options/registry is provided.');
+            }
+            serializers = options.registry.getSerializers();
+        }
         this.#partitionsRoundRobin = new NumericMap();
         this.#sequences = new NumericMap();
-        this.#keySerializer = options.serializers?.key ?? noopSerializer;
-        this.#valueSerializer = options.serializers?.value ?? noopSerializer;
-        this.#headerKeySerializer = options.serializers?.headerKey ?? noopSerializer;
-        this.#headerValueSerializer = options.serializers?.headerValue ?? noopSerializer;
+        this.#keySerializer = serializers?.key ?? noopSerializer;
+        this.#valueSerializer = serializers?.value ?? noopSerializer;
+        this.#headerKeySerializer = serializers?.headerKey ?? noopSerializer;
+        this.#headerValueSerializer = serializers?.headerValue ?? noopSerializer;
         this[kOptions].transactionalId ??= randomUUID();
-        this[kValidateOptions](options, producerOptionsValidator, '/options');
         if (this[kPrometheus]) {
             ensureMetric(this[kPrometheus], 'Gauge', 'kafka_producers', 'Number of active Kafka producers').inc();
             this.#metricsProducedMessages = ensureMetric(this[kPrometheus], 'Counter', 'kafka_produced_messages', 'Number of produced Kafka messages');
         }
+        if (options.registry) {
+            this.#sendOperation = this.#beforeSerialization.bind(this, options.registry.getBeforeSerializationHook());
+        }
+        else if (options.beforeSerialization) {
+            this.#sendOperation = this.#beforeSerialization.bind(this, options.beforeSerialization);
+        }
+        else {
+            this.#sendOperation = this.#send.bind(this);
+        }
         this[kAfterCreate]('producer');
     }
     get producerId() {
@@ -132,7 +153,7 @@ export class Producer extends Base {
             }
         }
         options.acks ??= idempotent ? ProduceAcks.ALL : ProduceAcks.LEADER;
-        producerSendsChannel.traceCallback(this.#send, 1, createDiagnosticContext({ client: this, operation: 'send', options }), this, options, callback);
+        producerSendsChannel.traceCallback(this.#sendOperation, 1, createDiagnosticContext({ client: this, operation: 'send', options }), this, options, callback);
         return callback[kCallbackPromise];
     }
     beginTransaction(options, callback) {
@@ -430,19 +451,28 @@ export class Producer extends Base {
         const messages = [];
         for (const message of options.messages) {
             const topic = message.topic;
+            let key;
+            let value;
             let headers = new Map();
             const serializedHeaders = new Map();
-            if (message.headers) {
-                headers =
-                    message.headers instanceof Map
-                        ? message.headers
-                        : new Map(Object.entries(message.headers));
-                for (const [key, value] of headers) {
-                    serializedHeaders.set(this.#headerKeySerializer(key), this.#headerValueSerializer(value));
+            const metadata = message.metadata;
+            try {
+                if (message.headers) {
+                    headers =
+                        message.headers instanceof Map
+                            ? message.headers
+                            : new Map(Object.entries(message.headers));
+                    for (const [key, value] of headers) {
+                        serializedHeaders.set(this.#headerKeySerializer(key, metadata), this.#headerValueSerializer(value, metadata));
+                    }
                 }
+                key = this.#keySerializer(message.key, headers, message);
+                value = this.#valueSerializer(message.value, headers, message);
+            }
+            catch (error) {
+                callback(new UserError('Failed to serialize a message.', { cause: error }));
+                return;
             }
-            const key = this.#keySerializer(message.key, headers);
-            const value = this.#valueSerializer(message.value, headers);
             let partition = 0;
             if (typeof message.partition !== 'number') {
                 if (partitioner) {
@@ -622,4 +652,34 @@ export class Producer extends Base {
             this.#transaction = undefined;
         }
     }
+    #beforeSerialization(hook, options, callback) {
+        // Create the pre-serialization requests
+        const requests = [];
+        for (const message of options.messages) {
+            requests.push([message.key, 'key', message]);
+            requests.push([message.value, 'value', message]);
+            if (typeof message.headers !== 'undefined') {
+                const headers = message.headers instanceof Map
+                    ? message.headers
+                    : new Map(Object.entries(message.headers));
+                for (const [headerKey, headerValue] of headers) {
+                    requests.push([headerKey, 'headerKey', message]);
+                    requests.push([headerValue, 'headerValue', message]);
+                }
+            }
+        }
+        runAsyncSeries((request, cb) => {
+            const [data, type, message] = request;
+            const result = hook(data, type, message, cb);
+            if (typeof result?.then === 'function') {
+                result.then(() => cb(null), cb);
+            }
+        }, requests, 0, error => {
+            if (error) {
+                callback(error);
+                return;
+            }
+            this.#send(options, callback);
+        });
+    }
 }

package/dist/clients/producer/types.d.ts

@@ -1,7 +1,8 @@
 import { type CompressionAlgorithmValue } from '../../protocol/compression.ts';
 import { type MessageToProduce } from '../../protocol/records.ts';
+import { type SchemaRegistry } from '../../registries/abstract.ts';
 import { type BaseOptions, type TopicWithPartitionAndOffset } from '../base/types.ts';
-import { type Serializers } from '../serde.ts';
+import { type BeforeSerializationHook, type Serializers } from '../serde.ts';
 export interface ProducerInfo {
     producerId: bigint;
     producerEpoch: number;
@@ -25,6 +26,8 @@ export interface ProduceOptions<Key, Value, HeaderKey, HeaderValue> {
 export type ProducerOptions<Key, Value, HeaderKey, HeaderValue> = BaseOptions & ProduceOptions<Key, Value, HeaderKey, HeaderValue> & {
     transactionalId?: string;
     serializers?: Partial<Serializers<Key, Value, HeaderKey, HeaderValue>>;
+    beforeSerialization?: BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;
+    registry?: SchemaRegistry<unknown, unknown, Key, Value, HeaderKey, HeaderValue>;
 };
 export type SendOptions<Key, Value, HeaderKey, HeaderValue> = {
     messages: MessageToProduce<Key, Value, HeaderKey, HeaderValue>[];

package/dist/clients/serde.d.ts

@@ -1,7 +1,9 @@
-export type Serializer<InputType = unknown> = (data?: InputType) => Buffer | undefined;
-export type Deserializer<OutputType = unknown> = (data?: Buffer) => OutputType | undefined;
-export type SerializerWithHeaders<InputType = unknown, HeaderKey = unknown, HeaderValue = unknown> = (data?: InputType, headers?: Map<HeaderKey, HeaderValue>) => Buffer | undefined;
-export type DeserializerWithHeaders<OutputType = unknown, HeaderKey = unknown, HeaderValue = unknown> = (data?: Buffer, headers?: Map<HeaderKey, HeaderValue>) => OutputType | undefined;
+import { type Callback } from '../apis/definitions.ts';
+import { type MessageToConsume, type MessageToProduce } from '../protocol/records.ts';
+export type Serializer<InputType = unknown> = (data?: InputType, metadata?: unknown) => Buffer | undefined;
+export type SerializerWithHeaders<InputType = unknown, HeaderKey = unknown, HeaderValue = unknown> = (data?: InputType, headers?: Map<HeaderKey, HeaderValue>, message?: MessageToProduce<unknown, unknown, unknown, unknown>) => Buffer | undefined;
+export type Deserializer<OutputType = unknown> = (data?: Buffer, message?: MessageToConsume) => OutputType | undefined;
+export type DeserializerWithHeaders<OutputType = unknown, HeaderKey = unknown, HeaderValue = unknown> = (data?: Buffer, headers?: Map<HeaderKey, HeaderValue>, message?: MessageToConsume) => OutputType | undefined;
 export interface Serializers<Key, Value, HeaderKey, HeaderValue> {
     key: SerializerWithHeaders<Key, HeaderKey, HeaderValue>;
     value: SerializerWithHeaders<Value, HeaderKey, HeaderValue>;
@@ -9,11 +11,14 @@ export interface Serializers<Key, Value, HeaderKey, HeaderValue> {
     headerValue: Serializer<HeaderValue>;
 }
 export interface Deserializers<Key, Value, HeaderKey, HeaderValue> {
-    key: DeserializerWithHeaders<Key>;
-    value: DeserializerWithHeaders<Value>;
+    key: DeserializerWithHeaders<Key, HeaderKey, HeaderValue>;
+    value: DeserializerWithHeaders<Value, HeaderKey, HeaderValue>;
     headerKey: Deserializer<HeaderKey>;
     headerValue: Deserializer<HeaderValue>;
 }
+export type BeforeHookPayloadType = 'key' | 'value' | 'headerKey' | 'headerValue';
+export type BeforeDeserializationHook = (payload: Buffer, type: BeforeHookPayloadType, message: MessageToConsume, callback: Callback<void>) => void | Promise<void>;
+export type BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue> = (payload: unknown, type: BeforeHookPayloadType, message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, callback: Callback<void>) => void | Promise<void>;
 export declare function stringSerializer(data?: string): Buffer | undefined;
 export declare function stringDeserializer(data?: string | Buffer): string | undefined;
 export declare function jsonSerializer<T = Record<string, any>>(data?: T): Buffer | undefined;

package/dist/errors.d.ts

@@ -2,7 +2,7 @@ import { type NullableString } from './protocol/definitions.ts';
 declare const kGenericError: unique symbol;
 declare const kMultipleErrors: unique symbol;
 export declare const ERROR_PREFIX = "PLT_KFK_";
-export declare const errorCodes: readonly ["PLT_KFK_AUTHENTICATION", "PLT_KFK_MULTIPLE", "PLT_KFK_NETWORK", "PLT_KFK_OUT_OF_BOUNDS", "PLT_KFK_PROTOCOL", "PLT_KFK_RESPONSE", "PLT_KFK_TIMEOUT", "PLT_KFK_UNEXPECTED_CORRELATION_ID", "PLT_KFK_UNFINISHED_WRITE_BUFFER", "PLT_KFK_UNSUPPORTED_API", "PLT_KFK_UNSUPPORTED_COMPRESSION", "PLT_KFK_UNSUPPORTED", "PLT_KFK_USER"];
+export declare const errorCodes: readonly ["PLT_KFK_AUTHENTICATION", "PLT_KFK_MULTIPLE", "PLT_KFK_NETWORK", "PLT_KFK_OUT_OF_BOUNDS", "PLT_KFK_PROTOCOL", "PLT_KFK_RESPONSE", "PLT_KFK_TIMEOUT", "PLT_KFK_UNEXPECTED_CORRELATION_ID", "PLT_KFK_UNFINISHED_WRITE_BUFFER", "PLT_KFK_UNSUPPORTED_API", "PLT_KFK_UNSUPPORTED_COMPRESSION", "PLT_KFK_UNSUPPORTED_FORMAT", "PLT_KFK_UNSUPPORTED", "PLT_KFK_USER"];
 export type ErrorCode = (typeof errorCodes)[number];
 export type ErrorProperties = {
     cause?: Error;
@@ -67,6 +67,10 @@ export declare class UnsupportedCompressionError extends GenericError {
     static code: ErrorCode;
     constructor(message: string, properties?: ErrorProperties);
 }
+export declare class UnsupportedFormatError extends GenericError {
+    static code: ErrorCode;
+    constructor(message: string, properties?: ErrorProperties);
+}
 export declare class UnsupportedError extends GenericError {
     static code: ErrorCode;
     constructor(message: string, properties?: ErrorProperties);

package/dist/errors.js

@@ -15,6 +15,7 @@ export const errorCodes = [
     'PLT_KFK_UNFINISHED_WRITE_BUFFER',
     'PLT_KFK_UNSUPPORTED_API',
     'PLT_KFK_UNSUPPORTED_COMPRESSION',
+    'PLT_KFK_UNSUPPORTED_FORMAT',
     'PLT_KFK_UNSUPPORTED',
     'PLT_KFK_USER'
 ];
@@ -166,6 +167,13 @@ export class UnsupportedCompressionError extends GenericError {
         super(UnsupportedCompressionError.code, message, { canRetry: false, ...properties });
     }
 }
+/* c8 ignore next 7 - Format libraries are always available in tests */
+export class UnsupportedFormatError extends GenericError {
+    static code = 'PLT_KFK_UNSUPPORTED_FORMAT';
+    constructor(message, properties = {}) {
+        super(UnsupportedFormatError.code, message, { canRetry: false, ...properties });
+    }
+}
 export class UnsupportedError extends GenericError {
     static code = 'PLT_KFK_UNSUPPORTED';
     constructor(message, properties = {}) {

package/dist/index.d.ts

@@ -6,3 +6,4 @@ export * from './network/index.ts';
 export * from './protocol/index.ts';
 export * from './apis/index.ts';
 export * from './clients/index.ts';
+export * from './registries/index.ts';

package/dist/index.js

@@ -10,3 +10,5 @@ export * from "./protocol/index.js";
 export * from "./apis/index.js";
 // Clients
 export * from "./clients/index.js";
+// Registries
+export * from "./registries/index.js";

package/dist/network/connection.d.ts

@@ -20,18 +20,19 @@ export interface ConnectionEvents extends TypedEvents {
     'sasl:authentication:extended': (authBytes?: Buffer) => void;
     drain: () => void;
 }
-export type SASLCredentialProvider<T = string> = () => T | Promise<T>;
+export type CredentialProvider<T = string> = () => T | Promise<T>;
+export type SASLCredentialProvider<T> = CredentialProvider<T>;
 export interface Broker {
     host: string;
     port: number;
 }
-export type SASLCustomAuthenticator = (mechanism: SASLMechanismValue, connection: Connection, authenticate: SASLAuthenticationAPI, usernameProvider: string | SASLCredentialProvider | undefined, passwordProvider: string | SASLCredentialProvider | undefined, tokenProvider: string | SASLCredentialProvider | undefined, callback: CallbackWithPromise<SaslAuthenticateResponse>) => void;
+export type SASLCustomAuthenticator = (mechanism: SASLMechanismValue, connection: Connection, authenticate: SASLAuthenticationAPI, usernameProvider: string | CredentialProvider | undefined, passwordProvider: string | CredentialProvider | undefined, tokenProvider: string | CredentialProvider | undefined, callback: CallbackWithPromise<SaslAuthenticateResponse>) => void;
 export interface SASLOptions {
     mechanism: SASLMechanismValue;
-    username?: string | SASLCredentialProvider;
-    password?: string | SASLCredentialProvider;
-    token?: string | SASLCredentialProvider;
-    oauthBearerExtensions?: Record<string, string> | SASLCredentialProvider<Record<string, string>>;
+    username?: string | CredentialProvider;
+    password?: string | CredentialProvider;
+    token?: string | CredentialProvider;
+    oauthBearerExtensions?: Record<string, string> | CredentialProvider<Record<string, string>>;
     authenticate?: SASLCustomAuthenticator;
     authBytesValidator?: (authBytes: Buffer, callback: CallbackWithPromise<Buffer>) => void;
 }