@platformatic/kafka 1.27.0 → 1.28.0

package/README.md

@@ -51,6 +51,7 @@ await producer.send({
 })
 
 // Close the producer when done
+// If you created producer streams with producer.asStream(), either close them first or use producer.close(true)
 await producer.close()
 ```
 

package/dist/clients/consumer/messages-stream.js

@@ -37,6 +37,9 @@ export class MessagesStream extends Readable {
     #mode;
     #fallbackMode;
     #paused;
+    #refreshOffsetsInflight;
+    #refreshOffsetsPending;
+    #refreshOffsetsDestroyOnError;
     #fetches;
     #maxFetches;
     #options;
@@ -99,6 +102,9 @@ export class MessagesStream extends Readable {
         this.#offsetsCommitted = new Map();
         this.#partitionsEpochs = new Map();
         this.#paused = false;
+        this.#refreshOffsetsInflight = false;
+        this.#refreshOffsetsPending = false;
+        this.#refreshOffsetsDestroyOnError = false;
         this.#fetches = 0;
         this.#maxFetches = maxFetches ?? 0;
         this.#topics = structuredClone(options.topics);
@@ -144,14 +150,7 @@ export class MessagesStream extends Readable {
         // having some.
         this.#consumer.on('consumer:group:join', () => {
             this.#offsetsCommitted.clear();
-            this.#refreshOffsets(error => {
-                /* c8 ignore next 4 - Hard to test */
-                if (error) {
-                    this.destroy(error);
-                    return;
-                }
-                this.#fetch();
-            });
+            this.#scheduleRefreshOffsetsAndFetch();
         });
         if (consumer[kPrometheus]) {
             this.#metricsConsumedMessages = ensureMetric(consumer[kPrometheus], 'Counter', 'kafka_consumed_messages', 'Number of consumed Kafka messages');
@@ -291,7 +290,7 @@ export class MessagesStream extends Readable {
             return;
         }
         // No need to fetch if nobody is consuming the data
-        if (this.readableFlowing === null || this.#paused) {
+        if (this.readableFlowing === null || this.#paused || this.#refreshOffsetsInflight || this.#refreshOffsetsPending) {
             return;
         }
         this.#consumer.metadata({ topics: this.#consumer.topics.current }, (error, metadata) => {
@@ -448,8 +447,11 @@ export class MessagesStream extends Readable {
                         try {
                             const headers = new Map();
                             for (const [headerKey, headerValue] of record.headers) {
-                                headers.set(headerKeyDeserializer(headerKey ?? undefined, messageToConsume), headerValueDeserializer(headerValue ?? undefined, messageToConsume));
+                                headers.set(
+                                /* c8 ignore next 2 - Hard to test */
+                                headerKeyDeserializer(headerKey ?? undefined, messageToConsume), headerValueDeserializer(headerValue ?? undefined, messageToConsume));
                             }
+                            /* c8 ignore next 2 - Hard to test */
                             const key = keyDeserializer(record.key ?? undefined, headers, messageToConsume);
                             const value = valueDeserializer(record.value ?? undefined, headers, messageToConsume);
                             this.#metricsConsumedMessages?.inc();
@@ -595,11 +597,37 @@ export class MessagesStream extends Readable {
             });
         });
     }
-    [kRefreshOffsetsAndFetch]() {
-        this.#refreshOffsets(() => {
+    #scheduleRefreshOffsetsAndFetch(destroyOnError = true) {
+        this.#refreshOffsetsDestroyOnError ||= destroyOnError;
+        if (this.#refreshOffsetsInflight) {
+            this.#refreshOffsetsPending = true;
+            return;
+        }
+        this.#refreshOffsetsInflight = true;
+        this.#refreshOffsets(error => {
+            const shouldDestroyOnError = this.#refreshOffsetsDestroyOnError;
+            this.#refreshOffsetsInflight = false;
+            this.#refreshOffsetsDestroyOnError = false;
+            this.#refreshOffsetsPending = false;
+            /* c8 ignore next 9 - Hard to test */
+            if (error) {
+                if (shouldDestroyOnError) {
+                    this.destroy(error);
+                }
+                return;
+            }
+            // A new one was scheduled while the previous one was inflight, we need to run it immediately
+            /* c8 ignore next 4 - Hard to test */
+            if (this.#refreshOffsetsPending) {
+                this.#scheduleRefreshOffsetsAndFetch(shouldDestroyOnError);
+                return;
+            }
             this.#fetch();
         });
     }
+    [kRefreshOffsetsAndFetch]() {
+        this.#scheduleRefreshOffsetsAndFetch(false);
+    }
     #assignOffsets(offsets, commits, callback) {
         for (const [topic, partitions] of offsets) {
             for (let i = 0; i < partitions.length; i++) {

package/dist/clients/producer/index.d.ts

@@ -1,3 +1,4 @@
 export * from './options.ts';
+export * from './producer-stream.ts';
 export * from './producer.ts';
 export * from './types.ts';

package/dist/clients/producer/index.js

@@ -1,3 +1,4 @@
 export * from "./options.js";
+export * from "./producer-stream.js";
 export * from "./producer.js";
 export * from "./types.js";

package/dist/clients/producer/options.d.ts

@@ -155,3 +155,66 @@ export declare const sendOptionsSchema: {
 export declare const sendOptionsValidator: import("ajv").ValidateFunction<{
     [x: string]: {};
 }>;
+export declare const defaultProducerStreamOptions: {
+    batchSize: number;
+    batchTime: number;
+    reportMode: "none";
+};
+export declare const producerStreamOptionsSchema: {
+    type: string;
+    properties: {
+        producerId: {
+            bigint: boolean;
+        };
+        producerEpoch: {
+            type: string;
+        };
+        idempotent: {
+            type: string;
+        };
+        transactionalId: {
+            type: string;
+        };
+        acks: {
+            type: string;
+            enumeration: {
+                allowed: (0 | 1 | -1)[];
+                errorMessage: string;
+            };
+        };
+        compression: {
+            type: string;
+            enumeration: {
+                allowed: import("../../protocol/compression.ts").CompressionAlgorithmValue[];
+                errorMessage: string;
+            };
+        };
+        partitioner: {
+            function: boolean;
+        };
+        autocreateTopics: {
+            type: string;
+        };
+        repeatOnStaleMetadata: {
+            type: string;
+        };
+        highWaterMark: {
+            type: string;
+            minimum: number;
+        };
+        batchSize: {
+            type: string;
+            minimum: number;
+        };
+        batchTime: {
+            type: string;
+            minimum: number;
+        };
+        reportMode: {
+            type: string;
+            enum: import("./types.ts").ProducerStreamReportModeValue[];
+        };
+    };
+    additionalProperties: boolean;
+};
+export declare const producerStreamOptionsValidator: import("ajv").ValidateFunction<unknown>;

package/dist/clients/producer/options.js

@@ -3,6 +3,7 @@ import { allowedCompressionsAlgorithms, compressionsAlgorithms } from "../../pro
 import { messageSchema } from "../../protocol/records.js";
 import { ajv, enumErrorMessage } from "../../utils.js";
 import { serdeProperties } from "../serde.js";
+import { allowedProducerStreamReportModes, ProducerStreamReportModes } from "./types.js";
 export const produceOptionsProperties = {
     producerId: { bigint: true },
     producerEpoch: { type: 'number' },
@@ -52,3 +53,20 @@ export const sendOptionsSchema = {
     additionalProperties: false
 };
 export const sendOptionsValidator = ajv.compile(sendOptionsSchema);
+export const defaultProducerStreamOptions = {
+    batchSize: 100,
+    batchTime: 10,
+    reportMode: ProducerStreamReportModes.NONE
+};
+export const producerStreamOptionsSchema = {
+    type: 'object',
+    properties: {
+        highWaterMark: { type: 'integer', minimum: 1 },
+        batchSize: { type: 'integer', minimum: 1 },
+        batchTime: { type: 'integer', minimum: 0 },
+        reportMode: { type: 'string', enum: allowedProducerStreamReportModes },
+        ...produceOptionsProperties
+    },
+    additionalProperties: false
+};
+export const producerStreamOptionsValidator = ajv.compile(producerStreamOptionsSchema);

package/dist/clients/producer/producer-stream.d.ts

@@ -0,0 +1,19 @@
+import { Writable } from 'node:stream';
+import { type CallbackWithPromise } from '../../apis/callbacks.ts';
+import type { MessageToProduce } from '../../protocol/records.ts';
+import type { Producer } from './producer.ts';
+import { type ProducerStreamOptions } from './types.ts';
+export declare class ProducerStream<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends Writable {
+    #private;
+    instance: number;
+    constructor(producer: Producer<Key, Value, HeaderKey, HeaderValue>, options: ProducerStreamOptions<Key, Value, HeaderKey, HeaderValue>);
+    get producer(): Producer<Key, Value, HeaderKey, HeaderValue>;
+    close(callback: CallbackWithPromise<void>): void;
+    close(): Promise<void>;
+    _write(message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, _: BufferEncoding, callback: (error?: Error | null) => void): void;
+    _writev(chunks: Array<{
+        chunk: MessageToProduce<Key, Value, HeaderKey, HeaderValue>;
+    }>, callback: (error?: Error | null) => void): void;
+    _final(callback: (error?: Error | null) => void): void;
+    _destroy(error: Error | null, callback: (error?: Error | null) => void): void;
+}

package/dist/clients/producer/producer-stream.js

@@ -0,0 +1,187 @@
+import { Writable } from 'node:stream';
+import { createPromisifiedCallback, kCallbackPromise } from "../../apis/callbacks.js";
+import { notifyCreation } from "../../diagnostic.js";
+import { defaultProducerStreamOptions } from "./options.js";
+import { ProducerStreamReportModes } from "./types.js";
+let currentInstance = 0;
+export class ProducerStream extends Writable {
+    #producer;
+    #batchSize;
+    #batchTime;
+    #reportMode;
+    #produceOptions;
+    #buffer;
+    #timer;
+    #flushing;
+    #batchId;
+    #pendingWriteCallback;
+    instance;
+    constructor(producer, options) {
+        super({
+            objectMode: true,
+            highWaterMark: options.highWaterMark
+        });
+        const { batchSize, batchTime, reportMode, highWaterMark: _highWaterMark, ...sendOptions } = options;
+        this.#producer = producer;
+        this.#batchSize = batchSize ?? defaultProducerStreamOptions.batchSize;
+        this.#batchTime = batchTime ?? defaultProducerStreamOptions.batchTime;
+        this.#reportMode = reportMode ?? defaultProducerStreamOptions.reportMode;
+        this.#produceOptions = sendOptions;
+        this.#buffer = [];
+        this.#timer = null;
+        this.#flushing = false;
+        this.#batchId = 0;
+        this.#pendingWriteCallback = null;
+        this.instance = currentInstance++;
+        notifyCreation('producer-stream', this);
+    }
+    get producer() {
+        return this.#producer;
+    }
+    close(callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        this.end((error) => callback(error ?? null));
+        return callback[kCallbackPromise];
+    }
+    _write(message, _, callback) {
+        this.#buffer.push(message);
+        const finalizeAfterFlush = this.#buffer.length >= this.writableHighWaterMark;
+        if (this.#buffer.length >= this.#batchSize) {
+            this.#clearTimer();
+            this.#flush();
+        }
+        else {
+            this.#scheduleFlush();
+        }
+        if (finalizeAfterFlush) {
+            this.#pendingWriteCallback = callback;
+            return;
+        }
+        callback(null);
+    }
+    _writev(chunks, callback) {
+        for (const { chunk } of chunks) {
+            this.#buffer.push(chunk);
+        }
+        const finalizeAfterFlush = this.#buffer.length >= this.writableHighWaterMark;
+        if (this.#buffer.length >= this.#batchSize) {
+            this.#clearTimer();
+            this.#flush();
+        }
+        else {
+            this.#scheduleFlush();
+        }
+        if (finalizeAfterFlush) {
+            this.#pendingWriteCallback = callback;
+            return;
+        }
+        callback(null);
+    }
+    _final(callback) {
+        this.#clearTimer();
+        if (!this.#buffer.length && !this.#flushing) {
+            callback(null);
+            return;
+        }
+        const self = this;
+        function onError(error) {
+            self.removeListener('error', onError);
+            self.removeListener('flush', onFlush);
+            callback(error);
+        }
+        function onFlush() {
+            if (self.#buffer.length || self.#flushing) {
+                return;
+            }
+            self.removeListener('error', onError);
+            self.removeListener('flush', onFlush);
+            callback(null);
+        }
+        this.on('error', onError);
+        this.on('flush', onFlush);
+        this.#flush();
+    }
+    _destroy(error, callback) {
+        this.#clearTimer();
+        this.#buffer = [];
+        this.#flushWriteCallbacks(error);
+        callback(error);
+    }
+    #scheduleFlush() {
+        if (this.#timer || this.#batchTime < 0) {
+            return;
+        }
+        this.#timer = setTimeout(() => {
+            this.#timer = null;
+            this.#flush();
+        }, this.#batchTime);
+    }
+    #clearTimer() {
+        if (this.#timer) {
+            clearTimeout(this.#timer);
+            this.#timer = null;
+        }
+    }
+    #flush() {
+        if (this.#flushing || this.destroyed) {
+            return;
+        }
+        /* c8 ignore next 3 - Hard to test */
+        if (this.#buffer.length === 0) {
+            return;
+        }
+        this.#flushing = true;
+        const pending = this.#buffer.splice(0, this.#batchSize);
+        const batchId = this.#batchId++;
+        const startedAt = Date.now();
+        this.#producer.send({ messages: pending, ...this.#produceOptions }, (error, result) => {
+            this.#flushing = false;
+            if (error) {
+                this.#flushWriteCallbacks(error);
+                this.destroy(error);
+                return;
+            }
+            this.#emitReports(this.#reportMode, batchId, pending, result);
+            this.emit('flush', { batchId, count: pending.length, duration: Date.now() - startedAt, result: result });
+            if (this.#buffer.length >= this.#batchSize) {
+                this.#flush();
+            }
+            else if (this.#buffer.length > 0) {
+                this.#scheduleFlush();
+            }
+            this.#flushWriteCallbacks();
+        });
+    }
+    #flushWriteCallbacks(error) {
+        if (!this.#pendingWriteCallback || this.#buffer.length >= this.writableHighWaterMark) {
+            return;
+        }
+        const callback = this.#pendingWriteCallback;
+        this.#pendingWriteCallback = null;
+        callback(error ?? null);
+    }
+    #emitReports(mode, batchId, pending, result) {
+        if (mode === ProducerStreamReportModes.NONE) {
+            return;
+        }
+        if (mode === ProducerStreamReportModes.BATCH) {
+            const report = {
+                batchId,
+                count: pending.length,
+                result
+            };
+            this.emit('delivery-report', report);
+            return;
+        }
+        for (let i = 0; i < pending.length; i++) {
+            const report = {
+                batchId,
+                index: i,
+                message: pending[i]
+            };
+            this.emit('delivery-report', report);
+        }
+    }
+}

package/dist/clients/producer/producer.d.ts

@@ -2,8 +2,9 @@ import { type CallbackWithPromise } from '../../apis/callbacks.ts';
 import { type Message } from '../../protocol/records.ts';
 import { kTransaction, kTransactionAddOffsets, kTransactionAddPartitions, kTransactionCancel, kTransactionCommitOffset, kTransactionEnd, kTransactionFindCoordinator } from '../../symbols.ts';
 import { Base } from '../base/base.ts';
+import { ProducerStream } from './producer-stream.ts';
 import { Transaction } from './transaction.ts';
-import { type ProduceOptions, type ProduceResult, type ProducerInfo, type ProducerOptions, type SendOptions } from './types.ts';
+import { type ProduceOptions, type ProduceResult, type ProducerInfo, type ProducerOptions, type ProducerStreamOptions, type SendOptions } from './types.ts';
 export declare function noopSerializer(data?: Buffer): Buffer | undefined;
 export declare class Producer<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends Base<ProducerOptions<Key, Value, HeaderKey, HeaderValue>> {
     #private;
@@ -12,8 +13,9 @@ export declare class Producer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
     get producerEpoch(): number | undefined;
     get transaction(): Transaction<Key, Value, HeaderKey, HeaderValue> | undefined;
     get coordinatorId(): number;
-    close(callback: CallbackWithPromise<void>): void;
-    close(): Promise<void>;
+    get streamsCount(): number;
+    close(force: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
+    close(force?: boolean): Promise<void>;
     initIdempotentProducer(options: ProduceOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<ProducerInfo>): void;
     initIdempotentProducer(options: ProduceOptions<Key, Value, HeaderKey, HeaderValue>): Promise<ProducerInfo>;
     send(options: SendOptions<Key, Value, HeaderKey, HeaderValue> & {
@@ -22,6 +24,7 @@ export declare class Producer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
     send(options: SendOptions<Key, Value, HeaderKey, HeaderValue> & {
         [kTransaction]?: number;
     }): Promise<ProduceResult>;
+    asStream(options?: ProducerStreamOptions<Key, Value, HeaderKey, HeaderValue>): ProducerStream<Key, Value, HeaderKey, HeaderValue>;
     beginTransaction(options: ProduceOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<Transaction<Key, Value, HeaderKey, HeaderValue>>): void;
     beginTransaction(options?: ProduceOptions<Key, Value, HeaderKey, HeaderValue>): Promise<Transaction<Key, Value, HeaderKey, HeaderValue>>;
     [kTransactionFindCoordinator](callback: CallbackWithPromise<void>): void;

package/dist/clients/producer/producer.js

@@ -9,7 +9,8 @@ import { kInstance, kTransaction, kTransactionAddOffsets, kTransactionAddPartiti
 import { emitExperimentalApiWarning, NumericMap } from "../../utils.js";
 import { Base, kAfterCreate, kCheckNotClosed, kClosed, kGetApi, kGetBootstrapConnection, kGetConnection, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kPrometheus, kValidateOptions } from "../base/base.js";
 import { ensureMetric } from "../metrics.js";
-import { produceOptionsValidator, producerOptionsValidator, sendOptionsValidator } from "./options.js";
+import { produceOptionsValidator, producerOptionsValidator, producerStreamOptionsValidator, sendOptionsValidator } from "./options.js";
+import { ProducerStream } from "./producer-stream.js";
 import { Transaction } from "./transaction.js";
 // Don't move this function as being in the same file will enable V8 to remove.
 // For futher info, ask Matteo.
@@ -29,6 +30,7 @@ export class Producer extends Base {
     #metricsProducedMessages;
     #coordinatorId;
     #transaction;
+    #streams;
     #sendOperation;
     constructor(options) {
         if (options.idempotent) {
@@ -64,6 +66,7 @@ export class Producer extends Base {
         this.#valueSerializer = serializers?.value ?? noopSerializer;
         this.#headerKeySerializer = serializers?.headerKey ?? noopSerializer;
         this.#headerValueSerializer = serializers?.headerValue ?? noopSerializer;
+        this.#streams = new Set();
         this[kOptions].transactionalId ??= randomUUID();
         if (this[kPrometheus]) {
             ensureMetric(this[kPrometheus], 'Gauge', 'kafka_producers', 'Number of active Kafka producers').inc();
@@ -92,7 +95,14 @@ export class Producer extends Base {
     get coordinatorId() {
         return this.#coordinatorId;
     }
-    close(callback) {
+    get streamsCount() {
+        return this.#streams.size;
+    }
+    close(force, callback) {
+        if (typeof force === 'function') {
+            callback = force;
+            force = false;
+        }
         if (!callback) {
             callback = createPromisifiedCallback();
         }
@@ -101,6 +111,36 @@ export class Producer extends Base {
             return callback[kCallbackPromise];
         }
         this[kClosed] = true;
+        this.#performClose(force, callback);
+        return callback[kCallbackPromise];
+    }
+    #performClose(force, callback) {
+        for (const stream of this.#streams) {
+            /* c8 ignore next 3 - Hard to test */
+            if (stream.closed || stream.destroyed) {
+                this.#streams.delete(stream);
+            }
+        }
+        if (this.#streams.size) {
+            if (!force) {
+                this[kClosed] = false;
+                callback(new UserError('Cannot close producer while producing messages.'));
+                return;
+            }
+            runConcurrentCallbacks('Closing streams failed.', this.#streams, (stream, concurrentCallback) => {
+                stream.close(concurrentCallback);
+            }, error => {
+                /* c8 ignore next 5 - Hard to test */
+                if (error) {
+                    this[kClosed] = false;
+                    callback(error);
+                    return;
+                }
+                this.#streams.clear();
+                this.#performClose(false, callback);
+            });
+            return;
+        }
         super.close(error => {
             if (error) {
                 this[kClosed] = false;
@@ -112,7 +152,6 @@ export class Producer extends Base {
             }
             callback(null);
         });
-        return callback[kCallbackPromise];
     }
     initIdempotentProducer(options, callback) {
         if (!callback) {
@@ -162,6 +201,19 @@ export class Producer extends Base {
         producerSendsChannel.traceCallback(this.#sendOperation, 1, createDiagnosticContext({ client: this, operation: 'send', options }), this, options, callback);
         return callback[kCallbackPromise];
     }
+    asStream(options) {
+        options ??= {};
+        const validationError = this[kValidateOptions](options, producerStreamOptionsValidator, '/options', false);
+        if (validationError) {
+            throw validationError;
+        }
+        const stream = new ProducerStream(this, options);
+        this.#streams.add(stream);
+        stream.once('close', () => {
+            this.#streams.delete(stream);
+        });
+        return stream;
+    }
     beginTransaction(options, callback) {
         if (!callback) {
             callback = createPromisifiedCallback();

package/dist/clients/producer/types.d.ts

@@ -13,6 +13,16 @@ export interface ProduceResult {
     unwritableNodes?: number[];
 }
 export type Partitioner<Key, Value, HeaderKey, HeaderValue> = (message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>) => number;
+export interface ProducerStreamReport {
+    batchId: number;
+    count: number;
+    result: ProduceResult;
+}
+export interface ProducerStreamMessageReport<Key, Value, HeaderKey, HeaderValue> {
+    batchId: number;
+    index: number;
+    message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>;
+}
 export interface ProduceOptions<Key, Value, HeaderKey, HeaderValue> {
     producerId?: bigint;
     producerEpoch?: number;
@@ -32,3 +42,17 @@ export type ProducerOptions<Key, Value, HeaderKey, HeaderValue> = BaseOptions &
 export type SendOptions<Key, Value, HeaderKey, HeaderValue> = {
     messages: MessageToProduce<Key, Value, HeaderKey, HeaderValue>[];
 } & ProduceOptions<Key, Value, HeaderKey, HeaderValue>;
+export declare const ProducerStreamReportModes: {
+    readonly NONE: "none";
+    readonly BATCH: "batch";
+    readonly MESSAGE: "message";
+};
+export declare const allowedProducerStreamReportModes: ProducerStreamReportModeValue[];
+export type ProducerStreamReportMode = keyof typeof ProducerStreamReportModes;
+export type ProducerStreamReportModeValue = (typeof ProducerStreamReportModes)[keyof typeof ProducerStreamReportModes];
+export type ProducerStreamOptions<Key, Value, HeaderKey, HeaderValue> = Omit<SendOptions<Key, Value, HeaderKey, HeaderValue>, 'messages'> & {
+    highWaterMark?: number;
+    batchSize?: number;
+    batchTime?: number;
+    reportMode?: ProducerStreamReportModeValue;
+};

package/dist/clients/producer/types.js

@@ -1 +1,6 @@
-export {};
+export const ProducerStreamReportModes = {
+    NONE: 'none',
+    BATCH: 'batch',
+    MESSAGE: 'message'
+};
+export const allowedProducerStreamReportModes = Object.values(ProducerStreamReportModes);

package/dist/diagnostic.d.ts

@@ -4,7 +4,7 @@ import { type ConnectionPool } from './network/connection-pool.ts';
 import { type Connection } from './network/connection.ts';
 export type ClientType = 'base' | 'producer' | 'consumer' | 'admin';
 export interface CreationEvent<InstanceType> {
-    type: ClientType | 'connection' | 'connectionPool';
+    type: ClientType | 'connection' | 'connection-pool' | 'messages-stream' | 'producer-stream';
     instance: InstanceType;
 }
 export type ConnectionDiagnosticEvent<Attributes = Record<string, unknown>> = {
@@ -29,7 +29,7 @@ export type DiagnosticContext<BaseContext = {}> = BaseContext & {
 };
 export declare const channelsNamespace: "plt:kafka";
 export declare function createDiagnosticContext<BaseContext = {}>(context: BaseContext): DiagnosticContext<BaseContext>;
-export declare function notifyCreation<InstanceType>(type: ClientType | 'connection' | 'connection-pool' | 'messages-stream', instance: InstanceType): void;
+export declare function notifyCreation<InstanceType>(type: ClientType | 'connection' | 'connection-pool' | 'messages-stream' | 'producer-stream', instance: InstanceType): void;
 export declare function createChannel<DiagnosticEvent extends object>(name: string): ChannelWithName<DiagnosticEvent>;
 export declare function createTracingChannel<DiagnosticEvent extends object>(name: string): TracingChannelWithName<DiagnosticEvent>;
 export declare const instancesChannel: ChannelWithName<object>;

package/dist/typescript-4/dist/clients/producer/index.d.ts

@@ -1,3 +1,4 @@
 export * from "./options";
+export * from "./producer-stream";
 export * from "./producer";
 export * from "./types";

package/dist/typescript-4/dist/clients/producer/options.d.ts

@@ -154,4 +154,67 @@ export declare const sendOptionsSchema: {
 };
 export declare const sendOptionsValidator: import("ajv").ValidateFunction<{
   [x: string]: {};
-}>;
+}>;
+export declare const defaultProducerStreamOptions: {
+  batchSize: number;
+  batchTime: number;
+  reportMode: "none";
+};
+export declare const producerStreamOptionsSchema: {
+  type: string;
+  properties: {
+    producerId: {
+      bigint: boolean;
+    };
+    producerEpoch: {
+      type: string;
+    };
+    idempotent: {
+      type: string;
+    };
+    transactionalId: {
+      type: string;
+    };
+    acks: {
+      type: string;
+      enumeration: {
+        allowed: (0 | 1 | -1)[];
+        errorMessage: string;
+      };
+    };
+    compression: {
+      type: string;
+      enumeration: {
+        allowed: import("../../protocol/compression").CompressionAlgorithmValue[];
+        errorMessage: string;
+      };
+    };
+    partitioner: {
+      function: boolean;
+    };
+    autocreateTopics: {
+      type: string;
+    };
+    repeatOnStaleMetadata: {
+      type: string;
+    };
+    highWaterMark: {
+      type: string;
+      minimum: number;
+    };
+    batchSize: {
+      type: string;
+      minimum: number;
+    };
+    batchTime: {
+      type: string;
+      minimum: number;
+    };
+    reportMode: {
+      type: string;
+      enum: import("./types").ProducerStreamReportModeValue[];
+    };
+  };
+  additionalProperties: boolean;
+};
+export declare const producerStreamOptionsValidator: import("ajv").ValidateFunction<unknown>;

package/dist/typescript-4/dist/clients/producer/producer-stream.d.ts

@@ -0,0 +1,19 @@
+import { Writable } from 'node:stream';
+import { type CallbackWithPromise } from "../../apis/callbacks";
+import type { MessageToProduce } from "../../protocol/records";
+import type { Producer } from "./producer";
+import { type ProducerStreamOptions } from "./types";
+export declare class ProducerStream<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends Writable {
+  #private;
+  instance: number;
+  constructor(producer: Producer<Key, Value, HeaderKey, HeaderValue>, options: ProducerStreamOptions<Key, Value, HeaderKey, HeaderValue>);
+  get producer(): Producer<Key, Value, HeaderKey, HeaderValue>;
+  close(callback: CallbackWithPromise<void>): void;
+  close(): Promise<void>;
+  _write(message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, _: BufferEncoding, callback: (error?: Error | null) => void): void;
+  _writev(chunks: Array<{
+    chunk: MessageToProduce<Key, Value, HeaderKey, HeaderValue>;
+  }>, callback: (error?: Error | null) => void): void;
+  _final(callback: (error?: Error | null) => void): void;
+  _destroy(error: Error | null, callback: (error?: Error | null) => void): void;
+}

package/dist/typescript-4/dist/clients/producer/producer.d.ts

@@ -2,8 +2,9 @@ import { type CallbackWithPromise } from "../../apis/callbacks";
 import { type Message } from "../../protocol/records";
 import { kTransaction, kTransactionAddOffsets, kTransactionAddPartitions, kTransactionCancel, kTransactionCommitOffset, kTransactionEnd, kTransactionFindCoordinator } from "../../symbols";
 import { Base } from "../base/base";
+import { ProducerStream } from "./producer-stream";
 import { Transaction } from "./transaction";
-import { type ProduceOptions, type ProduceResult, type ProducerInfo, type ProducerOptions, type SendOptions } from "./types";
+import { type ProduceOptions, type ProduceResult, type ProducerInfo, type ProducerOptions, type ProducerStreamOptions, type SendOptions } from "./types";
 export declare function noopSerializer(data?: Buffer): Buffer | undefined;
 export declare class Producer<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends Base<ProducerOptions<Key, Value, HeaderKey, HeaderValue>> {
   #private;
@@ -12,8 +13,9 @@ export declare class Producer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
   get producerEpoch(): number | undefined;
   get transaction(): Transaction<Key, Value, HeaderKey, HeaderValue> | undefined;
   get coordinatorId(): number;
-  close(callback: CallbackWithPromise<void>): void;
-  close(): Promise<void>;
+  get streamsCount(): number;
+  close(force: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
+  close(force?: boolean): Promise<void>;
   initIdempotentProducer(options: ProduceOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<ProducerInfo>): void;
   initIdempotentProducer(options: ProduceOptions<Key, Value, HeaderKey, HeaderValue>): Promise<ProducerInfo>;
   send(options: SendOptions<Key, Value, HeaderKey, HeaderValue> & {
@@ -22,6 +24,7 @@ export declare class Producer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
   send(options: SendOptions<Key, Value, HeaderKey, HeaderValue> & {
     [kTransaction]?: number;
   }): Promise<ProduceResult>;
+  asStream(options?: ProducerStreamOptions<Key, Value, HeaderKey, HeaderValue>): ProducerStream<Key, Value, HeaderKey, HeaderValue>;
   beginTransaction(options: ProduceOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<Transaction<Key, Value, HeaderKey, HeaderValue>>): void;
   beginTransaction(options?: ProduceOptions<Key, Value, HeaderKey, HeaderValue>): Promise<Transaction<Key, Value, HeaderKey, HeaderValue>>;
   [kTransactionFindCoordinator](callback: CallbackWithPromise<void>): void;

package/dist/typescript-4/dist/clients/producer/types.d.ts

@@ -13,6 +13,16 @@ export interface ProduceResult {
   unwritableNodes?: number[];
 }
 export type Partitioner<Key, Value, HeaderKey, HeaderValue> = (message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>) => number;
+export interface ProducerStreamReport {
+  batchId: number;
+  count: number;
+  result: ProduceResult;
+}
+export interface ProducerStreamMessageReport<Key, Value, HeaderKey, HeaderValue> {
+  batchId: number;
+  index: number;
+  message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>;
+}
 export interface ProduceOptions<Key, Value, HeaderKey, HeaderValue> {
   producerId?: bigint;
   producerEpoch?: number;
@@ -31,4 +41,18 @@ export type ProducerOptions<Key, Value, HeaderKey, HeaderValue> = BaseOptions &
 };
 export type SendOptions<Key, Value, HeaderKey, HeaderValue> = {
   messages: MessageToProduce<Key, Value, HeaderKey, HeaderValue>[];
-} & ProduceOptions<Key, Value, HeaderKey, HeaderValue>;
+} & ProduceOptions<Key, Value, HeaderKey, HeaderValue>;
+export declare const ProducerStreamReportModes: {
+  readonly NONE: "none";
+  readonly BATCH: "batch";
+  readonly MESSAGE: "message";
+};
+export declare const allowedProducerStreamReportModes: ProducerStreamReportModeValue[];
+export type ProducerStreamReportMode = keyof typeof ProducerStreamReportModes;
+export type ProducerStreamReportModeValue = (typeof ProducerStreamReportModes)[keyof typeof ProducerStreamReportModes];
+export type ProducerStreamOptions<Key, Value, HeaderKey, HeaderValue> = Omit<SendOptions<Key, Value, HeaderKey, HeaderValue>, 'messages'> & {
+  highWaterMark?: number;
+  batchSize?: number;
+  batchTime?: number;
+  reportMode?: ProducerStreamReportModeValue;
+};

package/dist/typescript-4/dist/diagnostic.d.ts

@@ -4,7 +4,7 @@ import { type ConnectionPool } from "./network/connection-pool";
 import { type Connection } from "./network/connection";
 export type ClientType = 'base' | 'producer' | 'consumer' | 'admin';
 export interface CreationEvent<InstanceType> {
-  type: ClientType | 'connection' | 'connectionPool';
+  type: ClientType | 'connection' | 'connection-pool' | 'messages-stream' | 'producer-stream';
   instance: InstanceType;
 }
 export type ConnectionDiagnosticEvent<Attributes = Record<string, unknown>> = {
@@ -29,7 +29,7 @@ export type DiagnosticContext<BaseContext = {}> = BaseContext & {
 };
 export declare const channelsNamespace: "plt:kafka";
 export declare function createDiagnosticContext<BaseContext = {}>(context: BaseContext): DiagnosticContext<BaseContext>;
-export declare function notifyCreation<InstanceType>(type: ClientType | 'connection' | 'connection-pool' | 'messages-stream', instance: InstanceType): void;
+export declare function notifyCreation<InstanceType>(type: ClientType | 'connection' | 'connection-pool' | 'messages-stream' | 'producer-stream', instance: InstanceType): void;
 export declare function createChannel<DiagnosticEvent extends object>(name: string): ChannelWithName<DiagnosticEvent>;
 export declare function createTracingChannel<DiagnosticEvent extends object>(name: string): TracingChannelWithName<DiagnosticEvent>;
 export declare const instancesChannel: ChannelWithName<object>;

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "1.27.0";
+export const version = "1.28.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "1.27.0",
+  "version": "1.28.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
@@ -28,6 +28,7 @@
     "@platformatic/dynamic-buffer": "^0.3.0",
     "@platformatic/wasm-utils": "^0.1.0",
     "ajv": "^8.17.1",
+    "avsc": "^5.7.9",
     "debug": "^4.4.3",
     "fastq": "^1.19.1",
     "mnemonist": "^0.40.3",
@@ -53,7 +54,6 @@
     "@types/node": "^22.18.8",
     "@types/semver": "^7.7.1",
     "@watchable/unpromise": "^1.0.2",
-    "avsc": "^5.7.9",
     "c8": "^10.1.3",
     "cleaner-spec-reporter": "^0.5.0",
     "cronometro": "^5.3.0",