kafka-ts 1.0.3 → 1.1.1

package/README.md

@@ -141,6 +141,7 @@ The existing high-level libraries (e.g. kafkajs) are missing a few crucial featu
 | bootstrapServers | TcpSocketConnectOpts[] | true     |         | List of kafka brokers for initial cluster discovery. |
 | sasl             | SASLProvider           | false    |         | SASL provider                                        |
 | ssl              | TLSSocketOptions       | false    |         | SSL configuration.                                   |
+| requestTimeout   | number                 | false    | 60000   | Request timeout in milliseconds.                     |
 
 #### Supported SASL mechanisms
 

package/dist/broker.d.ts

@@ -14,6 +14,7 @@ type BrokerOptions = {
     options: TcpSocketConnectOpts;
     sasl: SASLProvider | null;
     ssl: TLSSocketOptions | null;
+    requestTimeout: number;
 };
 export declare class Broker {
     private options;

package/dist/broker.js

@@ -14,6 +14,7 @@ class Broker {
             clientId: this.options.clientId,
             connection: this.options.options,
             ssl: this.options.ssl,
+            requestTimeout: this.options.requestTimeout,
         });
         this.sendRequest = this.connection.sendRequest.bind(this.connection);
     }

package/dist/client.d.ts

@@ -11,6 +11,7 @@ export type ClientOptions = {
     bootstrapServers: TcpSocketConnectOpts[];
     sasl?: SASLProvider | null;
     ssl?: TLSSocketOptions | null;
+    requestTimeout?: number;
 };
 export declare class Client {
     private options;

package/dist/client.js

@@ -12,6 +12,7 @@ class Client {
             clientId: options.clientId ?? null,
             sasl: options.sasl ?? null,
             ssl: options.ssl ?? null,
+            requestTimeout: options.requestTimeout ?? 60_000,
         };
     }
     async startConsumer(options) {
@@ -28,6 +29,7 @@ class Client {
             bootstrapServers: this.options.bootstrapServers,
             sasl: this.options.sasl,
             ssl: this.options.ssl,
+            requestTimeout: this.options.requestTimeout,
         });
     }
 }

package/dist/cluster.d.ts

@@ -9,6 +9,7 @@ type ClusterOptions = {
     bootstrapServers: TcpSocketConnectOpts[];
     sasl: SASLProvider | null;
     ssl: TLSSocketOptions | null;
+    requestTimeout: number;
 };
 export declare class Cluster {
     private options;

package/dist/cluster.js

@@ -54,6 +54,7 @@ class Cluster {
             clientId: this.options.clientId,
             sasl: this.options.sasl,
             ssl: this.options.ssl,
+            requestTimeout: this.options.requestTimeout,
             options: this.brokerMetadata[nodeId],
         });
         await broker.connect();
@@ -63,10 +64,11 @@ class Cluster {
         const randomizedBrokers = this.options.bootstrapServers.toSorted(() => Math.random() - 0.5);
         for (const options of randomizedBrokers) {
             try {
-                const broker = await new broker_1.Broker({
+                const broker = new broker_1.Broker({
                     clientId: this.options.clientId,
                     sasl: this.options.sasl,
                     ssl: this.options.ssl,
+                    requestTimeout: this.options.requestTimeout,
                     options,
                 });
                 await broker.connect();

package/dist/connection.d.ts

@@ -7,6 +7,7 @@ type ConnectionOptions = {
     clientId: string | null;
     connection: TcpSocketConnectOpts;
     ssl: TLSSocketOptions | null;
+    requestTimeout: number;
 };
 export declare class Connection {
     private options;

package/dist/connection.js

@@ -105,7 +105,7 @@ class Connection {
             timeout = setTimeout(() => {
                 delete this.queue[correlationId];
                 reject(new error_1.ConnectionError(`${apiName} timed out`));
-            }, 30_000);
+            }, this.options.requestTimeout);
             try {
                 this.queue[correlationId] = { resolve, reject };
                 await this.write(requestEncoder.value());

package/dist/producer/producer.d.ts

@@ -13,6 +13,7 @@ export declare class Producer {
     private producerEpoch;
     private sequences;
     private partition;
+    private lock;
     constructor(cluster: Cluster, options: ProducerOptions);
     send(messages: Message[], { acks }?: {
         acks?: -1 | 1;

package/dist/producer/producer.js

@@ -16,6 +16,7 @@ const partitioner_1 = require("../distributors/partitioner");
 const metadata_1 = require("../metadata");
 const delay_1 = require("../utils/delay");
 const error_1 = require("../utils/error");
+const lock_1 = require("../utils/lock");
 const tracer_1 = require("../utils/tracer");
 const trace = (0, tracer_1.createTracer)('Producer');
 class Producer {
@@ -26,6 +27,7 @@ class Producer {
     producerEpoch = 0;
     sequences = {};
     partition;
+    lock = new lock_1.Lock();
     constructor(cluster, options) {
         this.cluster = cluster;
         this.options = {
@@ -49,54 +51,57 @@ class Producer {
         const nodeTopicPartitionMessages = (0, messages_to_topic_partition_leaders_1.distributeMessagesToTopicPartitionLeaders)(partitionedMessages, this.metadata.getTopicPartitionLeaderIds());
         try {
             await Promise.all(Object.entries(nodeTopicPartitionMessages).map(async ([nodeId, topicPartitionMessages]) => {
-                const topicData = Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
-                    name: topic,
-                    partitionData: Object.entries(partitionMessages).map(([partition, messages]) => {
-                        const partitionIndex = parseInt(partition);
-                        let baseTimestamp;
-                        let maxTimestamp;
-                        messages.forEach(({ timestamp = defaultTimestamp }) => {
-                            if (!baseTimestamp || timestamp < baseTimestamp) {
-                                baseTimestamp = timestamp;
-                            }
-                            if (!maxTimestamp || timestamp > maxTimestamp) {
-                                maxTimestamp = timestamp;
-                            }
-                        });
-                        return {
-                            index: partitionIndex,
-                            baseOffset: 0n,
-                            partitionLeaderEpoch: -1,
-                            attributes: 0,
-                            lastOffsetDelta: messages.length - 1,
-                            baseTimestamp: baseTimestamp ?? 0n,
-                            maxTimestamp: maxTimestamp ?? 0n,
-                            producerId: this.producerId,
-                            producerEpoch: 0,
-                            baseSequence: this.getSequence(topic, partitionIndex),
-                            records: messages.map((message, index) => ({
+                const lockKeys = Object.entries(topicPartitionMessages).flatMap(([topic, partitionMessages]) => Object.entries(partitionMessages).map(([partition]) => `${topic}-${partition}`));
+                await this.lock.acquire(lockKeys, async () => {
+                    const topicData = Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
+                        name: topic,
+                        partitionData: Object.entries(partitionMessages).map(([partition, messages]) => {
+                            const partitionIndex = parseInt(partition);
+                            let baseTimestamp;
+                            let maxTimestamp;
+                            messages.forEach(({ timestamp = defaultTimestamp }) => {
+                                if (!baseTimestamp || timestamp < baseTimestamp) {
+                                    baseTimestamp = timestamp;
+                                }
+                                if (!maxTimestamp || timestamp > maxTimestamp) {
+                                    maxTimestamp = timestamp;
+                                }
+                            });
+                            return {
+                                index: partitionIndex,
+                                baseOffset: 0n,
+                                partitionLeaderEpoch: -1,
                                 attributes: 0,
-                                timestampDelta: (message.timestamp ?? defaultTimestamp) - (baseTimestamp ?? 0n),
-                                offsetDelta: index,
-                                key: message.key ?? null,
-                                value: message.value,
-                                headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
-                                    key,
-                                    value,
+                                lastOffsetDelta: messages.length - 1,
+                                baseTimestamp: baseTimestamp ?? 0n,
+                                maxTimestamp: maxTimestamp ?? 0n,
+                                producerId: this.producerId,
+                                producerEpoch: 0,
+                                baseSequence: this.getSequence(topic, partitionIndex),
+                                records: messages.map((message, index) => ({
+                                    attributes: 0,
+                                    timestampDelta: (message.timestamp ?? defaultTimestamp) - (baseTimestamp ?? 0n),
+                                    offsetDelta: index,
+                                    key: message.key ?? null,
+                                    value: message.value,
+                                    headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
+                                        key,
+                                        value,
+                                    })),
                                 })),
-                            })),
-                        };
-                    }),
-                }));
-                await this.cluster.sendRequestToNode(parseInt(nodeId))(api_1.API.PRODUCE, {
-                    transactionalId: null,
-                    acks,
-                    timeoutMs: 5000,
-                    topicData,
-                });
-                topicData.forEach(({ name, partitionData }) => {
-                    partitionData.forEach(({ index, records }) => {
-                        this.updateSequence(name, index, records.length);
+                            };
+                        }),
+                    }));
+                    await this.cluster.sendRequestToNode(parseInt(nodeId))(api_1.API.PRODUCE, {
+                        transactionalId: null,
+                        acks,
+                        timeoutMs: 5000,
+                        topicData,
+                    });
+                    topicData.forEach(({ name, partitionData }) => {
+                        partitionData.forEach(({ index, records }) => {
+                            this.updateSequence(name, index, records.length);
+                        });
                     });
                 });
             }));

package/dist/utils/lock.d.ts

@@ -0,0 +1,9 @@
+/// <reference types="node" />
+import EventEmitter from 'events';
+export declare class Lock extends EventEmitter {
+    private locks;
+    constructor();
+    acquire(keys: string[], callback: () => Promise<void>): Promise<void>;
+    private acquireKey;
+    private releaseKey;
+}

package/dist/utils/lock.js

@@ -0,0 +1,44 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Lock = void 0;
+const events_1 = __importDefault(require("events"));
+const logger_1 = require("./logger");
+class Lock extends events_1.default {
+    locks = {};
+    constructor() {
+        super();
+        this.setMaxListeners(Infinity);
+    }
+    async acquire(keys, callback) {
+        await Promise.all(keys.map((key) => this.acquireKey(key)));
+        try {
+            await callback();
+        }
+        finally {
+            keys.forEach((key) => this.releaseKey(key));
+        }
+    }
+    async acquireKey(key) {
+        while (this.locks[key]) {
+            await new Promise((resolve) => {
+                const timeout = setTimeout(() => {
+                    logger_1.log.warn(`Lock timed out`, { key });
+                    this.emit(`release:${key}`);
+                }, 10_000);
+                this.once(`release:${key}`, () => {
+                    clearTimeout(timeout);
+                    resolve();
+                });
+            });
+        }
+        this.locks[key] = true;
+    }
+    releaseKey(key) {
+        this.locks[key] = false;
+        this.emit(`release:${key}`);
+    }
+}
+exports.Lock = Lock;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kafka-ts",
-    "version": "1.0.3",
+    "version": "1.1.1",
     "main": "dist/index.js",
     "author": "Priit Käärd",
     "license": "MIT",