kafka-ts 1.1.0 → 1.1.2-beta.0

package/dist/cluster.js

@@ -50,6 +50,9 @@ class Cluster {
         return this.brokerById[nodeId].sendRequest(...args);
     };
     async acquireBroker(nodeId) {
+        if (!(nodeId in this.brokerMetadata)) {
+            throw new error_1.KafkaTSError(`Broker ${nodeId} is not available`);
+        }
         const broker = new broker_1.Broker({
             clientId: this.options.clientId,
             sasl: this.options.sasl,

package/dist/producer/producer.d.ts

@@ -13,6 +13,7 @@ export declare class Producer {
     private producerEpoch;
     private sequences;
     private partition;
+    private lock;
     constructor(cluster: Cluster, options: ProducerOptions);
     send(messages: Message[], { acks }?: {
         acks?: -1 | 1;

package/dist/producer/producer.js

@@ -16,6 +16,9 @@ const partitioner_1 = require("../distributors/partitioner");
 const metadata_1 = require("../metadata");
 const delay_1 = require("../utils/delay");
 const error_1 = require("../utils/error");
+const lock_1 = require("../utils/lock");
+const logger_1 = require("../utils/logger");
+const shared_1 = require("../utils/shared");
 const tracer_1 = require("../utils/tracer");
 const trace = (0, tracer_1.createTracer)('Producer');
 class Producer {
@@ -26,6 +29,7 @@ class Producer {
     producerEpoch = 0;
     sequences = {};
     partition;
+    lock = new lock_1.Lock();
     constructor(cluster, options) {
         this.cluster = cluster;
         this.options = {
@@ -49,54 +53,57 @@ class Producer {
         const nodeTopicPartitionMessages = (0, messages_to_topic_partition_leaders_1.distributeMessagesToTopicPartitionLeaders)(partitionedMessages, this.metadata.getTopicPartitionLeaderIds());
         try {
             await Promise.all(Object.entries(nodeTopicPartitionMessages).map(async ([nodeId, topicPartitionMessages]) => {
-                const topicData = Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
-                    name: topic,
-                    partitionData: Object.entries(partitionMessages).map(([partition, messages]) => {
-                        const partitionIndex = parseInt(partition);
-                        let baseTimestamp;
-                        let maxTimestamp;
-                        messages.forEach(({ timestamp = defaultTimestamp }) => {
-                            if (!baseTimestamp || timestamp < baseTimestamp) {
-                                baseTimestamp = timestamp;
-                            }
-                            if (!maxTimestamp || timestamp > maxTimestamp) {
-                                maxTimestamp = timestamp;
-                            }
-                        });
-                        return {
-                            index: partitionIndex,
-                            baseOffset: 0n,
-                            partitionLeaderEpoch: -1,
-                            attributes: 0,
-                            lastOffsetDelta: messages.length - 1,
-                            baseTimestamp: baseTimestamp ?? 0n,
-                            maxTimestamp: maxTimestamp ?? 0n,
-                            producerId: this.producerId,
-                            producerEpoch: 0,
-                            baseSequence: this.getSequence(topic, partitionIndex),
-                            records: messages.map((message, index) => ({
+                const lockKeys = Object.entries(topicPartitionMessages).flatMap(([topic, partitionMessages]) => Object.entries(partitionMessages).map(([partition]) => `${topic}-${partition}`));
+                await this.lock.acquire(lockKeys, async () => {
+                    const topicData = Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
+                        name: topic,
+                        partitionData: Object.entries(partitionMessages).map(([partition, messages]) => {
+                            const partitionIndex = parseInt(partition);
+                            let baseTimestamp;
+                            let maxTimestamp;
+                            messages.forEach(({ timestamp = defaultTimestamp }) => {
+                                if (!baseTimestamp || timestamp < baseTimestamp) {
+                                    baseTimestamp = timestamp;
+                                }
+                                if (!maxTimestamp || timestamp > maxTimestamp) {
+                                    maxTimestamp = timestamp;
+                                }
+                            });
+                            return {
+                                index: partitionIndex,
+                                baseOffset: 0n,
+                                partitionLeaderEpoch: -1,
                                 attributes: 0,
-                                timestampDelta: (message.timestamp ?? defaultTimestamp) - (baseTimestamp ?? 0n),
-                                offsetDelta: index,
-                                key: message.key ?? null,
-                                value: message.value,
-                                headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
-                                    key,
-                                    value,
+                                lastOffsetDelta: messages.length - 1,
+                                baseTimestamp: baseTimestamp ?? 0n,
+                                maxTimestamp: maxTimestamp ?? 0n,
+                                producerId: this.producerId,
+                                producerEpoch: 0,
+                                baseSequence: this.getSequence(topic, partitionIndex),
+                                records: messages.map((message, index) => ({
+                                    attributes: 0,
+                                    timestampDelta: (message.timestamp ?? defaultTimestamp) - (baseTimestamp ?? 0n),
+                                    offsetDelta: index,
+                                    key: message.key ?? null,
+                                    value: message.value,
+                                    headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
+                                        key,
+                                        value,
+                                    })),
                                 })),
-                            })),
-                        };
-                    }),
-                }));
-                await this.cluster.sendRequestToNode(parseInt(nodeId))(api_1.API.PRODUCE, {
-                    transactionalId: null,
-                    acks,
-                    timeoutMs: 5000,
-                    topicData,
-                });
-                topicData.forEach(({ name, partitionData }) => {
-                    partitionData.forEach(({ index, records }) => {
-                        this.updateSequence(name, index, records.length);
+                            };
+                        }),
+                    }));
+                    await this.cluster.sendRequestToNode(parseInt(nodeId))(api_1.API.PRODUCE, {
+                        transactionalId: null,
+                        acks,
+                        timeoutMs: 5000,
+                        topicData,
+                    });
+                    topicData.forEach(({ name, partitionData }) => {
+                        partitionData.forEach(({ index, records }) => {
+                            this.updateSequence(name, index, records.length);
+                        });
                     });
                 });
             }));
@@ -108,18 +115,26 @@ class Producer {
             if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.OUT_OF_ORDER_SEQUENCE_NUMBER) {
                 await this.initProducerId();
             }
+            logger_1.log.warn('Reconnecting producer due to an unhandled error', { error });
+            try {
+                await this.cluster.disconnect();
+                await this.cluster.connect();
+            }
+            catch (error) {
+                logger_1.log.warn('Failed to reconnect producer', { error });
+            }
             throw error;
         }
     }
     async close() {
         await this.cluster.disconnect();
     }
-    async ensureConnected() {
+    ensureConnected = (0, shared_1.shared)(async () => {
         await this.cluster.ensureConnected();
         if (!this.producerId) {
             await this.initProducerId();
         }
-    }
+    });
     async initProducerId() {
         try {
             const result = await this.cluster.sendRequest(api_1.API.INIT_PRODUCER_ID, {

package/dist/utils/lock.d.ts

@@ -0,0 +1,9 @@
+/// <reference types="node" />
+import EventEmitter from 'events';
+export declare class Lock extends EventEmitter {
+    private locks;
+    constructor();
+    acquire(keys: string[], callback: () => Promise<void>): Promise<void>;
+    private acquireKey;
+    private releaseKey;
+}

package/dist/utils/lock.js

@@ -0,0 +1,44 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Lock = void 0;
+const events_1 = __importDefault(require("events"));
+const logger_1 = require("./logger");
+class Lock extends events_1.default {
+    locks = {};
+    constructor() {
+        super();
+        this.setMaxListeners(Infinity);
+    }
+    async acquire(keys, callback) {
+        await Promise.all(keys.map((key) => this.acquireKey(key)));
+        try {
+            await callback();
+        }
+        finally {
+            keys.forEach((key) => this.releaseKey(key));
+        }
+    }
+    async acquireKey(key) {
+        while (this.locks[key]) {
+            await new Promise((resolve) => {
+                const timeout = setTimeout(() => {
+                    logger_1.log.warn(`Lock timed out`, { key });
+                    this.emit(`release:${key}`);
+                }, 10_000);
+                this.once(`release:${key}`, () => {
+                    clearTimeout(timeout);
+                    resolve();
+                });
+            });
+        }
+        this.locks[key] = true;
+    }
+    releaseKey(key) {
+        this.locks[key] = false;
+        this.emit(`release:${key}`);
+    }
+}
+exports.Lock = Lock;

package/dist/utils/shared.d.ts

@@ -0,0 +1 @@
+export declare const shared: <F extends () => Promise<any>>(func: F) => () => ReturnType<F>;

package/dist/utils/shared.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.shared = void 0;
+const shared = (func) => {
+    let promise;
+    return () => {
+        if (!promise) {
+            promise = func();
+            promise.finally(() => {
+                promise = undefined;
+            });
+        }
+        return promise;
+    };
+};
+exports.shared = shared;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kafka-ts",
-    "version": "1.1.0",
+    "version": "1.1.2-beta.0",
     "main": "dist/index.js",
     "author": "Priit Käärd",
     "license": "MIT",