kafka-ts 1.1.9 → 1.2.0

package/README.md

@@ -177,7 +177,7 @@ Custom SASL mechanisms can be implemented following the `SASLProvider` interface
 | allowTopicAutoCreation | boolean     | false    | false              | Allow kafka to auto-create topic when it doesn't exist                                  |
 | partitioner            | Partitioner | false    | defaultPartitioner | Custom partitioner function. By default, it uses a default java-compatible partitioner. |
 
-### `producer.send(messages: Message[], options?: { acks?: -1 | 1 })`
+### `producer.send(messages: Message[])`
 
 <!-- export type Message = {
     topic: string;

package/dist/consumer/consumer.js

@@ -15,7 +15,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.Consumer = void 0;
 const events_1 = __importDefault(require("events"));
 const api_1 = require("../api");
-const messages_to_topic_partition_leaders_1 = require("../distributors/messages-to-topic-partition-leaders");
+const group_by_leader_id_1 = require("../distributors/group-by-leader-id");
+const group_partitions_by_topic_1 = require("../distributors/group-partitions-by-topic");
 const delay_1 = require("../utils/delay");
 const error_1 = require("../utils/error");
 const logger_1 = require("../utils/logger");
@@ -112,12 +113,11 @@ class Consumer extends events_1.default {
             try {
                 await this.consumerGroup?.join();
                 // TODO: If leader is not available, find another read replica
-                const nodeAssignments = Object.entries((0, messages_to_topic_partition_leaders_1.distributeMessagesToTopicPartitionLeaders)(Object.entries(this.metadata.getAssignment()).flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition }))), this.metadata.getTopicPartitionLeaderIds())).map(([nodeId, assignment]) => ({
-                    nodeId: parseInt(nodeId),
-                    assignment: Object.fromEntries(Object.entries(assignment).map(([topic, partitions]) => [
-                        topic,
-                        Object.keys(partitions).map(Number),
-                    ])),
+                const topicPartitions = Object.entries(this.metadata.getAssignment()).flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition })));
+                const topicPartitionsByLeaderId = (0, group_by_leader_id_1.groupByLeaderId)(topicPartitions, this.metadata.getTopicPartitionLeaderIds());
+                const nodeAssignments = Object.entries(topicPartitionsByLeaderId).map(([leaderId, topicPartitions]) => ({
+                    nodeId: parseInt(leaderId),
+                    assignment: (0, group_partitions_by_topic_1.groupPartitionsByTopic)(topicPartitions),
                 }));
                 this.fetchManager = new fetch_manager_1.FetchManager({
                     fetch: this.fetch.bind(this),

package/dist/consumer/offset-manager.js

@@ -2,7 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.OffsetManager = void 0;
 const api_1 = require("../api");
-const messages_to_topic_partition_leaders_1 = require("../distributors/messages-to-topic-partition-leaders");
+const group_by_leader_id_1 = require("../distributors/group-by-leader-id");
+const group_partitions_by_topic_1 = require("../distributors/group-partitions-by-topic");
 const tracer_1 = require("../utils/tracer");
 const trace = (0, tracer_1.createTracer)('OffsetManager');
 class OffsetManager {
@@ -40,11 +41,11 @@ class OffsetManager {
     async fetchOffsets(options) {
         const { metadata } = this.options;
         const topicPartitions = Object.entries(metadata.getAssignment()).flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition })));
-        const nodeTopicPartitions = (0, messages_to_topic_partition_leaders_1.distributeMessagesToTopicPartitionLeaders)(topicPartitions, metadata.getTopicPartitionLeaderIds());
-        await Promise.all(Object.entries(nodeTopicPartitions).map(([nodeId, topicPartitions]) => this.listOffsets({
+        const topicPartitionsByLeaderId = (0, group_by_leader_id_1.groupByLeaderId)(topicPartitions, metadata.getTopicPartitionLeaderIds());
+        await Promise.all(Object.entries(topicPartitionsByLeaderId).map(([leaderId, topicPartitions]) => this.listOffsets({
             ...options,
-            nodeId: parseInt(nodeId),
-            nodeAssignment: Object.fromEntries(Object.entries(topicPartitions).map(([topicName, partitions]) => [topicName, Object.keys(partitions).map(Number)])),
+            nodeId: parseInt(leaderId),
+            nodeAssignment: (0, group_partitions_by_topic_1.groupPartitionsByTopic)(topicPartitions),
         })));
     }
     async listOffsets({ nodeId, nodeAssignment, fromTimestamp, }) {

package/dist/distributors/group-by-leader-id.d.ts

@@ -0,0 +1,10 @@
+export declare const groupByLeaderId: <T extends {
+    topic: string;
+    partition: number;
+}>(items: T[], leaderIdByTopicPartition: {
+    [topic: string]: {
+        [partition: number]: number;
+    };
+}) => {
+    [nodeId: number]: T[];
+};

package/dist/distributors/group-by-leader-id.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.groupByLeaderId = void 0;
+const groupByLeaderId = (items, leaderIdByTopicPartition) => {
+    const result = {};
+    items.forEach((item) => {
+        const leaderId = leaderIdByTopicPartition[item.topic][item.partition];
+        result[leaderId] ??= [];
+        result[leaderId].push(item);
+    });
+    return result;
+};
+exports.groupByLeaderId = groupByLeaderId;

package/dist/distributors/group-partitions-by-topic.d.ts

@@ -0,0 +1,6 @@
+export declare const groupPartitionsByTopic: <T extends {
+    topic: string;
+    partition: number;
+}>(items: T[]) => {
+    [topic: string]: number[];
+};

package/dist/distributors/group-partitions-by-topic.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.groupPartitionsByTopic = void 0;
+const groupPartitionsByTopic = (items) => {
+    const result = {};
+    items.forEach((item) => {
+        result[item.topic] ??= [];
+        result[item.topic].push(item.partition);
+    });
+    return result;
+};
+exports.groupPartitionsByTopic = groupPartitionsByTopic;

package/dist/metadata.d.ts

@@ -16,7 +16,7 @@ export declare class Metadata {
     getTopicIdByName(name: string): string;
     getTopicNameById(id: string): string;
     fetchMetadataIfNecessary({ topics, allowTopicAutoCreation, }: {
-        topics: string[] | Set<string>;
+        topics: string[];
         allowTopicAutoCreation: boolean;
     }): Promise<void>;
     fetchMetadata({ topics, allowTopicAutoCreation, }: {

package/dist/metadata.js

@@ -41,7 +41,7 @@ class Metadata {
         return this.topicNameById[id];
     }
     async fetchMetadataIfNecessary({ topics, allowTopicAutoCreation, }) {
-        const missingTopics = Array.from(topics).filter((topic) => !this.topicPartitions[topic]);
+        const missingTopics = topics.filter((topic) => !this.topicPartitions[topic]);
         if (!missingTopics.length) {
             return;
         }

package/dist/producer/producer-buffer.d.ts

@@ -0,0 +1,20 @@
+import { Cluster } from '../cluster';
+import { Message } from '../types';
+import { ProducerState } from './producer-state';
+type ProducerBufferOptions = {
+    nodeId: number;
+    maxBatchSize: number;
+    cluster: Cluster;
+    state: ProducerState;
+};
+export declare class ProducerBuffer {
+    private options;
+    private buffer;
+    private head;
+    private isFlushing;
+    constructor(options: ProducerBufferOptions);
+    enqueue(messages: Message[]): Promise<void>;
+    private flush;
+    private compactBuffer;
+}
+export {};

package/dist/producer/producer-buffer.js

@@ -0,0 +1,118 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ProducerBuffer = void 0;
+const api_1 = require("../api");
+class ProducerBuffer {
+    options;
+    buffer = [];
+    head = 0;
+    isFlushing = false;
+    constructor(options) {
+        this.options = options;
+    }
+    enqueue(messages) {
+        return new Promise((resolve, reject) => {
+            this.buffer.push({ messages, resolve, reject });
+            this.flush();
+        });
+    }
+    async flush() {
+        if (this.isFlushing)
+            return;
+        this.isFlushing = true;
+        const { cluster, state, nodeId, maxBatchSize } = this.options;
+        while (true) {
+            const batch = [];
+            const resolvers = [];
+            const rejecters = [];
+            while (this.head < this.buffer.length) {
+                const entry = this.buffer[this.head++];
+                batch.push(...entry.messages);
+                resolvers.push(entry.resolve);
+                rejecters.push(entry.reject);
+                const nextLength = this.buffer[this.head]?.messages.length ?? 0;
+                if (batch.length + nextLength > maxBatchSize) {
+                    break;
+                }
+            }
+            if (!batch.length)
+                break;
+            this.compactBuffer();
+            const topicPartitionMessages = {};
+            batch.forEach((message) => {
+                topicPartitionMessages[message.topic] ??= {};
+                topicPartitionMessages[message.topic][message.partition] ??= [];
+                topicPartitionMessages[message.topic][message.partition].push(message);
+            });
+            const defaultTimestamp = BigInt(Date.now());
+            const topicData = Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
+                name: topic,
+                partitionData: Object.entries(partitionMessages).map(([partition, messages]) => {
+                    const partitionIndex = parseInt(partition);
+                    let baseTimestamp;
+                    let maxTimestamp;
+                    messages.forEach(({ timestamp = defaultTimestamp }) => {
+                        if (!baseTimestamp || timestamp < baseTimestamp) {
+                            baseTimestamp = timestamp;
+                        }
+                        if (!maxTimestamp || timestamp > maxTimestamp) {
+                            maxTimestamp = timestamp;
+                        }
+                    });
+                    return {
+                        index: partitionIndex,
+                        baseOffset: 0n,
+                        partitionLeaderEpoch: -1,
+                        attributes: 0,
+                        lastOffsetDelta: messages.length - 1,
+                        baseTimestamp: baseTimestamp ?? 0n,
+                        maxTimestamp: maxTimestamp ?? 0n,
+                        producerId: state.producerId,
+                        producerEpoch: 0,
+                        baseSequence: state.getSequence(topic, partitionIndex),
+                        records: messages.map((message, index) => ({
+                            attributes: 0,
+                            timestampDelta: (message.timestamp ?? defaultTimestamp) - (baseTimestamp ?? 0n),
+                            offsetDelta: index,
+                            key: message.key ?? null,
+                            value: message.value,
+                            headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
+                                key,
+                                value,
+                            })),
+                        })),
+                    };
+                }),
+            }));
+            try {
+                await cluster.sendRequestToNode(nodeId)(api_1.API.PRODUCE, {
+                    transactionalId: null,
+                    acks: -1,
+                    timeoutMs: 30000,
+                    topicData,
+                });
+                topicData.forEach(({ name, partitionData }) => {
+                    partitionData.forEach(({ index, records }) => {
+                        state.updateSequence(name, index, records.length);
+                    });
+                });
+                resolvers.forEach((resolve) => resolve());
+            }
+            catch (error) {
+                rejecters.forEach((reject) => reject(error));
+            }
+        }
+        this.isFlushing = false;
+    }
+    compactBuffer() {
+        if (this.head >= this.buffer.length) {
+            this.buffer = [];
+            this.head = 0;
+        }
+        else if (this.head > 1000 && this.head > this.buffer.length / 2) {
+            this.buffer = this.buffer.slice(this.head);
+            this.head = 0;
+        }
+    }
+}
+exports.ProducerBuffer = ProducerBuffer;

package/dist/producer/producer-state.d.ts

@@ -0,0 +1,15 @@
+import { Cluster } from "../cluster";
+type ProducerStateOptions = {
+    cluster: Cluster;
+};
+export declare class ProducerState {
+    private options;
+    producerId: bigint;
+    private producerEpoch;
+    private sequences;
+    constructor(options: ProducerStateOptions);
+    initProducerId(): Promise<void>;
+    getSequence(topic: string, partition: number): number;
+    updateSequence(topic: string, partition: number, messagesCount: number): void;
+}
+export {};

package/dist/producer/producer-state.js

@@ -0,0 +1,33 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ProducerState = void 0;
+const api_1 = require("../api");
+class ProducerState {
+    options;
+    producerId = 0n;
+    producerEpoch = 0;
+    sequences = {};
+    constructor(options) {
+        this.options = options;
+    }
+    async initProducerId() {
+        const result = await this.options.cluster.sendRequest(api_1.API.INIT_PRODUCER_ID, {
+            transactionalId: null,
+            transactionTimeoutMs: 0,
+            producerId: this.producerId,
+            producerEpoch: this.producerEpoch,
+        });
+        this.producerId = result.producerId;
+        this.producerEpoch = result.producerEpoch;
+        this.sequences = {};
+    }
+    getSequence(topic, partition) {
+        return this.sequences[topic]?.[partition] ?? 0;
+    }
+    updateSequence(topic, partition, messagesCount) {
+        this.sequences[topic] ??= {};
+        this.sequences[topic][partition] ??= 0;
+        this.sequences[topic][partition] += messagesCount;
+    }
+}
+exports.ProducerState = ProducerState;

package/dist/producer/producer.d.ts

@@ -4,25 +4,20 @@ import { Message } from '../types';
 export type ProducerOptions = {
     allowTopicAutoCreation?: boolean;
     partitioner?: Partitioner;
+    maxBatchSize?: number;
 };
 export declare class Producer {
     private cluster;
     private options;
     private metadata;
-    private producerId;
-    private producerEpoch;
-    private sequences;
+    private state;
     private partition;
-    private lock;
+    private chain;
+    private bufferByNodeId;
     constructor(cluster: Cluster, options: ProducerOptions);
-    send(messages: Message[], { acks }?: {
-        acks?: -1 | 1;
-    }): Promise<void>;
+    send(messages: Message[]): Promise<void>;
     close(): Promise<void>;
     private ensureProducerInitialized;
-    private initProducerId;
-    private getSequence;
-    private updateSequence;
-    private fetchMetadata;
+    private fetchMetadataForTopics;
     private handleError;
 }

package/dist/producer/producer.js

@@ -11,107 +11,60 @@ var __metadata = (this && this.__metadata) || function (k, v) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Producer = void 0;
 const api_1 = require("../api");
-const messages_to_topic_partition_leaders_1 = require("../distributors/messages-to-topic-partition-leaders");
+const group_by_leader_id_1 = require("../distributors/group-by-leader-id");
 const partitioner_1 = require("../distributors/partitioner");
 const metadata_1 = require("../metadata");
 const error_1 = require("../utils/error");
-const lock_1 = require("../utils/lock");
 const logger_1 = require("../utils/logger");
-const retry_1 = require("../utils/retry");
+const promise_chain_1 = require("../utils/promise-chain");
 const shared_1 = require("../utils/shared");
 const tracer_1 = require("../utils/tracer");
+const producer_buffer_1 = require("./producer-buffer");
+const producer_state_1 = require("./producer-state");
 const trace = (0, tracer_1.createTracer)('Producer');
 class Producer {
     cluster;
     options;
     metadata;
-    producerId = 0n;
-    producerEpoch = 0;
-    sequences = {};
+    state;
     partition;
-    lock = new lock_1.Lock();
+    chain = new promise_chain_1.PromiseChain();
+    bufferByNodeId = {};
     constructor(cluster, options) {
         this.cluster = cluster;
         this.options = {
             ...options,
             allowTopicAutoCreation: options.allowTopicAutoCreation ?? false,
             partitioner: options.partitioner ?? partitioner_1.defaultPartitioner,
+            maxBatchSize: options.maxBatchSize ?? 500,
         };
         this.metadata = new metadata_1.Metadata({ cluster });
+        this.state = new producer_state_1.ProducerState({ cluster });
         this.partition = this.options.partitioner({ metadata: this.metadata });
     }
-    async send(messages, { acks = -1 } = {}) {
+    async send(messages) {
         await this.ensureProducerInitialized();
-        const { allowTopicAutoCreation } = this.options;
-        const defaultTimestamp = BigInt(Date.now());
-        const topics = new Set(messages.map((message) => message.topic));
-        await this.lock.acquire([...topics].map((topic) => `metadata:${topic}`), () => this.metadata.fetchMetadataIfNecessary({ topics, allowTopicAutoCreation }));
+        const topics = [...new Set(messages.map((message) => message.topic))];
+        await this.fetchMetadataForTopics(topics);
         const partitionedMessages = messages.map((message) => {
             message.partition = this.partition(message);
             return message;
         });
-        const nodeTopicPartitionMessages = (0, messages_to_topic_partition_leaders_1.distributeMessagesToTopicPartitionLeaders)(partitionedMessages, this.metadata.getTopicPartitionLeaderIds());
-        await Promise.all(Object.entries(nodeTopicPartitionMessages).map(async ([nodeId, topicPartitionMessages]) => {
+        const messagesByLeaderId = (0, group_by_leader_id_1.groupByLeaderId)(partitionedMessages, this.metadata.getTopicPartitionLeaderIds());
+        await Promise.all(Object.entries(messagesByLeaderId).map(async ([leaderId, messages]) => {
+            const nodeId = parseInt(leaderId);
+            const buffer = (this.bufferByNodeId[nodeId] ??= new producer_buffer_1.ProducerBuffer({
+                nodeId,
+                maxBatchSize: this.options.maxBatchSize,
+                cluster: this.cluster,
+                state: this.state,
+            }));
             try {
-                await this.lock.acquire([`node:${nodeId}`], async () => {
-                    const topicData = Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
-                        name: topic,
-                        partitionData: Object.entries(partitionMessages).map(([partition, messages]) => {
-                            const partitionIndex = parseInt(partition);
-                            let baseTimestamp;
-                            let maxTimestamp;
-                            messages.forEach(({ timestamp = defaultTimestamp }) => {
-                                if (!baseTimestamp || timestamp < baseTimestamp) {
-                                    baseTimestamp = timestamp;
-                                }
-                                if (!maxTimestamp || timestamp > maxTimestamp) {
-                                    maxTimestamp = timestamp;
-                                }
-                            });
-                            return {
-                                index: partitionIndex,
-                                baseOffset: 0n,
-                                partitionLeaderEpoch: -1,
-                                attributes: 0,
-                                lastOffsetDelta: messages.length - 1,
-                                baseTimestamp: baseTimestamp ?? 0n,
-                                maxTimestamp: maxTimestamp ?? 0n,
-                                producerId: this.producerId,
-                                producerEpoch: 0,
-                                baseSequence: this.getSequence(topic, partitionIndex),
-                                records: messages.map((message, index) => ({
-                                    attributes: 0,
-                                    timestampDelta: (message.timestamp ?? defaultTimestamp) - (baseTimestamp ?? 0n),
-                                    offsetDelta: index,
-                                    key: message.key ?? null,
-                                    value: message.value,
-                                    headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
-                                        key,
-                                        value,
-                                    })),
-                                })),
-                            };
-                        }),
-                    }));
-                    await this.cluster.sendRequestToNode(parseInt(nodeId))(api_1.API.PRODUCE, {
-                        transactionalId: null,
-                        acks,
-                        timeoutMs: 30000,
-                        topicData,
-                    });
-                    topicData.forEach(({ name, partitionData }) => {
-                        partitionData.forEach(({ index, records }) => {
-                            this.updateSequence(name, index, records.length);
-                        });
-                    });
-                });
+                await buffer.enqueue(messages);
             }
             catch (error) {
                 await this.handleError(error);
-                const messages = Object.values(topicPartitionMessages)
-                    .flatMap((partitionMessages) => Object.values(partitionMessages).flat())
-                    .map(({ partition, ...message }) => message);
-                return this.send(messages, { acks });
+                return this.send(messages);
             }
         }));
     }
@@ -120,47 +73,25 @@ class Producer {
     }
     ensureProducerInitialized = (0, shared_1.shared)(async () => {
         await this.cluster.ensureConnected();
-        if (!this.producerId) {
-            await this.initProducerId();
+        if (!this.state.producerId) {
+            await this.state.initProducerId();
         }
     });
-    async initProducerId() {
-        return (0, retry_1.withRetry)(this.handleError.bind(this))(async () => {
-            const result = await this.cluster.sendRequest(api_1.API.INIT_PRODUCER_ID, {
-                transactionalId: null,
-                transactionTimeoutMs: 0,
-                producerId: this.producerId,
-                producerEpoch: this.producerEpoch,
-            });
-            this.producerId = result.producerId;
-            this.producerEpoch = result.producerEpoch;
-            this.sequences = {};
-        });
-    }
-    getSequence(topic, partition) {
-        return this.sequences[topic]?.[partition] ?? 0;
-    }
-    updateSequence(topic, partition, messagesCount) {
-        this.sequences[topic] ??= {};
-        this.sequences[topic][partition] ??= 0;
-        this.sequences[topic][partition] += messagesCount;
-    }
-    async fetchMetadata(topics, allowTopicAutoCreation) {
-        return (0, retry_1.withRetry)(this.handleError.bind(this))(async () => {
-            await this.metadata.fetchMetadata({ topics, allowTopicAutoCreation });
-        });
-    }
+    fetchMetadataForTopics = (0, shared_1.shared)(async (topics) => {
+        const { allowTopicAutoCreation } = this.options;
+        await this.chain.run(topics.map((topic) => `metadata:${topic}`), () => this.metadata.fetchMetadataIfNecessary({ topics, allowTopicAutoCreation }));
+    });
     async handleError(error) {
         await (0, api_1.handleApiError)(error).catch(async (error) => {
             if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.NOT_LEADER_OR_FOLLOWER) {
                 logger_1.log.debug('Refreshing metadata', { reason: error.message });
                 const topics = Object.keys(this.metadata.getTopicPartitions());
-                await this.fetchMetadata(topics, false);
+                await this.metadata.fetchMetadata({ topics, allowTopicAutoCreation: false });
                 return;
             }
             if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.OUT_OF_ORDER_SEQUENCE_NUMBER) {
                 logger_1.log.debug('Out of order sequence number. Reinitializing producer ID');
-                await this.initProducerId();
+                await this.state.initProducerId();
                 return;
             }
             throw error;
@@ -171,6 +102,6 @@ exports.Producer = Producer;
 __decorate([
     trace(() => ({ root: true })),
     __metadata("design:type", Function),
-    __metadata("design:paramtypes", [Array, Object]),
+    __metadata("design:paramtypes", [Array]),
     __metadata("design:returntype", Promise)
 ], Producer.prototype, "send", null);

package/dist/utils/decoder.js

@@ -110,12 +110,16 @@ class Decoder {
     }
     readArray(callback) {
         const length = this.readInt32();
-        const results = Array.from({ length }).map(() => callback(this));
+        const results = new Array(length);
+        for (let i = 0; i < length; i++)
+            results[i] = callback(this);
         return results;
     }
     readCompactArray(callback) {
         const length = this.readUVarInt() - 1;
-        const results = Array.from({ length }).map(() => callback(this));
+        const results = new Array(length);
+        for (let i = 0; i < length; i++)
+            results[i] = callback(this);
         return results;
     }
     readVarIntArray(callback) {
@@ -125,15 +129,16 @@ class Decoder {
     }
     readRecords(callback) {
         const length = this.readInt32();
-        return Array.from({ length }).map(() => {
+        const results = [];
+        for (let i = 0; i < length; i++) {
             const size = this.readVarInt();
-            if (!size) {
-                return null;
-            }
+            if (!size)
+                continue;
             const child = new Decoder(this.buffer.subarray(this.offset, this.offset + size));
             this.offset += size;
-            return callback(child);
-        }).filter(x => x !== null);
+            results.push(callback(child));
+        }
+        return results;
     }
     read(length) {
         const value = this.buffer.subarray(this.offset, length !== undefined ? this.offset + length : undefined);

package/dist/utils/encoder.d.ts

@@ -1,9 +1,11 @@
 export declare class Encoder {
-    private chunks;
-    getChunks(): Buffer<ArrayBufferLike>[];
+    private buffer;
+    private offset;
+    constructor(initialCapacity?: number);
+    private ensure;
     getBufferLength(): number;
-    write(...buffers: Buffer[]): this;
-    writeEncoder(encoder: Encoder): this;
+    write(src: Buffer): this;
+    writeEncoder(other: Encoder): this;
     writeInt8(value: number): this;
     writeInt16(value: number): this;
     writeInt32(value: number): this;
@@ -18,10 +20,10 @@ export declare class Encoder {
     writeVarIntString(value: string | null): this;
     writeUUID(value: string | null): this;
     writeBoolean(value: boolean): this;
-    writeArray<T>(arr: T[], callback: (encoder: Encoder, item: T) => Encoder): this;
-    writeCompactArray<T>(arr: T[] | null, callback: (encoder: Encoder, item: T) => Encoder): this;
-    writeVarIntArray<T>(arr: T[], callback: (encoder: Encoder, item: T) => Encoder): this;
+    writeArray<T>(arr: T[], callback: (encoder: Encoder, item: T) => void): this;
+    writeCompactArray<T>(arr: T[] | null, callback: (encoder: Encoder, item: T) => void): this;
+    writeVarIntArray<T>(arr: T[], callback: (encoder: Encoder, item: T) => void): this;
     writeBytes(value: Buffer): this;
     writeCompactBytes(value: Buffer): this;
-    value(): Buffer<ArrayBuffer>;
+    value(): Buffer<ArrayBufferLike>;
 }

package/dist/utils/encoder.js

@@ -2,121 +2,158 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Encoder = void 0;
 class Encoder {
-    chunks = [];
-    getChunks() {
-        return this.chunks;
+    buffer;
+    offset = 0;
+    constructor(initialCapacity = 512) {
+        this.buffer = Buffer.allocUnsafe(initialCapacity);
+    }
+    ensure(extra) {
+        const need = this.offset + extra;
+        if (need <= this.buffer.length)
+            return;
+        let cap = this.buffer.length;
+        while (cap < need)
+            cap <<= 1;
+        const n = Buffer.allocUnsafe(cap);
+        this.buffer.copy(n, 0, 0, this.offset);
+        this.buffer = n;
     }
     getBufferLength() {
-        return this.chunks.reduce((acc, chunk) => acc + chunk.length, 0);
+        return this.offset;
     }
-    write(...buffers) {
-        this.chunks.push(...buffers);
+    write(src) {
+        this.ensure(src.length);
+        src.copy(this.buffer, this.offset);
+        this.offset += src.length;
         return this;
     }
-    writeEncoder(encoder) {
-        return this.write(...encoder.getChunks());
+    writeEncoder(other) {
+        this.write(other.buffer.subarray(0, other.offset));
+        return this;
     }
     writeInt8(value) {
-        const buffer = Buffer.allocUnsafe(1);
-        buffer.writeInt8(value);
-        return this.write(buffer);
+        this.ensure(1);
+        this.buffer.writeInt8(value, this.offset);
+        this.offset += 1;
+        return this;
     }
     writeInt16(value) {
-        const buffer = Buffer.allocUnsafe(2);
-        buffer.writeInt16BE(value);
-        return this.write(buffer);
+        this.ensure(2);
+        this.buffer.writeInt16BE(value, this.offset);
+        this.offset += 2;
+        return this;
     }
     writeInt32(value) {
-        const buffer = Buffer.allocUnsafe(4);
-        buffer.writeInt32BE(value);
-        return this.write(buffer);
+        this.ensure(4);
+        this.buffer.writeInt32BE(value, this.offset);
+        this.offset += 4;
+        return this;
     }
     writeUInt32(value) {
-        const buffer = Buffer.allocUnsafe(4);
-        buffer.writeUInt32BE(value);
-        return this.write(buffer);
+        this.ensure(4);
+        this.buffer.writeUInt32BE(value, this.offset);
+        this.offset += 4;
+        return this;
     }
     writeInt64(value) {
-        const buffer = Buffer.allocUnsafe(8);
-        buffer.writeBigInt64BE(value);
-        return this.write(buffer);
+        this.ensure(8);
+        this.buffer.writeBigInt64BE(value, this.offset);
+        this.offset += 8;
+        return this;
     }
     writeUVarInt(value) {
-        const byteArray = [];
-        while ((value & 0xffffff80) !== 0) {
-            byteArray.push((value & 0x7f) | 0x80);
+        this.ensure(5);
+        while (value & 0xffffff80) {
+            this.buffer[this.offset++] = (value & 0x7f) | 0x80;
             value >>>= 7;
         }
-        byteArray.push(value & 0x7f);
-        return this.write(Buffer.from(byteArray));
+        this.buffer[this.offset++] = value & 0x7f;
+        return this;
     }
     writeVarInt(value) {
-        const encodedValue = (value << 1) ^ (value >> 31);
-        return this.writeUVarInt(encodedValue);
+        return this.writeUVarInt((value << 1) ^ (value >> 31));
     }
     writeUVarLong(value) {
-        const byteArray = [];
-        while ((value & 0xffffffffffffff80n) !== 0n) {
-            byteArray.push(Number((value & BigInt(0x7f)) | BigInt(0x80)));
+        this.ensure(10);
+        while (value >= 0x80n) {
+            this.buffer[this.offset++] = Number((value & 0x7fn) | 0x80n);
             value >>= 7n;
         }
-        byteArray.push(Number(value & BigInt(0x7f)));
-        return this.write(Buffer.from(byteArray));
+        this.buffer[this.offset++] = Number(value);
+        return this;
     }
     writeVarLong(value) {
-        const encodedValue = (value << BigInt(1)) ^ (value >> BigInt(63));
-        return this.writeUVarLong(encodedValue);
+        return this.writeUVarLong((value << 1n) ^ (value >> 63n));
     }
     writeString(value) {
-        if (value === null) {
+        if (value === null)
             return this.writeInt16(-1);
-        }
         const buffer = Buffer.from(value, 'utf-8');
-        return this.writeInt16(buffer.length).write(buffer);
+        this.writeInt16(buffer.length);
+        this.write(buffer);
+        return this;
     }
     writeCompactString(value) {
-        if (value === null) {
+        if (value === null)
             return this.writeUVarInt(0);
-        }
-        const buffer = Buffer.from(value, 'utf-8');
-        return this.writeUVarInt(buffer.length + 1).write(buffer);
+        const b = Buffer.from(value, 'utf-8');
+        this.writeUVarInt(b.length + 1);
+        this.write(b);
+        return this;
     }
     writeVarIntString(value) {
-        if (value === null) {
+        if (value === null)
             return this.writeVarInt(-1);
-        }
-        const buffer = Buffer.from(value, 'utf-8');
-        return this.writeVarInt(buffer.length).write(buffer);
+        const b = Buffer.from(value, 'utf-8');
+        this.writeVarInt(b.length);
+        this.write(b);
+        return this;
     }
     writeUUID(value) {
         if (value === null) {
-            return this.write(Buffer.alloc(16));
+            this.ensure(16);
+            this.buffer.fill(0, this.offset, this.offset + 16);
+            this.offset += 16;
+            return this;
         }
-        return this.write(Buffer.from(value, 'hex'));
+        this.write(Buffer.from(value, 'hex'));
+        return this;
     }
     writeBoolean(value) {
         return this.writeInt8(value ? 1 : 0);
     }
     writeArray(arr, callback) {
-        return this.writeInt32(arr.length).write(...arr.flatMap((item) => callback(new Encoder(), item).getChunks()));
+        this.writeInt32(arr.length);
+        for (const it of arr)
+            callback(this, it);
+        return this;
     }
     writeCompactArray(arr, callback) {
-        if (arr === null) {
+        if (arr === null)
             return this.writeUVarInt(0);
-        }
-        return this.writeUVarInt(arr.length + 1).write(...arr.flatMap((item) => callback(new Encoder(), item).getChunks()));
+        this.writeUVarInt(arr.length + 1);
+        for (const it of arr)
+            callback(this, it);
+        return this;
     }
     writeVarIntArray(arr, callback) {
-        return this.writeVarInt(arr.length).write(...arr.flatMap((item) => callback(new Encoder(), item).getChunks()));
+        this.writeVarInt(arr.length);
+        for (const it of arr)
+            callback(this, it);
+        return this;
     }
     writeBytes(value) {
-        return this.writeInt32(value.length).write(value);
+        this.writeInt32(value.length);
+        this.write(value);
+        return this;
     }
     writeCompactBytes(value) {
-        return this.writeUVarInt(value.length + 1).write(value);
+        this.writeUVarInt(value.length + 1);
+        this.write(value);
+        return this;
     }
     value() {
-        return Buffer.concat(this.chunks);
+        return this.buffer.subarray(0, this.offset);
     }
 }
 exports.Encoder = Encoder;

package/dist/utils/promise-chain.d.ts

@@ -0,0 +1,5 @@
+export declare class PromiseChain {
+    private locks;
+    run(keys: string[], callback: () => Promise<void>): Promise<void>;
+    private acquire;
+}

package/dist/utils/promise-chain.js

@@ -0,0 +1,39 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PromiseChain = void 0;
+class PromiseChain {
+    locks = new Map();
+    async run(keys, callback) {
+        const orderedKeys = [...new Set(keys)].sort();
+        const releases = [];
+        for (const key of orderedKeys) {
+            const release = await this.acquire(key);
+            releases.push(release);
+        }
+        try {
+            await callback();
+        }
+        finally {
+            releases.reverse().forEach((release) => release());
+        }
+    }
+    async acquire(key) {
+        const previousTail = this.locks.get(key);
+        let release;
+        const currentTail = new Promise((resolve) => (release = resolve));
+        if (previousTail) {
+            this.locks.set(key, previousTail.then(() => currentTail));
+            await previousTail;
+        }
+        else {
+            this.locks.set(key, currentTail);
+        }
+        return () => {
+            release();
+            if (this.locks.get(key) === currentTail) {
+                this.locks.delete(key);
+            }
+        };
+    }
+}
+exports.PromiseChain = PromiseChain;

package/dist/utils/shared.d.ts

@@ -1 +1 @@
-export declare const shared: <F extends () => Promise<any>>(func: F) => () => ReturnType<F>;
+export declare const shared: <F extends (...args: any[]) => Promise<any>>(func: F) => (...args: Parameters<F>) => ReturnType<F>;

package/dist/utils/shared.js

@@ -2,15 +2,16 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.shared = void 0;
 const shared = (func) => {
-    let promise;
-    return () => {
-        if (!promise) {
-            promise = func();
-            promise.finally(() => {
-                promise = undefined;
+    let promises = {};
+    return (...args) => {
+        const key = JSON.stringify(args);
+        if (!promises[key]) {
+            promises[key] = func(...args);
+            promises[key].finally(() => {
+                delete promises[key];
             });
         }
-        return promise;
+        return promises[key];
     };
 };
 exports.shared = shared;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kafka-ts",
-    "version": "1.1.9",
+    "version": "1.2.0",
     "main": "dist/index.js",
     "author": "Priit Käärd",
     "license": "MIT",

package/dist/distributors/messages-to-topic-partition-leaders.d.ts

@@ -1,17 +0,0 @@
-type TopicPartitionLeader = {
-    [topicName: string]: {
-        [partitionId: number]: number;
-    };
-};
-type MessagesByNodeTopicPartition<T> = {
-    [nodeId: number]: {
-        [topicName: string]: {
-            [partitionId: number]: T[];
-        };
-    };
-};
-export declare const distributeMessagesToTopicPartitionLeaders: <T extends {
-    topic: string;
-    partition: number;
-}>(messages: T[], topicPartitionLeader: TopicPartitionLeader) => MessagesByNodeTopicPartition<T>;
-export {};

package/dist/distributors/messages-to-topic-partition-leaders.js

@@ -1,15 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.distributeMessagesToTopicPartitionLeaders = void 0;
-const distributeMessagesToTopicPartitionLeaders = (messages, topicPartitionLeader) => {
-    const result = {};
-    messages.forEach((message) => {
-        const leaderId = topicPartitionLeader[message.topic][message.partition];
-        result[leaderId] ??= {};
-        result[leaderId][message.topic] ??= {};
-        result[leaderId][message.topic][message.partition] ??= [];
-        result[leaderId][message.topic][message.partition].push(message);
-    });
-    return result;
-};
-exports.distributeMessagesToTopicPartitionLeaders = distributeMessagesToTopicPartitionLeaders;

package/dist/distributors/messages-to-topic-partition-leaders.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/distributors/messages-to-topic-partition-leaders.test.js

@@ -1,30 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const vitest_1 = require("vitest");
-const messages_to_topic_partition_leaders_1 = require("./messages-to-topic-partition-leaders");
-(0, vitest_1.describe)('Distribute messages to partition leader ids', () => {
-    (0, vitest_1.describe)('distributeMessagesToTopicPartitionLeaders', () => {
-        (0, vitest_1.it)('snoke', () => {
-            const result = (0, messages_to_topic_partition_leaders_1.distributeMessagesToTopicPartitionLeaders)([{ topic: 'topic', partition: 0, key: null, value: null, offset: 0n, timestamp: 0n, headers: {} }], { topic: { 0: 1 } });
-            (0, vitest_1.expect)(result).toMatchInlineSnapshot(`
-              {
-                "1": {
-                  "topic": {
-                    "0": [
-                      {
-                        "headers": {},
-                        "key": null,
-                        "offset": 0n,
-                        "partition": 0,
-                        "timestamp": 0n,
-                        "topic": "topic",
-                        "value": null,
-                      },
-                    ],
-                  },
-                },
-              }
-            `);
-        });
-    });
-});

package/dist/utils/lock.d.ts

@@ -1,8 +0,0 @@
-import EventEmitter from 'events';
-export declare class Lock extends EventEmitter {
-    private locks;
-    constructor();
-    acquire(keys: string[], callback: () => Promise<void>): Promise<void>;
-    private acquireKey;
-    private releaseKey;
-}

package/dist/utils/lock.js

@@ -1,44 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Lock = void 0;
-const events_1 = __importDefault(require("events"));
-const logger_1 = require("./logger");
-class Lock extends events_1.default {
-    locks = {};
-    constructor() {
-        super();
-        this.setMaxListeners(Infinity);
-    }
-    async acquire(keys, callback) {
-        await Promise.all(keys.map((key) => this.acquireKey(key)));
-        try {
-            await callback();
-        }
-        finally {
-            keys.forEach((key) => this.releaseKey(key));
-        }
-    }
-    async acquireKey(key) {
-        while (this.locks[key]) {
-            await new Promise((resolve) => {
-                const timeout = setTimeout(() => {
-                    logger_1.log.warn(`Lock timed out`, { key });
-                    this.releaseKey(key);
-                }, 60_000);
-                this.once(`release:${key}`, () => {
-                    clearTimeout(timeout);
-                    resolve();
-                });
-            });
-        }
-        this.locks[key] = true;
-    }
-    releaseKey(key) {
-        this.locks[key] = false;
-        this.emit(`release:${key}`);
-    }
-}
-exports.Lock = Lock;