kafka-ts 1.1.8 → 1.2.0

package/dist/producer/producer.d.ts

@@ -4,25 +4,20 @@ import { Message } from '../types';
 export type ProducerOptions = {
     allowTopicAutoCreation?: boolean;
     partitioner?: Partitioner;
+    maxBatchSize?: number;
 };
 export declare class Producer {
     private cluster;
     private options;
     private metadata;
-    private producerId;
-    private producerEpoch;
-    private sequences;
+    private state;
     private partition;
-    private lock;
+    private chain;
+    private bufferByNodeId;
     constructor(cluster: Cluster, options: ProducerOptions);
-    send(messages: Message[], { acks }?: {
-        acks?: -1 | 1;
-    }): Promise<void>;
+    send(messages: Message[]): Promise<void>;
     close(): Promise<void>;
     private ensureProducerInitialized;
-    private initProducerId;
-    private getSequence;
-    private updateSequence;
-    private fetchMetadata;
+    private fetchMetadataForTopics;
     private handleError;
 }

package/dist/producer/producer.js

@@ -11,106 +11,60 @@ var __metadata = (this && this.__metadata) || function (k, v) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Producer = void 0;
 const api_1 = require("../api");
-const messages_to_topic_partition_leaders_1 = require("../distributors/messages-to-topic-partition-leaders");
+const group_by_leader_id_1 = require("../distributors/group-by-leader-id");
 const partitioner_1 = require("../distributors/partitioner");
 const metadata_1 = require("../metadata");
 const error_1 = require("../utils/error");
-const lock_1 = require("../utils/lock");
 const logger_1 = require("../utils/logger");
+const promise_chain_1 = require("../utils/promise-chain");
 const shared_1 = require("../utils/shared");
 const tracer_1 = require("../utils/tracer");
+const producer_buffer_1 = require("./producer-buffer");
+const producer_state_1 = require("./producer-state");
 const trace = (0, tracer_1.createTracer)('Producer');
 class Producer {
     cluster;
     options;
     metadata;
-    producerId = 0n;
-    producerEpoch = 0;
-    sequences = {};
+    state;
     partition;
-    lock = new lock_1.Lock();
+    chain = new promise_chain_1.PromiseChain();
+    bufferByNodeId = {};
     constructor(cluster, options) {
         this.cluster = cluster;
         this.options = {
             ...options,
             allowTopicAutoCreation: options.allowTopicAutoCreation ?? false,
             partitioner: options.partitioner ?? partitioner_1.defaultPartitioner,
+            maxBatchSize: options.maxBatchSize ?? 500,
         };
         this.metadata = new metadata_1.Metadata({ cluster });
+        this.state = new producer_state_1.ProducerState({ cluster });
         this.partition = this.options.partitioner({ metadata: this.metadata });
     }
-    async send(messages, { acks = -1 } = {}) {
+    async send(messages) {
         await this.ensureProducerInitialized();
-        const { allowTopicAutoCreation } = this.options;
-        const defaultTimestamp = BigInt(Date.now());
-        const topics = new Set(messages.map((message) => message.topic));
-        await this.lock.acquire([...topics].map((topic) => `metadata:${topic}`), () => this.metadata.fetchMetadataIfNecessary({ topics, allowTopicAutoCreation }));
+        const topics = [...new Set(messages.map((message) => message.topic))];
+        await this.fetchMetadataForTopics(topics);
         const partitionedMessages = messages.map((message) => {
             message.partition = this.partition(message);
             return message;
         });
-        const nodeTopicPartitionMessages = (0, messages_to_topic_partition_leaders_1.distributeMessagesToTopicPartitionLeaders)(partitionedMessages, this.metadata.getTopicPartitionLeaderIds());
-        await Promise.all(Object.entries(nodeTopicPartitionMessages).map(async ([nodeId, topicPartitionMessages]) => {
+        const messagesByLeaderId = (0, group_by_leader_id_1.groupByLeaderId)(partitionedMessages, this.metadata.getTopicPartitionLeaderIds());
+        await Promise.all(Object.entries(messagesByLeaderId).map(async ([leaderId, messages]) => {
+            const nodeId = parseInt(leaderId);
+            const buffer = (this.bufferByNodeId[nodeId] ??= new producer_buffer_1.ProducerBuffer({
+                nodeId,
+                maxBatchSize: this.options.maxBatchSize,
+                cluster: this.cluster,
+                state: this.state,
+            }));
             try {
-                await this.lock.acquire([`node:${nodeId}`], async () => {
-                    const topicData = Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
-                        name: topic,
-                        partitionData: Object.entries(partitionMessages).map(([partition, messages]) => {
-                            const partitionIndex = parseInt(partition);
-                            let baseTimestamp;
-                            let maxTimestamp;
-                            messages.forEach(({ timestamp = defaultTimestamp }) => {
-                                if (!baseTimestamp || timestamp < baseTimestamp) {
-                                    baseTimestamp = timestamp;
-                                }
-                                if (!maxTimestamp || timestamp > maxTimestamp) {
-                                    maxTimestamp = timestamp;
-                                }
-                            });
-                            return {
-                                index: partitionIndex,
-                                baseOffset: 0n,
-                                partitionLeaderEpoch: -1,
-                                attributes: 0,
-                                lastOffsetDelta: messages.length - 1,
-                                baseTimestamp: baseTimestamp ?? 0n,
-                                maxTimestamp: maxTimestamp ?? 0n,
-                                producerId: this.producerId,
-                                producerEpoch: 0,
-                                baseSequence: this.getSequence(topic, partitionIndex),
-                                records: messages.map((message, index) => ({
-                                    attributes: 0,
-                                    timestampDelta: (message.timestamp ?? defaultTimestamp) - (baseTimestamp ?? 0n),
-                                    offsetDelta: index,
-                                    key: message.key ?? null,
-                                    value: message.value,
-                                    headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
-                                        key,
-                                        value,
-                                    })),
-                                })),
-                            };
-                        }),
-                    }));
-                    await this.cluster.sendRequestToNode(parseInt(nodeId))(api_1.API.PRODUCE, {
-                        transactionalId: null,
-                        acks,
-                        timeoutMs: 30000,
-                        topicData,
-                    });
-                    topicData.forEach(({ name, partitionData }) => {
-                        partitionData.forEach(({ index, records }) => {
-                            this.updateSequence(name, index, records.length);
-                        });
-                    });
-                });
+                await buffer.enqueue(messages);
             }
             catch (error) {
                 await this.handleError(error);
-                const messages = Object.values(topicPartitionMessages)
-                    .flatMap((partitionMessages) => Object.values(partitionMessages).flat())
-                    .map(({ partition, ...message }) => message);
-                return this.send(messages, { acks });
+                return this.send(messages);
             }
         }));
     }
@@ -119,55 +73,25 @@ class Producer {
     }
     ensureProducerInitialized = (0, shared_1.shared)(async () => {
         await this.cluster.ensureConnected();
-        if (!this.producerId) {
-            await this.initProducerId();
+        if (!this.state.producerId) {
+            await this.state.initProducerId();
         }
     });
-    async initProducerId() {
-        try {
-            const result = await this.cluster.sendRequest(api_1.API.INIT_PRODUCER_ID, {
-                transactionalId: null,
-                transactionTimeoutMs: 0,
-                producerId: this.producerId,
-                producerEpoch: this.producerEpoch,
-            });
-            this.producerId = result.producerId;
-            this.producerEpoch = result.producerEpoch;
-            this.sequences = {};
-        }
-        catch (error) {
-            await this.handleError(error);
-            return this.initProducerId();
-        }
-    }
-    getSequence(topic, partition) {
-        return this.sequences[topic]?.[partition] ?? 0;
-    }
-    updateSequence(topic, partition, messagesCount) {
-        this.sequences[topic] ??= {};
-        this.sequences[topic][partition] ??= 0;
-        this.sequences[topic][partition] += messagesCount;
-    }
-    async fetchMetadata(topics, allowTopicAutoCreation) {
-        try {
-            await this.metadata.fetchMetadata({ topics, allowTopicAutoCreation });
-        }
-        catch (error) {
-            await this.handleError(error);
-            return this.fetchMetadata(topics, allowTopicAutoCreation);
-        }
-    }
+    fetchMetadataForTopics = (0, shared_1.shared)(async (topics) => {
+        const { allowTopicAutoCreation } = this.options;
+        await this.chain.run(topics.map((topic) => `metadata:${topic}`), () => this.metadata.fetchMetadataIfNecessary({ topics, allowTopicAutoCreation }));
+    });
     async handleError(error) {
         await (0, api_1.handleApiError)(error).catch(async (error) => {
             if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.NOT_LEADER_OR_FOLLOWER) {
                 logger_1.log.debug('Refreshing metadata', { reason: error.message });
                 const topics = Object.keys(this.metadata.getTopicPartitions());
-                await this.fetchMetadata(topics, false);
+                await this.metadata.fetchMetadata({ topics, allowTopicAutoCreation: false });
                 return;
             }
             if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.OUT_OF_ORDER_SEQUENCE_NUMBER) {
                 logger_1.log.debug('Out of order sequence number. Reinitializing producer ID');
-                await this.initProducerId();
+                await this.state.initProducerId();
                 return;
             }
             throw error;
@@ -178,6 +102,6 @@ exports.Producer = Producer;
 __decorate([
     trace(() => ({ root: true })),
     __metadata("design:type", Function),
-    __metadata("design:paramtypes", [Array, Object]),
+    __metadata("design:paramtypes", [Array]),
     __metadata("design:returntype", Promise)
 ], Producer.prototype, "send", null);

package/dist/utils/decoder.js

@@ -110,12 +110,16 @@ class Decoder {
     }
     readArray(callback) {
         const length = this.readInt32();
-        const results = Array.from({ length }).map(() => callback(this));
+        const results = new Array(length);
+        for (let i = 0; i < length; i++)
+            results[i] = callback(this);
         return results;
     }
     readCompactArray(callback) {
         const length = this.readUVarInt() - 1;
-        const results = Array.from({ length }).map(() => callback(this));
+        const results = new Array(length);
+        for (let i = 0; i < length; i++)
+            results[i] = callback(this);
         return results;
     }
     readVarIntArray(callback) {
@@ -125,15 +129,16 @@ class Decoder {
     }
     readRecords(callback) {
         const length = this.readInt32();
-        return Array.from({ length }).map(() => {
+        const results = [];
+        for (let i = 0; i < length; i++) {
             const size = this.readVarInt();
-            if (!size) {
-                return null;
-            }
+            if (!size)
+                continue;
             const child = new Decoder(this.buffer.subarray(this.offset, this.offset + size));
             this.offset += size;
-            return callback(child);
-        }).filter(x => x !== null);
+            results.push(callback(child));
+        }
+        return results;
     }
     read(length) {
         const value = this.buffer.subarray(this.offset, length !== undefined ? this.offset + length : undefined);

package/dist/utils/encoder.d.ts

@@ -1,9 +1,11 @@
 export declare class Encoder {
-    private chunks;
-    getChunks(): Buffer<ArrayBufferLike>[];
+    private buffer;
+    private offset;
+    constructor(initialCapacity?: number);
+    private ensure;
     getBufferLength(): number;
-    write(...buffers: Buffer[]): this;
-    writeEncoder(encoder: Encoder): this;
+    write(src: Buffer): this;
+    writeEncoder(other: Encoder): this;
     writeInt8(value: number): this;
     writeInt16(value: number): this;
     writeInt32(value: number): this;
@@ -18,10 +20,10 @@ export declare class Encoder {
     writeVarIntString(value: string | null): this;
     writeUUID(value: string | null): this;
     writeBoolean(value: boolean): this;
-    writeArray<T>(arr: T[], callback: (encoder: Encoder, item: T) => Encoder): this;
-    writeCompactArray<T>(arr: T[] | null, callback: (encoder: Encoder, item: T) => Encoder): this;
-    writeVarIntArray<T>(arr: T[], callback: (encoder: Encoder, item: T) => Encoder): this;
+    writeArray<T>(arr: T[], callback: (encoder: Encoder, item: T) => void): this;
+    writeCompactArray<T>(arr: T[] | null, callback: (encoder: Encoder, item: T) => void): this;
+    writeVarIntArray<T>(arr: T[], callback: (encoder: Encoder, item: T) => void): this;
     writeBytes(value: Buffer): this;
     writeCompactBytes(value: Buffer): this;
-    value(): Buffer<ArrayBuffer>;
+    value(): Buffer<ArrayBufferLike>;
 }

package/dist/utils/encoder.js

@@ -2,121 +2,158 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Encoder = void 0;
 class Encoder {
-    chunks = [];
-    getChunks() {
-        return this.chunks;
+    buffer;
+    offset = 0;
+    constructor(initialCapacity = 512) {
+        this.buffer = Buffer.allocUnsafe(initialCapacity);
+    }
+    ensure(extra) {
+        const need = this.offset + extra;
+        if (need <= this.buffer.length)
+            return;
+        let cap = this.buffer.length;
+        while (cap < need)
+            cap <<= 1;
+        const n = Buffer.allocUnsafe(cap);
+        this.buffer.copy(n, 0, 0, this.offset);
+        this.buffer = n;
     }
     getBufferLength() {
-        return this.chunks.reduce((acc, chunk) => acc + chunk.length, 0);
+        return this.offset;
     }
-    write(...buffers) {
-        this.chunks.push(...buffers);
+    write(src) {
+        this.ensure(src.length);
+        src.copy(this.buffer, this.offset);
+        this.offset += src.length;
         return this;
     }
-    writeEncoder(encoder) {
-        return this.write(...encoder.getChunks());
+    writeEncoder(other) {
+        this.write(other.buffer.subarray(0, other.offset));
+        return this;
     }
     writeInt8(value) {
-        const buffer = Buffer.allocUnsafe(1);
-        buffer.writeInt8(value);
-        return this.write(buffer);
+        this.ensure(1);
+        this.buffer.writeInt8(value, this.offset);
+        this.offset += 1;
+        return this;
     }
     writeInt16(value) {
-        const buffer = Buffer.allocUnsafe(2);
-        buffer.writeInt16BE(value);
-        return this.write(buffer);
+        this.ensure(2);
+        this.buffer.writeInt16BE(value, this.offset);
+        this.offset += 2;
+        return this;
     }
     writeInt32(value) {
-        const buffer = Buffer.allocUnsafe(4);
-        buffer.writeInt32BE(value);
-        return this.write(buffer);
+        this.ensure(4);
+        this.buffer.writeInt32BE(value, this.offset);
+        this.offset += 4;
+        return this;
     }
     writeUInt32(value) {
-        const buffer = Buffer.allocUnsafe(4);
-        buffer.writeUInt32BE(value);
-        return this.write(buffer);
+        this.ensure(4);
+        this.buffer.writeUInt32BE(value, this.offset);
+        this.offset += 4;
+        return this;
     }
     writeInt64(value) {
-        const buffer = Buffer.allocUnsafe(8);
-        buffer.writeBigInt64BE(value);
-        return this.write(buffer);
+        this.ensure(8);
+        this.buffer.writeBigInt64BE(value, this.offset);
+        this.offset += 8;
+        return this;
     }
     writeUVarInt(value) {
-        const byteArray = [];
-        while ((value & 0xffffff80) !== 0) {
-            byteArray.push((value & 0x7f) | 0x80);
+        this.ensure(5);
+        while (value & 0xffffff80) {
+            this.buffer[this.offset++] = (value & 0x7f) | 0x80;
             value >>>= 7;
         }
-        byteArray.push(value & 0x7f);
-        return this.write(Buffer.from(byteArray));
+        this.buffer[this.offset++] = value & 0x7f;
+        return this;
     }
     writeVarInt(value) {
-        const encodedValue = (value << 1) ^ (value >> 31);
-        return this.writeUVarInt(encodedValue);
+        return this.writeUVarInt((value << 1) ^ (value >> 31));
     }
     writeUVarLong(value) {
-        const byteArray = [];
-        while ((value & 0xffffffffffffff80n) !== 0n) {
-            byteArray.push(Number((value & BigInt(0x7f)) | BigInt(0x80)));
+        this.ensure(10);
+        while (value >= 0x80n) {
+            this.buffer[this.offset++] = Number((value & 0x7fn) | 0x80n);
             value >>= 7n;
         }
-        byteArray.push(Number(value & BigInt(0x7f)));
-        return this.write(Buffer.from(byteArray));
+        this.buffer[this.offset++] = Number(value);
+        return this;
     }
     writeVarLong(value) {
-        const encodedValue = (value << BigInt(1)) ^ (value >> BigInt(63));
-        return this.writeUVarLong(encodedValue);
+        return this.writeUVarLong((value << 1n) ^ (value >> 63n));
     }
     writeString(value) {
-        if (value === null) {
+        if (value === null)
             return this.writeInt16(-1);
-        }
         const buffer = Buffer.from(value, 'utf-8');
-        return this.writeInt16(buffer.length).write(buffer);
+        this.writeInt16(buffer.length);
+        this.write(buffer);
+        return this;
     }
     writeCompactString(value) {
-        if (value === null) {
+        if (value === null)
             return this.writeUVarInt(0);
-        }
-        const buffer = Buffer.from(value, 'utf-8');
-        return this.writeUVarInt(buffer.length + 1).write(buffer);
+        const b = Buffer.from(value, 'utf-8');
+        this.writeUVarInt(b.length + 1);
+        this.write(b);
+        return this;
     }
     writeVarIntString(value) {
-        if (value === null) {
+        if (value === null)
             return this.writeVarInt(-1);
-        }
-        const buffer = Buffer.from(value, 'utf-8');
-        return this.writeVarInt(buffer.length).write(buffer);
+        const b = Buffer.from(value, 'utf-8');
+        this.writeVarInt(b.length);
+        this.write(b);
+        return this;
     }
     writeUUID(value) {
         if (value === null) {
-            return this.write(Buffer.alloc(16));
+            this.ensure(16);
+            this.buffer.fill(0, this.offset, this.offset + 16);
+            this.offset += 16;
+            return this;
         }
-        return this.write(Buffer.from(value, 'hex'));
+        this.write(Buffer.from(value, 'hex'));
+        return this;
     }
     writeBoolean(value) {
         return this.writeInt8(value ? 1 : 0);
     }
     writeArray(arr, callback) {
-        return this.writeInt32(arr.length).write(...arr.flatMap((item) => callback(new Encoder(), item).getChunks()));
+        this.writeInt32(arr.length);
+        for (const it of arr)
+            callback(this, it);
+        return this;
     }
     writeCompactArray(arr, callback) {
-        if (arr === null) {
+        if (arr === null)
             return this.writeUVarInt(0);
-        }
-        return this.writeUVarInt(arr.length + 1).write(...arr.flatMap((item) => callback(new Encoder(), item).getChunks()));
+        this.writeUVarInt(arr.length + 1);
+        for (const it of arr)
+            callback(this, it);
+        return this;
     }
     writeVarIntArray(arr, callback) {
-        return this.writeVarInt(arr.length).write(...arr.flatMap((item) => callback(new Encoder(), item).getChunks()));
+        this.writeVarInt(arr.length);
+        for (const it of arr)
+            callback(this, it);
+        return this;
     }
     writeBytes(value) {
-        return this.writeInt32(value.length).write(value);
+        this.writeInt32(value.length);
+        this.write(value);
+        return this;
     }
     writeCompactBytes(value) {
-        return this.writeUVarInt(value.length + 1).write(value);
+        this.writeUVarInt(value.length + 1);
+        this.write(value);
+        return this;
     }
     value() {
-        return Buffer.concat(this.chunks);
+        return this.buffer.subarray(0, this.offset);
     }
 }
 exports.Encoder = Encoder;

package/dist/utils/promise-chain.d.ts

@@ -0,0 +1,5 @@
+export declare class PromiseChain {
+    private locks;
+    run(keys: string[], callback: () => Promise<void>): Promise<void>;
+    private acquire;
+}

package/dist/utils/promise-chain.js

@@ -0,0 +1,39 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PromiseChain = void 0;
+class PromiseChain {
+    locks = new Map();
+    async run(keys, callback) {
+        const orderedKeys = [...new Set(keys)].sort();
+        const releases = [];
+        for (const key of orderedKeys) {
+            const release = await this.acquire(key);
+            releases.push(release);
+        }
+        try {
+            await callback();
+        }
+        finally {
+            releases.reverse().forEach((release) => release());
+        }
+    }
+    async acquire(key) {
+        const previousTail = this.locks.get(key);
+        let release;
+        const currentTail = new Promise((resolve) => (release = resolve));
+        if (previousTail) {
+            this.locks.set(key, previousTail.then(() => currentTail));
+            await previousTail;
+        }
+        else {
+            this.locks.set(key, currentTail);
+        }
+        return () => {
+            release();
+            if (this.locks.get(key) === currentTail) {
+                this.locks.delete(key);
+            }
+        };
+    }
+}
+exports.PromiseChain = PromiseChain;

package/dist/utils/retry.d.ts

@@ -0,0 +1 @@
+export declare const withRetry: (handleError: (error: unknown) => Promise<void>) => <T>(func: () => Promise<T>) => Promise<T>;

package/dist/utils/retry.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.withRetry = void 0;
+const logger_1 = require("./logger");
+const withRetry = (handleError) => async (func) => {
+    let lastError;
+    for (let i = 0; i < 15; i++) {
+        try {
+            return await func();
+        }
+        catch (error) {
+            await handleError(error);
+            lastError = error;
+        }
+    }
+    logger_1.log.warn('Retries exhausted', { lastError });
+    throw lastError;
+};
+exports.withRetry = withRetry;

package/dist/utils/shared.d.ts

@@ -1 +1 @@
-export declare const shared: <F extends () => Promise<any>>(func: F) => () => ReturnType<F>;
+export declare const shared: <F extends (...args: any[]) => Promise<any>>(func: F) => (...args: Parameters<F>) => ReturnType<F>;

package/dist/utils/shared.js

@@ -2,15 +2,16 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.shared = void 0;
 const shared = (func) => {
-    let promise;
-    return () => {
-        if (!promise) {
-            promise = func();
-            promise.finally(() => {
-                promise = undefined;
+    let promises = {};
+    return (...args) => {
+        const key = JSON.stringify(args);
+        if (!promises[key]) {
+            promises[key] = func(...args);
+            promises[key].finally(() => {
+                delete promises[key];
             });
         }
-        return promise;
+        return promises[key];
     };
 };
 exports.shared = shared;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kafka-ts",
-    "version": "1.1.8",
+    "version": "1.2.0",
     "main": "dist/index.js",
     "author": "Priit Käärd",
     "license": "MIT",

package/dist/consumer/metadata.d.ts

@@ -1,24 +0,0 @@
-import { IsolationLevel } from "../api/fetch";
-import { Assignment } from "../api/sync-group";
-import { Cluster } from "../cluster";
-import { OffsetManager } from "./offset-manager";
-export type Metadata = ReturnType<typeof createMetadata>;
-type MetadataOptions = {
-    cluster: Cluster;
-    topics?: string[];
-    isolationLevel?: IsolationLevel;
-    allowTopicAutoCreation?: boolean;
-    fromBeginning?: boolean;
-    offsetManager?: OffsetManager;
-};
-export declare const createMetadata: ({ cluster, topics, isolationLevel, allowTopicAutoCreation, fromBeginning, offsetManager, }: MetadataOptions) => {
-    init: () => Promise<void>;
-    getTopicPartitions: () => Record<string, number[]>;
-    getTopicIdByName: (name: string) => string;
-    getTopicNameById: (id: string) => string;
-    getAssignment: () => Assignment;
-    setAssignment: (newAssignment: Assignment) => void;
-    getLeaderIdByTopicPartition: (topic: string, partition: number) => number;
-    getIsrNodeIdsByTopicPartition: (topic: string, partition: number) => number[];
-};
-export {};