kafka-ts 1.1.8 → 1.1.9

package/dist/consumer/consumer-group.js

@@ -14,6 +14,7 @@ const api_1 = require("../api");
 const find_coordinator_1 = require("../api/find-coordinator");
 const error_1 = require("../utils/error");
 const logger_1 = require("../utils/logger");
+const retry_1 = require("../utils/retry");
 const tracer_1 = require("../utils/tracer");
 const trace = (0, tracer_1.createTracer)('ConsumerGroup');
 class ConsumerGroup {
@@ -62,7 +63,7 @@ class ConsumerGroup {
         }
     }
     async findCoordinator() {
-        try {
+        return (0, retry_1.withRetry)(this.handleError.bind(this))(async () => {
             const { coordinators } = await this.options.cluster.sendRequest(api_1.API.FIND_COORDINATOR, {
                 keyType: find_coordinator_1.KEY_TYPE.GROUP,
                 keys: [this.options.groupId],
@@ -70,15 +71,11 @@ class ConsumerGroup {
             this.coordinatorId = coordinators[0].nodeId;
             await this.options.cluster.setSeedBroker(this.coordinatorId);
             this.heartbeatError = null;
-        }
-        catch (error) {
-            await this.handleError(error);
-            return this.findCoordinator();
-        }
+        });
     }
     async joinGroup() {
-        const { cluster, groupId, groupInstanceId, sessionTimeoutMs, rebalanceTimeoutMs, topics } = this.options;
-        try {
+        return (0, retry_1.withRetry)(this.handleError.bind(this))(async () => {
+            const { cluster, groupId, groupInstanceId, sessionTimeoutMs, rebalanceTimeoutMs, topics } = this.options;
             const response = await cluster.sendRequest(api_1.API.JOIN_GROUP, {
                 groupId,
                 groupInstanceId,
@@ -93,28 +90,27 @@ class ConsumerGroup {
             this.generationId = response.generationId;
             this.leaderId = response.leader;
             this.memberIds = response.members.map((member) => member.memberId);
-        }
-        catch (error) {
-            await this.handleError(error);
-            return this.joinGroup();
-        }
+        });
     }
     async syncGroup() {
-        const { cluster, metadata, groupId, groupInstanceId } = this.options;
-        let assignments = [];
-        if (this.memberId === this.leaderId) {
-            const memberAssignments = Object.entries(metadata.getTopicPartitions())
-                .flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition })))
-                .reduce((acc, { topic, partition }, index) => {
-                const memberId = this.memberIds[index % this.memberIds.length];
-                acc[memberId] ??= {};
-                acc[memberId][topic] ??= [];
-                acc[memberId][topic].push(partition);
-                return acc;
-            }, {});
-            assignments = Object.entries(memberAssignments).map(([memberId, assignment]) => ({ memberId, assignment }));
-        }
-        try {
+        return (0, retry_1.withRetry)(this.handleError.bind(this))(async () => {
+            const { cluster, metadata, groupId, groupInstanceId } = this.options;
+            let assignments = [];
+            if (this.memberId === this.leaderId) {
+                const memberAssignments = Object.entries(metadata.getTopicPartitions())
+                    .flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition })))
+                    .reduce((acc, { topic, partition }, index) => {
+                    const memberId = this.memberIds[index % this.memberIds.length];
+                    acc[memberId] ??= {};
+                    acc[memberId][topic] ??= [];
+                    acc[memberId][topic].push(partition);
+                    return acc;
+                }, {});
+                assignments = Object.entries(memberAssignments).map(([memberId, assignment]) => ({
+                    memberId,
+                    assignment,
+                }));
+            }
             const response = await cluster.sendRequest(api_1.API.SYNC_GROUP, {
                 groupId,
                 groupInstanceId,
@@ -125,29 +121,25 @@ class ConsumerGroup {
                 assignments,
             });
             metadata.setAssignment(response.assignments);
-        }
-        catch (error) {
-            await this.handleError(error);
-            return this.syncGroup();
-        }
+        });
     }
     async offsetFetch() {
-        const { cluster, groupId, topics, metadata, offsetManager } = this.options;
-        const assignment = metadata.getAssignment();
-        const request = {
-            groups: [
-                {
-                    groupId,
-                    topics: topics
-                        .map((topic) => ({ name: topic, partitionIndexes: assignment[topic] ?? [] }))
-                        .filter(({ partitionIndexes }) => partitionIndexes.length),
-                },
-            ].filter(({ topics }) => topics.length),
-            requireStable: true,
-        };
-        if (!request.groups.length)
-            return;
-        try {
+        return (0, retry_1.withRetry)(this.handleError.bind(this))(async () => {
+            const { cluster, groupId, topics, metadata, offsetManager } = this.options;
+            const assignment = metadata.getAssignment();
+            const request = {
+                groups: [
+                    {
+                        groupId,
+                        topics: topics
+                            .map((topic) => ({ name: topic, partitionIndexes: assignment[topic] ?? [] }))
+                            .filter(({ partitionIndexes }) => partitionIndexes.length),
+                    },
+                ].filter(({ topics }) => topics.length),
+                requireStable: true,
+            };
+            if (!request.groups.length)
+                return;
             const response = await cluster.sendRequest(api_1.API.OFFSET_FETCH, request);
             const topicPartitions = {};
             response.groups.forEach((group) => {
@@ -162,44 +154,36 @@ class ConsumerGroup {
                 });
             });
             offsetManager.flush(topicPartitions);
-        }
-        catch (error) {
-            await this.handleError(error);
-            return this.offsetFetch();
-        }
+        });
     }
     async offsetCommit(topicPartitions) {
-        const { cluster, groupId, groupInstanceId, offsetManager, consumer } = this.options;
-        const request = {
-            groupId,
-            groupInstanceId,
-            memberId: this.memberId,
-            generationIdOrMemberEpoch: this.generationId,
-            topics: Object.entries(topicPartitions)
-                .filter(([topic]) => topic in offsetManager.pendingOffsets)
-                .map(([topic, partitions]) => ({
-                name: topic,
-                partitions: [...partitions]
-                    .filter((partition) => partition in offsetManager.pendingOffsets[topic])
-                    .map((partitionIndex) => ({
-                    partitionIndex,
-                    committedOffset: offsetManager.pendingOffsets[topic][partitionIndex],
-                    committedLeaderEpoch: -1,
-                    committedMetadata: null,
+        return (0, retry_1.withRetry)(this.handleError.bind(this))(async () => {
+            const { cluster, groupId, groupInstanceId, offsetManager, consumer } = this.options;
+            const request = {
+                groupId,
+                groupInstanceId,
+                memberId: this.memberId,
+                generationIdOrMemberEpoch: this.generationId,
+                topics: Object.entries(topicPartitions)
+                    .filter(([topic]) => topic in offsetManager.pendingOffsets)
+                    .map(([topic, partitions]) => ({
+                    name: topic,
+                    partitions: [...partitions]
+                        .filter((partition) => partition in offsetManager.pendingOffsets[topic])
+                        .map((partitionIndex) => ({
+                        partitionIndex,
+                        committedOffset: offsetManager.pendingOffsets[topic][partitionIndex],
+                        committedLeaderEpoch: -1,
+                        committedMetadata: null,
+                    })),
                 })),
-            })),
-        };
-        if (!request.topics.length) {
-            return;
-        }
-        try {
+            };
+            if (!request.topics.length) {
+                return;
+            }
             await cluster.sendRequest(api_1.API.OFFSET_COMMIT, request);
-        }
-        catch (error) {
-            await this.handleError(error);
-            return this.offsetCommit(topicPartitions);
-        }
-        consumer.emit('offsetCommit');
+            consumer.emit('offsetCommit');
+        });
     }
     async heartbeat() {
         const { cluster, groupId, groupInstanceId, consumer } = this.options;
@@ -212,24 +196,25 @@ class ConsumerGroup {
         consumer.emit('heartbeat');
     }
     async leaveGroup() {
-        if (this.coordinatorId === -1) {
-            return;
-        }
-        const { cluster, groupId, groupInstanceId } = this.options;
-        this.stopHeartbeater();
-        try {
-            await cluster.sendRequest(api_1.API.LEAVE_GROUP, {
-                groupId,
-                members: [{ memberId: this.memberId, groupInstanceId, reason: null }],
-            });
-        }
-        catch (error) {
-            if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.FENCED_INSTANCE_ID) {
+        return (0, retry_1.withRetry)(this.handleError.bind(this))(async () => {
+            if (this.coordinatorId === -1) {
                 return;
             }
-            await this.handleError(error);
-            return this.leaveGroup();
-        }
+            const { cluster, groupId, groupInstanceId } = this.options;
+            this.stopHeartbeater();
+            try {
+                await cluster.sendRequest(api_1.API.LEAVE_GROUP, {
+                    groupId,
+                    members: [{ memberId: this.memberId, groupInstanceId, reason: null }],
+                });
+            }
+            catch (error) {
+                if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.FENCED_INSTANCE_ID) {
+                    return;
+                }
+                throw error;
+            }
+        });
     }
     async handleError(error) {
         await (0, api_1.handleApiError)(error).catch(async (error) => {

package/dist/consumer/consumer.js

@@ -20,6 +20,7 @@ const delay_1 = require("../utils/delay");
 const error_1 = require("../utils/error");
 const logger_1 = require("../utils/logger");
 const retrier_1 = require("../utils/retrier");
+const retry_1 = require("../utils/retry");
 const tracer_1 = require("../utils/tracer");
 const consumer_group_1 = require("./consumer-group");
 const consumer_metadata_1 = require("./consumer-metadata");
@@ -100,7 +101,9 @@ class Consumer extends events_1.default {
                 await this.fetchManager?.stop();
             });
         }
-        await this.consumerGroup?.leaveGroup().catch((error) => logger_1.log.debug('Failed to leave group', { reason: error.message }));
+        await this.consumerGroup
+            ?.leaveGroup()
+            .catch((error) => logger_1.log.debug('Failed to leave group', { reason: error.message }));
         await this.cluster.disconnect().catch(() => { });
     }
     async startFetchManager() {
@@ -140,7 +143,7 @@ class Consumer extends events_1.default {
                 if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.NOT_COORDINATOR) {
                     logger_1.log.debug('Not coordinator. Searching for new coordinator...');
                     await this.consumerGroup?.findCoordinator();
-                    return;
+                    continue;
                 }
                 if (error instanceof error_1.ConnectionError) {
                     logger_1.log.debug(`${error.message}. Restarting consumer...`, { stack: error.stack });
@@ -210,10 +213,10 @@ class Consumer extends events_1.default {
         this.offsetManager.flush(topicPartitions);
     }
     async fetch(nodeId, assignment) {
-        const { rackId, maxWaitMs, minBytes, maxBytes, partitionMaxBytes, isolationLevel } = this.options;
-        this.consumerGroup?.handleLastHeartbeat();
-        try {
-            return await this.cluster.sendRequestToNode(nodeId)(api_1.API.FETCH, {
+        return (0, retry_1.withRetry)(this.handleError.bind(this))(async () => {
+            const { rackId, maxWaitMs, minBytes, maxBytes, partitionMaxBytes, isolationLevel } = this.options;
+            this.consumerGroup?.handleLastHeartbeat();
+            return this.cluster.sendRequestToNode(nodeId)(api_1.API.FETCH, {
                 maxWaitMs,
                 minBytes,
                 maxBytes,
@@ -234,31 +237,19 @@ class Consumer extends events_1.default {
                 forgottenTopicsData: [],
                 rackId,
             });
-        }
-        catch (error) {
-            await this.handleError(error);
-            return this.fetch(nodeId, assignment);
-        }
+        });
     }
     async fetchMetadata() {
-        const { topics, allowTopicAutoCreation } = this.options;
-        try {
+        return (0, retry_1.withRetry)(this.handleError.bind(this))(async () => {
+            const { topics, allowTopicAutoCreation } = this.options;
             await this.metadata.fetchMetadata({ topics, allowTopicAutoCreation });
-        }
-        catch (error) {
-            await this.handleError(error);
-            return this.fetchMetadata();
-        }
+        });
     }
     async fetchOffsets() {
-        const { fromTimestamp } = this.options;
-        try {
+        return (0, retry_1.withRetry)(this.handleError.bind(this))(async () => {
+            const { fromTimestamp } = this.options;
             await this.offsetManager.fetchOffsets({ fromTimestamp });
-        }
-        catch (error) {
-            await this.handleError(error);
-            return this.fetchOffsets();
-        }
+        });
     }
     async handleError(error) {
         await (0, api_1.handleApiError)(error).catch(async (error) => {

package/dist/producer/producer.js

@@ -17,6 +17,7 @@ const metadata_1 = require("../metadata");
 const error_1 = require("../utils/error");
 const lock_1 = require("../utils/lock");
 const logger_1 = require("../utils/logger");
+const retry_1 = require("../utils/retry");
 const shared_1 = require("../utils/shared");
 const tracer_1 = require("../utils/tracer");
 const trace = (0, tracer_1.createTracer)('Producer');
@@ -124,7 +125,7 @@ class Producer {
         }
     });
     async initProducerId() {
-        try {
+        return (0, retry_1.withRetry)(this.handleError.bind(this))(async () => {
             const result = await this.cluster.sendRequest(api_1.API.INIT_PRODUCER_ID, {
                 transactionalId: null,
                 transactionTimeoutMs: 0,
@@ -134,11 +135,7 @@ class Producer {
             this.producerId = result.producerId;
             this.producerEpoch = result.producerEpoch;
             this.sequences = {};
-        }
-        catch (error) {
-            await this.handleError(error);
-            return this.initProducerId();
-        }
+        });
     }
     getSequence(topic, partition) {
         return this.sequences[topic]?.[partition] ?? 0;
@@ -149,13 +146,9 @@ class Producer {
         this.sequences[topic][partition] += messagesCount;
     }
     async fetchMetadata(topics, allowTopicAutoCreation) {
-        try {
+        return (0, retry_1.withRetry)(this.handleError.bind(this))(async () => {
             await this.metadata.fetchMetadata({ topics, allowTopicAutoCreation });
-        }
-        catch (error) {
-            await this.handleError(error);
-            return this.fetchMetadata(topics, allowTopicAutoCreation);
-        }
+        });
     }
     async handleError(error) {
         await (0, api_1.handleApiError)(error).catch(async (error) => {

package/dist/utils/retry.d.ts

@@ -0,0 +1 @@
+export declare const withRetry: (handleError: (error: unknown) => Promise<void>) => <T>(func: () => Promise<T>) => Promise<T>;

package/dist/utils/retry.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.withRetry = void 0;
+const logger_1 = require("./logger");
+const withRetry = (handleError) => async (func) => {
+    let lastError;
+    for (let i = 0; i < 15; i++) {
+        try {
+            return await func();
+        }
+        catch (error) {
+            await handleError(error);
+            lastError = error;
+        }
+    }
+    logger_1.log.warn('Retries exhausted', { lastError });
+    throw lastError;
+};
+exports.withRetry = withRetry;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kafka-ts",
-    "version": "1.1.8",
+    "version": "1.1.9",
     "main": "dist/index.js",
     "author": "Priit Käärd",
     "license": "MIT",

package/dist/consumer/metadata.d.ts

@@ -1,24 +0,0 @@
-import { IsolationLevel } from "../api/fetch";
-import { Assignment } from "../api/sync-group";
-import { Cluster } from "../cluster";
-import { OffsetManager } from "./offset-manager";
-export type Metadata = ReturnType<typeof createMetadata>;
-type MetadataOptions = {
-    cluster: Cluster;
-    topics?: string[];
-    isolationLevel?: IsolationLevel;
-    allowTopicAutoCreation?: boolean;
-    fromBeginning?: boolean;
-    offsetManager?: OffsetManager;
-};
-export declare const createMetadata: ({ cluster, topics, isolationLevel, allowTopicAutoCreation, fromBeginning, offsetManager, }: MetadataOptions) => {
-    init: () => Promise<void>;
-    getTopicPartitions: () => Record<string, number[]>;
-    getTopicIdByName: (name: string) => string;
-    getTopicNameById: (id: string) => string;
-    getAssignment: () => Assignment;
-    setAssignment: (newAssignment: Assignment) => void;
-    getLeaderIdByTopicPartition: (topic: string, partition: number) => number;
-    getIsrNodeIdsByTopicPartition: (topic: string, partition: number) => number[];
-};
-export {};

package/dist/consumer/metadata.js

@@ -1,64 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createMetadata = void 0;
-const api_1 = require("../api");
-const createMetadata = ({ cluster, topics, isolationLevel = 0 /* IsolationLevel.READ_UNCOMMITTED */, allowTopicAutoCreation = true, fromBeginning = false, offsetManager, }) => {
-    let topicPartitions = {};
-    let topicNameById = {};
-    let topicIdByName = {};
-    let leaderIdByTopicPartition = {};
-    let isrNodesByTopicPartition;
-    let assignment = {};
-    const fetchMetadata = async () => {
-        const response = await cluster.sendRequest(api_1.API.METADATA, {
-            allowTopicAutoCreation,
-            includeTopicAuthorizedOperations: false,
-            topics: topics?.map((name) => ({ id: null, name })) ?? null,
-        });
-        topicPartitions = Object.fromEntries(response.topics.map((topic) => [topic.name, topic.partitions.map((partition) => partition.partitionIndex)]));
-        topicNameById = Object.fromEntries(response.topics.map((topic) => [topic.topicId, topic.name]));
-        topicIdByName = Object.fromEntries(response.topics.map((topic) => [topic.name, topic.topicId]));
-        leaderIdByTopicPartition = Object.fromEntries(response.topics.map((topic) => [
-            topic.name,
-            Object.fromEntries(topic.partitions.map((partition) => [partition.partitionIndex, partition.leaderId])),
-        ]));
-        isrNodesByTopicPartition = Object.fromEntries(response.topics.map((topic) => [
-            topic.name,
-            Object.fromEntries(topic.partitions.map((partition) => [partition.partitionIndex, partition.isrNodes])),
-        ]));
-        assignment = topicPartitions;
-    };
-    const listOffsets = async () => {
-        const offsets = await cluster.sendRequest(api_1.API.LIST_OFFSETS, {
-            replicaId: -1,
-            isolationLevel,
-            topics: Object.entries(assignment)
-                .flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition })))
-                .map(({ topic, partition }) => ({
-                name: topic,
-                partitions: [{ partitionIndex: partition, currentLeaderEpoch: -1, timestamp: -1n }],
-            })),
-        });
-        offsets.topics.forEach(({ name, partitions }) => {
-            partitions.forEach(({ partitionIndex, offset }) => {
-                offsetManager?.resolve(name, partitionIndex, fromBeginning ? 0n : offset);
-            });
-        });
-    };
-    return {
-        init: async () => {
-            await fetchMetadata();
-            await listOffsets();
-        },
-        getTopicPartitions: () => topicPartitions,
-        getTopicIdByName: (name) => topicIdByName[name],
-        getTopicNameById: (id) => topicNameById[id],
-        getAssignment: () => assignment,
-        setAssignment: (newAssignment) => {
-            assignment = newAssignment;
-        },
-        getLeaderIdByTopicPartition: (topic, partition) => leaderIdByTopicPartition[topic][partition],
-        getIsrNodeIdsByTopicPartition: (topic, partition) => isrNodesByTopicPartition[topic][partition],
-    };
-};
-exports.createMetadata = createMetadata;

package/dist/examples/src/replicator.js

@@ -1,34 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const kafkats_1 = require("kafkats");
-const json_1 = require("./utils/json");
-(async () => {
-    const brokers = [{ host: "localhost", port: 9092 }];
-    const topic = "playground-topic";
-    // const producer = createProducer({ brokers });
-    // const producerInterval = setInterval(async () => {
-    //     await producer.send([
-    //         {
-    //             topic,
-    //             partition: 0,
-    //             offset: 1n,
-    //             timestamp: BigInt(Date.now()),
-    //             key: null,
-    //             value: `PING ${Math.random()}`,
-    //             headers: { timestamp: Date.now().toString() }
-    //         }
-    //     ])
-    // }, 5000);
-    const consumer = await (0, kafkats_1.startConsumer)({
-        topics: [topic],
-        brokers,
-        onBatch: (messages) => {
-            console.log(JSON.stringify(messages, json_1.serializer, 2));
-        },
-    });
-    process.on("SIGINT", async () => {
-        await consumer.close();
-        // clearInterval(producerInterval);
-        // await producer.close();
-    });
-})();

package/dist/examples/src/utils/json.js

@@ -1,5 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.serializer = void 0;
-const serializer = (_, value) => (typeof value === "bigint" ? value.toString() : value);
-exports.serializer = serializer;

package/dist/request-handler.d.ts

@@ -1,16 +0,0 @@
-import { Connection } from "./connection";
-import { Api } from "./utils/api";
-type RequestHandlerOptions = {
-    clientId: string | null;
-};
-export declare class RequestHandler {
-    private connection;
-    private options;
-    private queue;
-    private currentBuffer;
-    constructor(connection: Connection, options: RequestHandlerOptions);
-    private handleData;
-    sendRequest<Request, Response>(api: Api<Request, Response>, args: Request): Promise<Response>;
-}
-export type SendRequest = typeof RequestHandler.prototype.sendRequest;
-export {};

package/dist/request-handler.js

@@ -1,67 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.RequestHandler = void 0;
-const node_assert_1 = __importDefault(require("node:assert"));
-const decoder_1 = require("./utils/decoder");
-const encoder_1 = require("./utils/encoder");
-class RequestHandler {
-    connection;
-    options;
-    queue = {};
-    currentBuffer = null;
-    constructor(connection, options) {
-        this.connection = connection;
-        this.options = options;
-        this.connection.on("data", this.handleData);
-    }
-    handleData(buffer) {
-        this.currentBuffer = this.currentBuffer ? Buffer.concat([this.currentBuffer, buffer]) : buffer;
-        if (this.currentBuffer.length < 4) {
-            return;
-        }
-        const decoder = (0, decoder_1.createDecoder)({ buffer: this.currentBuffer });
-        const size = decoder.readInt32();
-        if (size > decoder.buffer.length) {
-            return;
-        }
-        const correlationId = decoder.readInt32();
-        const request = this.queue[correlationId];
-        delete this.queue[correlationId];
-        request.callback(decoder);
-        // debug(handleData.name, 'Response offsets', {
-        //     offset: decoder.offset,
-        //     length: decoder.buffer.length,
-        //     rest: decoder.buffer.subarray(decoder.offset, decoder.buffer.length)?.toString(),
-        // });
-        (0, node_assert_1.default)(decoder.offset - 4 === size, `Buffer not correctly consumed: ${decoder.offset - 4} !== ${buffer.length}`);
-        this.currentBuffer = null;
-    }
-    async sendRequest(api, args) {
-        const correlationId = Math.floor(Math.random() * 1000000);
-        const encoder = (0, encoder_1.createEncoder)()
-            .writeInt16(api.apiKey)
-            .writeInt16(api.apiVersion)
-            .writeInt32(correlationId)
-            .writeString(this.options.clientId);
-        const request = api.request(encoder, args).value();
-        const buffer = (0, encoder_1.createEncoder)().writeInt32(request.length).write(request).value();
-        return new Promise(async (resolve, reject) => {
-            await this.connection.write(buffer);
-            this.queue[correlationId] = {
-                callback: (decoder) => {
-                    try {
-                        const response = api.response(decoder);
-                        resolve(response);
-                    }
-                    catch (error) {
-                        reject(error);
-                    }
-                },
-            };
-        });
-    }
-}
-exports.RequestHandler = RequestHandler;

package/dist/request-handler.test.d.ts

@@ -1 +0,0 @@
-export declare const kafka: import("./client").Client;