kafka-ts 0.0.3-beta → 0.0.4

package/dist/consumer/consumer-group.d.ts

@@ -0,0 +1,41 @@
+/// <reference types="node" />
+import EventEmitter from 'events';
+import { Cluster } from '../cluster';
+import { ConsumerMetadata } from './consumer-metadata';
+import { OffsetManager } from './offset-manager';
+type ConsumerGroupOptions = {
+    cluster: Cluster;
+    topics: string[];
+    groupId: string;
+    groupInstanceId: string | null;
+    sessionTimeoutMs: number;
+    rebalanceTimeoutMs: number;
+    metadata: ConsumerMetadata;
+    offsetManager: OffsetManager;
+};
+export declare class ConsumerGroup extends EventEmitter<{
+    offsetCommit: [];
+}> {
+    private options;
+    private coordinatorId;
+    private memberId;
+    private generationId;
+    private leaderId;
+    private memberIds;
+    private heartbeatInterval;
+    private heartbeatError;
+    constructor(options: ConsumerGroupOptions);
+    join(): Promise<void>;
+    private startHeartbeater;
+    private stopHeartbeater;
+    handleLastHeartbeat(): void;
+    resetHeartbeat(): void;
+    private findCoordinator;
+    private joinGroup;
+    private syncGroup;
+    private offsetFetch;
+    offsetCommit(topicPartitions: Record<string, Set<number>>): Promise<void>;
+    heartbeat(): Promise<void>;
+    leaveGroup(): Promise<void>;
+}
+export {};

package/dist/consumer/consumer-group.js

@@ -0,0 +1,215 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConsumerGroup = void 0;
+const events_1 = __importDefault(require("events"));
+const api_1 = require("../api");
+const find_coordinator_1 = require("../api/find-coordinator");
+const tracer_1 = require("../utils/tracer");
+const trace = (0, tracer_1.createTracer)('ConsumerGroup');
+class ConsumerGroup extends events_1.default {
+    options;
+    coordinatorId = -1;
+    memberId = '';
+    generationId = -1;
+    leaderId = '';
+    memberIds = [];
+    heartbeatInterval = null;
+    heartbeatError = null;
+    constructor(options) {
+        super();
+        this.options = options;
+    }
+    async join() {
+        await this.findCoordinator();
+        await this.options.cluster.setSeedBroker(this.coordinatorId);
+        this.memberId = '';
+        await this.joinGroup();
+        await this.syncGroup();
+        await this.offsetFetch();
+        this.startHeartbeater();
+    }
+    async startHeartbeater() {
+        this.heartbeatInterval = setInterval(async () => {
+            try {
+                await this.heartbeat();
+            }
+            catch (error) {
+                this.heartbeatError = error;
+            }
+        }, 5000);
+    }
+    async stopHeartbeater() {
+        if (this.heartbeatInterval) {
+            clearInterval(this.heartbeatInterval);
+            this.heartbeatInterval = null;
+        }
+    }
+    handleLastHeartbeat() {
+        if (this.heartbeatError) {
+            throw this.heartbeatError;
+        }
+    }
+    resetHeartbeat() {
+        this.heartbeatError = null;
+    }
+    async findCoordinator() {
+        const { coordinators } = await this.options.cluster.sendRequest(api_1.API.FIND_COORDINATOR, {
+            keyType: find_coordinator_1.KEY_TYPE.GROUP,
+            keys: [this.options.groupId],
+        });
+        this.coordinatorId = coordinators[0].nodeId;
+    }
+    async joinGroup() {
+        const { cluster, groupId, groupInstanceId, sessionTimeoutMs, rebalanceTimeoutMs, topics } = this.options;
+        try {
+            const response = await cluster.sendRequest(api_1.API.JOIN_GROUP, {
+                groupId,
+                groupInstanceId,
+                memberId: this.memberId,
+                sessionTimeoutMs,
+                rebalanceTimeoutMs,
+                protocolType: 'consumer',
+                protocols: [{ name: 'RoundRobinAssigner', metadata: { version: 0, topics } }],
+                reason: null,
+            });
+            this.memberId = response.memberId;
+            this.generationId = response.generationId;
+            this.leaderId = response.leader;
+            this.memberIds = response.members.map((member) => member.memberId);
+        }
+        catch (error) {
+            if (error.errorCode === api_1.API_ERROR.MEMBER_ID_REQUIRED) {
+                this.memberId = error.response.memberId;
+                return this.joinGroup();
+            }
+            throw error;
+        }
+    }
+    async syncGroup() {
+        const { cluster, metadata, groupId, groupInstanceId } = this.options;
+        let assignments = [];
+        if (this.memberId === this.leaderId) {
+            const memberAssignments = Object.entries(metadata.getTopicPartitions())
+                .flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition })))
+                .reduce((acc, { topic, partition }, index) => {
+                const memberId = this.memberIds[index % this.memberIds.length];
+                acc[memberId] ??= {};
+                acc[memberId][topic] ??= [];
+                acc[memberId][topic].push(partition);
+                return acc;
+            }, {});
+            assignments = Object.entries(memberAssignments).map(([memberId, assignment]) => ({ memberId, assignment }));
+        }
+        const response = await cluster.sendRequest(api_1.API.SYNC_GROUP, {
+            groupId,
+            groupInstanceId,
+            memberId: this.memberId,
+            generationId: this.generationId,
+            protocolType: 'consumer',
+            protocolName: 'RoundRobinAssigner',
+            assignments,
+        });
+        metadata.setAssignment(JSON.parse(response.assignments || '{}'));
+    }
+    async offsetFetch() {
+        const { cluster, groupId, topics, metadata, offsetManager } = this.options;
+        const assignment = metadata.getAssignment();
+        const request = {
+            groups: [
+                {
+                    groupId,
+                    topics: topics
+                        .map((topic) => ({ name: topic, partitionIndexes: assignment[topic] ?? [] }))
+                        .filter(({ partitionIndexes }) => partitionIndexes.length),
+                },
+            ].filter(({ topics }) => topics.length),
+            requireStable: true,
+        };
+        if (!request.groups.length)
+            return;
+        const response = await cluster.sendRequest(api_1.API.OFFSET_FETCH, request);
+        const topicPartitions = {};
+        response.groups.forEach((group) => {
+            group.topics.forEach((topic) => {
+                topicPartitions[topic.name] ??= new Set();
+                topic.partitions.forEach(({ partitionIndex, committedOffset }) => {
+                    if (committedOffset >= 0) {
+                        topicPartitions[topic.name].add(partitionIndex);
+                        offsetManager.resolve(topic.name, partitionIndex, committedOffset);
+                    }
+                });
+            });
+        });
+        offsetManager.flush(topicPartitions);
+    }
+    async offsetCommit(topicPartitions) {
+        const { cluster, groupId, groupInstanceId, offsetManager } = this.options;
+        const request = {
+            groupId,
+            groupInstanceId,
+            memberId: this.memberId,
+            generationIdOrMemberEpoch: this.generationId,
+            topics: Object.entries(topicPartitions).map(([topic, partitions]) => ({
+                name: topic,
+                partitions: [...partitions].map((partitionIndex) => ({
+                    partitionIndex,
+                    committedOffset: offsetManager.pendingOffsets[topic][partitionIndex],
+                    committedLeaderEpoch: -1,
+                    committedMetadata: null,
+                })),
+            })),
+        };
+        if (!request.topics.length) {
+            return;
+        }
+        await cluster.sendRequest(api_1.API.OFFSET_COMMIT, request);
+        this.emit('offsetCommit');
+    }
+    async heartbeat() {
+        const { cluster, groupId, groupInstanceId } = this.options;
+        await cluster.sendRequest(api_1.API.HEARTBEAT, {
+            groupId,
+            groupInstanceId,
+            memberId: this.memberId,
+            generationId: this.generationId,
+        });
+    }
+    async leaveGroup() {
+        if (this.coordinatorId === -1) {
+            return;
+        }
+        const { cluster, groupId, groupInstanceId } = this.options;
+        this.stopHeartbeater();
+        try {
+            await cluster.sendRequest(api_1.API.LEAVE_GROUP, {
+                groupId,
+                members: [{ memberId: this.memberId, groupInstanceId, reason: null }],
+            });
+        }
+        catch (error) {
+            if (error.errorCode === api_1.API_ERROR.FENCED_INSTANCE_ID) {
+                return;
+            }
+            throw error;
+        }
+    }
+}
+exports.ConsumerGroup = ConsumerGroup;
+__decorate([
+    trace(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", Promise)
+], ConsumerGroup.prototype, "join", null);

package/dist/consumer/consumer-metadata.d.ts

@@ -0,0 +1,7 @@
+import { Assignment } from '../api/sync-group';
+import { Metadata } from '../metadata';
+export declare class ConsumerMetadata extends Metadata {
+    private assignment;
+    getAssignment(): Assignment;
+    setAssignment(newAssignment: Assignment): void;
+}

package/dist/consumer/consumer-metadata.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConsumerMetadata = void 0;
+const metadata_1 = require("../metadata");
+class ConsumerMetadata extends metadata_1.Metadata {
+    assignment = {};
+    getAssignment() {
+        return this.assignment;
+    }
+    setAssignment(newAssignment) {
+        this.assignment = newAssignment;
+    }
+}
+exports.ConsumerMetadata = ConsumerMetadata;

package/dist/consumer/consumer.d.ts

@@ -0,0 +1,44 @@
+/// <reference types="node" />
+import EventEmitter from 'events';
+import { IsolationLevel } from '../api/fetch';
+import { Cluster } from '../cluster';
+import { Message } from '../types';
+import { BatchGranularity } from './fetch-manager';
+export type ConsumerOptions = {
+    topics: string[];
+    groupId?: string | null;
+    groupInstanceId?: string | null;
+    rackId?: string;
+    isolationLevel?: IsolationLevel;
+    sessionTimeoutMs?: number;
+    rebalanceTimeoutMs?: number;
+    maxWaitMs?: number;
+    minBytes?: number;
+    maxBytes?: number;
+    partitionMaxBytes?: number;
+    allowTopicAutoCreation?: boolean;
+    fromBeginning?: boolean;
+    batchGranularity?: BatchGranularity;
+    concurrency?: number;
+} & ({
+    onBatch: (messages: Required<Message>[]) => unknown;
+} | {
+    onMessage: (message: Required<Message>) => unknown;
+});
+export declare class Consumer extends EventEmitter<{
+    offsetCommit: [];
+}> {
+    private cluster;
+    private options;
+    private metadata;
+    private consumerGroup;
+    private offsetManager;
+    private fetchManager?;
+    private stopHook;
+    constructor(cluster: Cluster, options: ConsumerOptions);
+    start(): Promise<void>;
+    close(force?: boolean): Promise<void>;
+    private startFetchManager;
+    private process;
+    private fetch;
+}

package/dist/consumer/consumer.js

@@ -0,0 +1,225 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Consumer = void 0;
+const events_1 = __importDefault(require("events"));
+const api_1 = require("../api");
+const messages_to_topic_partition_leaders_1 = require("../distributors/messages-to-topic-partition-leaders");
+const delay_1 = require("../utils/delay");
+const error_1 = require("../utils/error");
+const logger_1 = require("../utils/logger");
+const tracer_1 = require("../utils/tracer");
+const consumer_group_1 = require("./consumer-group");
+const consumer_metadata_1 = require("./consumer-metadata");
+const fetch_manager_1 = require("./fetch-manager");
+const offset_manager_1 = require("./offset-manager");
+const trace = (0, tracer_1.createTracer)('Consumer');
+class Consumer extends events_1.default {
+    cluster;
+    options;
+    metadata;
+    consumerGroup;
+    offsetManager;
+    fetchManager;
+    stopHook;
+    constructor(cluster, options) {
+        super();
+        this.cluster = cluster;
+        this.options = {
+            ...options,
+            groupId: options.groupId ?? null,
+            groupInstanceId: options.groupInstanceId ?? null,
+            rackId: options.rackId ?? '',
+            sessionTimeoutMs: options.sessionTimeoutMs ?? 30_000,
+            rebalanceTimeoutMs: options.rebalanceTimeoutMs ?? 60_000,
+            maxWaitMs: options.maxWaitMs ?? 5000,
+            minBytes: options.minBytes ?? 1,
+            maxBytes: options.maxBytes ?? 1_048_576,
+            partitionMaxBytes: options.partitionMaxBytes ?? 1_048_576,
+            isolationLevel: options.isolationLevel ?? 0 /* IsolationLevel.READ_UNCOMMITTED */,
+            allowTopicAutoCreation: options.allowTopicAutoCreation ?? false,
+            fromBeginning: options.fromBeginning ?? false,
+            batchGranularity: options.batchGranularity ?? 'partition',
+            concurrency: options.concurrency ?? 1,
+        };
+        this.metadata = new consumer_metadata_1.ConsumerMetadata({ cluster: this.cluster });
+        this.offsetManager = new offset_manager_1.OffsetManager({
+            cluster: this.cluster,
+            metadata: this.metadata,
+            isolationLevel: this.options.isolationLevel,
+        });
+        this.consumerGroup = this.options.groupId
+            ? new consumer_group_1.ConsumerGroup({
+                cluster: this.cluster,
+                topics: this.options.topics,
+                groupId: this.options.groupId,
+                groupInstanceId: this.options.groupInstanceId,
+                sessionTimeoutMs: this.options.sessionTimeoutMs,
+                rebalanceTimeoutMs: this.options.rebalanceTimeoutMs,
+                metadata: this.metadata,
+                offsetManager: this.offsetManager,
+            })
+            : undefined;
+        this.consumerGroup?.on('offsetCommit', () => this.emit('offsetCommit'));
+    }
+    async start() {
+        const { topics, allowTopicAutoCreation, fromBeginning } = this.options;
+        this.stopHook = undefined;
+        try {
+            await this.cluster.connect();
+            await this.metadata.fetchMetadataIfNecessary({ topics, allowTopicAutoCreation });
+            this.metadata.setAssignment(this.metadata.getTopicPartitions());
+            await this.offsetManager.fetchOffsets({ fromBeginning });
+            await this.consumerGroup?.join();
+        }
+        catch (error) {
+            logger_1.log.warn('Failed to start consumer', error);
+            logger_1.log.debug(`Restarting consumer in 1 second...`);
+            await (0, delay_1.delay)(1000);
+            if (this.stopHook)
+                return this.stopHook();
+            return this.close(true).then(() => this.start());
+        }
+        this.startFetchManager();
+    }
+    async close(force = false) {
+        if (!force) {
+            await new Promise(async (resolve) => {
+                this.stopHook = resolve;
+                await this.fetchManager?.stop();
+            });
+        }
+        await this.consumerGroup?.leaveGroup().catch((error) => logger_1.log.debug(`Failed to leave group: ${error.message}`));
+        await this.cluster.disconnect().catch((error) => logger_1.log.debug(`Failed to disconnect: ${error.message}`));
+    }
+    async startFetchManager() {
+        const { batchGranularity, concurrency } = this.options;
+        while (!this.stopHook) {
+            this.consumerGroup?.resetHeartbeat();
+            // TODO: If leader is not available, find another read replica
+            const nodeAssignments = Object.entries((0, messages_to_topic_partition_leaders_1.distributeMessagesToTopicPartitionLeaders)(Object.entries(this.metadata.getAssignment()).flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition }))), this.metadata.getTopicPartitionLeaderIds())).map(([nodeId, assignment]) => ({
+                nodeId: parseInt(nodeId),
+                assignment: Object.fromEntries(Object.entries(assignment).map(([topic, partitions]) => [
+                    topic,
+                    Object.keys(partitions).map(Number),
+                ])),
+            }));
+            const numPartitions = Object.values(this.metadata.getAssignment()).flat().length;
+            const numProcessors = Math.min(concurrency, numPartitions);
+            this.fetchManager = new fetch_manager_1.FetchManager({
+                fetch: this.fetch.bind(this),
+                process: this.process.bind(this),
+                metadata: this.metadata,
+                consumerGroup: this.consumerGroup,
+                nodeAssignments,
+                batchGranularity,
+                concurrency: numProcessors,
+            });
+            try {
+                await this.fetchManager.start();
+                if (!nodeAssignments.length) {
+                    logger_1.log.debug('No partitions assigned. Waiting for reassignment...');
+                    await (0, delay_1.delay)(this.options.maxWaitMs);
+                    this.consumerGroup?.handleLastHeartbeat();
+                }
+            }
+            catch (error) {
+                await this.fetchManager.stop();
+                if (error.errorCode === api_1.API_ERROR.REBALANCE_IN_PROGRESS) {
+                    logger_1.log.debug('Rebalance in progress...');
+                    continue;
+                }
+                if (error.errorCode === api_1.API_ERROR.FENCED_INSTANCE_ID) {
+                    logger_1.log.debug('New consumer with the same groupInstanceId joined. Exiting the consumer...');
+                    this.close();
+                    break;
+                }
+                if (error instanceof error_1.ConnectionError ||
+                    (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.NOT_COORDINATOR)) {
+                    logger_1.log.debug(`${error.message}. Restarting consumer...`);
+                    this.close().then(() => this.start());
+                    break;
+                }
+                logger_1.log.error(error.message, error);
+                this.close();
+                break;
+            }
+        }
+        this.stopHook?.();
+    }
+    async process(messages) {
+        const { options } = this;
+        const topicPartitions = {};
+        for (const { topic, partition } of messages) {
+            topicPartitions[topic] ??= new Set();
+            topicPartitions[topic].add(partition);
+        }
+        if ('onBatch' in options) {
+            await options.onBatch(messages);
+            messages.forEach(({ topic, partition, offset }) => this.offsetManager.resolve(topic, partition, offset + 1n));
+        }
+        else if ('onMessage' in options) {
+            for (const message of messages) {
+                await options.onMessage(message);
+                const { topic, partition, offset } = message;
+                this.offsetManager.resolve(topic, partition, offset + 1n);
+            }
+        }
+        await this.consumerGroup?.offsetCommit(topicPartitions);
+        this.offsetManager.flush(topicPartitions);
+    }
+    fetch(nodeId, assignment) {
+        const { rackId, maxWaitMs, minBytes, maxBytes, partitionMaxBytes, isolationLevel } = this.options;
+        return this.cluster.sendRequestToNode(nodeId)(api_1.API.FETCH, {
+            maxWaitMs,
+            minBytes,
+            maxBytes,
+            isolationLevel,
+            sessionId: 0,
+            sessionEpoch: -1,
+            topics: Object.entries(assignment).map(([topic, partitions]) => ({
+                topicId: this.metadata.getTopicIdByName(topic),
+                partitions: partitions.map((partition) => ({
+                    partition,
+                    currentLeaderEpoch: -1,
+                    fetchOffset: this.offsetManager.getCurrentOffset(topic, partition),
+                    lastFetchedEpoch: -1,
+                    logStartOffset: 0n,
+                    partitionMaxBytes,
+                })),
+            })),
+            forgottenTopicsData: [],
+            rackId,
+        });
+    }
+}
+exports.Consumer = Consumer;
+__decorate([
+    trace(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", Promise)
+], Consumer.prototype, "start", null);
+__decorate([
+    trace(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Consumer.prototype, "close", null);
+__decorate([
+    trace((messages) => ({ count: messages.length })),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Array]),
+    __metadata("design:returntype", Promise)
+], Consumer.prototype, "process", null);

package/dist/consumer/fetch-manager.d.ts

@@ -0,0 +1,33 @@
+import { FetchResponse } from '../api/fetch';
+import { Assignment } from '../api/sync-group';
+import { Metadata } from '../metadata';
+import { Batch } from '../types';
+import { ConsumerGroup } from './consumer-group';
+export type BatchGranularity = 'partition' | 'topic' | 'broker';
+type FetchManagerOptions = {
+    fetch: (nodeId: number, assignment: Assignment) => Promise<FetchResponse>;
+    process: (batch: Batch) => Promise<void>;
+    metadata: Metadata;
+    consumerGroup?: ConsumerGroup;
+    nodeAssignments: {
+        nodeId: number;
+        assignment: Assignment;
+    }[];
+    batchGranularity: BatchGranularity;
+    concurrency: number;
+};
+export declare class FetchManager {
+    private options;
+    private queue;
+    private isRunning;
+    private fetchers;
+    private processors;
+    private pollQueue;
+    private fetcherCallbacks;
+    constructor(options: FetchManagerOptions);
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    poll(): Promise<Batch>;
+    private onResponse;
+}
+export {};