kafka-ts 0.0.3 → 0.0.4

package/src/connection.ts

@@ -1,157 +0,0 @@
-import assert from 'assert';
-import net, { isIP, Socket, TcpSocketConnectOpts } from 'net';
-import tls, { TLSSocketOptions } from 'tls';
-import { getApiName } from './api';
-import { Api } from './utils/api';
-import { Decoder } from './utils/decoder';
-import { Encoder } from './utils/encoder';
-import { ConnectionError } from './utils/error';
-import { log } from './utils/logger';
-import { createTracer } from './utils/tracer';
-
-const trace = createTracer('Connection');
-
-type ConnectionOptions = {
-    clientId: string | null;
-    connection: TcpSocketConnectOpts;
-    ssl: TLSSocketOptions | null;
-};
-
-type RawResonse = { responseDecoder: Decoder; responseSize: number };
-
-export class Connection {
-    private socket = new Socket();
-    private queue: {
-        [correlationId: number]: { resolve: (response: RawResonse) => void; reject: (error: Error) => void };
-    } = {};
-    private lastCorrelationId = 0;
-    private chunks: Buffer[] = [];
-
-    constructor(private options: ConnectionOptions) {}
-
-    @trace()
-    public async connect() {
-        this.queue = {};
-        this.chunks = [];
-
-        await new Promise<void>((resolve, reject) => {
-            const { ssl, connection } = this.options;
-
-            this.socket = ssl
-                ? tls.connect(
-                      {
-                          ...connection,
-                          ...ssl,
-                          ...(connection.host && !isIP(connection.host) && { servername: connection.host }),
-                      },
-                      resolve,
-                  )
-                : net.connect(connection, resolve);
-            this.socket.once('error', reject);
-        });
-        this.socket.removeAllListeners('error');
-
-        this.socket.on('data', (data) => this.handleData(data));
-        this.socket.once('close', async () => {
-            Object.values(this.queue).forEach(({ reject }) => {
-                reject(new ConnectionError('Socket closed unexpectedly'));
-            });
-            this.queue = {};
-        });
-    }
-
-    @trace()
-    public disconnect() {
-        this.socket.removeAllListeners();
-        return new Promise<void>((resolve) => {
-            if (this.socket.pending) {
-                return resolve();
-            }
-            this.socket.end(resolve);
-        });
-    }
-
-    @trace((api, body) => ({ message: getApiName(api), body }))
-    public async sendRequest<Request, Response>(api: Api<Request, Response>, body: Request): Promise<Response> {
-        const correlationId = this.nextCorrelationId();
-        const apiName = getApiName(api);
-
-        const encoder = new Encoder()
-            .writeInt16(api.apiKey)
-            .writeInt16(api.apiVersion)
-            .writeInt32(correlationId)
-            .writeString(this.options.clientId);
-
-        const request = api.request(encoder, body);
-        const requestEncoder = new Encoder().writeInt32(request.getByteLength()).writeEncoder(request);
-
-        let timeout: NodeJS.Timeout | undefined;
-        const { responseDecoder, responseSize } = await new Promise<RawResonse>(async (resolve, reject) => {
-            timeout = setTimeout(() => {
-                delete this.queue[correlationId];
-                reject(new ConnectionError(`${apiName} timed out`));
-            }, 30_000);
-
-            try {
-                this.queue[correlationId] = { resolve, reject };
-                await this.write(requestEncoder.value());
-            } catch (error) {
-                reject(error);
-            }
-        });
-        clearTimeout(timeout);
-        const response = await api.response(responseDecoder);
-
-        assert(
-            responseDecoder.getOffset() - 4 === responseSize,
-            `Buffer not correctly consumed: ${responseDecoder.getOffset() - 4} !== ${responseSize}`,
-        );
-
-        return response;
-    }
-
-    private write(buffer: Buffer) {
-        return new Promise<void>((resolve, reject) => {
-            const { stack } = new Error('Write error');
-            this.socket.write(buffer, (error) => {
-                if (error) {
-                    const err = new ConnectionError(error.message);
-                    err.stack += `\n${stack}`;
-                    return reject(err);
-                }
-                resolve();
-            });
-        });
-    }
-
-    private handleData(buffer: Buffer) {
-        this.chunks.push(buffer);
-
-        const decoder = new Decoder(Buffer.concat(this.chunks));
-        if (decoder.getBufferLength() < 4) {
-            return;
-        }
-
-        const size = decoder.readInt32();
-        if (size !== decoder.getBufferLength() - 4) {
-            return;
-        }
-
-        const correlationId = decoder.readInt32();
-
-        const context = this.queue[correlationId];
-        if (context) {
-            delete this.queue[correlationId];
-            context.resolve({ responseDecoder: decoder, responseSize: size });
-        } else {
-            log.debug('Could not find pending request for correlationId', { correlationId });
-        }
-        this.chunks = [];
-    }
-
-    private nextCorrelationId() {
-        return this.lastCorrelationId++;
-    }
-}
-
-export type SendRequest = typeof Connection.prototype.sendRequest;

package/src/consumer/consumer-group.ts

@@ -1,229 +0,0 @@
-import EventEmitter from 'events';
-import { API, API_ERROR } from '../api';
-import { KEY_TYPE } from '../api/find-coordinator';
-import { Assignment, MemberAssignment } from '../api/sync-group';
-import { Cluster } from '../cluster';
-import { KafkaTSApiError, KafkaTSError } from '../utils/error';
-import { createTracer } from '../utils/tracer';
-import { ConsumerMetadata } from './consumer-metadata';
-import { OffsetManager } from './offset-manager';
-
-const trace = createTracer('ConsumerGroup');
-
-type ConsumerGroupOptions = {
-    cluster: Cluster;
-    topics: string[];
-    groupId: string;
-    groupInstanceId: string | null;
-    sessionTimeoutMs: number;
-    rebalanceTimeoutMs: number;
-    metadata: ConsumerMetadata;
-    offsetManager: OffsetManager;
-};
-
-export class ConsumerGroup extends EventEmitter<{ offsetCommit: [] }> {
-    private coordinatorId = -1;
-    private memberId = '';
-    private generationId = -1;
-    private leaderId = '';
-    private memberIds: string[] = [];
-    private heartbeatInterval: NodeJS.Timeout | null = null;
-    private heartbeatError: KafkaTSError | null = null;
-
-    constructor(private options: ConsumerGroupOptions) {
-        super();
-    }
-
-    @trace()
-    public async join() {
-        await this.findCoordinator();
-        await this.options.cluster.setSeedBroker(this.coordinatorId);
-
-        this.memberId = '';
-        await this.joinGroup();
-        await this.syncGroup();
-        await this.offsetFetch();
-        this.startHeartbeater();
-    }
-
-    private async startHeartbeater() {
-        this.heartbeatInterval = setInterval(async () => {
-            try {
-                await this.heartbeat();
-            } catch (error) {
-                this.heartbeatError = error as KafkaTSError;
-            }
-        }, 5000);
-    }
-
-    private async stopHeartbeater() {
-        if (this.heartbeatInterval) {
-            clearInterval(this.heartbeatInterval);
-            this.heartbeatInterval = null;
-        }
-    }
-
-    public handleLastHeartbeat() {
-        if (this.heartbeatError) {
-            throw this.heartbeatError;
-        }
-    }
-
-    public resetHeartbeat() {
-        this.heartbeatError = null;
-    }
-
-    private async findCoordinator() {
-        const { coordinators } = await this.options.cluster.sendRequest(API.FIND_COORDINATOR, {
-            keyType: KEY_TYPE.GROUP,
-            keys: [this.options.groupId],
-        });
-        this.coordinatorId = coordinators[0].nodeId;
-    }
-
-    private async joinGroup(): Promise<void> {
-        const { cluster, groupId, groupInstanceId, sessionTimeoutMs, rebalanceTimeoutMs, topics } = this.options;
-        try {
-            const response = await cluster.sendRequest(API.JOIN_GROUP, {
-                groupId,
-                groupInstanceId,
-                memberId: this.memberId,
-                sessionTimeoutMs,
-                rebalanceTimeoutMs,
-                protocolType: 'consumer',
-                protocols: [{ name: 'RoundRobinAssigner', metadata: { version: 0, topics } }],
-                reason: null,
-            });
-            this.memberId = response.memberId;
-            this.generationId = response.generationId;
-            this.leaderId = response.leader;
-            this.memberIds = response.members.map((member) => member.memberId);
-        } catch (error) {
-            if ((error as KafkaTSApiError).errorCode === API_ERROR.MEMBER_ID_REQUIRED) {
-                this.memberId = (error as KafkaTSApiError).response.memberId;
-                return this.joinGroup();
-            }
-            throw error;
-        }
-    }
-
-    private async syncGroup() {
-        const { cluster, metadata, groupId, groupInstanceId } = this.options;
-
-        let assignments: MemberAssignment[] = [];
-        if (this.memberId === this.leaderId) {
-            const memberAssignments = Object.entries(metadata.getTopicPartitions())
-                .flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition })))
-                .reduce(
-                    (acc, { topic, partition }, index) => {
-                        const memberId = this.memberIds[index % this.memberIds.length];
-                        acc[memberId] ??= {};
-                        acc[memberId][topic] ??= [];
-                        acc[memberId][topic].push(partition);
-                        return acc;
-                    },
-                    {} as Record<string, Record<string, number[]>>,
-                );
-            assignments = Object.entries(memberAssignments).map(([memberId, assignment]) => ({ memberId, assignment }));
-        }
-
-        const response = await cluster.sendRequest(API.SYNC_GROUP, {
-            groupId,
-            groupInstanceId,
-            memberId: this.memberId,
-            generationId: this.generationId,
-            protocolType: 'consumer',
-            protocolName: 'RoundRobinAssigner',
-            assignments,
-        });
-        metadata.setAssignment(JSON.parse(response.assignments || '{}') as Assignment);
-    }
-
-    private async offsetFetch() {
-        const { cluster, groupId, topics, metadata, offsetManager } = this.options;
-
-        const assignment = metadata.getAssignment();
-        const request = {
-            groups: [
-                {
-                    groupId,
-                    topics: topics
-                        .map((topic) => ({ name: topic, partitionIndexes: assignment[topic] ?? [] }))
-                        .filter(({ partitionIndexes }) => partitionIndexes.length),
-                },
-            ].filter(({ topics }) => topics.length),
-            requireStable: true,
-        };
-        if (!request.groups.length) return;
-
-        const response = await cluster.sendRequest(API.OFFSET_FETCH, request);
-
-        const topicPartitions: Record<string, Set<number>> = {};
-        response.groups.forEach((group) => {
-            group.topics.forEach((topic) => {
-                topicPartitions[topic.name] ??= new Set();
-                topic.partitions.forEach(({ partitionIndex, committedOffset }) => {
-                    if (committedOffset >= 0) {
-                        topicPartitions[topic.name].add(partitionIndex);
-                        offsetManager.resolve(topic.name, partitionIndex, committedOffset);
-                    }
-                });
-            });
-        });
-        offsetManager.flush(topicPartitions);
-    }
-
-    public async offsetCommit(topicPartitions: Record<string, Set<number>>) {
-        const { cluster, groupId, groupInstanceId, offsetManager } = this.options;
-        const request = {
-            groupId,
-            groupInstanceId,
-            memberId: this.memberId,
-            generationIdOrMemberEpoch: this.generationId,
-            topics: Object.entries(topicPartitions).map(([topic, partitions]) => ({
-                name: topic,
-                partitions: [...partitions].map((partitionIndex) => ({
-                    partitionIndex,
-                    committedOffset: offsetManager.pendingOffsets[topic][partitionIndex],
-                    committedLeaderEpoch: -1,
-                    committedMetadata: null,
-                })),
-            })),
-        };
-        if (!request.topics.length) {
-            return;
-        }
-        await cluster.sendRequest(API.OFFSET_COMMIT, request);
-        this.emit('offsetCommit');
-    }
-
-    public async heartbeat() {
-        const { cluster, groupId, groupInstanceId } = this.options;
-        await cluster.sendRequest(API.HEARTBEAT, {
-            groupId,
-            groupInstanceId,
-            memberId: this.memberId,
-            generationId: this.generationId,
-        });
-    }
-
-    public async leaveGroup() {
-        if (this.coordinatorId === -1) {
-            return;
-        }
-
-        const { cluster, groupId, groupInstanceId } = this.options;
-        this.stopHeartbeater();
-        try {
-            await cluster.sendRequest(API.LEAVE_GROUP, {
-                groupId,
-                members: [{ memberId: this.memberId, groupInstanceId, reason: null }],
-            });
-        } catch (error) {
-            if ((error as KafkaTSApiError).errorCode === API_ERROR.FENCED_INSTANCE_ID) {
-                return;
-            }
-            throw error;
-        }
-    }
-}

package/src/consumer/consumer-metadata.ts

@@ -1,14 +0,0 @@
-import { Assignment } from '../api/sync-group';
-import { Metadata } from '../metadata';
-
-export class ConsumerMetadata extends Metadata {
-    private assignment: Assignment = {};
-
-    public getAssignment() {
-        return this.assignment;
-    }
-
-    public setAssignment(newAssignment: Assignment) {
-        this.assignment = newAssignment;
-    }
-}

package/src/consumer/consumer.ts

@@ -1,252 +0,0 @@
-import EventEmitter from 'events';
-import { API, API_ERROR } from '../api';
-import { IsolationLevel } from '../api/fetch';
-import { Assignment } from '../api/sync-group';
-import { Cluster } from '../cluster';
-import { distributeMessagesToTopicPartitionLeaders } from '../distributors/messages-to-topic-partition-leaders';
-import { Message } from '../types';
-import { delay } from '../utils/delay';
-import { ConnectionError, KafkaTSApiError } from '../utils/error';
-import { log } from '../utils/logger';
-import { createTracer } from '../utils/tracer';
-import { ConsumerGroup } from './consumer-group';
-import { ConsumerMetadata } from './consumer-metadata';
-import { BatchGranularity, FetchManager } from './fetch-manager';
-import { OffsetManager } from './offset-manager';
-
-const trace = createTracer('Consumer');
-
-export type ConsumerOptions = {
-    topics: string[];
-    groupId?: string | null;
-    groupInstanceId?: string | null;
-    rackId?: string;
-    isolationLevel?: IsolationLevel;
-    sessionTimeoutMs?: number;
-    rebalanceTimeoutMs?: number;
-    maxWaitMs?: number;
-    minBytes?: number;
-    maxBytes?: number;
-    partitionMaxBytes?: number;
-    allowTopicAutoCreation?: boolean;
-    fromBeginning?: boolean;
-    batchGranularity?: BatchGranularity;
-    concurrency?: number;
-} & ({ onBatch: (messages: Required<Message>[]) => unknown } | { onMessage: (message: Required<Message>) => unknown });
-
-export class Consumer extends EventEmitter<{ offsetCommit: [] }> {
-    private options: Required<ConsumerOptions>;
-    private metadata: ConsumerMetadata;
-    private consumerGroup: ConsumerGroup | undefined;
-    private offsetManager: OffsetManager;
-    private fetchManager?: FetchManager;
-    private stopHook: (() => void) | undefined;
-
-    constructor(
-        private cluster: Cluster,
-        options: ConsumerOptions,
-    ) {
-        super();
-
-        this.options = {
-            ...options,
-            groupId: options.groupId ?? null,
-            groupInstanceId: options.groupInstanceId ?? null,
-            rackId: options.rackId ?? '',
-            sessionTimeoutMs: options.sessionTimeoutMs ?? 30_000,
-            rebalanceTimeoutMs: options.rebalanceTimeoutMs ?? 60_000,
-            maxWaitMs: options.maxWaitMs ?? 5000,
-            minBytes: options.minBytes ?? 1,
-            maxBytes: options.maxBytes ?? 1_048_576,
-            partitionMaxBytes: options.partitionMaxBytes ?? 1_048_576,
-            isolationLevel: options.isolationLevel ?? IsolationLevel.READ_UNCOMMITTED,
-            allowTopicAutoCreation: options.allowTopicAutoCreation ?? false,
-            fromBeginning: options.fromBeginning ?? false,
-            batchGranularity: options.batchGranularity ?? 'partition',
-            concurrency: options.concurrency ?? 1,
-        };
-
-        this.metadata = new ConsumerMetadata({ cluster: this.cluster });
-        this.offsetManager = new OffsetManager({
-            cluster: this.cluster,
-            metadata: this.metadata,
-            isolationLevel: this.options.isolationLevel,
-        });
-        this.consumerGroup = this.options.groupId
-            ? new ConsumerGroup({
-                  cluster: this.cluster,
-                  topics: this.options.topics,
-                  groupId: this.options.groupId,
-                  groupInstanceId: this.options.groupInstanceId,
-                  sessionTimeoutMs: this.options.sessionTimeoutMs,
-                  rebalanceTimeoutMs: this.options.rebalanceTimeoutMs,
-                  metadata: this.metadata,
-                  offsetManager: this.offsetManager,
-              })
-            : undefined;
-        this.consumerGroup?.on('offsetCommit', () => this.emit('offsetCommit'));
-    }
-
-    @trace()
-    public async start(): Promise<void> {
-        const { topics, allowTopicAutoCreation, fromBeginning } = this.options;
-
-        this.stopHook = undefined;
-
-        try {
-            await this.cluster.connect();
-            await this.metadata.fetchMetadataIfNecessary({ topics, allowTopicAutoCreation });
-            this.metadata.setAssignment(this.metadata.getTopicPartitions());
-            await this.offsetManager.fetchOffsets({ fromBeginning });
-            await this.consumerGroup?.join();
-        } catch (error) {
-            log.warn('Failed to start consumer', error);
-            log.debug(`Restarting consumer in 1 second...`);
-            await delay(1000);
-
-            if (this.stopHook) return (this.stopHook as () => void)();
-            return this.close(true).then(() => this.start());
-        }
-        this.startFetchManager();
-    }
-
-    @trace()
-    public async close(force = false): Promise<void> {
-        if (!force) {
-            await new Promise<void>(async (resolve) => {
-                this.stopHook = resolve;
-                await this.fetchManager?.stop();
-            });
-        }
-        await this.consumerGroup?.leaveGroup().catch((error) => log.debug(`Failed to leave group: ${error.message}`));
-        await this.cluster.disconnect().catch((error) => log.debug(`Failed to disconnect: ${error.message}`));
-    }
-
-    private async startFetchManager() {
-        const { batchGranularity, concurrency } = this.options;
-
-        while (!this.stopHook) {
-            this.consumerGroup?.resetHeartbeat();
-
-            // TODO: If leader is not available, find another read replica
-            const nodeAssignments = Object.entries(
-                distributeMessagesToTopicPartitionLeaders(
-                    Object.entries(this.metadata.getAssignment()).flatMap(([topic, partitions]) =>
-                        partitions.map((partition) => ({ topic, partition })),
-                    ),
-                    this.metadata.getTopicPartitionLeaderIds(),
-                ),
-            ).map(([nodeId, assignment]) => ({
-                nodeId: parseInt(nodeId),
-                assignment: Object.fromEntries(
-                    Object.entries(assignment).map(([topic, partitions]) => [
-                        topic,
-                        Object.keys(partitions).map(Number),
-                    ]),
-                ),
-            }));
-
-            const numPartitions = Object.values(this.metadata.getAssignment()).flat().length;
-            const numProcessors = Math.min(concurrency, numPartitions);
-
-            this.fetchManager = new FetchManager({
-                fetch: this.fetch.bind(this),
-                process: this.process.bind(this),
-                metadata: this.metadata,
-                consumerGroup: this.consumerGroup,
-                nodeAssignments,
-                batchGranularity,
-                concurrency: numProcessors,
-            });
-
-            try {
-                await this.fetchManager.start();
-
-                if (!nodeAssignments.length) {
-                    log.debug('No partitions assigned. Waiting for reassignment...');
-                    await delay(this.options.maxWaitMs);
-                    this.consumerGroup?.handleLastHeartbeat();
-                }
-            } catch (error) {
-                await this.fetchManager.stop();
-
-                if ((error as KafkaTSApiError).errorCode === API_ERROR.REBALANCE_IN_PROGRESS) {
-                    log.debug('Rebalance in progress...');
-                    continue;
-                }
-                if ((error as KafkaTSApiError).errorCode === API_ERROR.FENCED_INSTANCE_ID) {
-                    log.debug('New consumer with the same groupInstanceId joined. Exiting the consumer...');
-                    this.close();
-                    break;
-                }
-                if (
-                    error instanceof ConnectionError ||
-                    (error instanceof KafkaTSApiError && error.errorCode === API_ERROR.NOT_COORDINATOR)
-                ) {
-                    log.debug(`${error.message}. Restarting consumer...`);
-                    this.close().then(() => this.start());
-                    break;
-                }
-                log.error((error as Error).message, error);
-                this.close();
-                break;
-            }
-        }
-        this.stopHook?.();
-    }
-
-    @trace((messages) => ({ count: messages.length }))
-    private async process(messages: Required<Message>[]) {
-        const { options } = this;
-
-        const topicPartitions: Record<string, Set<number>> = {};
-        for (const { topic, partition } of messages) {
-            topicPartitions[topic] ??= new Set();
-            topicPartitions[topic].add(partition);
-        }
-
-        if ('onBatch' in options) {
-            await options.onBatch(messages);
-
-            messages.forEach(({ topic, partition, offset }) =>
-                this.offsetManager.resolve(topic, partition, offset + 1n),
-            );
-        } else if ('onMessage' in options) {
-            for (const message of messages) {
-                await options.onMessage(message);
-
-                const { topic, partition, offset } = message;
-                this.offsetManager.resolve(topic, partition, offset + 1n);
-            }
-        }
-
-        await this.consumerGroup?.offsetCommit(topicPartitions);
-        this.offsetManager.flush(topicPartitions);
-    }
-
-    private fetch(nodeId: number, assignment: Assignment) {
-        const { rackId, maxWaitMs, minBytes, maxBytes, partitionMaxBytes, isolationLevel } = this.options;
-
-        return this.cluster.sendRequestToNode(nodeId)(API.FETCH, {
-            maxWaitMs,
-            minBytes,
-            maxBytes,
-            isolationLevel,
-            sessionId: 0,
-            sessionEpoch: -1,
-            topics: Object.entries(assignment).map(([topic, partitions]) => ({
-                topicId: this.metadata.getTopicIdByName(topic),
-                partitions: partitions.map((partition) => ({
-                    partition,
-                    currentLeaderEpoch: -1,
-                    fetchOffset: this.offsetManager.getCurrentOffset(topic, partition),
-                    lastFetchedEpoch: -1,
-                    logStartOffset: 0n,
-                    partitionMaxBytes,
-                })),
-            })),
-            forgottenTopicsData: [],
-            rackId,
-        });
-    }
-}