kafka-ts 0.0.3-beta → 0.0.4

package/src/api/sasl-handshake.ts

@@ -1,16 +0,0 @@
-import { createApi } from '../utils/api';
-import { KafkaTSApiError } from '../utils/error';
-
-export const SASL_HANDSHAKE = createApi({
-    apiKey: 17,
-    apiVersion: 1,
-    request: (encoder, data: { mechanism: string }) => encoder.writeString(data.mechanism),
-    response: (decoder) => {
-        const result = {
-            errorCode: decoder.readInt16(),
-            mechanisms: decoder.readArray((mechanism) => mechanism.readString()),
-        };
-        if (result.errorCode) throw new KafkaTSApiError(result.errorCode, null, result);
-        return result;
-    },
-});

package/src/api/sync-group.ts

@@ -1,54 +0,0 @@
-import { createApi } from '../utils/api';
-import { KafkaTSApiError } from '../utils/error';
-
-export type Assignment = { [topic: string]: number[] };
-
-export type MemberAssignment = {
-    memberId: string;
-    assignment: Assignment;
-};
-
-export const SYNC_GROUP = createApi({
-    apiKey: 14,
-    apiVersion: 5,
-    request: (
-        encoder,
-        data: {
-            groupId: string;
-            generationId: number;
-            memberId: string;
-            groupInstanceId: string | null;
-            protocolType: string | null;
-            protocolName: string | null;
-            assignments: MemberAssignment[];
-        },
-    ) =>
-        encoder
-            .writeUVarInt(0)
-            .writeCompactString(data.groupId)
-            .writeInt32(data.generationId)
-            .writeCompactString(data.memberId)
-            .writeCompactString(data.groupInstanceId)
-            .writeCompactString(data.protocolType)
-            .writeCompactString(data.protocolName)
-            .writeCompactArray(data.assignments, (encoder, assignment) =>
-                encoder
-                    .writeCompactString(assignment.memberId)
-                    .writeCompactString(JSON.stringify(assignment.assignment))
-                    .writeUVarInt(0),
-            )
-            .writeUVarInt(0),
-    response: (decoder) => {
-        const result = {
-            _tag: decoder.readTagBuffer(),
-            throttleTimeMs: decoder.readInt32(),
-            errorCode: decoder.readInt16(),
-            protocolType: decoder.readCompactString(),
-            protocolName: decoder.readCompactString(),
-            assignments: decoder.readCompactString()!,
-            _tag2: decoder.readTagBuffer(),
-        };
-        if (result.errorCode) throw new KafkaTSApiError(result.errorCode, null, result);
-        return result;
-    },
-});

package/src/broker.ts

@@ -1,74 +0,0 @@
-import { TcpSocketConnectOpts } from 'net';
-import { TLSSocketOptions } from 'tls';
-import { API } from './api';
-import { Connection, SendRequest } from './connection';
-import { KafkaTSError } from './utils/error';
-import { memo } from './utils/memo';
-
-export type SASLOptions = { mechanism: 'PLAIN'; username: string; password: string };
-
-type BrokerOptions = {
-    clientId: string | null;
-    options: TcpSocketConnectOpts;
-    sasl: SASLOptions | null;
-    ssl: TLSSocketOptions | null;
-};
-
-export class Broker {
-    private connection: Connection;
-    public sendRequest: SendRequest;
-
-    constructor(private options: BrokerOptions) {
-        this.connection = new Connection({
-            clientId: this.options.clientId,
-            connection: this.options.options,
-            ssl: this.options.ssl,
-        });
-        this.sendRequest = this.connection.sendRequest.bind(this.connection);
-    }
-
-    public async connect() {
-        await this.connection.connect();
-        await this.validateApiVersions();
-        await this.saslHandshake();
-        await this.saslAuthenticate();
-        return this;
-    }
-
-    public ensureConnected = memo(() => this.connect());
-
-    public async disconnect() {
-        await this.connection.disconnect();
-    }
-
-    private async validateApiVersions() {
-        const { versions } = await this.sendRequest(API.API_VERSIONS, {});
-
-        const apiByKey = Object.fromEntries(Object.values(API).map((api) => [api.apiKey, api]));
-        versions.forEach(({ apiKey, minVersion, maxVersion }) => {
-            if (!apiByKey[apiKey]) {
-                return;
-            }
-            const { apiVersion } = apiByKey[apiKey];
-            if (apiVersion < minVersion || apiVersion > maxVersion) {
-                throw new KafkaTSError(`API ${apiKey} version ${apiVersion} is not supported by the broker`);
-            }
-        });
-    }
-
-    private async saslHandshake() {
-        if (!this.options.sasl) {
-            return;
-        }
-        await this.sendRequest(API.SASL_HANDSHAKE, { mechanism: this.options.sasl.mechanism });
-    }
-
-    private async saslAuthenticate() {
-        if (this.options.sasl?.mechanism !== 'PLAIN') {
-            return;
-        }
-        const { username, password } = this.options.sasl;
-        const authBytes = [null, username, password].join('\u0000');
-        await this.sendRequest(API.SASL_AUTHENTICATE, { authBytes: Buffer.from(authBytes) });
-    }
-}

package/src/client.ts

@@ -1,47 +0,0 @@
-import { TcpSocketConnectOpts } from 'net';
-import { TLSSocketOptions } from 'tls';
-import { SASLOptions } from './broker';
-import { Cluster } from './cluster';
-import { Consumer, ConsumerOptions } from './consumer/consumer';
-import { Producer, ProducerOptions } from './producer/producer';
-
-type ClientOptions = {
-    clientId?: string | null;
-    bootstrapServers: TcpSocketConnectOpts[];
-    sasl?: SASLOptions | null;
-    ssl?: TLSSocketOptions | null;
-};
-
-export class Client {
-    private options: Required<ClientOptions>;
-
-    constructor(options: ClientOptions) {
-        this.options = {
-            ...options,
-            clientId: options.clientId ?? null,
-            sasl: options.sasl ?? null,
-            ssl: options.ssl ?? null,
-        };
-    }
-
-    public async startConsumer(options: ConsumerOptions) {
-        const consumer = new Consumer(this.createCluster(), options);
-        await consumer.start();
-        return consumer;
-    }
-
-    public createProducer(options: ProducerOptions) {
-        return new Producer(this.createCluster(), options);
-    }
-
-    public createCluster() {
-        return new Cluster({
-            clientId: this.options.clientId,
-            bootstrapServers: this.options.bootstrapServers,
-            sasl: this.options.sasl,
-            ssl: this.options.ssl,
-        });
-    }
-}
-
-export const createKafkaClient = (options: ClientOptions) => new Client(options);

package/src/cluster.ts

@@ -1,87 +0,0 @@
-import { TcpSocketConnectOpts } from 'net';
-import { TLSSocketOptions } from 'tls';
-import { API } from './api';
-import { Broker, SASLOptions } from './broker';
-import { SendRequest } from './connection';
-import { ConnectionError, KafkaTSError } from './utils/error';
-
-type ClusterOptions = {
-    clientId: string | null;
-    bootstrapServers: TcpSocketConnectOpts[];
-    sasl: SASLOptions | null;
-    ssl: TLSSocketOptions | null;
-};
-
-export class Cluster {
-    private seedBroker: Broker;
-    private brokerById: Record<number, Broker> = {};
-
-    constructor(private options: ClusterOptions) {
-        this.seedBroker = new Broker({
-            clientId: this.options.clientId,
-            sasl: this.options.sasl,
-            ssl: this.options.ssl,
-            options: this.options.bootstrapServers[0],
-        });
-    }
-
-    public async connect() {
-        await this.connectSeedBroker();
-        const metadata = await this.sendRequest(API.METADATA, {
-            allowTopicAutoCreation: false,
-            includeTopicAuthorizedOperations: false,
-            topics: [],
-        });
-
-        this.brokerById = Object.fromEntries(
-            metadata.brokers.map(({ nodeId, ...options }) => [
-                nodeId,
-                new Broker({
-                    clientId: this.options.clientId,
-                    sasl: this.options.sasl,
-                    ssl: this.options.ssl,
-                    options,
-                }),
-            ]),
-        );
-        return this;
-    }
-
-    public async disconnect() {
-        await Promise.all([
-            this.seedBroker.disconnect(),
-            ...Object.values(this.brokerById).map((broker) => broker.disconnect()),
-        ]);
-    }
-
-    public sendRequest: SendRequest = (...args) => this.seedBroker.sendRequest(...args);
-
-    public sendRequestToNode =
-        (nodeId: number): SendRequest =>
-        async (...args) => {
-            const broker = this.brokerById[nodeId];
-            if (!broker) {
-                throw new ConnectionError(`Broker ${nodeId} is not available`);
-            }
-            await broker.ensureConnected();
-            return broker.sendRequest(...args);
-        };
-
-    private async connectSeedBroker() {
-        const randomizedBrokers = this.options.bootstrapServers.toSorted(() => Math.random() - 0.5);
-        for (const options of randomizedBrokers) {
-            try {
-                this.seedBroker = await new Broker({
-                    clientId: this.options.clientId,
-                    sasl: this.options.sasl,
-                    ssl: this.options.ssl,
-                    options,
-                }).connect();
-                return;
-            } catch (error) {
-                console.warn(`Failed to connect to seed broker ${options.host}:${options.port}`, error);
-            }
-        }
-        throw new KafkaTSError('No seed brokers found');
-    }
-}

package/src/connection.ts

@@ -1,143 +0,0 @@
-import assert from 'assert';
-import net, { isIP, Socket, TcpSocketConnectOpts } from 'net';
-import tls, { TLSSocketOptions } from 'tls';
-import { getApiName } from './api';
-import { Api } from './utils/api';
-import { Decoder } from './utils/decoder';
-import { Encoder } from './utils/encoder';
-import { ConnectionError } from './utils/error';
-import { createTracer } from './utils/tracer';
-
-const trace = createTracer('Connection');
-
-export type ConnectionOptions = {
-    clientId: string | null;
-    connection: TcpSocketConnectOpts;
-    ssl: TLSSocketOptions | null;
-};
-
-type RawResonse = { responseDecoder: Decoder; responseSize: number };
-
-export class Connection {
-    private socket = new Socket();
-    private queue: {
-        [correlationId: number]: { resolve: (response: RawResonse) => void; reject: (error: Error) => void };
-    } = {};
-    private lastCorrelationId = 0;
-    private buffer: Buffer | null = null;
-
-    constructor(private options: ConnectionOptions) {}
-
-    @trace()
-    public async connect() {
-        this.queue = {};
-        this.buffer = null;
-
-        await new Promise<void>((resolve, reject) => {
-            const { ssl, connection } = this.options;
-
-            this.socket = ssl
-                ? tls.connect(
-                      {
-                          ...connection,
-                          ...ssl,
-                          ...(connection.host && !isIP(connection.host) && { servername: connection.host }),
-                      },
-                      resolve,
-                  )
-                : net.connect(connection, resolve);
-            this.socket.once('error', reject);
-        });
-        this.socket.removeAllListeners('error');
-
-        this.socket.on('data', (data) => this.handleData(data));
-        this.socket.once('close', async () => {
-            Object.values(this.queue).forEach(({ reject }) => {
-                reject(new ConnectionError('Socket closed unexpectedly'));
-            });
-            this.queue = {};
-        });
-    }
-
-    public disconnect() {
-        this.socket.removeAllListeners();
-        return new Promise<void>((resolve) => {
-            if (this.socket.pending) {
-                return resolve();
-            }
-            this.socket.end(resolve);
-        });
-    }
-
-    @trace((api, body) => ({ apiName: getApiName(api), body }))
-    public async sendRequest<Request, Response>(api: Api<Request, Response>, body: Request): Promise<Response> {
-        const correlationId = this.nextCorrelationId();
-
-        const encoder = new Encoder()
-            .writeInt16(api.apiKey)
-            .writeInt16(api.apiVersion)
-            .writeInt32(correlationId)
-            .writeString(this.options.clientId);
-
-        const request = api.request(encoder, body).value();
-        const requestEncoder = new Encoder().writeInt32(request.length).write(request);
-
-        const { responseDecoder, responseSize } = await new Promise<RawResonse>(async (resolve, reject) => {
-            try {
-                await this.write(requestEncoder.value());
-                this.queue[correlationId] = { resolve, reject };
-            } catch (error) {
-                reject(error);
-            }
-        });
-        const response = api.response(responseDecoder);
-
-        assert(
-            responseDecoder.getOffset() - 4 === responseSize,
-            `Buffer not correctly consumed: ${responseDecoder.getOffset() - 4} !== ${responseSize}`,
-        );
-
-        return response;
-    }
-
-    private write(buffer: Buffer) {
-        return new Promise<void>((resolve, reject) => {
-            const { stack } = new Error('Write error');
-            this.socket.write(buffer, (error) => {
-                if (error) {
-                    const err = new ConnectionError(error.message);
-                    err.stack += `\n${stack}`;
-                    return reject(err);
-                }
-                resolve();
-            });
-        });
-    }
-
-    private handleData(buffer: Buffer) {
-        this.buffer = this.buffer ? Buffer.concat([this.buffer, buffer]) : buffer;
-        if (this.buffer.length < 4) {
-            return;
-        }
-
-        const decoder = new Decoder(this.buffer);
-        const size = decoder.readInt32();
-        if (size !== decoder.getBufferLength() - 4) {
-            return;
-        }
-
-        const correlationId = decoder.readInt32();
-
-        const { resolve } = this.queue[correlationId];
-        delete this.queue[correlationId];
-
-        resolve({ responseDecoder: decoder, responseSize: size });
-        this.buffer = null;
-    }
-
-    private nextCorrelationId() {
-        return (this.lastCorrelationId = (this.lastCorrelationId + 1) % 2 ** 31);
-    }
-}
-
-export type SendRequest = typeof Connection.prototype.sendRequest;

package/src/consumer/consumer-group.ts

@@ -1,209 +0,0 @@
-import { API, API_ERROR } from '../api';
-import { KEY_TYPE } from '../api/find-coordinator';
-import { Assignment, MemberAssignment } from '../api/sync-group';
-import { Cluster } from '../cluster';
-import { KafkaTSApiError, KafkaTSError } from '../utils/error';
-import { ConsumerMetadata } from './consumer-metadata';
-import { OffsetManager } from './offset-manager';
-
-type ConsumerGroupOptions = {
-    cluster: Cluster;
-    topics: string[];
-    groupId: string;
-    groupInstanceId: string | null;
-    sessionTimeoutMs: number;
-    rebalanceTimeoutMs: number;
-    metadata: ConsumerMetadata;
-    offsetManager: OffsetManager;
-};
-
-export class ConsumerGroup {
-    private coordinatorId = -1;
-    private memberId = '';
-    private generationId = -1;
-    private leaderId = '';
-    private memberIds: string[] = [];
-    private heartbeatInterval: NodeJS.Timeout | null = null;
-    private heartbeatError: KafkaTSError | null = null;
-
-    constructor(private options: ConsumerGroupOptions) {}
-
-    public async join() {
-        await this.findCoordinator();
-        await this.joinGroup();
-        await this.syncGroup();
-        await this.offsetFetch();
-        this.startHeartbeater();
-    }
-
-    private async startHeartbeater() {
-        this.heartbeatInterval = setInterval(async () => {
-            try {
-                await this.heartbeat();
-            } catch (error) {
-                this.heartbeatError = error as KafkaTSError;
-            }
-        }, 5000);
-    }
-
-    private async stopHeartbeater() {
-        if (this.heartbeatInterval) {
-            clearInterval(this.heartbeatInterval);
-            this.heartbeatInterval = null;
-        }
-    }
-
-    public async handleLastHeartbeat() {
-        if (this.heartbeatError) {
-            throw this.heartbeatError;
-        }
-    }
-
-    private async findCoordinator() {
-        const { coordinators } = await this.options.cluster.sendRequest(API.FIND_COORDINATOR, {
-            keyType: KEY_TYPE.GROUP,
-            keys: [this.options.groupId],
-        });
-        this.coordinatorId = coordinators[0].nodeId;
-    }
-
-    private async joinGroup(): Promise<void> {
-        const { cluster, groupId, groupInstanceId, sessionTimeoutMs, rebalanceTimeoutMs, topics } = this.options;
-        try {
-            const response = await cluster.sendRequestToNode(this.coordinatorId)(API.JOIN_GROUP, {
-                groupId,
-                groupInstanceId,
-                memberId: this.memberId,
-                sessionTimeoutMs,
-                rebalanceTimeoutMs,
-                protocolType: 'consumer',
-                protocols: [{ name: 'RoundRobinAssigner', metadata: { version: 0, topics } }],
-                reason: null,
-            });
-            this.memberId = response.memberId;
-            this.generationId = response.generationId;
-            this.leaderId = response.leader;
-            this.memberIds = response.members.map((member) => member.memberId);
-        } catch (error) {
-            if ((error as KafkaTSApiError).errorCode === API_ERROR.MEMBER_ID_REQUIRED) {
-                this.memberId = (error as KafkaTSApiError).response.memberId;
-                return this.joinGroup();
-            }
-            throw error;
-        }
-    }
-
-    private async syncGroup() {
-        const { cluster, metadata, groupId, groupInstanceId } = this.options;
-
-        let assignments: MemberAssignment[] = [];
-        if (this.memberId === this.leaderId) {
-            const memberAssignments = Object.entries(metadata.getTopicPartitions())
-                .flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition })))
-                .reduce(
-                    (acc, { topic, partition }, index) => {
-                        const memberId = this.memberIds[index % this.memberIds.length];
-                        acc[memberId] ??= {};
-                        acc[memberId][topic] ??= [];
-                        acc[memberId][topic].push(partition);
-                        return acc;
-                    },
-                    {} as Record<string, Record<string, number[]>>,
-                );
-            assignments = Object.entries(memberAssignments).map(([memberId, assignment]) => ({ memberId, assignment }));
-        }
-
-        const response = await cluster.sendRequestToNode(this.coordinatorId)(API.SYNC_GROUP, {
-            groupId,
-            groupInstanceId,
-            memberId: this.memberId,
-            generationId: this.generationId,
-            protocolType: 'consumer',
-            protocolName: 'RoundRobinAssigner',
-            assignments,
-        });
-        metadata.setAssignment(JSON.parse(response.assignments || '{}') as Assignment);
-    }
-
-    private async offsetFetch() {
-        const { cluster, groupId, topics, metadata, offsetManager } = this.options;
-
-        const assignment = metadata.getAssignment();
-        const request = {
-            groups: [
-                {
-                    groupId,
-                    memberId: this.memberId,
-                    memberEpoch: -1,
-                    topics: topics
-                        .map((topic) => ({ name: topic, partitionIndexes: assignment[topic] ?? [] }))
-                        .filter(({ partitionIndexes }) => partitionIndexes.length),
-                },
-            ].filter(({ topics }) => topics.length),
-            requireStable: true,
-        };
-        if (!request.groups.length) return;
-
-        const response = await cluster.sendRequestToNode(this.coordinatorId)(API.OFFSET_FETCH, request);
-        response.groups.forEach((group) => {
-            group.topics.forEach((topic) => {
-                topic.partitions
-                    .filter(({ committedOffset }) => committedOffset >= 0)
-                    .forEach(({ partitionIndex, committedOffset }) =>
-                        offsetManager.resolve(topic.name, partitionIndex, committedOffset),
-                    );
-            });
-        });
-        offsetManager.flush();
-    }
-
-    public async offsetCommit() {
-        const { cluster, groupId, groupInstanceId, offsetManager } = this.options;
-        const request = {
-            groupId,
-            groupInstanceId,
-            memberId: this.memberId,
-            generationIdOrMemberEpoch: this.generationId,
-            topics: Object.entries(offsetManager.pendingOffsets).map(([topic, partitions]) => ({
-                name: topic,
-                partitions: Object.entries(partitions).map(([partition, offset]) => ({
-                    partitionIndex: parseInt(partition),
-                    committedOffset: offset,
-                    committedLeaderEpoch: -1,
-                    committedMetadata: null,
-                })),
-            })),
-        };
-        if (!request.topics.length) {
-            return;
-        }
-        await cluster.sendRequestToNode(this.coordinatorId)(API.OFFSET_COMMIT, request);
-        offsetManager.flush();
-    }
-
-    public async heartbeat() {
-        const { cluster, groupId, groupInstanceId } = this.options;
-        await cluster.sendRequestToNode(this.coordinatorId)(API.HEARTBEAT, {
-            groupId,
-            groupInstanceId,
-            memberId: this.memberId,
-            generationId: this.generationId,
-        });
-    }
-
-    public async leaveGroup() {
-        const { cluster, groupId, groupInstanceId } = this.options;
-        this.stopHeartbeater();
-        try {
-            await cluster.sendRequestToNode(this.coordinatorId)(API.LEAVE_GROUP, {
-                groupId,
-                members: [{ memberId: this.memberId, groupInstanceId, reason: null }],
-            });
-        } catch (error) {
-            if ((error as KafkaTSApiError).errorCode === API_ERROR.FENCED_INSTANCE_ID) {
-                return;
-            }
-            throw error;
-        }
-    }
-}

package/src/consumer/consumer-metadata.ts

@@ -1,14 +0,0 @@
-import { Assignment } from '../api/sync-group';
-import { Metadata } from '../metadata';
-
-export class ConsumerMetadata extends Metadata {
-    private assignment: Assignment = {};
-
-    public getAssignment() {
-        return this.assignment;
-    }
-
-    public setAssignment(newAssignment: Assignment) {
-        this.assignment = newAssignment;
-    }
-}