kafka-ts 0.0.3 → 0.0.5

package/src/broker.ts

@@ -1,72 +0,0 @@
-import { TcpSocketConnectOpts } from 'net';
-import { TLSSocketOptions } from 'tls';
-import { API } from './api';
-import { Connection, SendRequest } from './connection';
-import { KafkaTSError } from './utils/error';
-import { memo } from './utils/memo';
-
-export type SASLProvider = {
-    mechanism: string;
-    authenticate: (context: { sendRequest: SendRequest }) => Promise<void>;
-};
-
-type BrokerOptions = {
-    clientId: string | null;
-    options: TcpSocketConnectOpts;
-    sasl: SASLProvider | null;
-    ssl: TLSSocketOptions | null;
-};
-
-export class Broker {
-    private connection: Connection;
-    public sendRequest: SendRequest;
-
-    constructor(private options: BrokerOptions) {
-        this.connection = new Connection({
-            clientId: this.options.clientId,
-            connection: this.options.options,
-            ssl: this.options.ssl,
-        });
-        this.sendRequest = this.connection.sendRequest.bind(this.connection);
-    }
-
-    public async connect() {
-        await this.connection.connect();
-        await this.validateApiVersions();
-        await this.saslHandshake();
-        await this.saslAuthenticate();
-        return this;
-    }
-
-    public ensureConnected = memo(() => this.connect());
-
-    public async disconnect() {
-        await this.connection.disconnect();
-    }
-
-    private async validateApiVersions() {
-        const { versions } = await this.sendRequest(API.API_VERSIONS, {});
-
-        const apiByKey = Object.fromEntries(Object.values(API).map((api) => [api.apiKey, api]));
-        versions.forEach(({ apiKey, minVersion, maxVersion }) => {
-            if (!apiByKey[apiKey]) {
-                return;
-            }
-            const { apiVersion } = apiByKey[apiKey];
-            if (apiVersion < minVersion || apiVersion > maxVersion) {
-                throw new KafkaTSError(`API ${apiKey} version ${apiVersion} is not supported by the broker (minVersion=${minVersion}, maxVersion=${maxVersion})`);
-            }
-        });
-    }
-
-    private async saslHandshake() {
-        if (!this.options.sasl) {
-            return;
-        }
-        await this.sendRequest(API.SASL_HANDSHAKE, { mechanism: this.options.sasl.mechanism });
-    }
-
-    private async saslAuthenticate() {
-        await this.options.sasl?.authenticate({ sendRequest: this.sendRequest });
-    }
-}

package/src/client.ts

@@ -1,47 +0,0 @@
-import { TcpSocketConnectOpts } from 'net';
-import { TLSSocketOptions } from 'tls';
-import { SASLProvider } from './broker';
-import { Cluster } from './cluster';
-import { Consumer, ConsumerOptions } from './consumer/consumer';
-import { Producer, ProducerOptions } from './producer/producer';
-
-type ClientOptions = {
-    clientId?: string | null;
-    bootstrapServers: TcpSocketConnectOpts[];
-    sasl?: SASLProvider | null;
-    ssl?: TLSSocketOptions | null;
-};
-
-export class Client {
-    private options: Required<ClientOptions>;
-
-    constructor(options: ClientOptions) {
-        this.options = {
-            ...options,
-            clientId: options.clientId ?? null,
-            sasl: options.sasl ?? null,
-            ssl: options.ssl ?? null,
-        };
-    }
-
-    public async startConsumer(options: ConsumerOptions) {
-        const consumer = new Consumer(this.createCluster(), options);
-        await consumer.start();
-        return consumer;
-    }
-
-    public createProducer(options: ProducerOptions) {
-        return new Producer(this.createCluster(), options);
-    }
-
-    public createCluster() {
-        return new Cluster({
-            clientId: this.options.clientId,
-            bootstrapServers: this.options.bootstrapServers,
-            sasl: this.options.sasl,
-            ssl: this.options.ssl,
-        });
-    }
-}
-
-export const createKafkaClient = (options: ClientOptions) => new Client(options);

package/src/cluster.test.ts

@@ -1,371 +0,0 @@
-import { randomBytes } from 'crypto';
-import { readFileSync } from 'fs';
-import { afterAll, beforeAll, describe, expect, it } from 'vitest';
-import { API } from './api';
-import { KEY_TYPE } from './api/find-coordinator';
-import { saslPlain } from './auth';
-import { createKafkaClient } from './client';
-import { Cluster } from './cluster';
-import { KafkaTSApiError } from './utils/error';
-
-const kafka = createKafkaClient({
-    clientId: 'kafka-ts',
-    bootstrapServers: [{ host: 'localhost', port: 9092 }],
-    sasl: saslPlain({ username: 'admin', password: 'admin' }),
-    ssl: { ca: readFileSync('./certs/ca.crt').toString() },
-});
-
-describe.sequential('Low-level API', () => {
-    const groupId = randomBytes(16).toString('hex');
-
-    let cluster: Cluster;
-
-    beforeAll(async () => {
-        cluster = await kafka.createCluster();
-        await cluster.connect();
-
-        const metadataResult = await cluster.sendRequest(API.METADATA, {
-            topics: null,
-            allowTopicAutoCreation: false,
-            includeTopicAuthorizedOperations: false,
-        });
-        if (metadataResult.topics.some((topic) => topic.name === 'kafka-ts-test-topic')) {
-            await cluster.sendRequest(API.DELETE_TOPICS, {
-                topics: [{ name: 'kafka-ts-test-topic', topicId: null }],
-                timeoutMs: 10000,
-            });
-        }
-    });
-
-    afterAll(async () => {
-        await cluster.disconnect();
-    });
-
-    it('should request api versions', async () => {
-        const result = await cluster.sendRequest(API.API_VERSIONS, {});
-        expect(result).toMatchSnapshot();
-    });
-
-    let topicId: string = 'd6718d178e1b47c886441ad2d19faea5';
-
-    it('should create topics', async () => {
-        const result = await cluster.sendRequest(API.CREATE_TOPICS, {
-            topics: [
-                {
-                    name: 'kafka-ts-test-topic',
-                    numPartitions: 10,
-                    replicationFactor: 3,
-                    assignments: [],
-                    configs: [],
-                },
-            ],
-            timeoutMs: 10000,
-            validateOnly: false,
-        });
-        topicId = result.topics[0].topicId;
-        result.topics.forEach((topic) => {
-            topic.topicId = 'Any<UUID>';
-        });
-        expect(result).toMatchSnapshot();
-
-        await new Promise((resolve) => setTimeout(resolve, 1000));
-    });
-
-    it('should request metadata for all topics', async () => {
-        const result = await cluster.sendRequest(API.METADATA, {
-            topics: null,
-            allowTopicAutoCreation: false,
-            includeTopicAuthorizedOperations: false,
-        });
-        result.controllerId = 0;
-        result.topics = result.topics.filter((topic) => topic.name !== '__consumer_offsets');
-        result.topics.forEach((topic) => {
-            topic.topicId = 'Any<UUID>';
-            topic.partitions.forEach((partition) => {
-                partition.leaderId = 0;
-                partition.isrNodes = [0];
-                partition.replicaNodes = [0];
-            });
-        });
-        expect(result).toMatchSnapshot();
-    });
-
-    let partitionIndex = 0;
-    let leaderId = 0;
-
-    it('should request metadata for a topic', async () => {
-        const result = await cluster.sendRequest(API.METADATA, {
-            topics: [{ id: topicId, name: 'kafka-ts-test-topic' }],
-            allowTopicAutoCreation: false,
-            includeTopicAuthorizedOperations: false,
-        });
-        partitionIndex = result.topics[0].partitions[0].partitionIndex;
-        leaderId = result.topics[0].partitions[0].leaderId;
-        result.controllerId = 0;
-        result.topics.forEach((topic) => {
-            topic.topicId = 'Any<UUID>';
-            topic.partitions.forEach((partition) => {
-                partition.leaderId = 0;
-                partition.isrNodes = [0];
-                partition.replicaNodes = [0];
-            });
-        });
-        expect(result).toMatchSnapshot();
-    });
-
-    let producerId = 9n;
-
-    it('should init producer id', async () => {
-        const result = await cluster.sendRequest(API.INIT_PRODUCER_ID, {
-            transactionalId: null,
-            transactionTimeoutMs: 0,
-            producerId,
-            producerEpoch: 0,
-        });
-        result.producerId = 0n;
-        expect(result).toMatchSnapshot();
-    });
-
-    it('should produce messages', async () => {
-        const now = Date.now();
-        const result = await cluster.sendRequestToNode(leaderId)(API.PRODUCE, {
-            transactionalId: null,
-            timeoutMs: 10000,
-            acks: 1,
-            topicData: [
-                {
-                    name: 'kafka-ts-test-topic',
-                    partitionData: [
-                        {
-                            index: partitionIndex,
-                            baseOffset: 0n,
-                            partitionLeaderEpoch: 0,
-                            attributes: 0,
-                            baseSequence: 0,
-                            baseTimestamp: BigInt(now),
-                            lastOffsetDelta: 0,
-                            maxTimestamp: BigInt(now),
-                            producerEpoch: 0,
-                            producerId,
-                            records: [
-                                {
-                                    attributes: 0,
-                                    offsetDelta: 0,
-                                    timestampDelta: 0n,
-                                    key: Buffer.from('key'),
-                                    value: Buffer.from('value'),
-                                    headers: [
-                                        {
-                                            key: Buffer.from('header-key'),
-                                            value: Buffer.from('header-value'),
-                                        },
-                                    ],
-                                },
-                            ],
-                        },
-                    ],
-                },
-            ],
-        });
-        expect(result).toMatchSnapshot();
-    });
-
-    it('should fetch messages', async () => {
-        const result = await cluster.sendRequestToNode(leaderId)(API.FETCH, {
-            maxWaitMs: 100,
-            minBytes: 1,
-            maxBytes: 10485760,
-            isolationLevel: 1,
-            sessionId: 0,
-            sessionEpoch: -1,
-            topics: [
-                {
-                    topicId,
-                    partitions: [
-                        {
-                            partition: partitionIndex,
-                            currentLeaderEpoch: -1,
-                            fetchOffset: 0n,
-                            lastFetchedEpoch: 0,
-                            logStartOffset: -1n,
-                            partitionMaxBytes: 10485760,
-                        },
-                    ],
-                },
-            ],
-            forgottenTopicsData: [],
-            rackId: '',
-        });
-        result.responses.forEach((response) => {
-            response.topicId = 'Any<UUID>';
-            response.partitions.forEach((partition) => {
-                partition.records.forEach((record) => {
-                    expect(record.baseTimestamp).toBeGreaterThan(1721926744730n);
-                    expect(record.maxTimestamp).toBeGreaterThan(1721926744730n);
-                    expect(record.crc).toBeGreaterThan(0);
-
-                    record.baseTimestamp = 0n;
-                    record.maxTimestamp = 0n;
-                    record.crc = 0;
-                });
-            });
-        });
-        expect(result).toMatchSnapshot();
-    });
-
-    let coordinatorId = -1;
-
-    it('should find coordinator', async () => {
-        const result = await cluster.sendRequest(API.FIND_COORDINATOR, { keyType: KEY_TYPE.GROUP, keys: [groupId] });
-        result.coordinators.forEach((coordinator) => {
-            coordinator.key = 'Any<String>';
-        });
-        coordinatorId = result.coordinators[0].nodeId;
-        result.coordinators.forEach((coordinator) => {
-            coordinator.nodeId = 1;
-            coordinator.port = 9093;
-        });
-        expect(result).toMatchSnapshot();
-    });
-
-    let memberId = '';
-
-    it('should fail join group request with new memberId', async () => {
-        try {
-            const result = await cluster.sendRequestToNode(coordinatorId)(API.JOIN_GROUP, {
-                groupId,
-                sessionTimeoutMs: 30000,
-                rebalanceTimeoutMs: 60000,
-                memberId,
-                groupInstanceId: null,
-                protocolType: 'consumer',
-                protocols: [
-                    {
-                        name: 'RoundRobinAssigner',
-                        metadata: { version: 0, topics: ['kafka-ts-test-topic'] },
-                    },
-                ],
-                reason: null,
-            });
-            expect(false, 'Should throw an error').toBe(true);
-        } catch (error) {
-            const { response } = error as KafkaTSApiError;
-            memberId = response.memberId;
-            response.memberId = 'Any<UUID>';
-            expect(response).toMatchSnapshot();
-        }
-    });
-
-    it('should join group', async () => {
-        const result = await cluster.sendRequestToNode(coordinatorId)(API.JOIN_GROUP, {
-            groupId,
-            sessionTimeoutMs: 30000,
-            rebalanceTimeoutMs: 60000,
-            memberId,
-            groupInstanceId: null,
-            protocolType: 'consumer',
-            protocols: [
-                {
-                    name: 'RoundRobinAssigner',
-                    metadata: { version: 0, topics: ['kafka-ts-test-topic'] },
-                },
-            ],
-            reason: null,
-        });
-        result.memberId = 'Any<UUID>';
-        result.leader = 'Any<UUID>';
-        result.members.forEach((member) => {
-            member.memberId = 'Any<UUID>';
-        });
-        expect(result).toMatchSnapshot();
-    });
-
-    it('should sync group', async () => {
-        const result = await cluster.sendRequestToNode(coordinatorId)(API.SYNC_GROUP, {
-            groupId,
-            generationId: 1,
-            memberId,
-            groupInstanceId: null,
-            protocolType: 'consumer',
-            protocolName: 'RoundRobinAssigner',
-            assignments: [
-                {
-                    memberId,
-                    assignment: { 'kafka-test-topic': [0] },
-                },
-            ],
-        });
-        expect(result).toMatchSnapshot();
-    });
-
-    it('should commit offsets', async () => {
-        const result = await cluster.sendRequestToNode(coordinatorId)(API.OFFSET_COMMIT, {
-            groupId,
-            generationIdOrMemberEpoch: 1,
-            memberId,
-            groupInstanceId: null,
-            topics: [
-                {
-                    name: 'kafka-ts-test-topic',
-                    partitions: [
-                        { partitionIndex: 0, committedOffset: 1n, committedLeaderEpoch: 0, committedMetadata: null },
-                    ],
-                },
-            ],
-        });
-        expect(result).toMatchSnapshot();
-    });
-
-    it('should fetch offsets', async () => {
-        const result = await cluster.sendRequestToNode(coordinatorId)(API.OFFSET_FETCH, {
-            groups: [
-                {
-                    groupId,
-                    topics: [
-                        {
-                            name: 'kafka-ts-test-topic',
-                            partitionIndexes: [0],
-                        },
-                    ],
-                },
-            ],
-            requireStable: false,
-        });
-        result.groups.forEach((group) => {
-            group.groupId = 'Any<String>';
-        });
-        expect(result).toMatchSnapshot();
-    });
-
-    it('should heartbeat', async () => {
-        const result = await cluster.sendRequestToNode(coordinatorId)(API.HEARTBEAT, {
-            groupId,
-            generationId: 1,
-            memberId,
-            groupInstanceId: null,
-        });
-        expect(result).toMatchSnapshot();
-    });
-
-    it('should leave group', async () => {
-        const result = await cluster.sendRequestToNode(coordinatorId)(API.LEAVE_GROUP, {
-            groupId,
-            members: [{ memberId, groupInstanceId: null, reason: null }],
-        });
-        result.members.forEach((member) => {
-            member.memberId = 'Any<UUID>';
-        });
-        expect(result).toMatchSnapshot();
-    });
-
-    it('should delete topics', async () => {
-        const result = await cluster.sendRequest(API.DELETE_TOPICS, {
-            topics: [{ name: 'kafka-ts-test-topic', topicId: null }],
-            timeoutMs: 10000,
-        });
-        result.responses.forEach((response) => {
-            response.topicId = 'Any<UUID>';
-        });
-        expect(result).toMatchSnapshot();
-    });
-});

package/src/cluster.ts

@@ -1,85 +0,0 @@
-import { TcpSocketConnectOpts } from 'net';
-import { TLSSocketOptions } from 'tls';
-import { API } from './api';
-import { Metadata } from './api/metadata';
-import { Broker, SASLProvider } from './broker';
-import { SendRequest } from './connection';
-import { KafkaTSError } from './utils/error';
-import { log } from './utils/logger';
-
-type ClusterOptions = {
-    clientId: string | null;
-    bootstrapServers: TcpSocketConnectOpts[];
-    sasl: SASLProvider | null;
-    ssl: TLSSocketOptions | null;
-};
-
-export class Cluster {
-    private seedBroker = new Broker({ clientId: null, sasl: null, ssl: null, options: { port: 9092 } });
-    private brokerById: Record<number, Broker> = {};
-    private brokerMetadata: Record<number, Metadata['brokers'][number]> = {};
-
-    constructor(private options: ClusterOptions) {}
-
-    public async connect() {
-        this.seedBroker = await this.findSeedBroker();
-        this.brokerById = {};
-
-        const metadata = await this.sendRequest(API.METADATA, {
-            allowTopicAutoCreation: false,
-            includeTopicAuthorizedOperations: false,
-            topics: [],
-        });
-        this.brokerMetadata = Object.fromEntries(metadata.brokers.map((options) => [options.nodeId, options]));
-    }
-
-    public async disconnect() {
-        await Promise.all([this.seedBroker.disconnect(), ...Object.values(this.brokerById).map((x) => x.disconnect())]);
-    }
-
-    public setSeedBroker = async (nodeId: number) => {
-        await this.seedBroker.disconnect();
-        this.seedBroker = await this.acquireBroker(nodeId);
-    };
-
-    public sendRequest: SendRequest = (...args) => this.seedBroker.sendRequest(...args);
-
-    public sendRequestToNode =
-        (nodeId: number): SendRequest =>
-        async (...args) => {
-            if (!this.brokerById[nodeId]) {
-                this.brokerById[nodeId] = await this.acquireBroker(nodeId);
-            }
-            return this.brokerById[nodeId].sendRequest(...args);
-        };
-
-    public async acquireBroker(nodeId: number) {
-        const broker = new Broker({
-            clientId: this.options.clientId,
-            sasl: this.options.sasl,
-            ssl: this.options.ssl,
-            options: this.brokerMetadata[nodeId],
-        });
-        await broker.connect();
-        return broker;
-    }
-
-    private async findSeedBroker() {
-        const randomizedBrokers = this.options.bootstrapServers.toSorted(() => Math.random() - 0.5);
-        for (const options of randomizedBrokers) {
-            try {
-                const broker = await new Broker({
-                    clientId: this.options.clientId,
-                    sasl: this.options.sasl,
-                    ssl: this.options.ssl,
-                    options,
-                });
-                await broker.connect();
-                return broker;
-            } catch (error) {
-                log.warn(`Failed to connect to seed broker ${options.host}:${options.port}`, error);
-            }
-        }
-        throw new KafkaTSError('No seed brokers found');
-    }
-}

package/src/codecs/gzip.ts

@@ -1,9 +0,0 @@
-import { gzip, unzip } from 'zlib';
-import { Codec } from './types';
-
-export const GZIP: Codec = {
-    compress: async (data) =>
-        new Promise<Buffer>((resolve, reject) => gzip(data, (err, result) => (err ? reject(err) : resolve(result)))),
-    decompress: async (data) =>
-        new Promise<Buffer>((resolve, reject) => unzip(data, (err, result) => (err ? reject(err) : resolve(result)))),
-};

package/src/codecs/index.ts

@@ -1,16 +0,0 @@
-import { GZIP } from './gzip';
-import { NONE } from './none';
-import { Codec } from './types';
-
-const codecs: Record<number, Codec> = {
-    0: NONE,
-    1: GZIP,
-};
-
-export const findCodec = (type: number) => {
-    const codec = codecs[type];
-    if (!codec) {
-        throw new Error(`Unsupported codec: ${type}`);
-    }
-    return codec;
-};

package/src/codecs/none.ts

@@ -1,6 +0,0 @@
-import { Codec } from './types';
-
-export const NONE: Codec = {
-    compress: (data: Buffer) => Promise.resolve(data),
-    decompress: (data: Buffer) => Promise.resolve(data),
-};

package/src/codecs/types.ts

@@ -1,4 +0,0 @@
-export type Codec = {
-    compress: (data: Buffer) => Promise<Buffer>;
-    decompress: (data: Buffer) => Promise<Buffer>;
-};