kafka-ts 0.0.2-beta → 0.0.2

package/src/cluster.test.ts

@@ -1,35 +1,37 @@
-import { randomBytes } from "crypto";
-import { readFileSync } from "fs";
-import { afterAll, beforeAll, describe, expect, it } from "vitest";
-import { API } from "./api";
-import { KEY_TYPE } from "./api/find-coordinator";
-import { createKafkaClient } from "./client";
-import { Cluster } from "./cluster";
-import { KafkaTSApiError } from "./utils/error";
+import { randomBytes } from 'crypto';
+import { readFileSync } from 'fs';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { API } from './api';
+import { KEY_TYPE } from './api/find-coordinator';
+import { saslPlain } from './auth';
+import { createKafkaClient } from './client';
+import { Cluster } from './cluster';
+import { KafkaTSApiError } from './utils/error';
 
-export const kafka = createKafkaClient({
-    clientId: "kafka-ts",
-    bootstrapServers: [{ host: "localhost", port: 9092 }],
-    sasl: { mechanism: "PLAIN", username: "admin", password: "admin" },
-    ssl: { ca: readFileSync("./certs/ca.crt").toString() },
+const kafka = createKafkaClient({
+    clientId: 'kafka-ts',
+    bootstrapServers: [{ host: 'localhost', port: 9092 }],
+    sasl: saslPlain({ username: 'admin', password: 'admin' }),
+    ssl: { ca: readFileSync('./certs/ca.crt').toString() },
 });
 
-describe.sequential("Request handler", () => {
-    const groupId = randomBytes(16).toString("hex");
+describe.sequential('Low-level API', () => {
+    const groupId = randomBytes(16).toString('hex');
 
     let cluster: Cluster;
 
     beforeAll(async () => {
-        cluster = await kafka.createCluster().connect();
+        cluster = await kafka.createCluster();
+        await cluster.connect();
 
         const metadataResult = await cluster.sendRequest(API.METADATA, {
             topics: null,
             allowTopicAutoCreation: false,
             includeTopicAuthorizedOperations: false,
         });
-        if (metadataResult.topics.some((topic) => topic.name === "kafka-ts-test-topic")) {
+        if (metadataResult.topics.some((topic) => topic.name === 'kafka-ts-test-topic')) {
             await cluster.sendRequest(API.DELETE_TOPICS, {
-                topics: [{ name: "kafka-ts-test-topic", topicId: null }],
+                topics: [{ name: 'kafka-ts-test-topic', topicId: null }],
                 timeoutMs: 10000,
             });
         }
@@ -39,20 +41,20 @@ describe.sequential("Request handler", () => {
         await cluster.disconnect();
     });
 
-    it("should request api versions", async () => {
+    it('should request api versions', async () => {
         const result = await cluster.sendRequest(API.API_VERSIONS, {});
         expect(result).toMatchSnapshot();
     });
 
-    let topicId: string = "d6718d178e1b47c886441ad2d19faea5";
+    let topicId: string = 'd6718d178e1b47c886441ad2d19faea5';
 
-    it("should create topics", async () => {
+    it('should create topics', async () => {
         const result = await cluster.sendRequest(API.CREATE_TOPICS, {
             topics: [
                 {
-                    name: "kafka-ts-test-topic",
-                    numPartitions: 1,
-                    replicationFactor: 1,
+                    name: 'kafka-ts-test-topic',
+                    numPartitions: 10,
+                    replicationFactor: 3,
                     assignments: [],
                     configs: [],
                 },
@@ -62,23 +64,23 @@ describe.sequential("Request handler", () => {
         });
         topicId = result.topics[0].topicId;
         result.topics.forEach((topic) => {
-            topic.topicId = "Any<UUID>";
+            topic.topicId = 'Any<UUID>';
         });
         expect(result).toMatchSnapshot();
 
         await new Promise((resolve) => setTimeout(resolve, 1000));
     });
 
-    it("should request metadata for all topics", async () => {
+    it('should request metadata for all topics', async () => {
         const result = await cluster.sendRequest(API.METADATA, {
             topics: null,
             allowTopicAutoCreation: false,
             includeTopicAuthorizedOperations: false,
         });
         result.controllerId = 0;
-        result.topics = result.topics.filter((topic) => topic.name !== "__consumer_offsets");
+        result.topics = result.topics.filter((topic) => topic.name !== '__consumer_offsets');
         result.topics.forEach((topic) => {
-            topic.topicId = "Any<UUID>";
+            topic.topicId = 'Any<UUID>';
             topic.partitions.forEach((partition) => {
                 partition.leaderId = 0;
                 partition.isrNodes = [0];
@@ -88,18 +90,20 @@ describe.sequential("Request handler", () => {
         expect(result).toMatchSnapshot();
     });
 
+    let partitionIndex = 0;
     let leaderId = 0;
 
-    it("should request metadata for a topic", async () => {
+    it('should request metadata for a topic', async () => {
         const result = await cluster.sendRequest(API.METADATA, {
-            topics: [{ id: topicId, name: "kafka-ts-test-topic" }],
+            topics: [{ id: topicId, name: 'kafka-ts-test-topic' }],
             allowTopicAutoCreation: false,
             includeTopicAuthorizedOperations: false,
         });
+        partitionIndex = result.topics[0].partitions[0].partitionIndex;
         leaderId = result.topics[0].partitions[0].leaderId;
         result.controllerId = 0;
         result.topics.forEach((topic) => {
-            topic.topicId = "Any<UUID>";
+            topic.topicId = 'Any<UUID>';
             topic.partitions.forEach((partition) => {
                 partition.leaderId = 0;
                 partition.isrNodes = [0];
@@ -111,7 +115,7 @@ describe.sequential("Request handler", () => {
 
     let producerId = 9n;
 
-    it("should init producer id", async () => {
+    it('should init producer id', async () => {
         const result = await cluster.sendRequest(API.INIT_PRODUCER_ID, {
             transactionalId: null,
             transactionTimeoutMs: 0,
@@ -122,7 +126,7 @@ describe.sequential("Request handler", () => {
         expect(result).toMatchSnapshot();
     });
 
-    it("should produce messages", async () => {
+    it('should produce messages', async () => {
         const now = Date.now();
         const result = await cluster.sendRequestToNode(leaderId)(API.PRODUCE, {
             transactionalId: null,
@@ -130,10 +134,10 @@ describe.sequential("Request handler", () => {
             acks: 1,
             topicData: [
                 {
-                    name: "kafka-ts-test-topic",
+                    name: 'kafka-ts-test-topic',
                     partitionData: [
                         {
-                            index: 0,
+                            index: partitionIndex,
                             baseOffset: 0n,
                             partitionLeaderEpoch: 0,
                             attributes: 0,
@@ -148,12 +152,12 @@ describe.sequential("Request handler", () => {
                                     attributes: 0,
                                     offsetDelta: 0,
                                     timestampDelta: 0n,
-                                    key: "key",
-                                    value: "value",
+                                    key: Buffer.from('key'),
+                                    value: Buffer.from('value'),
                                     headers: [
                                         {
-                                            key: "header-key",
-                                            value: "header-value",
+                                            key: Buffer.from('header-key'),
+                                            value: Buffer.from('header-value'),
                                         },
                                     ],
                                 },
@@ -166,7 +170,7 @@ describe.sequential("Request handler", () => {
         expect(result).toMatchSnapshot();
     });
 
-    it("should fetch messages", async () => {
+    it('should fetch messages', async () => {
         const result = await cluster.sendRequestToNode(leaderId)(API.FETCH, {
             maxWaitMs: 100,
             minBytes: 1,
@@ -179,7 +183,7 @@ describe.sequential("Request handler", () => {
                     topicId,
                     partitions: [
                         {
-                            partition: 0,
+                            partition: partitionIndex,
                             currentLeaderEpoch: -1,
                             fetchOffset: 0n,
                             lastFetchedEpoch: 0,
@@ -190,10 +194,10 @@ describe.sequential("Request handler", () => {
                 },
             ],
             forgottenTopicsData: [],
-            rackId: "",
+            rackId: '',
         });
         result.responses.forEach((response) => {
-            response.topicId = "Any<UUID>";
+            response.topicId = 'Any<UUID>';
             response.partitions.forEach((partition) => {
                 partition.records.forEach((record) => {
                     expect(record.baseTimestamp).toBeGreaterThan(1721926744730n);
@@ -211,10 +215,10 @@ describe.sequential("Request handler", () => {
 
     let coordinatorId = -1;
 
-    it("should find coordinator", async () => {
+    it('should find coordinator', async () => {
         const result = await cluster.sendRequest(API.FIND_COORDINATOR, { keyType: KEY_TYPE.GROUP, keys: [groupId] });
         result.coordinators.forEach((coordinator) => {
-            coordinator.key = "Any<String>";
+            coordinator.key = 'Any<String>';
         });
         coordinatorId = result.coordinators[0].nodeId;
         result.coordinators.forEach((coordinator) => {
@@ -224,9 +228,9 @@ describe.sequential("Request handler", () => {
         expect(result).toMatchSnapshot();
     });
 
-    let memberId = "";
+    let memberId = '';
 
-    it("should fail join group request with new memberId", async () => {
+    it('should fail join group request with new memberId', async () => {
         try {
             const result = await cluster.sendRequestToNode(coordinatorId)(API.JOIN_GROUP, {
                 groupId,
@@ -234,67 +238,67 @@ describe.sequential("Request handler", () => {
                 rebalanceTimeoutMs: 60000,
                 memberId,
                 groupInstanceId: null,
-                protocolType: "consumer",
+                protocolType: 'consumer',
                 protocols: [
                     {
-                        name: "RoundRobinAssigner",
-                        metadata: { version: 0, topics: ["kafka-ts-test-topic"] },
+                        name: 'RoundRobinAssigner',
+                        metadata: { version: 0, topics: ['kafka-ts-test-topic'] },
                     },
                 ],
                 reason: null,
             });
-            expect(false, "Should throw an error").toBe(true);
+            expect(false, 'Should throw an error').toBe(true);
         } catch (error) {
             const { response } = error as KafkaTSApiError;
             memberId = response.memberId;
-            response.memberId = "Any<UUID>";
+            response.memberId = 'Any<UUID>';
             expect(response).toMatchSnapshot();
         }
     });
 
-    it("should join group", async () => {
+    it('should join group', async () => {
         const result = await cluster.sendRequestToNode(coordinatorId)(API.JOIN_GROUP, {
             groupId,
             sessionTimeoutMs: 30000,
             rebalanceTimeoutMs: 60000,
             memberId,
             groupInstanceId: null,
-            protocolType: "consumer",
+            protocolType: 'consumer',
             protocols: [
                 {
-                    name: "RoundRobinAssigner",
-                    metadata: { version: 0, topics: ["kafka-ts-test-topic"] },
+                    name: 'RoundRobinAssigner',
+                    metadata: { version: 0, topics: ['kafka-ts-test-topic'] },
                 },
             ],
             reason: null,
         });
-        result.memberId = "Any<UUID>";
-        result.leader = "Any<UUID>";
+        result.memberId = 'Any<UUID>';
+        result.leader = 'Any<UUID>';
         result.members.forEach((member) => {
-            member.memberId = "Any<UUID>";
+            member.memberId = 'Any<UUID>';
         });
         expect(result).toMatchSnapshot();
     });
 
-    it("should sync group", async () => {
+    it('should sync group', async () => {
         const result = await cluster.sendRequestToNode(coordinatorId)(API.SYNC_GROUP, {
             groupId,
             generationId: 1,
             memberId,
             groupInstanceId: null,
-            protocolType: "consumer",
-            protocolName: "RoundRobinAssigner",
+            protocolType: 'consumer',
+            protocolName: 'RoundRobinAssigner',
             assignments: [
                 {
                     memberId,
-                    assignment: { "kafka-test-topic": [0] },
+                    assignment: { 'kafka-test-topic': [0] },
                 },
             ],
         });
         expect(result).toMatchSnapshot();
     });
 
-    it("should commit offsets", async () => {
+    it('should commit offsets', async () => {
         const result = await cluster.sendRequestToNode(coordinatorId)(API.OFFSET_COMMIT, {
             groupId,
             generationIdOrMemberEpoch: 1,
@@ -302,7 +306,7 @@ describe.sequential("Request handler", () => {
             groupInstanceId: null,
             topics: [
                 {
-                    name: "kafka-ts-test-topic",
+                    name: 'kafka-ts-test-topic',
                     partitions: [
                         { partitionIndex: 0, committedOffset: 1n, committedLeaderEpoch: 0, committedMetadata: null },
                     ],
@@ -312,7 +316,7 @@ describe.sequential("Request handler", () => {
         expect(result).toMatchSnapshot();
     });
 
-    it("should fetch offsets", async () => {
+    it('should fetch offsets', async () => {
         const result = await cluster.sendRequestToNode(coordinatorId)(API.OFFSET_FETCH, {
             groups: [
                 {
@@ -321,7 +325,7 @@ describe.sequential("Request handler", () => {
                     memberEpoch: 0,
                     topics: [
                         {
-                            name: "kafka-ts-test-topic",
+                            name: 'kafka-ts-test-topic',
                             partitionIndexes: [0],
                         },
                     ],
@@ -330,12 +334,12 @@ describe.sequential("Request handler", () => {
             requireStable: false,
         });
         result.groups.forEach((group) => {
-            group.groupId = "Any<String>";
+            group.groupId = 'Any<String>';
         });
         expect(result).toMatchSnapshot();
     });
 
-    it("should heartbeat", async () => {
+    it('should heartbeat', async () => {
         const result = await cluster.sendRequestToNode(coordinatorId)(API.HEARTBEAT, {
             groupId,
             generationId: 1,
@@ -345,24 +349,24 @@ describe.sequential("Request handler", () => {
         expect(result).toMatchSnapshot();
     });
 
-    it("should leave group", async () => {
+    it('should leave group', async () => {
         const result = await cluster.sendRequestToNode(coordinatorId)(API.LEAVE_GROUP, {
             groupId,
             members: [{ memberId, groupInstanceId: null, reason: null }],
         });
         result.members.forEach((member) => {
-            member.memberId = "Any<UUID>";
+            member.memberId = 'Any<UUID>';
         });
         expect(result).toMatchSnapshot();
     });
 
-    it("should delete topics", async () => {
+    it('should delete topics', async () => {
         const result = await cluster.sendRequest(API.DELETE_TOPICS, {
-            topics: [{ name: "kafka-ts-test-topic", topicId: null }],
+            topics: [{ name: 'kafka-ts-test-topic', topicId: null }],
             timeoutMs: 10000,
         });
         result.responses.forEach((response) => {
-            response.topicId = "Any<UUID>";
+            response.topicId = 'Any<UUID>';
         });
         expect(result).toMatchSnapshot();
     });

package/src/cluster.ts

@@ -1,87 +1,85 @@
-import { TcpSocketConnectOpts } from "net";
-import { TLSSocketOptions } from "tls";
-import { API } from "./api";
-import { Broker, SASLOptions } from "./broker";
-import { SendRequest } from "./connection";
-import { ConnectionError, KafkaTSError } from "./utils/error";
+import { TcpSocketConnectOpts } from 'net';
+import { TLSSocketOptions } from 'tls';
+import { API } from './api';
+import { Metadata } from './api/metadata';
+import { Broker, SASLProvider } from './broker';
+import { SendRequest } from './connection';
+import { KafkaTSError } from './utils/error';
+import { log } from './utils/logger';
 
 type ClusterOptions = {
     clientId: string | null;
     bootstrapServers: TcpSocketConnectOpts[];
-    sasl: SASLOptions | null;
+    sasl: SASLProvider | null;
     ssl: TLSSocketOptions | null;
 };
 
 export class Cluster {
-    private seedBroker: Broker;
+    private seedBroker = new Broker({ clientId: null, sasl: null, ssl: null, options: { port: 9092 } });
     private brokerById: Record<number, Broker> = {};
+    private brokerMetadata: Record<number, Metadata['brokers'][number]> = {};
 
-    constructor(private options: ClusterOptions) {
-        this.seedBroker = new Broker({
-            clientId: this.options.clientId,
-            sasl: this.options.sasl,
-            ssl: this.options.ssl,
-            options: this.options.bootstrapServers[0],
-        });
-    }
+    constructor(private options: ClusterOptions) {}
 
     public async connect() {
-        await this.connectSeedBroker();
+        this.seedBroker = await this.findSeedBroker();
+        this.brokerById = {};
+
         const metadata = await this.sendRequest(API.METADATA, {
             allowTopicAutoCreation: false,
             includeTopicAuthorizedOperations: false,
             topics: [],
         });
-
-        this.brokerById = Object.fromEntries(
-            metadata.brokers.map(({ nodeId, ...options }) => [
-                nodeId,
-                new Broker({
-                    clientId: this.options.clientId,
-                    sasl: this.options.sasl,
-                    ssl: this.options.ssl,
-                    options,
-                }),
-            ]),
-        );
-        return this;
+        this.brokerMetadata = Object.fromEntries(metadata.brokers.map((options) => [options.nodeId, options]));
     }
 
     public async disconnect() {
-        await Promise.all([
-            this.seedBroker.disconnect(),
-            ...Object.values(this.brokerById).map((broker) => broker.disconnect()),
-        ]);
+        await Promise.all([this.seedBroker.disconnect(), ...Object.values(this.brokerById).map((x) => x.disconnect())]);
     }
 
+    public setSeedBroker = async (nodeId: number) => {
+        await this.seedBroker.disconnect();
+        this.seedBroker = await this.acquireBroker(nodeId);
+    };
+
     public sendRequest: SendRequest = (...args) => this.seedBroker.sendRequest(...args);
 
     public sendRequestToNode =
         (nodeId: number): SendRequest =>
         async (...args) => {
-            const broker = this.brokerById[nodeId];
-            if (!broker) {
-                throw new ConnectionError(`Broker ${nodeId} is not available`);
+            if (!this.brokerById[nodeId]) {
+                this.brokerById[nodeId] = await this.acquireBroker(nodeId);
             }
-            await broker.ensureConnected();
-            return broker.sendRequest(...args);
+            return this.brokerById[nodeId].sendRequest(...args);
         };
 
-    private async connectSeedBroker() {
+    public async acquireBroker(nodeId: number) {
+        const broker = new Broker({
+            clientId: this.options.clientId,
+            sasl: this.options.sasl,
+            ssl: this.options.ssl,
+            options: this.brokerMetadata[nodeId],
+        });
+        await broker.connect();
+        return broker;
+    }
+
+    private async findSeedBroker() {
         const randomizedBrokers = this.options.bootstrapServers.toSorted(() => Math.random() - 0.5);
         for (const options of randomizedBrokers) {
             try {
-                this.seedBroker = await new Broker({
+                const broker = await new Broker({
                     clientId: this.options.clientId,
                     sasl: this.options.sasl,
                     ssl: this.options.ssl,
                     options,
-                }).connect();
-                return;
+                });
+                await broker.connect();
+                return broker;
             } catch (error) {
-                console.warn(`Failed to connect to seed broker ${options.host}:${options.port}`, error);
+                log.warn(`Failed to connect to seed broker ${options.host}:${options.port}`, error);
             }
         }
-        throw new KafkaTSError("No seed brokers found");
+        throw new KafkaTSError('No seed brokers found');
     }
 }

package/src/codecs/gzip.ts

@@ -0,0 +1,9 @@
+import { gzip, unzip } from 'zlib';
+import { Codec } from './types';
+
+export const GZIP: Codec = {
+    compress: async (data) =>
+        new Promise<Buffer>((resolve, reject) => gzip(data, (err, result) => (err ? reject(err) : resolve(result)))),
+    decompress: async (data) =>
+        new Promise<Buffer>((resolve, reject) => unzip(data, (err, result) => (err ? reject(err) : resolve(result)))),
+};

package/src/codecs/index.ts

@@ -0,0 +1,16 @@
+import { GZIP } from './gzip';
+import { NONE } from './none';
+import { Codec } from './types';
+
+const codecs: Record<number, Codec> = {
+    0: NONE,
+    1: GZIP,
+};
+
+export const findCodec = (type: number) => {
+    const codec = codecs[type];
+    if (!codec) {
+        throw new Error(`Unsupported codec: ${type}`);
+    }
+    return codec;
+};

package/src/codecs/none.ts

@@ -0,0 +1,6 @@
+import { Codec } from './types';
+
+export const NONE: Codec = {
+    compress: (data: Buffer) => Promise.resolve(data),
+    decompress: (data: Buffer) => Promise.resolve(data),
+};

package/src/codecs/types.ts

@@ -0,0 +1,4 @@
+export type Codec = {
+    compress: (data: Buffer) => Promise<Buffer>;
+    decompress: (data: Buffer) => Promise<Buffer>;
+};