kafka-ts 0.0.1-beta.4 → 0.0.1-beta.6

package/src/__snapshots__/cluster.test.ts.snap

@@ -1,6 +1,6 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
-exports[`Request handler > should commit offsets 1`] = `
+exports[`Low-level API > should commit offsets 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -21,7 +21,7 @@ exports[`Request handler > should commit offsets 1`] = `
 }
 `;
 
-exports[`Request handler > should create topics 1`] = `
+exports[`Low-level API > should create topics 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -290,7 +290,7 @@ exports[`Request handler > should create topics 1`] = `
 }
 `;
 
-exports[`Request handler > should delete topics 1`] = `
+exports[`Low-level API > should delete topics 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -307,7 +307,7 @@ exports[`Request handler > should delete topics 1`] = `
 }
 `;
 
-exports[`Request handler > should fail join group request with new memberId 1`] = `
+exports[`Low-level API > should fail join group request with new memberId 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -323,7 +323,7 @@ exports[`Request handler > should fail join group request with new memberId 1`]
 }
 `;
 
-exports[`Request handler > should fetch messages 1`] = `
+exports[`Low-level API > should fetch messages 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -339,7 +339,7 @@ exports[`Request handler > should fetch messages 1`] = `
           "highWatermark": 1n,
           "lastStableOffset": 1n,
           "logStartOffset": 0n,
-          "partitionIndex": 0,
+          "partitionIndex": 6,
           "preferredReadReplica": -1,
           "records": [
             {
@@ -433,7 +433,7 @@ exports[`Request handler > should fetch messages 1`] = `
 }
 `;
 
-exports[`Request handler > should fetch offsets 1`] = `
+exports[`Low-level API > should fetch offsets 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -464,7 +464,7 @@ exports[`Request handler > should fetch offsets 1`] = `
 }
 `;
 
-exports[`Request handler > should find coordinator 1`] = `
+exports[`Low-level API > should find coordinator 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -483,7 +483,7 @@ exports[`Request handler > should find coordinator 1`] = `
 }
 `;
 
-exports[`Request handler > should heartbeat 1`] = `
+exports[`Low-level API > should heartbeat 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -492,7 +492,7 @@ exports[`Request handler > should heartbeat 1`] = `
 }
 `;
 
-exports[`Request handler > should init producer id 1`] = `
+exports[`Low-level API > should init producer id 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -503,7 +503,7 @@ exports[`Request handler > should init producer id 1`] = `
 }
 `;
 
-exports[`Request handler > should join group 1`] = `
+exports[`Low-level API > should join group 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -561,7 +561,7 @@ exports[`Request handler > should join group 1`] = `
 }
 `;
 
-exports[`Request handler > should leave group 1`] = `
+exports[`Low-level API > should leave group 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -578,7 +578,7 @@ exports[`Request handler > should leave group 1`] = `
 }
 `;
 
-exports[`Request handler > should produce messages 1`] = `
+exports[`Low-level API > should produce messages 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -592,7 +592,7 @@ exports[`Request handler > should produce messages 1`] = `
           "baseOffset": 0n,
           "errorCode": 0,
           "errorMessage": null,
-          "index": 0,
+          "index": 6,
           "logAppendTime": -1n,
           "logStartOffset": 0n,
           "recordErrors": [],
@@ -604,7 +604,7 @@ exports[`Request handler > should produce messages 1`] = `
 }
 `;
 
-exports[`Request handler > should request api versions 1`] = `
+exports[`Low-level API > should request api versions 1`] = `
 {
   "errorCode": 0,
   "throttleTimeMs": 0,
@@ -898,7 +898,7 @@ exports[`Request handler > should request api versions 1`] = `
 }
 `;
 
-exports[`Request handler > should request metadata for a topic 1`] = `
+exports[`Low-level API > should request metadata for a topic 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -925,7 +925,7 @@ exports[`Request handler > should request metadata for a topic 1`] = `
       "rack": null,
     },
   ],
-  "clusterId": "4L6g3nShT-eMCtK--X86sw",
+  "clusterId": "Some(4L6g3nShT-eMCtK--X86sw)",
   "controllerId": 0,
   "throttleTimeMs": 0,
   "topics": [
@@ -944,7 +944,7 @@ exports[`Request handler > should request metadata for a topic 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 0,
+          "partitionIndex": 6,
           "replicaNodes": [
             0,
           ],
@@ -958,7 +958,7 @@ exports[`Request handler > should request metadata for a topic 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 8,
+          "partitionIndex": 0,
           "replicaNodes": [
             0,
           ],
@@ -972,7 +972,7 @@ exports[`Request handler > should request metadata for a topic 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 1,
+          "partitionIndex": 2,
           "replicaNodes": [
             0,
           ],
@@ -986,7 +986,7 @@ exports[`Request handler > should request metadata for a topic 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 2,
+          "partitionIndex": 1,
           "replicaNodes": [
             0,
           ],
@@ -1000,7 +1000,7 @@ exports[`Request handler > should request metadata for a topic 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 6,
+          "partitionIndex": 9,
           "replicaNodes": [
             0,
           ],
@@ -1014,7 +1014,7 @@ exports[`Request handler > should request metadata for a topic 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 5,
+          "partitionIndex": 8,
           "replicaNodes": [
             0,
           ],
@@ -1028,7 +1028,7 @@ exports[`Request handler > should request metadata for a topic 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 7,
+          "partitionIndex": 5,
           "replicaNodes": [
             0,
           ],
@@ -1056,7 +1056,7 @@ exports[`Request handler > should request metadata for a topic 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 9,
+          "partitionIndex": 3,
           "replicaNodes": [
             0,
           ],
@@ -1070,7 +1070,7 @@ exports[`Request handler > should request metadata for a topic 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 3,
+          "partitionIndex": 7,
           "replicaNodes": [
             0,
           ],
@@ -1083,7 +1083,7 @@ exports[`Request handler > should request metadata for a topic 1`] = `
 }
 `;
 
-exports[`Request handler > should request metadata for all topics 1`] = `
+exports[`Low-level API > should request metadata for all topics 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -1110,7 +1110,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
       "rack": null,
     },
   ],
-  "clusterId": "4L6g3nShT-eMCtK--X86sw",
+  "clusterId": "Some(4L6g3nShT-eMCtK--X86sw)",
   "controllerId": 0,
   "throttleTimeMs": 0,
   "topics": [
@@ -1118,22 +1118,8 @@ exports[`Request handler > should request metadata for all topics 1`] = `
       "_tag": undefined,
       "errorCode": 0,
       "isInternal": false,
-      "name": "my-topic",
+      "name": "kafka-ts-test-topic",
       "partitions": [
-        {
-          "_tag": undefined,
-          "errorCode": 0,
-          "isrNodes": [
-            0,
-          ],
-          "leaderEpoch": 0,
-          "leaderId": 0,
-          "offlineReplicas": [],
-          "partitionIndex": 9,
-          "replicaNodes": [
-            0,
-          ],
-        },
         {
           "_tag": undefined,
           "errorCode": 0,
@@ -1148,128 +1134,6 @@ exports[`Request handler > should request metadata for all topics 1`] = `
             0,
           ],
         },
-        {
-          "_tag": undefined,
-          "errorCode": 0,
-          "isrNodes": [
-            0,
-          ],
-          "leaderEpoch": 0,
-          "leaderId": 0,
-          "offlineReplicas": [],
-          "partitionIndex": 1,
-          "replicaNodes": [
-            0,
-          ],
-        },
-        {
-          "_tag": undefined,
-          "errorCode": 0,
-          "isrNodes": [
-            0,
-          ],
-          "leaderEpoch": 0,
-          "leaderId": 0,
-          "offlineReplicas": [],
-          "partitionIndex": 2,
-          "replicaNodes": [
-            0,
-          ],
-        },
-        {
-          "_tag": undefined,
-          "errorCode": 0,
-          "isrNodes": [
-            0,
-          ],
-          "leaderEpoch": 0,
-          "leaderId": 0,
-          "offlineReplicas": [],
-          "partitionIndex": 5,
-          "replicaNodes": [
-            0,
-          ],
-        },
-        {
-          "_tag": undefined,
-          "errorCode": 0,
-          "isrNodes": [
-            0,
-          ],
-          "leaderEpoch": 0,
-          "leaderId": 0,
-          "offlineReplicas": [],
-          "partitionIndex": 3,
-          "replicaNodes": [
-            0,
-          ],
-        },
-        {
-          "_tag": undefined,
-          "errorCode": 0,
-          "isrNodes": [
-            0,
-          ],
-          "leaderEpoch": 0,
-          "leaderId": 0,
-          "offlineReplicas": [],
-          "partitionIndex": 8,
-          "replicaNodes": [
-            0,
-          ],
-        },
-        {
-          "_tag": undefined,
-          "errorCode": 0,
-          "isrNodes": [
-            0,
-          ],
-          "leaderEpoch": 0,
-          "leaderId": 0,
-          "offlineReplicas": [],
-          "partitionIndex": 0,
-          "replicaNodes": [
-            0,
-          ],
-        },
-        {
-          "_tag": undefined,
-          "errorCode": 0,
-          "isrNodes": [
-            0,
-          ],
-          "leaderEpoch": 0,
-          "leaderId": 0,
-          "offlineReplicas": [],
-          "partitionIndex": 4,
-          "replicaNodes": [
-            0,
-          ],
-        },
-        {
-          "_tag": undefined,
-          "errorCode": 0,
-          "isrNodes": [
-            0,
-          ],
-          "leaderEpoch": 0,
-          "leaderId": 0,
-          "offlineReplicas": [],
-          "partitionIndex": 7,
-          "replicaNodes": [
-            0,
-          ],
-        },
-      ],
-      "topicAuthorizedOperations": -2147483648,
-      "topicId": "Any<UUID>",
-    },
-    {
-      "_tag": undefined,
-      "errorCode": 0,
-      "isInternal": false,
-      "name": "kafka-ts-test-topic",
-      "partitions": [
         {
           "_tag": undefined,
           "errorCode": 0,
@@ -1293,7 +1157,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 8,
+          "partitionIndex": 2,
           "replicaNodes": [
             0,
           ],
@@ -1321,7 +1185,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 2,
+          "partitionIndex": 9,
           "replicaNodes": [
             0,
           ],
@@ -1335,7 +1199,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 6,
+          "partitionIndex": 8,
           "replicaNodes": [
             0,
           ],
@@ -1354,20 +1218,6 @@ exports[`Request handler > should request metadata for all topics 1`] = `
             0,
           ],
         },
-        {
-          "_tag": undefined,
-          "errorCode": 0,
-          "isrNodes": [
-            0,
-          ],
-          "leaderEpoch": 0,
-          "leaderId": 0,
-          "offlineReplicas": [],
-          "partitionIndex": 7,
-          "replicaNodes": [
-            0,
-          ],
-        },
         {
           "_tag": undefined,
           "errorCode": 0,
@@ -1391,7 +1241,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 9,
+          "partitionIndex": 3,
           "replicaNodes": [
             0,
           ],
@@ -1405,7 +1255,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 3,
+          "partitionIndex": 7,
           "replicaNodes": [
             0,
           ],
@@ -1418,7 +1268,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
 }
 `;
 
-exports[`Request handler > should sync group 1`] = `
+exports[`Low-level API > should sync group 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,

package/src/api/fetch.ts

@@ -8,6 +8,8 @@ export const enum IsolationLevel {
     READ_COMMITTED = 1,
 }
 
+export type FetchResponse = Awaited<ReturnType<(typeof FETCH)['response']>>;
+
 export const FETCH = createApi({
     apiKey: 1,
     apiVersion: 16,
@@ -138,9 +140,12 @@ const decodeRecordBatch = (decoder: Decoder) => {
     const recordBatchDecoder = new Decoder(decoder.read(size));
 
     const results = [];
-    while (recordBatchDecoder.getBufferLength() > recordBatchDecoder.getOffset()) {
+    while (recordBatchDecoder.getBufferLength() > recordBatchDecoder.getOffset() + 12) {
         const baseOffset = recordBatchDecoder.readInt64();
         const batchLength = recordBatchDecoder.readInt32();
+        if (!batchLength) {
+            continue;
+        }
 
         const batchDecoder = new Decoder(recordBatchDecoder.read(batchLength));
 

package/src/api/index.ts

@@ -37,7 +37,9 @@ export const API = {
     SYNC_GROUP,
 };
 
-export const getApiName = (api: Api<unknown, unknown>) => Object.entries(API).find(([, v]) => v === api)?.[0];
+const apiNameByKey = Object.fromEntries(Object.entries(API).map(([k, v]) => [v.apiKey, k]));
+
+export const getApiName = <Request, Response>(api: Api<Request, Response>) => apiNameByKey[api.apiKey];
 
 export const API_ERROR = {
     UNKNOWN_SERVER_ERROR: -1,

package/src/api/metadata.ts

@@ -1,7 +1,7 @@
 import { createApi } from '../utils/api';
 import { KafkaTSApiError } from '../utils/error';
 
-export type Metadata = ReturnType<(typeof METADATA)['response']>;
+export type Metadata = Awaited<ReturnType<(typeof METADATA)['response']>>;
 
 export const METADATA = createApi({
     apiKey: 3,

package/src/api/produce.ts

@@ -65,10 +65,9 @@ export const PRODUCE = createApi({
                                     .writeVarIntBuffer(record.value)
                                     .writeVarIntArray(record.headers, (encoder, header) =>
                                         encoder.writeVarIntBuffer(header.key).writeVarIntBuffer(header.value),
-                                    )
-                                    .value();
+                                    );
 
-                                return encoder.writeVarInt(recordBody.length).write(recordBody);
+                                return encoder.writeVarInt(recordBody.getByteLength()).writeEncoder(recordBody);
                             })
                             .value();
 
@@ -76,19 +75,17 @@ export const PRODUCE = createApi({
                             .writeInt32(partition.partitionLeaderEpoch)
                             .writeInt8(2) // magic byte
                             .writeUInt32(unsigned(crc32C(batchBody)))
-                            .write(batchBody)
-                            .value();
+                            .write(batchBody);
 
                         const batch = new Encoder()
                             .writeInt64(partition.baseOffset)
-                            .writeInt32(batchHeader.length)
-                            .write(batchHeader)
-                            .value();
+                            .writeInt32(batchHeader.getByteLength())
+                            .writeEncoder(batchHeader);
 
                         return encoder
                             .writeInt32(partition.index)
-                            .writeUVarInt(batch.length + 1) // batch size
-                            .write(batch)
+                            .writeUVarInt(batch.getByteLength() + 1) // batch size
+                            .writeEncoder(batch)
                             .writeUVarInt(0);
                     })
                     .writeUVarInt(0),

package/src/cluster.test.ts

@@ -8,14 +8,14 @@ import { createKafkaClient } from './client';
 import { Cluster } from './cluster';
 import { KafkaTSApiError } from './utils/error';
 
-export const kafka = createKafkaClient({
+const kafka = createKafkaClient({
     clientId: 'kafka-ts',
     bootstrapServers: [{ host: 'localhost', port: 9092 }],
     sasl: saslPlain({ username: 'admin', password: 'admin' }),
     ssl: { ca: readFileSync('./certs/ca.crt').toString() },
 });
 
-describe.sequential('Request handler', () => {
+describe.sequential('Low-level API', () => {
     const groupId = randomBytes(16).toString('hex');
 
     let cluster: Cluster;

package/src/cluster.ts

@@ -1,6 +1,7 @@
 import { TcpSocketConnectOpts } from 'net';
 import { TLSSocketOptions } from 'tls';
 import { API } from './api';
+import { Metadata } from './api/metadata';
 import { Broker, SASLProvider } from './broker';
 import { SendRequest } from './connection';
 import { KafkaTSError } from './utils/error';
@@ -15,14 +16,14 @@ type ClusterOptions = {
 
 export class Cluster {
     private seedBroker = new Broker({ clientId: null, sasl: null, ssl: null, options: { port: 9092 } });
-    private brokers: { nodeId: number; broker: Broker }[] = [];
-    private brokerMetadata: Record<number, Awaited<ReturnType<(typeof API.METADATA)['response']>>['brokers'][number]> =
-        {};
+    private brokerById: Record<number, Broker> = {};
+    private brokerMetadata: Record<number, Metadata['brokers'][number]> = {};
 
     constructor(private options: ClusterOptions) {}
 
     public async connect() {
         this.seedBroker = await this.findSeedBroker();
+        this.brokerById = {};
 
         const metadata = await this.sendRequest(API.METADATA, {
             allowTopicAutoCreation: false,
@@ -33,11 +34,11 @@ export class Cluster {
     }
 
     public async disconnect() {
-        await Promise.all(this.brokers.map((x) => x.broker.disconnect()));
+        await Promise.all([this.seedBroker.disconnect(), ...Object.values(this.brokerById).map((x) => x.disconnect())]);
     }
 
     public setSeedBroker = async (nodeId: number) => {
-        await this.releaseBroker(this.seedBroker);
+        await this.seedBroker.disconnect();
         this.seedBroker = await this.acquireBroker(nodeId);
     };
 
@@ -46,11 +47,10 @@ export class Cluster {
     public sendRequestToNode =
         (nodeId: number): SendRequest =>
         async (...args) => {
-            let broker = this.brokers.find((x) => x.nodeId === nodeId)?.broker;
-            if (!broker) {
-                broker = await this.acquireBroker(nodeId);
+            if (!this.brokerById[nodeId]) {
+                this.brokerById[nodeId] = await this.acquireBroker(nodeId);
             }
-            return broker.sendRequest(...args);
+            return this.brokerById[nodeId].sendRequest(...args);
         };
 
     public async acquireBroker(nodeId: number) {
@@ -60,16 +60,10 @@ export class Cluster {
             ssl: this.options.ssl,
             options: this.brokerMetadata[nodeId],
         });
-        this.brokers.push({ nodeId, broker });
         await broker.connect();
         return broker;
     }
 
-    public async releaseBroker(broker: Broker) {
-        await broker.disconnect();
-        this.brokers = this.brokers.filter((x) => x.broker !== broker);
-    };
-
     private async findSeedBroker() {
         const randomizedBrokers = this.options.bootstrapServers.toSorted(() => Math.random() - 0.5);
         for (const options of randomizedBrokers) {
@@ -81,7 +75,6 @@ export class Cluster {
                     options,
                 });
                 await broker.connect();
-                this.brokers.push({ nodeId: -1, broker });
                 return broker;
             } catch (error) {
                 log.warn(`Failed to connect to seed broker ${options.host}:${options.port}`, error);