kafka-ts 0.0.1-beta.3 → 0.0.1-beta.6

package/src/__snapshots__/cluster.test.ts.snap

@@ -1,6 +1,6 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
-exports[`Request handler > should commit offsets 1`] = `
+exports[`Low-level API > should commit offsets 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -21,7 +21,7 @@ exports[`Request handler > should commit offsets 1`] = `
 }
 `;
 
-exports[`Request handler > should create topics 1`] = `
+exports[`Low-level API > should create topics 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -282,15 +282,15 @@ exports[`Request handler > should create topics 1`] = `
       "errorCode": 0,
       "errorMessage": null,
       "name": "kafka-ts-test-topic",
-      "numPartitions": 1,
-      "replicationFactor": 1,
+      "numPartitions": 10,
+      "replicationFactor": 3,
       "topicId": "Any<UUID>",
     },
   ],
 }
 `;
 
-exports[`Request handler > should delete topics 1`] = `
+exports[`Low-level API > should delete topics 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -307,7 +307,7 @@ exports[`Request handler > should delete topics 1`] = `
 }
 `;
 
-exports[`Request handler > should fail join group request with new memberId 1`] = `
+exports[`Low-level API > should fail join group request with new memberId 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -323,7 +323,7 @@ exports[`Request handler > should fail join group request with new memberId 1`]
 }
 `;
 
-exports[`Request handler > should fetch messages 1`] = `
+exports[`Low-level API > should fetch messages 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -339,7 +339,7 @@ exports[`Request handler > should fetch messages 1`] = `
           "highWatermark": 1n,
           "lastStableOffset": 1n,
           "logStartOffset": 0n,
-          "partitionIndex": 0,
+          "partitionIndex": 6,
           "preferredReadReplica": -1,
           "records": [
             {
@@ -348,7 +348,11 @@ exports[`Request handler > should fetch messages 1`] = `
               "baseSequence": 0,
               "baseTimestamp": 0n,
               "batchLength": 94,
+              "compression": 0,
               "crc": 0,
+              "hasDeleteHorizonMs": false,
+              "isControlBatch": false,
+              "isTransactional": false,
               "lastOffsetDelta": 0,
               "magic": 2,
               "maxTimestamp": 0n,
@@ -416,6 +420,7 @@ exports[`Request handler > should fetch messages 1`] = `
                   },
                 },
               ],
+              "timestampType": "CreateTime",
             },
           ],
         },
@@ -428,7 +433,7 @@ exports[`Request handler > should fetch messages 1`] = `
 }
 `;
 
-exports[`Request handler > should fetch offsets 1`] = `
+exports[`Low-level API > should fetch offsets 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -459,7 +464,7 @@ exports[`Request handler > should fetch offsets 1`] = `
 }
 `;
 
-exports[`Request handler > should find coordinator 1`] = `
+exports[`Low-level API > should find coordinator 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -478,7 +483,7 @@ exports[`Request handler > should find coordinator 1`] = `
 }
 `;
 
-exports[`Request handler > should heartbeat 1`] = `
+exports[`Low-level API > should heartbeat 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -487,7 +492,7 @@ exports[`Request handler > should heartbeat 1`] = `
 }
 `;
 
-exports[`Request handler > should init producer id 1`] = `
+exports[`Low-level API > should init producer id 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -498,7 +503,7 @@ exports[`Request handler > should init producer id 1`] = `
 }
 `;
 
-exports[`Request handler > should join group 1`] = `
+exports[`Low-level API > should join group 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -556,7 +561,7 @@ exports[`Request handler > should join group 1`] = `
 }
 `;
 
-exports[`Request handler > should leave group 1`] = `
+exports[`Low-level API > should leave group 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -573,7 +578,7 @@ exports[`Request handler > should leave group 1`] = `
 }
 `;
 
-exports[`Request handler > should produce messages 1`] = `
+exports[`Low-level API > should produce messages 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -587,7 +592,7 @@ exports[`Request handler > should produce messages 1`] = `
           "baseOffset": 0n,
           "errorCode": 0,
           "errorMessage": null,
-          "index": 0,
+          "index": 6,
           "logAppendTime": -1n,
           "logStartOffset": 0n,
           "recordErrors": [],
@@ -599,7 +604,7 @@ exports[`Request handler > should produce messages 1`] = `
 }
 `;
 
-exports[`Request handler > should request api versions 1`] = `
+exports[`Low-level API > should request api versions 1`] = `
 {
   "errorCode": 0,
   "throttleTimeMs": 0,
@@ -893,7 +898,7 @@ exports[`Request handler > should request api versions 1`] = `
 }
 `;
 
-exports[`Request handler > should request metadata for a topic 1`] = `
+exports[`Low-level API > should request metadata for a topic 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -920,7 +925,7 @@ exports[`Request handler > should request metadata for a topic 1`] = `
       "rack": null,
     },
   ],
-  "clusterId": "4L6g3nShT-eMCtK--X86sw",
+  "clusterId": "Some(4L6g3nShT-eMCtK--X86sw)",
   "controllerId": 0,
   "throttleTimeMs": 0,
   "topics": [
@@ -930,6 +935,20 @@ exports[`Request handler > should request metadata for a topic 1`] = `
       "isInternal": false,
       "name": "kafka-ts-test-topic",
       "partitions": [
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 6,
+          "replicaNodes": [
+            0,
+          ],
+        },
         {
           "_tag": undefined,
           "errorCode": 0,
@@ -944,6 +963,118 @@ exports[`Request handler > should request metadata for a topic 1`] = `
             0,
           ],
         },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 2,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 1,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 9,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 8,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 5,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 4,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 3,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 7,
+          "replicaNodes": [
+            0,
+          ],
+        },
       ],
       "topicAuthorizedOperations": -2147483648,
       "topicId": "Any<UUID>",
@@ -952,7 +1083,7 @@ exports[`Request handler > should request metadata for a topic 1`] = `
 }
 `;
 
-exports[`Request handler > should request metadata for all topics 1`] = `
+exports[`Low-level API > should request metadata for all topics 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,
@@ -979,7 +1110,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
       "rack": null,
     },
   ],
-  "clusterId": "4L6g3nShT-eMCtK--X86sw",
+  "clusterId": "Some(4L6g3nShT-eMCtK--X86sw)",
   "controllerId": 0,
   "throttleTimeMs": 0,
   "topics": [
@@ -987,22 +1118,8 @@ exports[`Request handler > should request metadata for all topics 1`] = `
       "_tag": undefined,
       "errorCode": 0,
       "isInternal": false,
-      "name": "my-topic",
+      "name": "kafka-ts-test-topic",
       "partitions": [
-        {
-          "_tag": undefined,
-          "errorCode": 0,
-          "isrNodes": [
-            0,
-          ],
-          "leaderEpoch": 0,
-          "leaderId": 0,
-          "offlineReplicas": [],
-          "partitionIndex": 5,
-          "replicaNodes": [
-            0,
-          ],
-        },
         {
           "_tag": undefined,
           "errorCode": 0,
@@ -1054,7 +1171,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 7,
+          "partitionIndex": 1,
           "replicaNodes": [
             0,
           ],
@@ -1068,7 +1185,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 1,
+          "partitionIndex": 9,
           "replicaNodes": [
             0,
           ],
@@ -1096,7 +1213,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 9,
+          "partitionIndex": 5,
           "replicaNodes": [
             0,
           ],
@@ -1110,7 +1227,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 3,
+          "partitionIndex": 4,
           "replicaNodes": [
             0,
           ],
@@ -1124,21 +1241,11 @@ exports[`Request handler > should request metadata for all topics 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 4,
+          "partitionIndex": 3,
           "replicaNodes": [
             0,
           ],
         },
-      ],
-      "topicAuthorizedOperations": -2147483648,
-      "topicId": "Any<UUID>",
-    },
-    {
-      "_tag": undefined,
-      "errorCode": 0,
-      "isInternal": false,
-      "name": "kafka-ts-test-topic",
-      "partitions": [
         {
           "_tag": undefined,
           "errorCode": 0,
@@ -1148,7 +1255,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 0,
+          "partitionIndex": 7,
           "replicaNodes": [
             0,
           ],
@@ -1161,7 +1268,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
 }
 `;
 
-exports[`Request handler > should sync group 1`] = `
+exports[`Low-level API > should sync group 1`] = `
 {
   "_tag": undefined,
   "_tag2": undefined,

package/src/api/fetch.ts

@@ -1,3 +1,4 @@
+import { findCodec } from '../codecs';
 import { createApi } from '../utils/api';
 import { Decoder } from '../utils/decoder';
 import { KafkaTSApiError } from '../utils/error';
@@ -7,6 +8,8 @@ export const enum IsolationLevel {
     READ_COMMITTED = 1,
 }
 
+export type FetchResponse = Awaited<ReturnType<(typeof FETCH)['response']>>;
+
 export const FETCH = createApi({
     apiKey: 1,
     apiVersion: 16,
@@ -68,7 +71,7 @@ export const FETCH = createApi({
             )
             .writeCompactString(data.rackId)
             .writeUVarInt(0),
-    response: (decoder) => {
+    response: async (decoder) => {
         const result = {
             _tag: decoder.readTagBuffer(),
             throttleTimeMs: decoder.readInt32(),
@@ -88,56 +91,108 @@ export const FETCH = createApi({
                         _tag: abortedTransaction.readTagBuffer(),
                     })),
                     preferredReadReplica: partition.readInt32(),
-                    records: decodeRecords(partition),
+                    records: decodeRecordBatch(partition),
                     _tag: partition.readTagBuffer(),
                 })),
                 _tag: response.readTagBuffer(),
             })),
             _tag2: decoder.readTagBuffer(),
         };
+
         if (result.errorCode) throw new KafkaTSApiError(result.errorCode, null, result);
         result.responses.forEach((response) => {
             response.partitions.forEach((partition) => {
                 if (partition.errorCode) throw new KafkaTSApiError(partition.errorCode, null, result);
             });
         });
-        return result;
+
+        const decompressedResponses = await Promise.all(
+            result.responses.map(async (response) => ({
+                ...response,
+                partitions: await Promise.all(
+                    response.partitions.map(async (partition) => ({
+                        ...partition,
+                        records: await Promise.all(
+                            partition.records.map(async ({ recordsLength, compressedRecords, ...record }) => {
+                                const { decompress } = findCodec(record.compression);
+                                const decompressedRecords = await decompress(compressedRecords);
+                                const decompressedDecoder = new Decoder(
+                                    Buffer.concat([recordsLength, decompressedRecords]),
+                                );
+                                return { ...record, records: decodeRecord(decompressedDecoder) };
+                            }),
+                        ),
+                    })),
+                ),
+            })),
+        );
+
+        return { ...result, responses: decompressedResponses };
     },
 });
 
-const decodeRecords = (decoder: Decoder) => {
+const decodeRecordBatch = (decoder: Decoder) => {
     const size = decoder.readUVarInt() - 1;
     if (size <= 0) {
         return [];
     }
 
+    const recordBatchDecoder = new Decoder(decoder.read(size));
+
     const results = [];
-    while (decoder.getBufferLength() > decoder.getOffset() + 49) {
+    while (recordBatchDecoder.getBufferLength() > recordBatchDecoder.getOffset() + 12) {
+        const baseOffset = recordBatchDecoder.readInt64();
+        const batchLength = recordBatchDecoder.readInt32();
+        if (!batchLength) {
+            continue;
+        }
+
+        const batchDecoder = new Decoder(recordBatchDecoder.read(batchLength));
+
+        const result = {
+            baseOffset,
+            batchLength,
+            partitionLeaderEpoch: batchDecoder.readInt32(),
+            magic: batchDecoder.readInt8(),
+            crc: batchDecoder.readUInt32(),
+            attributes: batchDecoder.readInt16(),
+            lastOffsetDelta: batchDecoder.readInt32(),
+            baseTimestamp: batchDecoder.readInt64(),
+            maxTimestamp: batchDecoder.readInt64(),
+            producerId: batchDecoder.readInt64(),
+            producerEpoch: batchDecoder.readInt16(),
+            baseSequence: batchDecoder.readInt32(),
+            recordsLength: batchDecoder.read(4),
+            compressedRecords: batchDecoder.read(),
+        };
+
+        const compression = result.attributes & 0x07;
+        const timestampType = (result.attributes & 0x08) >> 3 ? 'LogAppendTime' : 'CreateTime';
+        const isTransactional = !!((result.attributes & 0x10) >> 4);
+        const isControlBatch = !!((result.attributes & 0x20) >> 5);
+        const hasDeleteHorizonMs = !!((result.attributes & 0x40) >> 6);
+
         results.push({
-            baseOffset: decoder.readInt64(),
-            batchLength: decoder.readInt32(),
-            partitionLeaderEpoch: decoder.readInt32(),
-            magic: decoder.readInt8(),
-            crc: decoder.readUInt32(),
-            attributes: decoder.readInt16(),
-            lastOffsetDelta: decoder.readInt32(),
-            baseTimestamp: decoder.readInt64(),
-            maxTimestamp: decoder.readInt64(),
-            producerId: decoder.readInt64(),
-            producerEpoch: decoder.readInt16(),
-            baseSequence: decoder.readInt32(),
-            records: decoder.readRecords((record) => ({
-                attributes: record.readInt8(),
-                timestampDelta: record.readVarLong(),
-                offsetDelta: record.readVarInt(),
-                key: record.readVarIntBuffer(),
-                value: record.readVarIntBuffer(),
-                headers: record.readCompactArray((header) => ({
-                    key: header.readVarIntBuffer(),
-                    value: header.readVarIntBuffer(),
-                })),
-            })),
+            ...result,
+            compression,
+            timestampType,
+            isTransactional,
+            isControlBatch,
+            hasDeleteHorizonMs,
         });
     }
     return results;
 };
+
+const decodeRecord = (decoder: Decoder) =>
+    decoder.readRecords((record) => ({
+        attributes: record.readInt8(),
+        timestampDelta: record.readVarLong(),
+        offsetDelta: record.readVarInt(),
+        key: record.readVarIntBuffer(),
+        value: record.readVarIntBuffer(),
+        headers: record.readVarIntArray((header) => ({
+            key: header.readVarIntBuffer(),
+            value: header.readVarIntBuffer(),
+        })),
+    }));

package/src/api/index.ts

@@ -37,7 +37,9 @@ export const API = {
     SYNC_GROUP,
 };
 
-export const getApiName = (api: Api<unknown, unknown>) => Object.entries(API).find(([, v]) => v === api)?.[0];
+const apiNameByKey = Object.fromEntries(Object.entries(API).map(([k, v]) => [v.apiKey, k]));
+
+export const getApiName = <Request, Response>(api: Api<Request, Response>) => apiNameByKey[api.apiKey];
 
 export const API_ERROR = {
     UNKNOWN_SERVER_ERROR: -1,

package/src/api/metadata.ts

@@ -1,7 +1,7 @@
 import { createApi } from '../utils/api';
 import { KafkaTSApiError } from '../utils/error';
 
-export type Metadata = ReturnType<(typeof METADATA)['response']>;
+export type Metadata = Awaited<ReturnType<(typeof METADATA)['response']>>;
 
 export const METADATA = createApi({
     apiKey: 3,

package/src/api/produce.ts

@@ -65,10 +65,9 @@ export const PRODUCE = createApi({
                                     .writeVarIntBuffer(record.value)
                                     .writeVarIntArray(record.headers, (encoder, header) =>
                                         encoder.writeVarIntBuffer(header.key).writeVarIntBuffer(header.value),
-                                    )
-                                    .value();
+                                    );
 
-                                return encoder.writeVarInt(recordBody.length).write(recordBody);
+                                return encoder.writeVarInt(recordBody.getByteLength()).writeEncoder(recordBody);
                             })
                             .value();
 
@@ -76,19 +75,17 @@ export const PRODUCE = createApi({
                             .writeInt32(partition.partitionLeaderEpoch)
                             .writeInt8(2) // magic byte
                             .writeUInt32(unsigned(crc32C(batchBody)))
-                            .write(batchBody)
-                            .value();
+                            .write(batchBody);
 
                         const batch = new Encoder()
                             .writeInt64(partition.baseOffset)
-                            .writeInt32(batchHeader.length)
-                            .write(batchHeader)
-                            .value();
+                            .writeInt32(batchHeader.getByteLength())
+                            .writeEncoder(batchHeader);
 
                         return encoder
                             .writeInt32(partition.index)
-                            .writeUVarInt(batch.length + 1) // batch size
-                            .write(batch)
+                            .writeUVarInt(batch.getByteLength() + 1) // batch size
+                            .writeEncoder(batch)
                             .writeUVarInt(0);
                     })
                     .writeUVarInt(0),