kafka-ts 0.0.1-beta.3 → 0.0.1-beta.4

package/examples/src/consumer.ts

@@ -1,12 +1,18 @@
+import { jsonSerializer, log } from 'kafka-ts';
 import { kafka } from './client';
+import { delay } from '../../dist/utils/delay';
 
 (async () => {
     const consumer = await kafka.startConsumer({
         groupId: 'example-group',
         groupInstanceId: 'example-group-instance',
         topics: ['my-topic'],
+        allowTopicAutoCreation: true,
         onBatch: (batch) => {
-            console.log(batch);
+            log.info(
+                `Received batch: ${JSON.stringify(batch.map((message) => ({ ...message, value: message.value?.toString() })), jsonSerializer)}`,
+            );
+            log.info(`Latency: ${Date.now() - parseInt(batch[0].timestamp.toString())}ms`)
         },
         batchGranularity: 'broker',
         concurrency: 10,

package/examples/src/create-topic.ts

@@ -1,4 +1,4 @@
-import { API, API_ERROR, KafkaTSApiError } from 'kafka-ts';
+import { API, API_ERROR, KafkaTSApiError, log } from 'kafka-ts';
 import { kafka } from './client';
 
 (async () => {
@@ -19,7 +19,7 @@ import { kafka } from './client';
                 {
                     name: 'my-topic',
                     numPartitions: 10,
-                    replicationFactor: 3,
+                    replicationFactor: 1,
                     assignments: [],
                     configs: [],
                 },
@@ -37,7 +37,7 @@ import { kafka } from './client';
         topics: [{ id: null, name: 'my-topic' }],
     });
 
-    console.log(metadata);
+    log.info('Metadata', metadata);
 
     await cluster.disconnect();
 })();

package/examples/src/producer.ts

@@ -9,10 +9,10 @@ process.stdout.write('> ');
 rl.on('line', async (line) => {
     await producer.send([
         {
-            topic: 'example-topic-f',
+            topic: 'my-topic',
             value: Buffer.from(line),
         },
-    ]);
+    ], { acks: -1 });
     process.stdout.write('> ');
 });
 

package/examples/src/replicator.ts

@@ -1,3 +1,4 @@
+import { log } from 'kafka-ts';
 import { kafka } from './client';
 
 (async () => {
@@ -15,7 +16,7 @@ import { kafka } from './client';
                     offset: 0n,
                 })),
             );
-            console.log(`Replicated ${messages.length} messages`);
+            log.info(`Replicated ${messages.length} messages`);
         },
     });
     process.on('SIGINT', async () => {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kafka-ts",
-    "version": "0.0.1-beta.3",
+    "version": "0.0.1-beta.4",
     "main": "dist/index.js",
     "author": "Priit Käärd",
     "license": "MIT",

package/src/__snapshots__/cluster.test.ts.snap

@@ -282,8 +282,8 @@ exports[`Request handler > should create topics 1`] = `
       "errorCode": 0,
       "errorMessage": null,
       "name": "kafka-ts-test-topic",
-      "numPartitions": 1,
-      "replicationFactor": 1,
+      "numPartitions": 10,
+      "replicationFactor": 3,
       "topicId": "Any<UUID>",
     },
   ],
@@ -348,7 +348,11 @@ exports[`Request handler > should fetch messages 1`] = `
               "baseSequence": 0,
               "baseTimestamp": 0n,
               "batchLength": 94,
+              "compression": 0,
               "crc": 0,
+              "hasDeleteHorizonMs": false,
+              "isControlBatch": false,
+              "isTransactional": false,
               "lastOffsetDelta": 0,
               "magic": 2,
               "maxTimestamp": 0n,
@@ -416,6 +420,7 @@ exports[`Request handler > should fetch messages 1`] = `
                   },
                 },
               ],
+              "timestampType": "CreateTime",
             },
           ],
         },
@@ -944,6 +949,132 @@ exports[`Request handler > should request metadata for a topic 1`] = `
             0,
           ],
         },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 8,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 1,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 2,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 6,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 5,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 7,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 4,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 9,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 3,
+          "replicaNodes": [
+            0,
+          ],
+        },
       ],
       "topicAuthorizedOperations": -2147483648,
       "topicId": "Any<UUID>",
@@ -998,7 +1129,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 5,
+          "partitionIndex": 9,
           "replicaNodes": [
             0,
           ],
@@ -1026,7 +1157,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 0,
+          "partitionIndex": 1,
           "replicaNodes": [
             0,
           ],
@@ -1054,7 +1185,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 7,
+          "partitionIndex": 5,
           "replicaNodes": [
             0,
           ],
@@ -1068,7 +1199,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 1,
+          "partitionIndex": 3,
           "replicaNodes": [
             0,
           ],
@@ -1096,7 +1227,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 9,
+          "partitionIndex": 0,
           "replicaNodes": [
             0,
           ],
@@ -1110,7 +1241,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 3,
+          "partitionIndex": 4,
           "replicaNodes": [
             0,
           ],
@@ -1124,7 +1255,7 @@ exports[`Request handler > should request metadata for all topics 1`] = `
           "leaderEpoch": 0,
           "leaderId": 0,
           "offlineReplicas": [],
-          "partitionIndex": 4,
+          "partitionIndex": 7,
           "replicaNodes": [
             0,
           ],
@@ -1153,6 +1284,132 @@ exports[`Request handler > should request metadata for all topics 1`] = `
             0,
           ],
         },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 8,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 1,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 2,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 6,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 5,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 7,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 4,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 9,
+          "replicaNodes": [
+            0,
+          ],
+        },
+        {
+          "_tag": undefined,
+          "errorCode": 0,
+          "isrNodes": [
+            0,
+          ],
+          "leaderEpoch": 0,
+          "leaderId": 0,
+          "offlineReplicas": [],
+          "partitionIndex": 3,
+          "replicaNodes": [
+            0,
+          ],
+        },
       ],
       "topicAuthorizedOperations": -2147483648,
       "topicId": "Any<UUID>",

package/src/api/fetch.ts

@@ -1,3 +1,4 @@
+import { findCodec } from '../codecs';
 import { createApi } from '../utils/api';
 import { Decoder } from '../utils/decoder';
 import { KafkaTSApiError } from '../utils/error';
@@ -68,7 +69,7 @@ export const FETCH = createApi({
             )
             .writeCompactString(data.rackId)
             .writeUVarInt(0),
-    response: (decoder) => {
+    response: async (decoder) => {
         const result = {
             _tag: decoder.readTagBuffer(),
             throttleTimeMs: decoder.readInt32(),
@@ -88,56 +89,105 @@ export const FETCH = createApi({
                         _tag: abortedTransaction.readTagBuffer(),
                     })),
                     preferredReadReplica: partition.readInt32(),
-                    records: decodeRecords(partition),
+                    records: decodeRecordBatch(partition),
                     _tag: partition.readTagBuffer(),
                 })),
                 _tag: response.readTagBuffer(),
             })),
             _tag2: decoder.readTagBuffer(),
         };
+
         if (result.errorCode) throw new KafkaTSApiError(result.errorCode, null, result);
         result.responses.forEach((response) => {
             response.partitions.forEach((partition) => {
                 if (partition.errorCode) throw new KafkaTSApiError(partition.errorCode, null, result);
             });
         });
-        return result;
+
+        const decompressedResponses = await Promise.all(
+            result.responses.map(async (response) => ({
+                ...response,
+                partitions: await Promise.all(
+                    response.partitions.map(async (partition) => ({
+                        ...partition,
+                        records: await Promise.all(
+                            partition.records.map(async ({ recordsLength, compressedRecords, ...record }) => {
+                                const { decompress } = findCodec(record.compression);
+                                const decompressedRecords = await decompress(compressedRecords);
+                                const decompressedDecoder = new Decoder(
+                                    Buffer.concat([recordsLength, decompressedRecords]),
+                                );
+                                return { ...record, records: decodeRecord(decompressedDecoder) };
+                            }),
+                        ),
+                    })),
+                ),
+            })),
+        );
+
+        return { ...result, responses: decompressedResponses };
     },
 });
 
-const decodeRecords = (decoder: Decoder) => {
+const decodeRecordBatch = (decoder: Decoder) => {
     const size = decoder.readUVarInt() - 1;
     if (size <= 0) {
         return [];
     }
 
+    const recordBatchDecoder = new Decoder(decoder.read(size));
+
     const results = [];
-    while (decoder.getBufferLength() > decoder.getOffset() + 49) {
+    while (recordBatchDecoder.getBufferLength() > recordBatchDecoder.getOffset()) {
+        const baseOffset = recordBatchDecoder.readInt64();
+        const batchLength = recordBatchDecoder.readInt32();
+
+        const batchDecoder = new Decoder(recordBatchDecoder.read(batchLength));
+
+        const result = {
+            baseOffset,
+            batchLength,
+            partitionLeaderEpoch: batchDecoder.readInt32(),
+            magic: batchDecoder.readInt8(),
+            crc: batchDecoder.readUInt32(),
+            attributes: batchDecoder.readInt16(),
+            lastOffsetDelta: batchDecoder.readInt32(),
+            baseTimestamp: batchDecoder.readInt64(),
+            maxTimestamp: batchDecoder.readInt64(),
+            producerId: batchDecoder.readInt64(),
+            producerEpoch: batchDecoder.readInt16(),
+            baseSequence: batchDecoder.readInt32(),
+            recordsLength: batchDecoder.read(4),
+            compressedRecords: batchDecoder.read(),
+        };
+
+        const compression = result.attributes & 0x07;
+        const timestampType = (result.attributes & 0x08) >> 3 ? 'LogAppendTime' : 'CreateTime';
+        const isTransactional = !!((result.attributes & 0x10) >> 4);
+        const isControlBatch = !!((result.attributes & 0x20) >> 5);
+        const hasDeleteHorizonMs = !!((result.attributes & 0x40) >> 6);
+
         results.push({
-            baseOffset: decoder.readInt64(),
-            batchLength: decoder.readInt32(),
-            partitionLeaderEpoch: decoder.readInt32(),
-            magic: decoder.readInt8(),
-            crc: decoder.readUInt32(),
-            attributes: decoder.readInt16(),
-            lastOffsetDelta: decoder.readInt32(),
-            baseTimestamp: decoder.readInt64(),
-            maxTimestamp: decoder.readInt64(),
-            producerId: decoder.readInt64(),
-            producerEpoch: decoder.readInt16(),
-            baseSequence: decoder.readInt32(),
-            records: decoder.readRecords((record) => ({
-                attributes: record.readInt8(),
-                timestampDelta: record.readVarLong(),
-                offsetDelta: record.readVarInt(),
-                key: record.readVarIntBuffer(),
-                value: record.readVarIntBuffer(),
-                headers: record.readCompactArray((header) => ({
-                    key: header.readVarIntBuffer(),
-                    value: header.readVarIntBuffer(),
-                })),
-            })),
+            ...result,
+            compression,
+            timestampType,
+            isTransactional,
+            isControlBatch,
+            hasDeleteHorizonMs,
         });
     }
     return results;
 };
+
+const decodeRecord = (decoder: Decoder) =>
+    decoder.readRecords((record) => ({
+        attributes: record.readInt8(),
+        timestampDelta: record.readVarLong(),
+        offsetDelta: record.readVarInt(),
+        key: record.readVarIntBuffer(),
+        value: record.readVarIntBuffer(),
+        headers: record.readVarIntArray((header) => ({
+            key: header.readVarIntBuffer(),
+            value: header.readVarIntBuffer(),
+        })),
+    }));

package/src/cluster.test.ts

@@ -21,7 +21,8 @@ describe.sequential('Request handler', () => {
     let cluster: Cluster;
 
     beforeAll(async () => {
-        cluster = await kafka.createCluster().connect();
+        cluster = await kafka.createCluster();
+        await cluster.connect();
 
         const metadataResult = await cluster.sendRequest(API.METADATA, {
             topics: null,
@@ -52,8 +53,8 @@ describe.sequential('Request handler', () => {
             topics: [
                 {
                     name: 'kafka-ts-test-topic',
-                    numPartitions: 1,
-                    replicationFactor: 1,
+                    numPartitions: 10,
+                    replicationFactor: 3,
                     assignments: [],
                     configs: [],
                 },
@@ -89,6 +90,7 @@ describe.sequential('Request handler', () => {
         expect(result).toMatchSnapshot();
     });
 
+    let partitionIndex = 0;
     let leaderId = 0;
 
     it('should request metadata for a topic', async () => {
@@ -97,6 +99,7 @@ describe.sequential('Request handler', () => {
             allowTopicAutoCreation: false,
             includeTopicAuthorizedOperations: false,
         });
+        partitionIndex = result.topics[0].partitions[0].partitionIndex;
         leaderId = result.topics[0].partitions[0].leaderId;
         result.controllerId = 0;
         result.topics.forEach((topic) => {
@@ -134,7 +137,7 @@ describe.sequential('Request handler', () => {
                     name: 'kafka-ts-test-topic',
                     partitionData: [
                         {
-                            index: 0,
+                            index: partitionIndex,
                             baseOffset: 0n,
                             partitionLeaderEpoch: 0,
                             attributes: 0,
@@ -180,7 +183,7 @@ describe.sequential('Request handler', () => {
                     topicId,
                     partitions: [
                         {
-                            partition: 0,
+                            partition: partitionIndex,
                             currentLeaderEpoch: -1,
                             fetchOffset: 0n,
                             lastFetchedEpoch: 0,