kafka-ts 0.0.17-beta.2 → 0.0.17-beta.3

package/dist/api/fetch.d.ts

@@ -41,17 +41,17 @@ export declare const FETCH: import("../utils/api").Api<{
                         value: string;
                     }[];
                 }[];
-                compression: number;
-                timestampType: string;
-                isTransactional: boolean;
-                isControlBatch: boolean;
-                hasDeleteHorizonMs: boolean;
                 baseOffset: bigint;
                 batchLength: number;
                 partitionLeaderEpoch: number;
                 magic: number;
                 crc: number;
                 attributes: number;
+                compression: number;
+                timestampType: string;
+                isTransactional: boolean;
+                isControlBatch: boolean;
+                hasDeleteHorizonMs: boolean;
                 lastOffsetDelta: number;
                 baseTimestamp: bigint;
                 maxTimestamp: bigint;

package/dist/api/fetch.js

@@ -5,6 +5,7 @@ const codecs_1 = require("../codecs");
 const api_1 = require("../utils/api");
 const decoder_1 = require("../utils/decoder");
 const error_1 = require("../utils/error");
+const logger_1 = require("../utils/logger");
 exports.FETCH = (0, api_1.createApi)({
     apiKey: 1,
     apiVersion: 15,
@@ -90,41 +91,62 @@ const decodeRecordBatch = (decoder) => {
     }
     const recordBatchDecoder = new decoder_1.Decoder(decoder.read(size));
     const results = [];
-    while (recordBatchDecoder.getBufferLength() > recordBatchDecoder.getOffset() + 12) {
+    while (recordBatchDecoder.canReadBytes(12)) {
         const baseOffset = recordBatchDecoder.readInt64();
         const batchLength = recordBatchDecoder.readInt32();
         if (!batchLength) {
             continue;
         }
+        if (!recordBatchDecoder.canReadBytes(batchLength)) {
+            // likely running into maxBytes limit
+            logger_1.log.debug('Record batch is incomplete, skipping last batch.');
+            recordBatchDecoder.read();
+            continue;
+        }
         const batchDecoder = new decoder_1.Decoder(recordBatchDecoder.read(batchLength));
-        const result = {
+        const partitionLeaderEpoch = batchDecoder.readInt32();
+        const magic = batchDecoder.readInt8();
+        if (magic !== 2) {
+            throw new error_1.KafkaTSError(`Unsupported magic byte: ${magic}`);
+        }
+        const crc = batchDecoder.readInt32();
+        const attributes = batchDecoder.readInt16();
+        const compression = attributes & 0x07;
+        const timestampType = (attributes & 0x08) >> 3 ? 'LogAppendTime' : 'CreateTime';
+        const isTransactional = !!((attributes & 0x10) >> 4);
+        const isControlBatch = !!((attributes & 0x20) >> 5);
+        const hasDeleteHorizonMs = !!((attributes & 0x40) >> 6);
+        if (compression !== 0) {
+            throw new error_1.KafkaTSError(`Unsupported compression: ${compression}`);
+        }
+        const lastOffsetDelta = batchDecoder.readInt32();
+        const baseTimestamp = batchDecoder.readInt64();
+        const maxTimestamp = batchDecoder.readInt64();
+        const producerId = batchDecoder.readInt64();
+        const producerEpoch = batchDecoder.readInt16();
+        const baseSequence = batchDecoder.readInt32();
+        const recordsLength = batchDecoder.read(4);
+        const compressedRecords = batchDecoder.read();
+        results.push({
             baseOffset,
             batchLength,
-            partitionLeaderEpoch: batchDecoder.readInt32(),
-            magic: batchDecoder.readInt8(),
-            crc: batchDecoder.readUInt32(),
-            attributes: batchDecoder.readInt16(),
-            lastOffsetDelta: batchDecoder.readInt32(),
-            baseTimestamp: batchDecoder.readInt64(),
-            maxTimestamp: batchDecoder.readInt64(),
-            producerId: batchDecoder.readInt64(),
-            producerEpoch: batchDecoder.readInt16(),
-            baseSequence: batchDecoder.readInt32(),
-            recordsLength: batchDecoder.read(4),
-            compressedRecords: batchDecoder.read(),
-        };
-        const compression = result.attributes & 0x07;
-        const timestampType = (result.attributes & 0x08) >> 3 ? 'LogAppendTime' : 'CreateTime';
-        const isTransactional = !!((result.attributes & 0x10) >> 4);
-        const isControlBatch = !!((result.attributes & 0x20) >> 5);
-        const hasDeleteHorizonMs = !!((result.attributes & 0x40) >> 6);
-        results.push({
-            ...result,
+            partitionLeaderEpoch,
+            magic,
+            crc,
+            attributes,
             compression,
             timestampType,
             isTransactional,
             isControlBatch,
             hasDeleteHorizonMs,
+            lastOffsetDelta,
+            baseTimestamp,
+            maxTimestamp,
+            producerId,
+            producerEpoch,
+            baseSequence,
+            recordsLength,
+            compressedRecords,
         });
     }
     return results;

package/dist/api/index.d.ts

@@ -104,17 +104,17 @@ export declare const API: {
                             value: string;
                         }[];
                     }[];
-                    compression: number;
-                    timestampType: string;
-                    isTransactional: boolean;
-                    isControlBatch: boolean;
-                    hasDeleteHorizonMs: boolean;
                     baseOffset: bigint;
                     batchLength: number;
                     partitionLeaderEpoch: number;
                     magic: number;
                     crc: number;
                     attributes: number;
+                    compression: number;
+                    timestampType: string;
+                    isTransactional: boolean;
+                    isControlBatch: boolean;
+                    hasDeleteHorizonMs: boolean;
                     lastOffsetDelta: number;
                     baseTimestamp: bigint;
                     maxTimestamp: bigint;

package/dist/connection.js

@@ -144,7 +144,7 @@ class Connection {
     handleData(buffer) {
         this.chunks.push(buffer);
         const decoder = new decoder_1.Decoder(Buffer.concat(this.chunks));
-        if (decoder.getBufferLength() < 4) {
+        if (!decoder.canReadBytes(4)) {
             return;
         }
         const size = decoder.readInt32();

package/dist/utils/decoder.d.ts

@@ -5,6 +5,7 @@ export declare class Decoder {
     constructor(buffer: Buffer);
     getOffset(): number;
     getBufferLength(): number;
+    canReadBytes(bytes: number): boolean;
     readInt8(): number;
     readInt16(): number;
     readInt32(): number;

package/dist/utils/decoder.js

@@ -13,6 +13,9 @@ class Decoder {
     getBufferLength() {
         return this.buffer.length;
     }
+    canReadBytes(bytes) {
+        return this.getBufferLength() - this.getOffset() >= bytes;
+    }
     readInt8() {
         const value = this.buffer.readInt8(this.offset);
         this.offset += 1;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kafka-ts",
-    "version": "0.0.17-beta.2",
+    "version": "0.0.17-beta.3",
     "main": "dist/index.js",
     "author": "Priit Käärd",
     "license": "MIT",