@platformatic/kafka 1.15.0 → 1.16.0

package/dist/apis/consumer/fetch-v15.js

@@ -1,4 +1,4 @@
-import { ResponseError } from "../../errors.js";
+import { OutOfBoundsError, ResponseError } from "../../errors.js";
 import { Reader } from "../../protocol/reader.js";
 import { readRecordsBatch } from "../../protocol/records.js";
 import { Writer } from "../../protocol/writer.js";
@@ -99,16 +99,27 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
                         }),
                         preferredReadReplica: r.readInt32()
                     };
-                    let recordsSize = r.readUnsignedVarInt();
                     if (partition.errorCode !== 0) {
                         errors.push([`/responses/${i}/partitions/${j}`, partition.errorCode]);
                     }
-                    if (recordsSize > 1) {
-                        recordsSize--;
+                    // We need to reduce the size by one to follow the COMPACT_RECORDS specification.
+                    const recordsSize = r.readUnsignedVarInt() - 1;
+                    if (recordsSize > 0) {
                         const recordsBatchesReader = Reader.from(r.buffer.subarray(r.position, r.position + recordsSize));
                         partition.records = [];
                         do {
-                            partition.records.push(readRecordsBatch(recordsBatchesReader));
+                            try {
+                                partition.records.push(readRecordsBatch(recordsBatchesReader));
+                            }
+                            catch (err) {
+                                // Contrary to other places in the protocol, records batches CAN BE truncated due to maxBytes argument.
+                                // In that case we just ignore the error.
+                                if (err.code === OutOfBoundsError.code) {
+                                    break;
+                                }
+                                /* c8 ignore next 3 - Hard to test */
+                                throw err;
+                            }
                         } while (recordsBatchesReader.position < recordsSize);
                         r.skip(recordsSize);
                     }

package/dist/apis/consumer/fetch-v16.js

@@ -1,4 +1,4 @@
-import { ResponseError } from "../../errors.js";
+import { OutOfBoundsError, ResponseError } from "../../errors.js";
 import { Reader } from "../../protocol/reader.js";
 import { readRecordsBatch } from "../../protocol/records.js";
 import { Writer } from "../../protocol/writer.js";
@@ -99,16 +99,27 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
                         }),
                         preferredReadReplica: r.readInt32()
                     };
-                    let recordsSize = r.readUnsignedVarInt();
                     if (partition.errorCode !== 0) {
                         errors.push([`/responses/${i}/partitions/${j}`, partition.errorCode]);
                     }
-                    if (recordsSize > 1) {
-                        recordsSize--;
+                    // We need to reduce the size by one to follow the COMPACT_RECORDS specification.
+                    const recordsSize = r.readUnsignedVarInt() - 1;
+                    if (recordsSize > 0) {
                         const recordsBatchesReader = Reader.from(r.buffer.subarray(r.position, r.position + recordsSize));
                         partition.records = [];
                         do {
-                            partition.records.push(readRecordsBatch(recordsBatchesReader));
+                            try {
+                                partition.records.push(readRecordsBatch(recordsBatchesReader));
+                            }
+                            catch (err) {
+                                // Contrary to other places in the protocol, records batches CAN BE truncated due to maxBytes argument.
+                                // In that case we just ignore the error.
+                                if (err.code === OutOfBoundsError.code) {
+                                    break;
+                                }
+                                /* c8 ignore next 3 - Hard to test */
+                                throw err;
+                            }
                         } while (recordsBatchesReader.position < recordsSize);
                         r.skip(recordsSize);
                     }

package/dist/apis/consumer/fetch-v17.js

@@ -1,4 +1,4 @@
-import { ResponseError } from "../../errors.js";
+import { OutOfBoundsError, ResponseError } from "../../errors.js";
 import { Reader } from "../../protocol/reader.js";
 import { readRecordsBatch } from "../../protocol/records.js";
 import { Writer } from "../../protocol/writer.js";
@@ -99,16 +99,27 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
                         }),
                         preferredReadReplica: r.readInt32()
                     };
-                    let recordsSize = r.readUnsignedVarInt();
                     if (partition.errorCode !== 0) {
                         errors.push([`/responses/${i}/partitions/${j}`, partition.errorCode]);
                     }
-                    if (recordsSize > 1) {
-                        recordsSize--;
+                    // We need to reduce the size by one to follow the COMPACT_RECORDS specification.
+                    const recordsSize = r.readUnsignedVarInt() - 1;
+                    if (recordsSize > 0) {
                         const recordsBatchesReader = Reader.from(r.buffer.subarray(r.position, r.position + recordsSize));
                         partition.records = [];
                         do {
-                            partition.records.push(readRecordsBatch(recordsBatchesReader));
+                            try {
+                                partition.records.push(readRecordsBatch(recordsBatchesReader));
+                            }
+                            catch (err) {
+                                // Contrary to other places in the protocol, records batches CAN BE truncated due to maxBytes argument.
+                                // In that case we just ignore the error.
+                                if (err.code === OutOfBoundsError.code) {
+                                    break;
+                                }
+                                /* c8 ignore next 3 - Hard to test */
+                                throw err;
+                            }
                         } while (recordsBatchesReader.position < recordsSize);
                         r.skip(recordsSize);
                     }

package/dist/errors.d.ts

@@ -1,7 +1,7 @@
 declare const kGenericError: unique symbol;
 declare const kMultipleErrors: unique symbol;
 export declare const ERROR_PREFIX = "PLT_KFK_";
-export declare const errorCodes: readonly ["PLT_KFK_AUTHENTICATION", "PLT_KFK_MULTIPLE", "PLT_KFK_NETWORK", "PLT_KFK_PROTOCOL", "PLT_KFK_RESPONSE", "PLT_KFK_TIMEOUT", "PLT_KFK_UNEXPECTED_CORRELATION_ID", "PLT_KFK_UNFINISHED_WRITE_BUFFER", "PLT_KFK_UNSUPPORTED_API", "PLT_KFK_UNSUPPORTED_COMPRESSION", "PLT_KFK_UNSUPPORTED", "PLT_KFK_USER"];
+export declare const errorCodes: readonly ["PLT_KFK_AUTHENTICATION", "PLT_KFK_MULTIPLE", "PLT_KFK_NETWORK", "PLT_KFK_OUT_OF_BOUNDS", "PLT_KFK_PROTOCOL", "PLT_KFK_RESPONSE", "PLT_KFK_TIMEOUT", "PLT_KFK_UNEXPECTED_CORRELATION_ID", "PLT_KFK_UNFINISHED_WRITE_BUFFER", "PLT_KFK_UNSUPPORTED_API", "PLT_KFK_UNSUPPORTED_COMPRESSION", "PLT_KFK_UNSUPPORTED", "PLT_KFK_USER"];
 export type ErrorCode = (typeof errorCodes)[number];
 export type ErrorProperties = {
     cause?: Error;
@@ -38,6 +38,10 @@ export declare class ProtocolError extends GenericError {
     static code: ErrorCode;
     constructor(codeOrId: string | number, properties?: ErrorProperties, response?: unknown);
 }
+export declare class OutOfBoundsError extends GenericError {
+    static code: ErrorCode;
+    constructor(message: string, properties?: ErrorProperties);
+}
 export declare class ResponseError extends MultipleErrors {
     static code: ErrorCode;
     constructor(apiName: number, apiVersion: number, errors: Record<string, number>, response: unknown, properties?: ErrorProperties);

package/dist/errors.js

@@ -7,6 +7,7 @@ export const errorCodes = [
     'PLT_KFK_AUTHENTICATION',
     'PLT_KFK_MULTIPLE',
     'PLT_KFK_NETWORK',
+    'PLT_KFK_OUT_OF_BOUNDS',
     'PLT_KFK_PROTOCOL',
     'PLT_KFK_RESPONSE',
     'PLT_KFK_TIMEOUT',
@@ -117,6 +118,12 @@ export class ProtocolError extends GenericError {
         });
     }
 }
+export class OutOfBoundsError extends GenericError {
+    static code = 'PLT_KFK_OUT_OF_BOUNDS';
+    constructor(message, properties = {}) {
+        super(OutOfBoundsError.code, message, { isOut: true, ...properties });
+    }
+}
 export class ResponseError extends MultipleErrors {
     static code = 'PLT_KFK_RESPONSE';
     constructor(apiName, apiVersion, errors, response, properties = {}) {

package/dist/protocol/crc32c.d.ts

@@ -1,2 +1,4 @@
 import { DynamicBuffer } from './dynamic-buffer.ts';
-export declare function crc32c(data: Buffer | Uint8Array | DynamicBuffer): number;
+export declare function loadNativeCRC32C(): typeof jsCRC32C | null;
+export declare function jsCRC32C(data: Buffer | Uint8Array | DynamicBuffer): number;
+export declare const crc32c: typeof jsCRC32C;

package/dist/protocol/crc32c.js

@@ -1,4 +1,5 @@
 // Based on the work from: https://github.com/tulios/kafkajs/blob/master/src/protocol/recordBatch/crc32C/crc32C.js
+import { createRequire } from 'node:module';
 import { DynamicBuffer } from "./dynamic-buffer.js";
 /* prettier-ignore */
 const CRC = [
@@ -67,7 +68,22 @@ const CRC = [
     0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e,
     0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351
 ];
-export function crc32c(data) {
+export function loadNativeCRC32C() {
+    try {
+        const require = createRequire(import.meta.url);
+        const { crc32c: nativeImplementation } = require('@node-rs/crc32');
+        return function nativeCRC32C(data) {
+            const bytes = DynamicBuffer.isDynamicBuffer(data)
+                ? data.buffer
+                : new Uint8Array(data);
+            return nativeImplementation(bytes);
+        };
+    }
+    catch (error) {
+        return null;
+    }
+}
+export function jsCRC32C(data) {
     const bytes = DynamicBuffer.isDynamicBuffer(data)
         ? data.buffer
         : new Uint8Array(data);
@@ -77,3 +93,4 @@ export function crc32c(data) {
     }
     return (crc ^ 0xffffffff) >>> 0;
 }
+export const crc32c = loadNativeCRC32C() ?? jsCRC32C;

package/dist/protocol/dynamic-buffer.js

@@ -1,4 +1,4 @@
-import { UserError } from "../errors.js";
+import { OutOfBoundsError } from "../errors.js";
 import { EMPTY_BUFFER, INT16_SIZE, INT32_SIZE, INT64_SIZE, INT8_SIZE } from "./definitions.js";
 import { BITS_8PLUS_MASK, BITS_8PLUS_MASK_64, int64ZigZagDecode, int64ZigZagEncode, intZigZagDecode, intZigZagEncode, LEAST_SIGNIFICANT_7_BITS, LEAST_SIGNIFICANT_7_BITS_64, MOST_SIGNIFICANT_BIT_FLAG, MOST_SIGNIFICANT_BIT_FLAG_64, sizeOfUnsignedVarInt, sizeOfUnsignedVarInt64 } from "./varint.js";
 const instanceIdentifier = Symbol('plt.kafka.dynamicBuffer.instanceIdentifier');
@@ -62,7 +62,7 @@ export class DynamicBuffer {
             end = this.length;
         }
         if (start < 0 || start > this.length || end > this.length) {
-            throw new UserError('Out of bounds.');
+            throw new OutOfBoundsError('Out of bounds.');
         }
         if (this.buffers.length === 0) {
             return new DynamicBuffer(EMPTY_BUFFER);
@@ -93,13 +93,13 @@ export class DynamicBuffer {
             end = this.length;
         }
         if (start < 0 || start > this.length || end > this.length) {
-            throw new UserError('Out of bounds.');
+            throw new OutOfBoundsError('Out of bounds.');
         }
         if (this.buffers.length === 0) {
             return EMPTY_BUFFER;
         }
         else if (this.buffers.length === 1) {
-            return this.buffers[0].slice(start, end);
+            return this.buffers[0].subarray(start, end);
         }
         let position = 0;
         let length = end - start;
@@ -133,7 +133,7 @@ export class DynamicBuffer {
     }
     consume(offset) {
         if (offset < 0 || offset > this.length) {
-            throw new UserError('Out of bounds.');
+            throw new OutOfBoundsError('Out of bounds.');
         }
         if (offset === 0) {
             return this;
@@ -159,14 +159,14 @@ export class DynamicBuffer {
     }
     get(offset) {
         if (offset < 0 || offset >= this.length) {
-            throw new UserError('Out of bounds.');
+            throw new OutOfBoundsError('Out of bounds.');
         }
         const [finalIndex, current] = this.#findInitialBuffer(offset);
         return this.buffers[current][finalIndex];
     }
     readUInt8(offset = 0) {
         if (offset < 0 || offset >= this.length) {
-            throw new UserError('Out of bounds.');
+            throw new OutOfBoundsError('Out of bounds.');
         }
         const [finalIndex, current] = this.#findInitialBuffer(offset);
         this.#readBuffer[0] = this.buffers[current][finalIndex];
@@ -202,7 +202,7 @@ export class DynamicBuffer {
         let value = 0;
         let read = 0;
         if (offset < 0 || offset >= this.length) {
-            throw new UserError('Out of bounds.');
+            throw new OutOfBoundsError('Out of bounds.');
         }
         // Find the initial buffer
         let [startOffset, current] = this.#findInitialBuffer(offset);
@@ -224,7 +224,7 @@ export class DynamicBuffer {
         let value = 0n;
         let read = 0;
         if (offset < 0 || offset >= this.length) {
-            throw new UserError('Out of bounds.');
+            throw new OutOfBoundsError('Out of bounds.');
         }
         // Find the initial buffer
         let [startOffset, current] = this.#findInitialBuffer(offset);
@@ -242,7 +242,7 @@ export class DynamicBuffer {
     }
     readInt8(offset = 0) {
         if (offset < 0 || offset >= this.length) {
-            throw new UserError('Out of bounds.');
+            throw new OutOfBoundsError('Out of bounds.');
         }
         const [finalIndex, current] = this.#findInitialBuffer(offset);
         this.#readBuffer[0] = this.buffers[current][finalIndex];
@@ -543,7 +543,7 @@ export class DynamicBuffer {
     }
     #readMultiple(index, length) {
         if (index < 0 || index + length > this.length) {
-            throw new UserError('Out of bounds.');
+            throw new OutOfBoundsError('Out of bounds.');
         }
         let [startOffset, current] = this.#findInitialBuffer(index);
         for (let i = 0; i < length; i++) {

package/dist/protocol/records.js

@@ -1,6 +1,7 @@
 import { UnsupportedCompressionError } from "../errors.js";
 import { compressionsAlgorithms, compressionsAlgorithmsByBitmask } from "./compression.js";
 import { crc32c } from "./crc32c.js";
+import { INT32_SIZE, INT64_SIZE } from "./definitions.js";
 import { DynamicBuffer } from "./dynamic-buffer.js";
 import { Reader } from "./reader.js";
 import { Writer } from "./writer.js";
@@ -117,6 +118,7 @@ export function createRecordsBatch(messages, options = {}) {
         .appendInt64(0n, false));
 }
 export function readRecordsBatch(reader) {
+    const initialPosition = reader.position;
     const batch = {
         firstOffset: reader.readInt64(),
         length: reader.readInt32(),
@@ -139,9 +141,12 @@ export function readRecordsBatch(reader) {
         if (!algorithm) {
             throw new UnsupportedCompressionError(`Unsupported compression algorithm with bitmask ${compression}`);
         }
-        const buffer = algorithm.decompressSync(reader.buffer.slice(reader.position, reader.buffer.length));
-        // Move the original reader to the end
-        reader.skip(reader.buffer.length - reader.position);
+        // The length of all headers immediately following Length up to the length of the Records array
+        const headersLength = reader.position - initialPosition - INT32_SIZE - INT64_SIZE;
+        const compressedDataLen = batch.length - headersLength;
+        const buffer = algorithm.decompressSync(reader.buffer.slice(reader.position, reader.position + compressedDataLen));
+        // Move the original reader to the end of the compressed data
+        reader.skip(compressedDataLen);
         // Replace the reader with the decompressed buffer
         reader = Reader.from(buffer);
     }

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "1.15.0";
+export const version = "1.16.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "1.15.0",
+  "version": "1.16.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
@@ -32,6 +32,7 @@
     "scule": "^1.3.0"
   },
   "optionalDependencies": {
+    "@node-rs/crc32": "^1.10.6",
     "lz4-napi": "^2.9.0",
     "snappy": "^7.3.3"
   },