@platformatic/kafka 1.15.1 → 1.17.0

package/dist/apis/consumer/fetch-v15.js

@@ -1,6 +1,6 @@
 import { ResponseError } from "../../errors.js";
 import { Reader } from "../../protocol/reader.js";
-import { readRecordsBatch } from "../../protocol/records.js";
+import { readRecordsBatches } from "../../protocol/records.js";
 import { Writer } from "../../protocol/writer.js";
 import { createAPI } from "../definitions.js";
 /*
@@ -99,17 +99,13 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
                         }),
                         preferredReadReplica: r.readInt32()
                     };
-                    let recordsSize = r.readUnsignedVarInt();
                     if (partition.errorCode !== 0) {
                         errors.push([`/responses/${i}/partitions/${j}`, partition.errorCode]);
                     }
-                    if (recordsSize > 1) {
-                        recordsSize--;
-                        const recordsBatchesReader = Reader.from(r.buffer.subarray(r.position, r.position + recordsSize));
-                        partition.records = [];
-                        do {
-                            partition.records.push(readRecordsBatch(recordsBatchesReader));
-                        } while (recordsBatchesReader.position < recordsSize);
+                    // We need to reduce the size by one to follow the COMPACT_RECORDS specification.
+                    const recordsSize = r.readUnsignedVarInt() - 1;
+                    if (recordsSize > 0) {
+                        partition.records = readRecordsBatches(Reader.from(r.buffer.subarray(r.position, r.position + recordsSize)));
                         r.skip(recordsSize);
                     }
                     return partition;

package/dist/apis/consumer/fetch-v16.js

@@ -1,6 +1,6 @@
 import { ResponseError } from "../../errors.js";
 import { Reader } from "../../protocol/reader.js";
-import { readRecordsBatch } from "../../protocol/records.js";
+import { readRecordsBatches } from "../../protocol/records.js";
 import { Writer } from "../../protocol/writer.js";
 import { createAPI } from "../definitions.js";
 /*
@@ -99,17 +99,13 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
                         }),
                         preferredReadReplica: r.readInt32()
                     };
-                    let recordsSize = r.readUnsignedVarInt();
                     if (partition.errorCode !== 0) {
                         errors.push([`/responses/${i}/partitions/${j}`, partition.errorCode]);
                     }
-                    if (recordsSize > 1) {
-                        recordsSize--;
-                        const recordsBatchesReader = Reader.from(r.buffer.subarray(r.position, r.position + recordsSize));
-                        partition.records = [];
-                        do {
-                            partition.records.push(readRecordsBatch(recordsBatchesReader));
-                        } while (recordsBatchesReader.position < recordsSize);
+                    // We need to reduce the size by one to follow the COMPACT_RECORDS specification.
+                    const recordsSize = r.readUnsignedVarInt() - 1;
+                    if (recordsSize > 0) {
+                        partition.records = readRecordsBatches(Reader.from(r.buffer.subarray(r.position, r.position + recordsSize)));
                         r.skip(recordsSize);
                     }
                     return partition;

package/dist/apis/consumer/fetch-v17.js

@@ -1,6 +1,6 @@
 import { ResponseError } from "../../errors.js";
 import { Reader } from "../../protocol/reader.js";
-import { readRecordsBatch } from "../../protocol/records.js";
+import { readRecordsBatches } from "../../protocol/records.js";
 import { Writer } from "../../protocol/writer.js";
 import { createAPI } from "../definitions.js";
 /*
@@ -102,14 +102,10 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
                     if (partition.errorCode !== 0) {
                         errors.push([`/responses/${i}/partitions/${j}`, partition.errorCode]);
                     }
-                    // We need to reduce the size by one to follow the COMPACT_RECORDS specification
+                    // We need to reduce the size by one to follow the COMPACT_RECORDS specification.
                     const recordsSize = r.readUnsignedVarInt() - 1;
                     if (recordsSize > 0) {
-                        const recordsBatchesReader = Reader.from(r.buffer.subarray(r.position, r.position + recordsSize));
-                        partition.records = [];
-                        do {
-                            partition.records.push(readRecordsBatch(recordsBatchesReader));
-                        } while (recordsBatchesReader.position < recordsSize);
+                        partition.records = readRecordsBatches(Reader.from(r.buffer.subarray(r.position, r.position + recordsSize)));
                         r.skip(recordsSize);
                     }
                     return partition;

package/dist/clients/base/options.d.ts

@@ -74,6 +74,10 @@ export declare const baseOptionsSchema: {
             type: string;
             minimum: number;
         };
+        handleBackPressure: {
+            type: string;
+            default: boolean;
+        };
         tls: {
             type: string;
             additionalProperties: boolean;

package/dist/clients/base/options.js

@@ -31,6 +31,7 @@ export const baseOptionsSchema = {
         retries: { oneOf: [{ type: 'number', minimum: 0 }, { type: 'boolean' }] },
         retryDelay: { type: 'number', minimum: 0 },
         maxInflights: { type: 'number', minimum: 0 },
+        handleBackPressure: { type: 'boolean', default: false },
         tls: { type: 'object', additionalProperties: true }, // No validation as they come from Node.js
         tlsServerName: { oneOf: [{ type: 'boolean' }, { type: 'string' }] },
         sasl: {

package/dist/errors.d.ts

@@ -1,7 +1,7 @@
 declare const kGenericError: unique symbol;
 declare const kMultipleErrors: unique symbol;
 export declare const ERROR_PREFIX = "PLT_KFK_";
-export declare const errorCodes: readonly ["PLT_KFK_AUTHENTICATION", "PLT_KFK_MULTIPLE", "PLT_KFK_NETWORK", "PLT_KFK_PROTOCOL", "PLT_KFK_RESPONSE", "PLT_KFK_TIMEOUT", "PLT_KFK_UNEXPECTED_CORRELATION_ID", "PLT_KFK_UNFINISHED_WRITE_BUFFER", "PLT_KFK_UNSUPPORTED_API", "PLT_KFK_UNSUPPORTED_COMPRESSION", "PLT_KFK_UNSUPPORTED", "PLT_KFK_USER"];
+export declare const errorCodes: readonly ["PLT_KFK_AUTHENTICATION", "PLT_KFK_MULTIPLE", "PLT_KFK_NETWORK", "PLT_KFK_OUT_OF_BOUNDS", "PLT_KFK_PROTOCOL", "PLT_KFK_RESPONSE", "PLT_KFK_TIMEOUT", "PLT_KFK_UNEXPECTED_CORRELATION_ID", "PLT_KFK_UNFINISHED_WRITE_BUFFER", "PLT_KFK_UNSUPPORTED_API", "PLT_KFK_UNSUPPORTED_COMPRESSION", "PLT_KFK_UNSUPPORTED", "PLT_KFK_USER"];
 export type ErrorCode = (typeof errorCodes)[number];
 export type ErrorProperties = {
     cause?: Error;
@@ -38,6 +38,10 @@ export declare class ProtocolError extends GenericError {
     static code: ErrorCode;
     constructor(codeOrId: string | number, properties?: ErrorProperties, response?: unknown);
 }
+export declare class OutOfBoundsError extends GenericError {
+    static code: ErrorCode;
+    constructor(message: string, properties?: ErrorProperties);
+}
 export declare class ResponseError extends MultipleErrors {
     static code: ErrorCode;
     constructor(apiName: number, apiVersion: number, errors: Record<string, number>, response: unknown, properties?: ErrorProperties);

package/dist/errors.js

@@ -7,6 +7,7 @@ export const errorCodes = [
     'PLT_KFK_AUTHENTICATION',
     'PLT_KFK_MULTIPLE',
     'PLT_KFK_NETWORK',
+    'PLT_KFK_OUT_OF_BOUNDS',
     'PLT_KFK_PROTOCOL',
     'PLT_KFK_RESPONSE',
     'PLT_KFK_TIMEOUT',
@@ -117,6 +118,12 @@ export class ProtocolError extends GenericError {
         });
     }
 }
+export class OutOfBoundsError extends GenericError {
+    static code = 'PLT_KFK_OUT_OF_BOUNDS';
+    constructor(message, properties = {}) {
+        super(OutOfBoundsError.code, message, { isOut: true, ...properties });
+    }
+}
 export class ResponseError extends MultipleErrors {
     static code = 'PLT_KFK_RESPONSE';
     constructor(apiName, apiVersion, errors, response, properties = {}) {

package/dist/network/connection.d.ts

@@ -24,14 +24,15 @@ export interface ConnectionOptions {
     tlsServerName?: string | boolean;
     sasl?: SASLOptions;
     ownerId?: number;
+    handleBackPressure?: boolean;
 }
 export interface Request {
     correlationId: number;
     apiKey: number;
     apiVersion: number;
-    hasRequestHeaderTaggedFields: boolean;
     hasResponseHeaderTaggedFields: boolean;
-    payload: () => Writer;
+    noResponse: boolean;
+    payload: Buffer;
     parser: ResponseParser<unknown>;
     callback: Callback<any>;
     diagnostic: Record<string, unknown>;
@@ -62,5 +63,5 @@ export declare class Connection extends EventEmitter {
     ready(): Promise<void>;
     close(callback: CallbackWithPromise<void>): void;
     close(): Promise<void>;
-    send<ReturnType>(apiKey: number, apiVersion: number, payload: () => Writer, responseParser: ResponseParser<ReturnType>, hasRequestHeaderTaggedFields: boolean, hasResponseHeaderTaggedFields: boolean, callback: Callback<ReturnType>): void;
+    send<ReturnType>(apiKey: number, apiVersion: number, createPayload: () => Writer, responseParser: ResponseParser<ReturnType>, hasRequestHeaderTaggedFields: boolean, hasResponseHeaderTaggedFields: boolean, callback: Callback<ReturnType>): void;
 }

package/dist/network/connection.js

@@ -38,6 +38,7 @@ export class Connection extends EventEmitter {
     #clientId;
     // @ts-ignore This is used just for debugging
     #ownerId;
+    #handleBackPressure;
     #correlationId;
     #nextMessage;
     #afterDrainRequests;
@@ -56,6 +57,7 @@ export class Connection extends EventEmitter {
         this.#status = ConnectionStatuses.NONE;
         this.#clientId = clientId;
         this.#ownerId = options.ownerId;
+        this.#handleBackPressure = options.handleBackPressure ?? false;
         this.#correlationId = 0;
         this.#nextMessage = 0;
         this.#afterDrainRequests = [];
@@ -134,7 +136,9 @@ export class Connection extends EventEmitter {
                 this.#socket.removeListener('error', connectionErrorHandler);
                 this.#socket.on('error', this.#onError.bind(this));
                 this.#socket.on('data', this.#onData.bind(this));
-                this.#socket.on('drain', this.#onDrain.bind(this));
+                if (this.#handleBackPressure) {
+                    this.#socket.on('drain', this.#onDrain.bind(this));
+                }
                 this.#socket.on('close', this.#onClose.bind(this));
                 this.#socket.setTimeout(0);
                 if (this.#options.sasl) {
@@ -205,26 +209,43 @@ export class Connection extends EventEmitter {
         this.#socket.end();
         return callback[kCallbackPromise];
     }
-    send(apiKey, apiVersion, payload, responseParser, hasRequestHeaderTaggedFields, hasResponseHeaderTaggedFields, callback) {
+    send(apiKey, apiVersion, createPayload, responseParser, hasRequestHeaderTaggedFields, hasResponseHeaderTaggedFields, callback) {
+        const correlationId = ++this.#correlationId;
+        const diagnostic = createDiagnosticContext({
+            connection: this,
+            operation: 'send',
+            apiKey,
+            apiVersion,
+            correlationId
+        });
+        const writer = Writer.create();
+        writer.appendInt16(apiKey).appendInt16(apiVersion).appendInt32(correlationId).appendString(this.#clientId, false);
+        if (hasRequestHeaderTaggedFields) {
+            writer.appendTaggedFields();
+        }
+        let payload;
+        try {
+            payload = createPayload();
+        }
+        catch (err) {
+            diagnostic.error = err;
+            connectionsApiChannel.error.publish(diagnostic);
+            throw err;
+        }
+        writer.appendFrom(payload).prependLength();
+        const request = {
+            correlationId,
+            apiKey,
+            apiVersion,
+            parser: responseParser,
+            payload: writer.buffer,
+            callback: null, // Will be set later
+            hasResponseHeaderTaggedFields,
+            noResponse: payload.context.noResponse ?? false,
+            diagnostic
+        };
         this.#requestsQueue.push(fastQueueCallback => {
-            const correlationId = ++this.#correlationId;
-            const request = {
-                correlationId,
-                apiKey,
-                apiVersion,
-                hasRequestHeaderTaggedFields,
-                hasResponseHeaderTaggedFields,
-                parser: responseParser,
-                payload,
-                callback: fastQueueCallback,
-                diagnostic: createDiagnosticContext({
-                    connection: this,
-                    operation: 'send',
-                    apiKey,
-                    apiVersion,
-                    correlationId
-                })
-            };
+            request.callback = fastQueueCallback;
             if (this.#socketMustBeDrained) {
                 this.#afterDrainRequests.push(request);
                 return false;
@@ -274,37 +295,31 @@ export class Connection extends EventEmitter {
                 request.callback(new NetworkError('Connection closed'), undefined);
                 return false;
             }
-            const writer = Writer.create();
-            writer
-                .appendInt16(request.apiKey)
-                .appendInt16(request.apiVersion)
-                .appendInt32(request.correlationId)
-                .appendString(this.#clientId, false);
-            if (request.hasRequestHeaderTaggedFields) {
-                writer.appendTaggedFields();
-            }
-            const payload = request.payload();
-            writer.appendFrom(payload).prependLength();
-            const expectResponse = !payload.context.noResponse;
-            if (expectResponse)
+            if (!request.noResponse) {
                 this.#inflightRequests.set(request.correlationId, request);
-            const canWrite = this.#socket.write(writer.buffer);
-            if (!canWrite)
+            }
+            let canWrite = this.#socket.write(request.payload);
+            if (!this.#handleBackPressure) {
+                canWrite = true;
+            }
+            if (!canWrite) {
                 this.#socketMustBeDrained = true;
-            if (!expectResponse)
+            }
+            if (request.noResponse) {
                 request.callback(null, canWrite);
+            }
             loggers.protocol('Sending request.', {
                 apiKey: protocolAPIsById[request.apiKey],
                 correlationId: request.correlationId,
                 request
             });
             return canWrite;
+            /* c8 ignore next 8 - Hard to test */
         }
         catch (err) {
             request.diagnostic.error = err;
             connectionsApiChannel.error.publish(request.diagnostic);
             throw err;
-            /* c8 ignore next 3 - Hard to test */
         }
         finally {
             connectionsApiChannel.end.publish(request.diagnostic);
@@ -453,8 +468,7 @@ export class Connection extends EventEmitter {
         this.emit('close');
         const error = new NetworkError('Connection closed');
         for (const request of this.#afterDrainRequests) {
-            const payload = request.payload();
-            if (!payload.context.noResponse) {
+            if (!request.noResponse) {
                 request.callback(error, undefined);
             }
         }

package/dist/protocol/crc32c.d.ts

@@ -1,2 +1,4 @@
 import { DynamicBuffer } from './dynamic-buffer.ts';
-export declare function crc32c(data: Buffer | Uint8Array | DynamicBuffer): number;
+export declare function loadNativeCRC32C(): typeof jsCRC32C | null;
+export declare function jsCRC32C(data: Buffer | Uint8Array | DynamicBuffer): number;
+export declare const crc32c: typeof jsCRC32C;

package/dist/protocol/crc32c.js

@@ -1,4 +1,5 @@
 // Based on the work from: https://github.com/tulios/kafkajs/blob/master/src/protocol/recordBatch/crc32C/crc32C.js
+import { createRequire } from 'node:module';
 import { DynamicBuffer } from "./dynamic-buffer.js";
 /* prettier-ignore */
 const CRC = [
@@ -67,7 +68,23 @@ const CRC = [
     0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e,
     0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351
 ];
-export function crc32c(data) {
+export function loadNativeCRC32C() {
+    try {
+        const require = createRequire(import.meta.url);
+        const { crc32c: nativeImplementation } = require('@node-rs/crc32');
+        return function nativeCRC32C(data) {
+            const bytes = DynamicBuffer.isDynamicBuffer(data)
+                ? data.buffer
+                : new Uint8Array(data);
+            return nativeImplementation(bytes);
+        };
+        /* c8 ignore next 3 - Hard to test */
+    }
+    catch (error) {
+        return null;
+    }
+}
+export function jsCRC32C(data) {
     const bytes = DynamicBuffer.isDynamicBuffer(data)
         ? data.buffer
         : new Uint8Array(data);
@@ -77,3 +94,5 @@ export function crc32c(data) {
     }
     return (crc ^ 0xffffffff) >>> 0;
 }
+/* c8 ignore next - Hard to test */
+export const crc32c = loadNativeCRC32C() ?? jsCRC32C;

package/dist/protocol/dynamic-buffer.js

@@ -1,4 +1,4 @@
-import { UserError } from "../errors.js";
+import { OutOfBoundsError } from "../errors.js";
 import { EMPTY_BUFFER, INT16_SIZE, INT32_SIZE, INT64_SIZE, INT8_SIZE } from "./definitions.js";
 import { BITS_8PLUS_MASK, BITS_8PLUS_MASK_64, int64ZigZagDecode, int64ZigZagEncode, intZigZagDecode, intZigZagEncode, LEAST_SIGNIFICANT_7_BITS, LEAST_SIGNIFICANT_7_BITS_64, MOST_SIGNIFICANT_BIT_FLAG, MOST_SIGNIFICANT_BIT_FLAG_64, sizeOfUnsignedVarInt, sizeOfUnsignedVarInt64 } from "./varint.js";
 const instanceIdentifier = Symbol('plt.kafka.dynamicBuffer.instanceIdentifier');
@@ -48,12 +48,18 @@ export class DynamicBuffer {
         return this;
     }
     appendFrom(DynamicBuffer) {
-        this.buffers.push(...DynamicBuffer.buffers);
+        const buffers = DynamicBuffer.buffers;
+        for (let i = 0; i < buffers.length; i++) {
+            this.buffers.push(buffers[i]);
+        }
         this.length += DynamicBuffer.length;
         return this;
     }
     prependFrom(DynamicBuffer) {
-        this.buffers.unshift(...DynamicBuffer.buffers);
+        const buffers = DynamicBuffer.buffers;
+        for (let i = buffers.length - 1; i >= 0; i--) {
+            this.buffers.unshift(buffers[i]);
+        }
         this.length += DynamicBuffer.length;
         return this;
     }
@@ -62,7 +68,7 @@ export class DynamicBuffer {
             end = this.length;
         }
         if (start < 0 || start > this.length || end > this.length) {
-            throw new UserError('Out of bounds.');
+            throw new OutOfBoundsError('Out of bounds.');
         }
         if (this.buffers.length === 0) {
             return new DynamicBuffer(EMPTY_BUFFER);
@@ -93,13 +99,13 @@ export class DynamicBuffer {
             end = this.length;
         }
         if (start < 0 || start > this.length || end > this.length) {
-            throw new UserError('Out of bounds.');
+            throw new OutOfBoundsError('Out of bounds.');
         }
         if (this.buffers.length === 0) {
             return EMPTY_BUFFER;
         }
         else if (this.buffers.length === 1) {
-            return this.buffers[0].slice(start, end);
+            return this.buffers[0].subarray(start, end);
         }
         let position = 0;
         let length = end - start;
@@ -133,7 +139,7 @@ export class DynamicBuffer {
     }
     consume(offset) {
         if (offset < 0 || offset > this.length) {
-            throw new UserError('Out of bounds.');
+            throw new OutOfBoundsError('Out of bounds.');
         }
         if (offset === 0) {
             return this;
@@ -159,14 +165,14 @@ export class DynamicBuffer {
     }
     get(offset) {
         if (offset < 0 || offset >= this.length) {
-            throw new UserError('Out of bounds.');
+            throw new OutOfBoundsError('Out of bounds.');
         }
         const [finalIndex, current] = this.#findInitialBuffer(offset);
         return this.buffers[current][finalIndex];
     }
     readUInt8(offset = 0) {
         if (offset < 0 || offset >= this.length) {
-            throw new UserError('Out of bounds.');
+            throw new OutOfBoundsError('Out of bounds.');
         }
         const [finalIndex, current] = this.#findInitialBuffer(offset);
         this.#readBuffer[0] = this.buffers[current][finalIndex];
@@ -202,7 +208,7 @@ export class DynamicBuffer {
         let value = 0;
         let read = 0;
         if (offset < 0 || offset >= this.length) {
-            throw new UserError('Out of bounds.');
+            throw new OutOfBoundsError('Out of bounds.');
         }
         // Find the initial buffer
         let [startOffset, current] = this.#findInitialBuffer(offset);
@@ -224,7 +230,7 @@ export class DynamicBuffer {
         let value = 0n;
         let read = 0;
         if (offset < 0 || offset >= this.length) {
-            throw new UserError('Out of bounds.');
+            throw new OutOfBoundsError('Out of bounds.');
         }
         // Find the initial buffer
         let [startOffset, current] = this.#findInitialBuffer(offset);
@@ -242,7 +248,7 @@ export class DynamicBuffer {
     }
     readInt8(offset = 0) {
         if (offset < 0 || offset >= this.length) {
-            throw new UserError('Out of bounds.');
+            throw new OutOfBoundsError('Out of bounds.');
         }
         const [finalIndex, current] = this.#findInitialBuffer(offset);
         this.#readBuffer[0] = this.buffers[current][finalIndex];
@@ -374,7 +380,7 @@ export class DynamicBuffer {
         return this;
     }
     writeUnsignedVarInt(value, append = true) {
-        const buffer = Buffer.alloc(sizeOfUnsignedVarInt(value));
+        const buffer = Buffer.allocUnsafe(sizeOfUnsignedVarInt(value));
         let position = 0;
         while ((value & BITS_8PLUS_MASK) !== 0) {
             buffer.writeUInt8((value & LEAST_SIGNIFICANT_7_BITS) | MOST_SIGNIFICANT_BIT_FLAG, position);
@@ -390,7 +396,7 @@ export class DynamicBuffer {
         }
     }
     writeUnsignedVarInt64(value, append = true) {
-        const buffer = Buffer.alloc(sizeOfUnsignedVarInt64(value));
+        const buffer = Buffer.allocUnsafe(sizeOfUnsignedVarInt64(value));
         let position = 0;
         while ((value & BITS_8PLUS_MASK_64) !== 0n) {
             buffer.writeUInt8(Number((value & LEAST_SIGNIFICANT_7_BITS_64) | MOST_SIGNIFICANT_BIT_FLAG_64), position);
@@ -543,7 +549,7 @@ export class DynamicBuffer {
     }
     #readMultiple(index, length) {
         if (index < 0 || index + length > this.length) {
-            throw new UserError('Out of bounds.');
+            throw new OutOfBoundsError('Out of bounds.');
         }
         let [startOffset, current] = this.#findInitialBuffer(index);
         for (let i = 0; i < length; i++) {

package/dist/protocol/reader.d.ts

@@ -5,28 +5,30 @@ declare const instanceIdentifier: unique symbol;
 export declare class Reader {
     buffer: DynamicBuffer;
     position: number;
+    length: number;
     [instanceIdentifier]: boolean;
     static isReader(target: any): boolean;
     static from(buffer: Buffer | DynamicBuffer | Writer): Reader;
     constructor(buffer: DynamicBuffer);
+    get remaining(): number;
     reset(buffer?: Buffer | DynamicBuffer): void;
     inspect(): string;
     skip(length: number): this;
-    peekUnsignedInt8(): number;
-    peekUnsignedInt16(): number;
-    peekUnsignedInt32(): number;
-    peekUnsignedInt64(): bigint;
-    peekUnsignedVarInt(): number;
-    peekUnsignedVarInt64(): bigint;
-    peekInt8(): number;
-    peekInt16(): number;
-    peekInt32(): number;
-    peekInt64(): bigint;
-    peekFloat64(): number;
-    peekVarInt(): number;
-    peekVarInt64(): bigint;
-    peekBoolean(): boolean;
-    peekUUID(): string;
+    peekUnsignedInt8(position?: number): number;
+    peekUnsignedInt16(position?: number): number;
+    peekUnsignedInt32(position?: number): number;
+    peekUnsignedInt64(position?: number): bigint;
+    peekUnsignedVarInt(position?: number): number;
+    peekUnsignedVarInt64(position?: number): bigint;
+    peekInt8(position?: number): number;
+    peekInt16(position?: number): number;
+    peekInt32(position?: number): number;
+    peekInt64(position?: number): bigint;
+    peekFloat64(position?: number): number;
+    peekVarInt(position?: number): number;
+    peekVarInt64(position?: number): bigint;
+    peekBoolean(position?: number): boolean;
+    peekUUID(position?: number): string;
     readUnsignedInt8(): number;
     readUnsignedInt16(): number;
     readUnsignedInt32(): number;

package/dist/protocol/reader.js

@@ -5,6 +5,7 @@ const instanceIdentifier = Symbol('plt.kafka.reader.instanceIdentifier');
 export class Reader {
     buffer;
     position;
+    length;
     [instanceIdentifier];
     static isReader(target) {
         return target?.[instanceIdentifier] === true;
@@ -21,8 +22,12 @@ export class Reader {
     constructor(buffer) {
         this.buffer = buffer;
         this.position = 0;
+        this.length = this.buffer.length;
         this[instanceIdentifier] = true;
     }
+    get remaining() {
+        return this.length - this.position;
+    }
     reset(buffer) {
         if (buffer) {
             if (Buffer.isBuffer(buffer)) {
@@ -43,50 +48,51 @@ export class Reader {
         this.position += length;
         return this;
     }
-    peekUnsignedInt8() {
-        return this.buffer.readUInt8(this.position);
+    peekUnsignedInt8(position) {
+        return this.buffer.readUInt8(position ?? this.position);
     }
-    peekUnsignedInt16() {
-        return this.buffer.readUInt16BE(this.position);
+    peekUnsignedInt16(position) {
+        return this.buffer.readUInt16BE(position ?? this.position);
     }
-    peekUnsignedInt32() {
-        return this.buffer.readUInt32BE(this.position);
+    peekUnsignedInt32(position) {
+        return this.buffer.readUInt32BE(position ?? this.position);
     }
-    peekUnsignedInt64() {
-        return this.buffer.readBigUInt64BE(this.position);
+    peekUnsignedInt64(position) {
+        return this.buffer.readBigUInt64BE(position ?? this.position);
     }
-    peekUnsignedVarInt() {
-        return this.buffer.readUnsignedVarInt(this.position)[0];
+    peekUnsignedVarInt(position) {
+        return this.buffer.readUnsignedVarInt(position ?? this.position)[0];
     }
-    peekUnsignedVarInt64() {
-        return this.buffer.readUnsignedVarInt64(this.position)[0];
+    peekUnsignedVarInt64(position) {
+        return this.buffer.readUnsignedVarInt64(position ?? this.position)[0];
     }
-    peekInt8() {
-        return this.buffer.readInt8(this.position);
+    peekInt8(position) {
+        return this.buffer.readInt8(position ?? this.position);
     }
-    peekInt16() {
-        return this.buffer.readInt16BE(this.position);
+    peekInt16(position) {
+        return this.buffer.readInt16BE(position ?? this.position);
     }
-    peekInt32() {
-        return this.buffer.readInt32BE(this.position);
+    peekInt32(position) {
+        return this.buffer.readInt32BE(position ?? this.position);
     }
-    peekInt64() {
-        return this.buffer.readBigInt64BE(this.position);
+    peekInt64(position) {
+        return this.buffer.readBigInt64BE(position ?? this.position);
     }
-    peekFloat64() {
-        return this.buffer.readDoubleBE(this.position);
+    peekFloat64(position) {
+        return this.buffer.readDoubleBE(position ?? this.position);
     }
-    peekVarInt() {
-        return this.buffer.readVarInt(this.position)[0];
+    peekVarInt(position) {
+        return this.buffer.readVarInt(position ?? this.position)[0];
     }
-    peekVarInt64() {
-        return this.buffer.readVarInt64(this.position)[0];
+    peekVarInt64(position) {
+        return this.buffer.readVarInt64(position ?? this.position)[0];
     }
-    peekBoolean() {
-        return this.buffer.readInt8(this.position) === 1;
+    peekBoolean(position) {
+        return this.buffer.readInt8(position ?? this.position) === 1;
     }
-    peekUUID() {
-        return this.buffer.toString('hex', this.position, this.position + UUID_SIZE);
+    peekUUID(position) {
+        position ??= this.position;
+        return this.buffer.toString('hex', position, position + UUID_SIZE);
     }
     readUnsignedInt8() {
         const value = this.peekUnsignedInt8();

package/dist/protocol/records.d.ts

@@ -108,3 +108,4 @@ export declare function createRecord(message: MessageRecord, offsetDelta: number
 export declare function readRecord(reader: Reader): KafkaRecord;
 export declare function createRecordsBatch(messages: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Writer;
 export declare function readRecordsBatch(reader: Reader): RecordsBatch;
+export declare function readRecordsBatches(reader: Reader): RecordsBatch[];

package/dist/protocol/records.js

@@ -8,6 +8,7 @@ import { Writer } from "./writer.js";
 const CURRENT_RECORD_VERSION = 2;
 const IS_TRANSACTIONAL = 0b10000; // Bit 4 set
 const IS_COMPRESSED = 0b111; // Bits 0, 1 and/or 2 set
+const BATCH_HEAD = INT64_SIZE + INT32_SIZE; // FirstOffset + Length
 export const messageSchema = {
     type: 'object',
     properties: {
@@ -117,6 +118,7 @@ export function createRecordsBatch(messages, options = {}) {
         // FirstOffset is 0
         .appendInt64(0n, false));
 }
+// TODO: Early bail out if there are not enough bytes to read all the records as it might be truncated
 export function readRecordsBatch(reader) {
     const initialPosition = reader.position;
     const batch = {
@@ -155,3 +157,11 @@ export function readRecordsBatch(reader) {
     }
     return batch;
 }
+export function readRecordsBatches(reader) {
+    const batches = [];
+    while (reader.remaining >= BATCH_HEAD &&
+        reader.remaining - BATCH_HEAD >= reader.peekInt32(reader.position + INT64_SIZE)) {
+        batches.push(readRecordsBatch(reader));
+    }
+    return batches;
+}

package/dist/protocol/writer.js

@@ -1,5 +1,5 @@
 import { humanize } from "../utils.js";
-import { EMPTY_UUID } from "./definitions.js";
+import { EMPTY_TAGGED_FIELDS_BUFFER, EMPTY_UUID } from "./definitions.js";
 import { DynamicBuffer } from "./dynamic-buffer.js";
 const instanceIdentifier = Symbol('plt.kafka.writer.instanceIdentifier');
 export class Writer {
@@ -212,7 +212,7 @@ export class Writer {
     }
     // TODO(ShogunPanda): Tagged fields are not supported yet
     appendTaggedFields(_ = []) {
-        return this.appendInt8(0);
+        return this.append(EMPTY_TAGGED_FIELDS_BUFFER);
     }
     prependLength() {
         return this.appendInt32(this.length, false);

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "1.15.1";
+export const version = "1.17.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "1.15.1",
+  "version": "1.17.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
@@ -32,10 +32,12 @@
     "scule": "^1.3.0"
   },
   "optionalDependencies": {
+    "@node-rs/crc32": "^1.10.6",
     "lz4-napi": "^2.9.0",
     "snappy": "^7.3.3"
   },
   "devDependencies": {
+    "@confluentinc/kafka-javascript": "^1.5.0",
     "@platformatic/rdkafka": "^4.0.0",
     "@types/debug": "^4.1.12",
     "@types/node": "^22.18.5",
@@ -48,10 +50,9 @@
     "eslint": "^9.35.0",
     "fast-jwt": "^6.0.2",
     "hwp": "^0.4.1",
-    "json5": "^2.2.3",
     "kafkajs": "^2.2.4",
+    "json5": "^2.2.3",
     "neostandard": "^0.12.2",
-    "node-rdkafka": "^3.5.0",
     "parse5": "^7.3.0",
     "prettier": "^3.6.2",
     "prettier-plugin-space-before-function-paren": "^0.0.8",