@platformatic/kafka 1.16.0 → 1.17.0

package/dist/apis/consumer/fetch-v15.js

@@ -1,6 +1,6 @@
-import { OutOfBoundsError, ResponseError } from "../../errors.js";
+import { ResponseError } from "../../errors.js";
 import { Reader } from "../../protocol/reader.js";
-import { readRecordsBatch } from "../../protocol/records.js";
+import { readRecordsBatches } from "../../protocol/records.js";
 import { Writer } from "../../protocol/writer.js";
 import { createAPI } from "../definitions.js";
 /*
@@ -105,22 +105,7 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
                     // We need to reduce the size by one to follow the COMPACT_RECORDS specification.
                     const recordsSize = r.readUnsignedVarInt() - 1;
                     if (recordsSize > 0) {
-                        const recordsBatchesReader = Reader.from(r.buffer.subarray(r.position, r.position + recordsSize));
-                        partition.records = [];
-                        do {
-                            try {
-                                partition.records.push(readRecordsBatch(recordsBatchesReader));
-                            }
-                            catch (err) {
-                                // Contrary to other places in the protocol, records batches CAN BE truncated due to maxBytes argument.
-                                // In that case we just ignore the error.
-                                if (err.code === OutOfBoundsError.code) {
-                                    break;
-                                }
-                                /* c8 ignore next 3 - Hard to test */
-                                throw err;
-                            }
-                        } while (recordsBatchesReader.position < recordsSize);
+                        partition.records = readRecordsBatches(Reader.from(r.buffer.subarray(r.position, r.position + recordsSize)));
                         r.skip(recordsSize);
                     }
                     return partition;

package/dist/apis/consumer/fetch-v16.js

@@ -1,6 +1,6 @@
-import { OutOfBoundsError, ResponseError } from "../../errors.js";
+import { ResponseError } from "../../errors.js";
 import { Reader } from "../../protocol/reader.js";
-import { readRecordsBatch } from "../../protocol/records.js";
+import { readRecordsBatches } from "../../protocol/records.js";
 import { Writer } from "../../protocol/writer.js";
 import { createAPI } from "../definitions.js";
 /*
@@ -105,22 +105,7 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
                     // We need to reduce the size by one to follow the COMPACT_RECORDS specification.
                     const recordsSize = r.readUnsignedVarInt() - 1;
                     if (recordsSize > 0) {
-                        const recordsBatchesReader = Reader.from(r.buffer.subarray(r.position, r.position + recordsSize));
-                        partition.records = [];
-                        do {
-                            try {
-                                partition.records.push(readRecordsBatch(recordsBatchesReader));
-                            }
-                            catch (err) {
-                                // Contrary to other places in the protocol, records batches CAN BE truncated due to maxBytes argument.
-                                // In that case we just ignore the error.
-                                if (err.code === OutOfBoundsError.code) {
-                                    break;
-                                }
-                                /* c8 ignore next 3 - Hard to test */
-                                throw err;
-                            }
-                        } while (recordsBatchesReader.position < recordsSize);
+                        partition.records = readRecordsBatches(Reader.from(r.buffer.subarray(r.position, r.position + recordsSize)));
                         r.skip(recordsSize);
                     }
                     return partition;

package/dist/apis/consumer/fetch-v17.js

@@ -1,6 +1,6 @@
-import { OutOfBoundsError, ResponseError } from "../../errors.js";
+import { ResponseError } from "../../errors.js";
 import { Reader } from "../../protocol/reader.js";
-import { readRecordsBatch } from "../../protocol/records.js";
+import { readRecordsBatches } from "../../protocol/records.js";
 import { Writer } from "../../protocol/writer.js";
 import { createAPI } from "../definitions.js";
 /*
@@ -105,22 +105,7 @@ export function parseResponse(_correlationId, apiKey, apiVersion, reader) {
                     // We need to reduce the size by one to follow the COMPACT_RECORDS specification.
                     const recordsSize = r.readUnsignedVarInt() - 1;
                     if (recordsSize > 0) {
-                        const recordsBatchesReader = Reader.from(r.buffer.subarray(r.position, r.position + recordsSize));
-                        partition.records = [];
-                        do {
-                            try {
-                                partition.records.push(readRecordsBatch(recordsBatchesReader));
-                            }
-                            catch (err) {
-                                // Contrary to other places in the protocol, records batches CAN BE truncated due to maxBytes argument.
-                                // In that case we just ignore the error.
-                                if (err.code === OutOfBoundsError.code) {
-                                    break;
-                                }
-                                /* c8 ignore next 3 - Hard to test */
-                                throw err;
-                            }
-                        } while (recordsBatchesReader.position < recordsSize);
+                        partition.records = readRecordsBatches(Reader.from(r.buffer.subarray(r.position, r.position + recordsSize)));
                         r.skip(recordsSize);
                     }
                     return partition;

package/dist/clients/base/options.d.ts

@@ -74,6 +74,10 @@ export declare const baseOptionsSchema: {
             type: string;
             minimum: number;
         };
+        handleBackPressure: {
+            type: string;
+            default: boolean;
+        };
         tls: {
             type: string;
             additionalProperties: boolean;

package/dist/clients/base/options.js

@@ -31,6 +31,7 @@ export const baseOptionsSchema = {
         retries: { oneOf: [{ type: 'number', minimum: 0 }, { type: 'boolean' }] },
         retryDelay: { type: 'number', minimum: 0 },
         maxInflights: { type: 'number', minimum: 0 },
+        handleBackPressure: { type: 'boolean', default: false },
         tls: { type: 'object', additionalProperties: true }, // No validation as they come from Node.js
         tlsServerName: { oneOf: [{ type: 'boolean' }, { type: 'string' }] },
         sasl: {

package/dist/network/connection.d.ts

@@ -24,14 +24,15 @@ export interface ConnectionOptions {
     tlsServerName?: string | boolean;
     sasl?: SASLOptions;
     ownerId?: number;
+    handleBackPressure?: boolean;
 }
 export interface Request {
     correlationId: number;
     apiKey: number;
     apiVersion: number;
-    hasRequestHeaderTaggedFields: boolean;
     hasResponseHeaderTaggedFields: boolean;
-    payload: () => Writer;
+    noResponse: boolean;
+    payload: Buffer;
     parser: ResponseParser<unknown>;
     callback: Callback<any>;
     diagnostic: Record<string, unknown>;
@@ -62,5 +63,5 @@ export declare class Connection extends EventEmitter {
     ready(): Promise<void>;
     close(callback: CallbackWithPromise<void>): void;
     close(): Promise<void>;
-    send<ReturnType>(apiKey: number, apiVersion: number, payload: () => Writer, responseParser: ResponseParser<ReturnType>, hasRequestHeaderTaggedFields: boolean, hasResponseHeaderTaggedFields: boolean, callback: Callback<ReturnType>): void;
+    send<ReturnType>(apiKey: number, apiVersion: number, createPayload: () => Writer, responseParser: ResponseParser<ReturnType>, hasRequestHeaderTaggedFields: boolean, hasResponseHeaderTaggedFields: boolean, callback: Callback<ReturnType>): void;
 }

package/dist/network/connection.js

@@ -38,6 +38,7 @@ export class Connection extends EventEmitter {
     #clientId;
     // @ts-ignore This is used just for debugging
     #ownerId;
+    #handleBackPressure;
     #correlationId;
     #nextMessage;
     #afterDrainRequests;
@@ -56,6 +57,7 @@ export class Connection extends EventEmitter {
         this.#status = ConnectionStatuses.NONE;
         this.#clientId = clientId;
         this.#ownerId = options.ownerId;
+        this.#handleBackPressure = options.handleBackPressure ?? false;
         this.#correlationId = 0;
         this.#nextMessage = 0;
         this.#afterDrainRequests = [];
@@ -134,7 +136,9 @@ export class Connection extends EventEmitter {
                 this.#socket.removeListener('error', connectionErrorHandler);
                 this.#socket.on('error', this.#onError.bind(this));
                 this.#socket.on('data', this.#onData.bind(this));
-                this.#socket.on('drain', this.#onDrain.bind(this));
+                if (this.#handleBackPressure) {
+                    this.#socket.on('drain', this.#onDrain.bind(this));
+                }
                 this.#socket.on('close', this.#onClose.bind(this));
                 this.#socket.setTimeout(0);
                 if (this.#options.sasl) {
@@ -205,26 +209,43 @@ export class Connection extends EventEmitter {
         this.#socket.end();
         return callback[kCallbackPromise];
     }
-    send(apiKey, apiVersion, payload, responseParser, hasRequestHeaderTaggedFields, hasResponseHeaderTaggedFields, callback) {
+    send(apiKey, apiVersion, createPayload, responseParser, hasRequestHeaderTaggedFields, hasResponseHeaderTaggedFields, callback) {
+        const correlationId = ++this.#correlationId;
+        const diagnostic = createDiagnosticContext({
+            connection: this,
+            operation: 'send',
+            apiKey,
+            apiVersion,
+            correlationId
+        });
+        const writer = Writer.create();
+        writer.appendInt16(apiKey).appendInt16(apiVersion).appendInt32(correlationId).appendString(this.#clientId, false);
+        if (hasRequestHeaderTaggedFields) {
+            writer.appendTaggedFields();
+        }
+        let payload;
+        try {
+            payload = createPayload();
+        }
+        catch (err) {
+            diagnostic.error = err;
+            connectionsApiChannel.error.publish(diagnostic);
+            throw err;
+        }
+        writer.appendFrom(payload).prependLength();
+        const request = {
+            correlationId,
+            apiKey,
+            apiVersion,
+            parser: responseParser,
+            payload: writer.buffer,
+            callback: null, // Will be set later
+            hasResponseHeaderTaggedFields,
+            noResponse: payload.context.noResponse ?? false,
+            diagnostic
+        };
         this.#requestsQueue.push(fastQueueCallback => {
-            const correlationId = ++this.#correlationId;
-            const request = {
-                correlationId,
-                apiKey,
-                apiVersion,
-                hasRequestHeaderTaggedFields,
-                hasResponseHeaderTaggedFields,
-                parser: responseParser,
-                payload,
-                callback: fastQueueCallback,
-                diagnostic: createDiagnosticContext({
-                    connection: this,
-                    operation: 'send',
-                    apiKey,
-                    apiVersion,
-                    correlationId
-                })
-            };
+            request.callback = fastQueueCallback;
             if (this.#socketMustBeDrained) {
                 this.#afterDrainRequests.push(request);
                 return false;
@@ -274,37 +295,31 @@ export class Connection extends EventEmitter {
                 request.callback(new NetworkError('Connection closed'), undefined);
                 return false;
             }
-            const writer = Writer.create();
-            writer
-                .appendInt16(request.apiKey)
-                .appendInt16(request.apiVersion)
-                .appendInt32(request.correlationId)
-                .appendString(this.#clientId, false);
-            if (request.hasRequestHeaderTaggedFields) {
-                writer.appendTaggedFields();
-            }
-            const payload = request.payload();
-            writer.appendFrom(payload).prependLength();
-            const expectResponse = !payload.context.noResponse;
-            if (expectResponse)
+            if (!request.noResponse) {
                 this.#inflightRequests.set(request.correlationId, request);
-            const canWrite = this.#socket.write(writer.buffer);
-            if (!canWrite)
+            }
+            let canWrite = this.#socket.write(request.payload);
+            if (!this.#handleBackPressure) {
+                canWrite = true;
+            }
+            if (!canWrite) {
                 this.#socketMustBeDrained = true;
-            if (!expectResponse)
+            }
+            if (request.noResponse) {
                 request.callback(null, canWrite);
+            }
             loggers.protocol('Sending request.', {
                 apiKey: protocolAPIsById[request.apiKey],
                 correlationId: request.correlationId,
                 request
             });
             return canWrite;
+            /* c8 ignore next 8 - Hard to test */
         }
         catch (err) {
             request.diagnostic.error = err;
             connectionsApiChannel.error.publish(request.diagnostic);
             throw err;
-            /* c8 ignore next 3 - Hard to test */
         }
         finally {
             connectionsApiChannel.end.publish(request.diagnostic);
@@ -453,8 +468,7 @@ export class Connection extends EventEmitter {
         this.emit('close');
         const error = new NetworkError('Connection closed');
         for (const request of this.#afterDrainRequests) {
-            const payload = request.payload();
-            if (!payload.context.noResponse) {
+            if (!request.noResponse) {
                 request.callback(error, undefined);
             }
         }

package/dist/protocol/crc32c.js

@@ -78,6 +78,7 @@ export function loadNativeCRC32C() {
                 : new Uint8Array(data);
             return nativeImplementation(bytes);
         };
+        /* c8 ignore next 3 - Hard to test */
     }
     catch (error) {
         return null;
@@ -93,4 +94,5 @@ export function jsCRC32C(data) {
     }
     return (crc ^ 0xffffffff) >>> 0;
 }
+/* c8 ignore next - Hard to test */
 export const crc32c = loadNativeCRC32C() ?? jsCRC32C;

package/dist/protocol/dynamic-buffer.js

@@ -48,12 +48,18 @@ export class DynamicBuffer {
         return this;
     }
     appendFrom(DynamicBuffer) {
-        this.buffers.push(...DynamicBuffer.buffers);
+        const buffers = DynamicBuffer.buffers;
+        for (let i = 0; i < buffers.length; i++) {
+            this.buffers.push(buffers[i]);
+        }
         this.length += DynamicBuffer.length;
         return this;
     }
     prependFrom(DynamicBuffer) {
-        this.buffers.unshift(...DynamicBuffer.buffers);
+        const buffers = DynamicBuffer.buffers;
+        for (let i = buffers.length - 1; i >= 0; i--) {
+            this.buffers.unshift(buffers[i]);
+        }
         this.length += DynamicBuffer.length;
         return this;
     }
@@ -374,7 +380,7 @@ export class DynamicBuffer {
         return this;
     }
     writeUnsignedVarInt(value, append = true) {
-        const buffer = Buffer.alloc(sizeOfUnsignedVarInt(value));
+        const buffer = Buffer.allocUnsafe(sizeOfUnsignedVarInt(value));
         let position = 0;
         while ((value & BITS_8PLUS_MASK) !== 0) {
             buffer.writeUInt8((value & LEAST_SIGNIFICANT_7_BITS) | MOST_SIGNIFICANT_BIT_FLAG, position);
@@ -390,7 +396,7 @@ export class DynamicBuffer {
         }
     }
     writeUnsignedVarInt64(value, append = true) {
-        const buffer = Buffer.alloc(sizeOfUnsignedVarInt64(value));
+        const buffer = Buffer.allocUnsafe(sizeOfUnsignedVarInt64(value));
         let position = 0;
         while ((value & BITS_8PLUS_MASK_64) !== 0n) {
             buffer.writeUInt8(Number((value & LEAST_SIGNIFICANT_7_BITS_64) | MOST_SIGNIFICANT_BIT_FLAG_64), position);

package/dist/protocol/reader.d.ts

@@ -5,28 +5,30 @@ declare const instanceIdentifier: unique symbol;
 export declare class Reader {
     buffer: DynamicBuffer;
     position: number;
+    length: number;
     [instanceIdentifier]: boolean;
     static isReader(target: any): boolean;
     static from(buffer: Buffer | DynamicBuffer | Writer): Reader;
     constructor(buffer: DynamicBuffer);
+    get remaining(): number;
     reset(buffer?: Buffer | DynamicBuffer): void;
     inspect(): string;
     skip(length: number): this;
-    peekUnsignedInt8(): number;
-    peekUnsignedInt16(): number;
-    peekUnsignedInt32(): number;
-    peekUnsignedInt64(): bigint;
-    peekUnsignedVarInt(): number;
-    peekUnsignedVarInt64(): bigint;
-    peekInt8(): number;
-    peekInt16(): number;
-    peekInt32(): number;
-    peekInt64(): bigint;
-    peekFloat64(): number;
-    peekVarInt(): number;
-    peekVarInt64(): bigint;
-    peekBoolean(): boolean;
-    peekUUID(): string;
+    peekUnsignedInt8(position?: number): number;
+    peekUnsignedInt16(position?: number): number;
+    peekUnsignedInt32(position?: number): number;
+    peekUnsignedInt64(position?: number): bigint;
+    peekUnsignedVarInt(position?: number): number;
+    peekUnsignedVarInt64(position?: number): bigint;
+    peekInt8(position?: number): number;
+    peekInt16(position?: number): number;
+    peekInt32(position?: number): number;
+    peekInt64(position?: number): bigint;
+    peekFloat64(position?: number): number;
+    peekVarInt(position?: number): number;
+    peekVarInt64(position?: number): bigint;
+    peekBoolean(position?: number): boolean;
+    peekUUID(position?: number): string;
     readUnsignedInt8(): number;
     readUnsignedInt16(): number;
     readUnsignedInt32(): number;

package/dist/protocol/reader.js

@@ -5,6 +5,7 @@ const instanceIdentifier = Symbol('plt.kafka.reader.instanceIdentifier');
 export class Reader {
     buffer;
     position;
+    length;
     [instanceIdentifier];
     static isReader(target) {
         return target?.[instanceIdentifier] === true;
@@ -21,8 +22,12 @@ export class Reader {
     constructor(buffer) {
         this.buffer = buffer;
         this.position = 0;
+        this.length = this.buffer.length;
         this[instanceIdentifier] = true;
     }
+    get remaining() {
+        return this.length - this.position;
+    }
     reset(buffer) {
         if (buffer) {
             if (Buffer.isBuffer(buffer)) {
@@ -43,50 +48,51 @@ export class Reader {
         this.position += length;
         return this;
     }
-    peekUnsignedInt8() {
-        return this.buffer.readUInt8(this.position);
+    peekUnsignedInt8(position) {
+        return this.buffer.readUInt8(position ?? this.position);
     }
-    peekUnsignedInt16() {
-        return this.buffer.readUInt16BE(this.position);
+    peekUnsignedInt16(position) {
+        return this.buffer.readUInt16BE(position ?? this.position);
     }
-    peekUnsignedInt32() {
-        return this.buffer.readUInt32BE(this.position);
+    peekUnsignedInt32(position) {
+        return this.buffer.readUInt32BE(position ?? this.position);
     }
-    peekUnsignedInt64() {
-        return this.buffer.readBigUInt64BE(this.position);
+    peekUnsignedInt64(position) {
+        return this.buffer.readBigUInt64BE(position ?? this.position);
     }
-    peekUnsignedVarInt() {
-        return this.buffer.readUnsignedVarInt(this.position)[0];
+    peekUnsignedVarInt(position) {
+        return this.buffer.readUnsignedVarInt(position ?? this.position)[0];
     }
-    peekUnsignedVarInt64() {
-        return this.buffer.readUnsignedVarInt64(this.position)[0];
+    peekUnsignedVarInt64(position) {
+        return this.buffer.readUnsignedVarInt64(position ?? this.position)[0];
     }
-    peekInt8() {
-        return this.buffer.readInt8(this.position);
+    peekInt8(position) {
+        return this.buffer.readInt8(position ?? this.position);
     }
-    peekInt16() {
-        return this.buffer.readInt16BE(this.position);
+    peekInt16(position) {
+        return this.buffer.readInt16BE(position ?? this.position);
     }
-    peekInt32() {
-        return this.buffer.readInt32BE(this.position);
+    peekInt32(position) {
+        return this.buffer.readInt32BE(position ?? this.position);
     }
-    peekInt64() {
-        return this.buffer.readBigInt64BE(this.position);
+    peekInt64(position) {
+        return this.buffer.readBigInt64BE(position ?? this.position);
     }
-    peekFloat64() {
-        return this.buffer.readDoubleBE(this.position);
+    peekFloat64(position) {
+        return this.buffer.readDoubleBE(position ?? this.position);
     }
-    peekVarInt() {
-        return this.buffer.readVarInt(this.position)[0];
+    peekVarInt(position) {
+        return this.buffer.readVarInt(position ?? this.position)[0];
     }
-    peekVarInt64() {
-        return this.buffer.readVarInt64(this.position)[0];
+    peekVarInt64(position) {
+        return this.buffer.readVarInt64(position ?? this.position)[0];
     }
-    peekBoolean() {
-        return this.buffer.readInt8(this.position) === 1;
+    peekBoolean(position) {
+        return this.buffer.readInt8(position ?? this.position) === 1;
     }
-    peekUUID() {
-        return this.buffer.toString('hex', this.position, this.position + UUID_SIZE);
+    peekUUID(position) {
+        position ??= this.position;
+        return this.buffer.toString('hex', position, position + UUID_SIZE);
     }
     readUnsignedInt8() {
         const value = this.peekUnsignedInt8();

package/dist/protocol/records.d.ts

@@ -108,3 +108,4 @@ export declare function createRecord(message: MessageRecord, offsetDelta: number
 export declare function readRecord(reader: Reader): KafkaRecord;
 export declare function createRecordsBatch(messages: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Writer;
 export declare function readRecordsBatch(reader: Reader): RecordsBatch;
+export declare function readRecordsBatches(reader: Reader): RecordsBatch[];

package/dist/protocol/records.js

@@ -8,6 +8,7 @@ import { Writer } from "./writer.js";
 const CURRENT_RECORD_VERSION = 2;
 const IS_TRANSACTIONAL = 0b10000; // Bit 4 set
 const IS_COMPRESSED = 0b111; // Bits 0, 1 and/or 2 set
+const BATCH_HEAD = INT64_SIZE + INT32_SIZE; // FirstOffset + Length
 export const messageSchema = {
     type: 'object',
     properties: {
@@ -117,6 +118,7 @@ export function createRecordsBatch(messages, options = {}) {
         // FirstOffset is 0
         .appendInt64(0n, false));
 }
+// TODO: Early bail out if there are not enough bytes to read all the records as it might be truncated
 export function readRecordsBatch(reader) {
     const initialPosition = reader.position;
     const batch = {
@@ -155,3 +157,11 @@ export function readRecordsBatch(reader) {
     }
     return batch;
 }
+export function readRecordsBatches(reader) {
+    const batches = [];
+    while (reader.remaining >= BATCH_HEAD &&
+        reader.remaining - BATCH_HEAD >= reader.peekInt32(reader.position + INT64_SIZE)) {
+        batches.push(readRecordsBatch(reader));
+    }
+    return batches;
+}

package/dist/protocol/writer.js

@@ -1,5 +1,5 @@
 import { humanize } from "../utils.js";
-import { EMPTY_UUID } from "./definitions.js";
+import { EMPTY_TAGGED_FIELDS_BUFFER, EMPTY_UUID } from "./definitions.js";
 import { DynamicBuffer } from "./dynamic-buffer.js";
 const instanceIdentifier = Symbol('plt.kafka.writer.instanceIdentifier');
 export class Writer {
@@ -212,7 +212,7 @@ export class Writer {
     }
     // TODO(ShogunPanda): Tagged fields are not supported yet
     appendTaggedFields(_ = []) {
-        return this.appendInt8(0);
+        return this.append(EMPTY_TAGGED_FIELDS_BUFFER);
     }
     prependLength() {
         return this.appendInt32(this.length, false);

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "1.16.0";
+export const version = "1.17.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "1.16.0",
+  "version": "1.17.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
@@ -37,6 +37,7 @@
     "snappy": "^7.3.3"
   },
   "devDependencies": {
+    "@confluentinc/kafka-javascript": "^1.5.0",
     "@platformatic/rdkafka": "^4.0.0",
     "@types/debug": "^4.1.12",
     "@types/node": "^22.18.5",
@@ -49,10 +50,9 @@
     "eslint": "^9.35.0",
     "fast-jwt": "^6.0.2",
     "hwp": "^0.4.1",
-    "json5": "^2.2.3",
     "kafkajs": "^2.2.4",
+    "json5": "^2.2.3",
     "neostandard": "^0.12.2",
-    "node-rdkafka": "^3.5.0",
     "parse5": "^7.3.0",
     "prettier": "^3.6.2",
     "prettier-plugin-space-before-function-paren": "^0.0.8",