kafka-ts 0.0.6-beta.2 → 0.0.6-beta.4

package/dist/api/fetch.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 export declare const enum IsolationLevel {
     READ_UNCOMMITTED = 0,
     READ_COMMITTED = 1
@@ -35,11 +34,11 @@ export declare const FETCH: import("../utils/api").Api<{
                     attributes: number;
                     timestampDelta: bigint;
                     offsetDelta: number;
-                    key: Buffer | null;
-                    value: Buffer | null;
+                    key: string | null;
+                    value: string | null;
                     headers: {
-                        key: Buffer | null;
-                        value: Buffer | null;
+                        key: string;
+                        value: string;
                     }[];
                 }[];
                 compression: number;

package/dist/api/fetch.js

@@ -133,10 +133,10 @@ const decodeRecord = (decoder) => decoder.readRecords((record) => ({
     attributes: record.readInt8(),
     timestampDelta: record.readVarLong(),
     offsetDelta: record.readVarInt(),
-    key: record.readVarIntBuffer(),
-    value: record.readVarIntBuffer(),
+    key: record.readVarIntString(),
+    value: record.readVarIntString(),
     headers: record.readVarIntArray((header) => ({
-        key: header.readVarIntBuffer(),
-        value: header.readVarIntBuffer(),
+        key: header.readVarIntString(),
+        value: header.readVarIntString(),
     })),
 }));

package/dist/api/index.d.ts

@@ -97,11 +97,11 @@ export declare const API: {
                         attributes: number;
                         timestampDelta: bigint;
                         offsetDelta: number;
-                        key: Buffer | null;
-                        value: Buffer | null;
+                        key: string | null;
+                        value: string | null;
                         headers: {
-                            key: Buffer | null;
-                            value: Buffer | null;
+                            key: string;
+                            value: string;
                         }[];
                     }[];
                     compression: number;
@@ -384,11 +384,11 @@ export declare const API: {
                     attributes: number;
                     timestampDelta: bigint;
                     offsetDelta: number;
-                    key: Buffer | null;
-                    value: Buffer | null;
+                    key: string | null;
+                    value: string | null;
                     headers: {
-                        key: Buffer;
-                        value: Buffer;
+                        key: string;
+                        value: string;
                     }[];
                 }[];
             }[];

package/dist/api/produce.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 export declare const PRODUCE: import("../utils/api.js").Api<{
     transactionalId: string | null;
     acks: number;
@@ -20,11 +19,11 @@ export declare const PRODUCE: import("../utils/api.js").Api<{
                 attributes: number;
                 timestampDelta: bigint;
                 offsetDelta: number;
-                key: Buffer | null;
-                value: Buffer | null;
+                key: string | null;
+                value: string | null;
                 headers: {
-                    key: Buffer;
-                    value: Buffer;
+                    key: string;
+                    value: string;
                 }[];
             }[];
         }[];

package/dist/api/produce.js

@@ -28,10 +28,10 @@ exports.PRODUCE = (0, api_js_1.createApi)({
                 .writeInt8(record.attributes)
                 .writeVarLong(record.timestampDelta)
                 .writeVarInt(record.offsetDelta)
-                .writeVarIntBuffer(record.key)
-                .writeVarIntBuffer(record.value)
-                .writeVarIntArray(record.headers, (encoder, header) => encoder.writeVarIntBuffer(header.key).writeVarIntBuffer(header.value));
-            return encoder.writeVarInt(recordBody.getByteLength()).writeEncoder(recordBody);
+                .writeVarIntString(record.key)
+                .writeVarIntString(record.value)
+                .writeVarIntArray(record.headers, (encoder, header) => encoder.writeVarIntString(header.key).writeVarIntString(header.value));
+            return encoder.writeVarInt(recordBody.getBufferLength()).writeEncoder(recordBody);
         })
             .value();
         const batchHeader = new encoder_js_1.Encoder()
@@ -41,11 +41,11 @@ exports.PRODUCE = (0, api_js_1.createApi)({
             .write(batchBody);
         const batch = new encoder_js_1.Encoder()
             .writeInt64(partition.baseOffset)
-            .writeInt32(batchHeader.getByteLength())
+            .writeInt32(batchHeader.getBufferLength())
             .writeEncoder(batchHeader);
         return encoder
             .writeInt32(partition.index)
-            .writeUVarInt(batch.getByteLength() + 1) // batch size
+            .writeUVarInt(batch.getBufferLength() + 1)
             .writeEncoder(batch)
             .writeUVarInt(0);
     })

package/dist/cluster.test.js

@@ -136,12 +136,12 @@ vitest_1.describe.sequential('Low-level API', () => {
                                     attributes: 0,
                                     offsetDelta: 0,
                                     timestampDelta: 0n,
-                                    key: Buffer.from('key'),
-                                    value: Buffer.from('value'),
+                                    key: 'key',
+                                    value: 'value',
                                     headers: [
                                         {
-                                            key: Buffer.from('header-key'),
-                                            value: Buffer.from('header-value'),
+                                            key: 'header-key',
+                                            value: 'header-value',
                                         },
                                     ],
                                 },

package/dist/connection.js

@@ -96,7 +96,7 @@ class Connection {
             .writeInt32(correlationId)
             .writeString(this.options.clientId);
         const request = api.request(encoder, body);
-        const requestEncoder = new encoder_1.Encoder().writeInt32(request.getByteLength()).writeEncoder(request);
+        const requestEncoder = new encoder_1.Encoder().writeInt32(request.getBufferLength()).writeEncoder(request);
         let timeout;
         const { responseDecoder, responseSize } = await new Promise(async (resolve, reject) => {
             timeout = setTimeout(() => {
@@ -119,7 +119,7 @@ class Connection {
         }
         catch (error) {
             if (error instanceof error_1.KafkaTSApiError) {
-                error.request = JSON.stringify(body, logger_1.jsonSerializer);
+                error.request = body;
             }
             throw error;
         }

package/dist/distributors/partitioner.js

@@ -15,7 +15,7 @@ const defaultPartitioner = ({ metadata }) => {
         const partitions = metadata.getTopicPartitions()[topic];
         const numPartitions = partitions.length;
         if (key) {
-            return (0, murmur2_1.toPositive)((0, murmur2_1.murmur2)(key)) % numPartitions;
+            return (0, murmur2_1.toPositive)((0, murmur2_1.murmur2)(Buffer.from(key))) % numPartitions;
         }
         return (0, murmur2_1.toPositive)(getNextValue(topic)) % numPartitions;
     };

package/dist/producer/producer.d.ts

@@ -20,6 +20,6 @@ export declare class Producer {
     close(): Promise<void>;
     private ensureConnected;
     private initProducerId;
-    private nextBaseSequence;
-    private revertBaseSequence;
+    private getSequence;
+    private updateSequence;
 }

package/dist/producer/producer.js

@@ -58,7 +58,6 @@ class Producer {
                             maxTimestamp = timestamp;
                         }
                     });
-                    const baseSequence = this.nextBaseSequence(topic, partitionIndex, messages.length);
                     return {
                         index: partitionIndex,
                         baseOffset: 0n,
@@ -69,37 +68,29 @@ class Producer {
                         maxTimestamp: maxTimestamp ?? 0n,
                         producerId: this.producerId,
                         producerEpoch: 0,
-                        baseSequence,
+                        baseSequence: this.getSequence(topic, partitionIndex),
                         records: messages.map((message, index) => ({
                             attributes: 0,
                             timestampDelta: (message.timestamp ?? defaultTimestamp) - (baseTimestamp ?? 0n),
                             offsetDelta: index,
                             key: message.key ?? null,
                             value: message.value,
-                            headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
-                                key: Buffer.from(key),
-                                value: Buffer.from(value),
-                            })),
+                            headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({ key, value })),
                         })),
                     };
                 }),
             }));
-            try {
-                return await this.cluster.sendRequestToNode(parseInt(nodeId))(api_1.API.PRODUCE, {
-                    transactionalId: null,
-                    acks,
-                    timeoutMs: 5000,
-                    topicData,
-                });
-            }
-            catch (error) {
-                topicData.forEach(({ name, partitionData }) => {
-                    partitionData.forEach(({ index, records }) => {
-                        this.revertBaseSequence(name, index, records.length);
-                    });
+            await this.cluster.sendRequestToNode(parseInt(nodeId))(api_1.API.PRODUCE, {
+                transactionalId: null,
+                acks,
+                timeoutMs: 5000,
+                topicData,
+            });
+            topicData.forEach(({ name, partitionData }) => {
+                partitionData.forEach(({ index, records }) => {
+                    this.updateSequence(name, index, records.length);
                 });
-                throw error;
-            }
+            });
         }));
     }
     async close() {
@@ -129,15 +120,13 @@ class Producer {
             throw error;
         }
     }
-    nextBaseSequence(topic, partition, messagesCount) {
+    getSequence(topic, partition) {
+        return this.sequences[topic]?.[partition] ?? 0;
+    }
+    updateSequence(topic, partition, messagesCount) {
         this.sequences[topic] ??= {};
         this.sequences[topic][partition] ??= 0;
-        const baseSequence = this.sequences[topic][partition];
         this.sequences[topic][partition] += messagesCount;
-        return baseSequence;
-    }
-    revertBaseSequence(topic, partition, messagesCount) {
-        this.sequences[topic][partition] -= messagesCount;
     }
 }
 exports.Producer = Producer;

package/dist/types.d.ts

@@ -1,11 +1,10 @@
-/// <reference types="node" />
 export type Message = {
     topic: string;
     partition?: number;
     offset?: bigint;
     timestamp?: bigint;
-    key?: Buffer | null;
-    value: Buffer | null;
+    key?: string | null;
+    value: string | null;
     headers?: Record<string, string>;
 };
 export type Batch = Required<Message>[];

package/dist/utils/decoder.d.ts

@@ -16,7 +16,7 @@ export declare class Decoder {
     readVarLong(): bigint;
     readString(): string | null;
     readCompactString(): string | null;
-    readVarIntBuffer(): Buffer | null;
+    readVarIntString(): string | null;
     readUUID(): string;
     readBoolean(): boolean;
     readArray<T>(callback: (opts: Decoder) => T): T[];

package/dist/utils/decoder.js

@@ -86,12 +86,12 @@ class Decoder {
         this.offset += length;
         return value;
     }
-    readVarIntBuffer() {
+    readVarIntString() {
         const length = this.readVarInt();
         if (length < 0) {
             return null;
         }
-        const value = this.buffer.subarray(this.offset, this.offset + length);
+        const value = this.buffer.toString('utf-8', this.offset, this.offset + length);
         this.offset += length;
         return value;
     }
@@ -131,7 +131,7 @@ class Decoder {
     }
     read(length) {
         const value = this.buffer.subarray(this.offset, length !== undefined ? this.offset + length : undefined);
-        this.offset += Buffer.byteLength(value);
+        this.offset += value.length;
         return value;
     }
     readBytes() {

package/dist/utils/encoder.d.ts

@@ -2,7 +2,7 @@
 export declare class Encoder {
     private chunks;
     getChunks(): Buffer[];
-    getByteLength(): number;
+    getBufferLength(): number;
     write(...buffers: Buffer[]): this;
     writeEncoder(encoder: Encoder): this;
     writeInt8(value: number): this;
@@ -16,7 +16,7 @@ export declare class Encoder {
     writeVarLong(value: bigint): this;
     writeString(value: string | null): this;
     writeCompactString(value: string | null): this;
-    writeVarIntBuffer(buffer: Buffer | null): this;
+    writeVarIntString(value: string | null): this;
     writeUUID(value: string | null): this;
     writeBoolean(value: boolean): this;
     writeArray<T>(arr: T[], callback: (encoder: Encoder, item: T) => Encoder): this;

package/dist/utils/encoder.js

@@ -6,8 +6,8 @@ class Encoder {
     getChunks() {
         return this.chunks;
     }
-    getByteLength() {
-        return this.chunks.reduce((acc, chunk) => acc + chunk.byteLength, 0);
+    getBufferLength() {
+        return this.chunks.reduce((acc, chunk) => acc + chunk.length, 0);
     }
     write(...buffers) {
         this.chunks.push(...buffers);
@@ -71,25 +71,22 @@ class Encoder {
         if (value === null) {
             return this.writeInt16(-1);
         }
-        const byteLength = Buffer.byteLength(value, 'utf-8');
-        const buffer = Buffer.allocUnsafe(byteLength);
-        buffer.write(value, 0, byteLength, 'utf-8');
-        return this.writeInt16(byteLength).write(buffer);
+        const buffer = Buffer.from(value, 'utf-8');
+        return this.writeInt16(buffer.length).write(buffer);
     }
     writeCompactString(value) {
         if (value === null) {
             return this.writeUVarInt(0);
         }
-        const byteLength = Buffer.byteLength(value, 'utf-8');
-        const buffer = Buffer.allocUnsafe(byteLength);
-        buffer.write(value, 0, byteLength, 'utf-8');
-        return this.writeUVarInt(byteLength + 1).write(buffer);
+        const buffer = Buffer.from(value, 'utf-8');
+        return this.writeUVarInt(buffer.length + 1).write(buffer);
     }
-    writeVarIntBuffer(buffer) {
-        if (buffer === null) {
+    writeVarIntString(value) {
+        if (value === null) {
             return this.writeVarInt(-1);
         }
-        return this.writeVarInt(buffer.byteLength).write(buffer);
+        const buffer = Buffer.from(value, 'utf-8');
+        return this.writeVarInt(buffer.length).write(buffer);
     }
     writeUUID(value) {
         if (value === null) {

package/dist/utils/error.d.ts

@@ -5,7 +5,7 @@ export declare class KafkaTSApiError<T = any> extends KafkaTSError {
     errorCode: number;
     errorMessage: string | null;
     response: T;
-    request: string | undefined;
+    request: unknown | undefined;
     constructor(errorCode: number, errorMessage: string | null, response: T);
 }
 export declare class ConnectionError extends KafkaTSError {

package/dist/utils/retrier.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.defaultRetrier = exports.createExponentialBackoffRetrier = void 0;
 const delay_1 = require("./delay");
+const logger_1 = require("./logger");
 const createExponentialBackoffRetrier = ({ retries = 5, initialDelayMs = 100, maxDelayMs = 3000, multiplier = 2, onFailure = (error) => {
     throw error;
 }, } = {}) => async (func) => {
@@ -16,6 +17,7 @@ const createExponentialBackoffRetrier = ({ retries = 5, initialDelayMs = 100, ma
         catch (error) {
             lastError = error;
         }
+        logger_1.log.debug(`Failed to process batch (retriesLeft: ${retriesLeft})`);
         if (--retriesLeft < 1)
             break;
         await (0, delay_1.delay)(delayMs);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kafka-ts",
-    "version": "0.0.6-beta.2",
+    "version": "0.0.6-beta.4",
     "main": "dist/index.js",
     "author": "Priit Käärd",
     "license": "MIT",