kafka-ts 0.0.6-beta.3 → 0.0.6-beta.5

package/dist/api/fetch.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 export declare const enum IsolationLevel {
     READ_UNCOMMITTED = 0,
     READ_COMMITTED = 1
@@ -35,11 +34,11 @@ export declare const FETCH: import("../utils/api").Api<{
                     attributes: number;
                     timestampDelta: bigint;
                     offsetDelta: number;
-                    key: Buffer | null;
-                    value: Buffer | null;
+                    key: string | null;
+                    value: string | null;
                     headers: {
-                        key: Buffer | null;
-                        value: Buffer | null;
+                        key: string;
+                        value: string;
                     }[];
                 }[];
                 compression: number;

package/dist/api/fetch.js

@@ -133,10 +133,10 @@ const decodeRecord = (decoder) => decoder.readRecords((record) => ({
     attributes: record.readInt8(),
     timestampDelta: record.readVarLong(),
     offsetDelta: record.readVarInt(),
-    key: record.readVarIntBuffer(),
-    value: record.readVarIntBuffer(),
+    key: record.readVarIntString(),
+    value: record.readVarIntString(),
     headers: record.readVarIntArray((header) => ({
-        key: header.readVarIntBuffer(),
-        value: header.readVarIntBuffer(),
+        key: header.readVarIntString(),
+        value: header.readVarIntString(),
     })),
 }));

package/dist/api/index.d.ts

@@ -97,11 +97,11 @@ export declare const API: {
                         attributes: number;
                         timestampDelta: bigint;
                         offsetDelta: number;
-                        key: Buffer | null;
-                        value: Buffer | null;
+                        key: string | null;
+                        value: string | null;
                         headers: {
-                            key: Buffer | null;
-                            value: Buffer | null;
+                            key: string;
+                            value: string;
                         }[];
                     }[];
                     compression: number;
@@ -384,11 +384,11 @@ export declare const API: {
                     attributes: number;
                     timestampDelta: bigint;
                     offsetDelta: number;
-                    key: Buffer | null;
-                    value: Buffer | null;
+                    key: string | null;
+                    value: string | null;
                     headers: {
-                        key: Buffer;
-                        value: Buffer;
+                        key: string;
+                        value: string;
                     }[];
                 }[];
             }[];

package/dist/api/produce.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 export declare const PRODUCE: import("../utils/api.js").Api<{
     transactionalId: string | null;
     acks: number;
@@ -20,11 +19,11 @@ export declare const PRODUCE: import("../utils/api.js").Api<{
                 attributes: number;
                 timestampDelta: bigint;
                 offsetDelta: number;
-                key: Buffer | null;
-                value: Buffer | null;
+                key: string | null;
+                value: string | null;
                 headers: {
-                    key: Buffer;
-                    value: Buffer;
+                    key: string;
+                    value: string;
                 }[];
             }[];
         }[];

package/dist/api/produce.js

@@ -28,10 +28,10 @@ exports.PRODUCE = (0, api_js_1.createApi)({
                 .writeInt8(record.attributes)
                 .writeVarLong(record.timestampDelta)
                 .writeVarInt(record.offsetDelta)
-                .writeVarIntBuffer(record.key)
-                .writeVarIntBuffer(record.value)
-                .writeVarIntArray(record.headers, (encoder, header) => encoder.writeVarIntBuffer(header.key).writeVarIntBuffer(header.value));
-            return encoder.writeVarInt(recordBody.getByteLength()).writeEncoder(recordBody);
+                .writeVarIntString(record.key)
+                .writeVarIntString(record.value)
+                .writeVarIntArray(record.headers, (encoder, header) => encoder.writeVarIntString(header.key).writeVarIntString(header.value));
+            return encoder.writeVarInt(recordBody.getBufferLength()).writeEncoder(recordBody);
         })
             .value();
         const batchHeader = new encoder_js_1.Encoder()
@@ -41,11 +41,11 @@ exports.PRODUCE = (0, api_js_1.createApi)({
             .write(batchBody);
         const batch = new encoder_js_1.Encoder()
             .writeInt64(partition.baseOffset)
-            .writeInt32(batchHeader.getByteLength())
+            .writeInt32(batchHeader.getBufferLength())
             .writeEncoder(batchHeader);
         return encoder
             .writeInt32(partition.index)
-            .writeUVarInt(batch.getByteLength() + 1) // batch size
+            .writeUVarInt(batch.getBufferLength() + 1)
             .writeEncoder(batch)
             .writeUVarInt(0);
     })

package/dist/cluster.test.js

@@ -136,12 +136,12 @@ vitest_1.describe.sequential('Low-level API', () => {
                                     attributes: 0,
                                     offsetDelta: 0,
                                     timestampDelta: 0n,
-                                    key: Buffer.from('key'),
-                                    value: Buffer.from('value'),
+                                    key: 'key',
+                                    value: 'value',
                                     headers: [
                                         {
-                                            key: Buffer.from('header-key'),
-                                            value: Buffer.from('header-value'),
+                                            key: 'header-key',
+                                            value: 'header-value',
                                         },
                                     ],
                                 },

package/dist/connection.js

@@ -96,7 +96,7 @@ class Connection {
             .writeInt32(correlationId)
             .writeString(this.options.clientId);
         const request = api.request(encoder, body);
-        const requestEncoder = new encoder_1.Encoder().writeInt32(request.getByteLength()).writeEncoder(request);
+        const requestEncoder = new encoder_1.Encoder().writeInt32(request.getBufferLength()).writeEncoder(request);
         let timeout;
         const { responseDecoder, responseSize } = await new Promise(async (resolve, reject) => {
             timeout = setTimeout(() => {

package/dist/distributors/partitioner.js

@@ -15,7 +15,7 @@ const defaultPartitioner = ({ metadata }) => {
         const partitions = metadata.getTopicPartitions()[topic];
         const numPartitions = partitions.length;
         if (key) {
-            return (0, murmur2_1.toPositive)((0, murmur2_1.murmur2)(key)) % numPartitions;
+            return (0, murmur2_1.toPositive)((0, murmur2_1.murmur2)(Buffer.from(key))) % numPartitions;
         }
         return (0, murmur2_1.toPositive)(getNextValue(topic)) % numPartitions;
     };

package/dist/producer/producer.d.ts

@@ -20,6 +20,6 @@ export declare class Producer {
     close(): Promise<void>;
     private ensureConnected;
     private initProducerId;
-    private nextBaseSequence;
-    private revertBaseSequence;
+    private getSequence;
+    private updateSequence;
 }

package/dist/producer/producer.js

@@ -15,6 +15,7 @@ const messages_to_topic_partition_leaders_1 = require("../distributors/messages-
 const partitioner_1 = require("../distributors/partitioner");
 const metadata_1 = require("../metadata");
 const delay_1 = require("../utils/delay");
+const error_1 = require("../utils/error");
 const memo_1 = require("../utils/memo");
 const tracer_1 = require("../utils/tracer");
 const trace = (0, tracer_1.createTracer)('Producer');
@@ -43,64 +44,63 @@ class Producer {
         const topics = Array.from(new Set(messages.map((message) => message.topic)));
         await this.metadata.fetchMetadataIfNecessary({ topics, allowTopicAutoCreation });
         const nodeTopicPartitionMessages = (0, messages_to_topic_partition_leaders_1.distributeMessagesToTopicPartitionLeaders)(messages.map((message) => ({ ...message, partition: this.partition(message) })), this.metadata.getTopicPartitionLeaderIds());
-        await Promise.all(Object.entries(nodeTopicPartitionMessages).map(async ([nodeId, topicPartitionMessages]) => {
-            const topicData = Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
-                name: topic,
-                partitionData: Object.entries(partitionMessages).map(([partition, messages]) => {
-                    const partitionIndex = parseInt(partition);
-                    let baseTimestamp;
-                    let maxTimestamp;
-                    messages.forEach(({ timestamp = defaultTimestamp }) => {
-                        if (!baseTimestamp || timestamp < baseTimestamp) {
-                            baseTimestamp = timestamp;
-                        }
-                        if (!maxTimestamp || timestamp > maxTimestamp) {
-                            maxTimestamp = timestamp;
-                        }
-                    });
-                    const baseSequence = this.nextBaseSequence(topic, partitionIndex, messages.length);
-                    return {
-                        index: partitionIndex,
-                        baseOffset: 0n,
-                        partitionLeaderEpoch: -1,
-                        attributes: 0,
-                        lastOffsetDelta: messages.length - 1,
-                        baseTimestamp: baseTimestamp ?? 0n,
-                        maxTimestamp: maxTimestamp ?? 0n,
-                        producerId: this.producerId,
-                        producerEpoch: 0,
-                        baseSequence,
-                        records: messages.map((message, index) => ({
+        try {
+            await Promise.all(Object.entries(nodeTopicPartitionMessages).map(async ([nodeId, topicPartitionMessages]) => {
+                const topicData = Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
+                    name: topic,
+                    partitionData: Object.entries(partitionMessages).map(([partition, messages]) => {
+                        const partitionIndex = parseInt(partition);
+                        let baseTimestamp;
+                        let maxTimestamp;
+                        messages.forEach(({ timestamp = defaultTimestamp }) => {
+                            if (!baseTimestamp || timestamp < baseTimestamp) {
+                                baseTimestamp = timestamp;
+                            }
+                            if (!maxTimestamp || timestamp > maxTimestamp) {
+                                maxTimestamp = timestamp;
+                            }
+                        });
+                        return {
+                            index: partitionIndex,
+                            baseOffset: 0n,
+                            partitionLeaderEpoch: -1,
                             attributes: 0,
-                            timestampDelta: (message.timestamp ?? defaultTimestamp) - (baseTimestamp ?? 0n),
-                            offsetDelta: index,
-                            key: message.key ?? null,
-                            value: message.value,
-                            headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
-                                key: Buffer.from(key),
-                                value: Buffer.from(value),
+                            lastOffsetDelta: messages.length - 1,
+                            baseTimestamp: baseTimestamp ?? 0n,
+                            maxTimestamp: maxTimestamp ?? 0n,
+                            producerId: this.producerId,
+                            producerEpoch: 0,
+                            baseSequence: this.getSequence(topic, partitionIndex),
+                            records: messages.map((message, index) => ({
+                                attributes: 0,
+                                timestampDelta: (message.timestamp ?? defaultTimestamp) - (baseTimestamp ?? 0n),
+                                offsetDelta: index,
+                                key: message.key ?? null,
+                                value: message.value,
+                                headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({ key, value })),
                             })),
-                        })),
-                    };
-                }),
-            }));
-            try {
-                return await this.cluster.sendRequestToNode(parseInt(nodeId))(api_1.API.PRODUCE, {
+                        };
+                    }),
+                }));
+                await this.cluster.sendRequestToNode(parseInt(nodeId))(api_1.API.PRODUCE, {
                     transactionalId: null,
                     acks,
                     timeoutMs: 5000,
                     topicData,
                 });
-            }
-            catch (error) {
                 topicData.forEach(({ name, partitionData }) => {
                     partitionData.forEach(({ index, records }) => {
-                        this.revertBaseSequence(name, index, records.length);
+                        this.updateSequence(name, index, records.length);
                     });
                 });
-                throw error;
+            }));
+        }
+        catch (error) {
+            if ((error instanceof error_1.KafkaTSApiError) && error.errorCode === api_1.API_ERROR.OUT_OF_ORDER_SEQUENCE_NUMBER) {
+                await this.initProducerId();
             }
-        }));
+            throw error;
+        }
     }
     async close() {
         await this.cluster.disconnect();
@@ -129,15 +129,13 @@ class Producer {
             throw error;
         }
     }
-    nextBaseSequence(topic, partition, messagesCount) {
+    getSequence(topic, partition) {
+        return this.sequences[topic]?.[partition] ?? 0;
+    }
+    updateSequence(topic, partition, messagesCount) {
         this.sequences[topic] ??= {};
         this.sequences[topic][partition] ??= 0;
-        const baseSequence = this.sequences[topic][partition];
         this.sequences[topic][partition] += messagesCount;
-        return baseSequence;
-    }
-    revertBaseSequence(topic, partition, messagesCount) {
-        this.sequences[topic][partition] -= messagesCount;
     }
 }
 exports.Producer = Producer;

package/dist/types.d.ts

@@ -1,11 +1,10 @@
-/// <reference types="node" />
 export type Message = {
     topic: string;
     partition?: number;
     offset?: bigint;
     timestamp?: bigint;
-    key?: Buffer | null;
-    value: Buffer | null;
+    key?: string | null;
+    value: string | null;
     headers?: Record<string, string>;
 };
 export type Batch = Required<Message>[];

package/dist/utils/decoder.d.ts

@@ -16,7 +16,7 @@ export declare class Decoder {
     readVarLong(): bigint;
     readString(): string | null;
     readCompactString(): string | null;
-    readVarIntBuffer(): Buffer | null;
+    readVarIntString(): string | null;
     readUUID(): string;
     readBoolean(): boolean;
     readArray<T>(callback: (opts: Decoder) => T): T[];

package/dist/utils/decoder.js

@@ -86,12 +86,12 @@ class Decoder {
         this.offset += length;
         return value;
     }
-    readVarIntBuffer() {
+    readVarIntString() {
         const length = this.readVarInt();
         if (length < 0) {
             return null;
         }
-        const value = this.buffer.subarray(this.offset, this.offset + length);
+        const value = this.buffer.toString('utf-8', this.offset, this.offset + length);
         this.offset += length;
         return value;
     }
@@ -131,7 +131,7 @@ class Decoder {
     }
     read(length) {
         const value = this.buffer.subarray(this.offset, length !== undefined ? this.offset + length : undefined);
-        this.offset += Buffer.byteLength(value);
+        this.offset += value.length;
         return value;
     }
     readBytes() {

package/dist/utils/encoder.d.ts

@@ -2,7 +2,7 @@
 export declare class Encoder {
     private chunks;
     getChunks(): Buffer[];
-    getByteLength(): number;
+    getBufferLength(): number;
     write(...buffers: Buffer[]): this;
     writeEncoder(encoder: Encoder): this;
     writeInt8(value: number): this;
@@ -16,7 +16,7 @@ export declare class Encoder {
     writeVarLong(value: bigint): this;
     writeString(value: string | null): this;
     writeCompactString(value: string | null): this;
-    writeVarIntBuffer(buffer: Buffer | null): this;
+    writeVarIntString(value: string | null): this;
     writeUUID(value: string | null): this;
     writeBoolean(value: boolean): this;
     writeArray<T>(arr: T[], callback: (encoder: Encoder, item: T) => Encoder): this;

package/dist/utils/encoder.js

@@ -6,8 +6,8 @@ class Encoder {
     getChunks() {
         return this.chunks;
     }
-    getByteLength() {
-        return this.chunks.reduce((acc, chunk) => acc + chunk.byteLength, 0);
+    getBufferLength() {
+        return this.chunks.reduce((acc, chunk) => acc + chunk.length, 0);
     }
     write(...buffers) {
         this.chunks.push(...buffers);
@@ -71,25 +71,22 @@ class Encoder {
         if (value === null) {
             return this.writeInt16(-1);
         }
-        const byteLength = Buffer.byteLength(value, 'utf-8');
-        const buffer = Buffer.allocUnsafe(byteLength);
-        buffer.write(value, 0, byteLength, 'utf-8');
-        return this.writeInt16(byteLength).write(buffer);
+        const buffer = Buffer.from(value, 'utf-8');
+        return this.writeInt16(buffer.length).write(buffer);
     }
     writeCompactString(value) {
         if (value === null) {
             return this.writeUVarInt(0);
         }
-        const byteLength = Buffer.byteLength(value, 'utf-8');
-        const buffer = Buffer.allocUnsafe(byteLength);
-        buffer.write(value, 0, byteLength, 'utf-8');
-        return this.writeUVarInt(byteLength + 1).write(buffer);
+        const buffer = Buffer.from(value, 'utf-8');
+        return this.writeUVarInt(buffer.length + 1).write(buffer);
     }
-    writeVarIntBuffer(buffer) {
-        if (buffer === null) {
+    writeVarIntString(value) {
+        if (value === null) {
             return this.writeVarInt(-1);
         }
-        return this.writeVarInt(buffer.byteLength).write(buffer);
+        const buffer = Buffer.from(value, 'utf-8');
+        return this.writeVarInt(buffer.length).write(buffer);
     }
     writeUUID(value) {
         if (value === null) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kafka-ts",
-    "version": "0.0.6-beta.3",
+    "version": "0.0.6-beta.5",
     "main": "dist/index.js",
     "author": "Priit Käärd",
     "license": "MIT",