kafka-ts 1.3.1-beta.1 → 1.3.1-beta.2

package/dist/cluster.test.js

@@ -15,6 +15,7 @@ const kafka = (0, client_1.createKafkaClient)({
 });
 vitest_1.describe.sequential('Low-level API', () => {
     const groupId = (0, crypto_1.randomBytes)(16).toString('hex');
+    const topicName = 'kafka-ts-test-topic';
     let cluster;
     (0, vitest_1.beforeAll)(async () => {
         cluster = await kafka.createCluster();
@@ -24,9 +25,9 @@ vitest_1.describe.sequential('Low-level API', () => {
             allowTopicAutoCreation: false,
             includeTopicAuthorizedOperations: false,
         });
-        if (metadataResult.topics.some((topic) => topic.name === 'kafka-ts-test-topic')) {
+        if (metadataResult.topics.some((topic) => topic.name === topicName)) {
             await cluster.sendRequest(api_1.API.DELETE_TOPICS, {
-                topics: [{ name: 'kafka-ts-test-topic', topicId: null }],
+                topics: [{ name: topicName, topicId: null }],
                 timeoutMs: 10000,
             });
         }
@@ -43,7 +44,7 @@ vitest_1.describe.sequential('Low-level API', () => {
         const result = await cluster.sendRequest(api_1.API.CREATE_TOPICS, {
             topics: [
                 {
-                    name: 'kafka-ts-test-topic',
+                    name: topicName,
                     numPartitions: 10,
                     replicationFactor: 3,
                     assignments: [],
@@ -53,9 +54,9 @@ vitest_1.describe.sequential('Low-level API', () => {
             timeoutMs: 10000,
             validateOnly: false,
         });
-        topicId = result.topics[0].topicId;
+        topicId = result.topics[0]._topicId;
         result.topics.forEach((topic) => {
-            topic.topicId = 'Any<UUID>';
+            topic._topicId = 'Any<UUID>';
         });
         (0, vitest_1.expect)(result).toMatchSnapshot();
         await new Promise((resolve) => setTimeout(resolve, 1000));
@@ -82,7 +83,7 @@ vitest_1.describe.sequential('Low-level API', () => {
     let leaderId = 0;
     (0, vitest_1.it)('should request metadata for a topic', async () => {
         const result = await cluster.sendRequest(api_1.API.METADATA, {
-            topics: [{ id: topicId, name: 'kafka-ts-test-topic' }],
+            topics: [{ id: topicId, name: topicName }],
             allowTopicAutoCreation: false,
             includeTopicAuthorizedOperations: false,
         });
@@ -118,7 +119,7 @@ vitest_1.describe.sequential('Low-level API', () => {
             acks: 1,
             topicData: [
                 {
-                    name: 'kafka-ts-test-topic',
+                    name: topicName,
                     partitionData: [
                         {
                             index: partitionIndex,
@@ -164,6 +165,7 @@ vitest_1.describe.sequential('Low-level API', () => {
             topics: [
                 {
                     topicId,
+                    topicName,
                     partitions: [
                         {
                             partition: partitionIndex,
@@ -180,7 +182,8 @@ vitest_1.describe.sequential('Low-level API', () => {
             rackId: '',
         });
         result.responses.forEach((response) => {
-            response.topicId = 'Any<UUID>';
+            if ('topicId' in response)
+                response.topicId = 'Any<UUID>';
             response.partitions.forEach((partition) => {
                 partition.records.forEach((record) => {
                     (0, vitest_1.expect)(record.baseTimestamp).toBeGreaterThan(1721926744730n);
@@ -220,7 +223,7 @@ vitest_1.describe.sequential('Low-level API', () => {
                 protocols: [
                     {
                         name: 'RoundRobinAssigner',
-                        metadata: { version: 0, topics: ['kafka-ts-test-topic'] },
+                        metadata: { version: 0, topics: [topicName] },
                     },
                 ],
                 reason: null,
@@ -245,7 +248,7 @@ vitest_1.describe.sequential('Low-level API', () => {
             protocols: [
                 {
                     name: 'RoundRobinAssigner',
-                    metadata: { version: 0, topics: ['kafka-ts-test-topic'] },
+                    metadata: { version: 0, topics: [topicName] },
                 },
             ],
             reason: null,
@@ -282,7 +285,7 @@ vitest_1.describe.sequential('Low-level API', () => {
             groupInstanceId: null,
             topics: [
                 {
-                    name: 'kafka-ts-test-topic',
+                    name: topicName,
                     partitions: [
                         { partitionIndex: 0, committedOffset: 1n, committedLeaderEpoch: 0, committedMetadata: null },
                     ],
@@ -298,7 +301,7 @@ vitest_1.describe.sequential('Low-level API', () => {
                     groupId,
                     topics: [
                         {
-                            name: 'kafka-ts-test-topic',
+                            name: topicName,
                             partitionIndexes: [0],
                         },
                     ],
@@ -332,11 +335,11 @@ vitest_1.describe.sequential('Low-level API', () => {
     });
     (0, vitest_1.it)('should delete topics', async () => {
         const result = await cluster.sendRequest(api_1.API.DELETE_TOPICS, {
-            topics: [{ name: 'kafka-ts-test-topic', topicId: null }],
+            topics: [{ name: topicName, topicId: null }],
             timeoutMs: 10000,
         });
         result.responses.forEach((response) => {
-            response.topicId = 'Any<UUID>';
+            response._topicId = 'Any<UUID>';
         });
         (0, vitest_1.expect)(result).toMatchSnapshot();
     });

package/dist/connection.d.ts

@@ -7,17 +7,27 @@ type ConnectionOptions = {
     ssl: TLSSocketOptions | null;
     requestTimeout: number;
 };
+type Versions = {
+    [apiKey: number]: {
+        minVersion: number;
+        maxVersion: number;
+    };
+};
 export declare class Connection {
     private options;
     private socket;
     private queue;
     private lastCorrelationId;
     private chunks;
+    private versions;
     constructor(options: ConnectionOptions);
     isConnected(): boolean;
     connect(): Promise<void>;
     disconnect(): Promise<void>;
-    sendRequest<Request, Response>(api: Api<Request, Response>, body: Request): Promise<Response>;
+    setVersions(versions: Versions): void;
+    private validateVersion;
+    private validateVersionCached;
+    sendRequest<Request, Response>(apiLatest: Api<Request, Response>, body: Request): Promise<Response>;
     private write;
     private handleData;
     private nextCorrelationId;

package/dist/connection.js

@@ -50,6 +50,7 @@ const assert_1 = __importDefault(require("assert"));
 const net_1 = __importStar(require("net"));
 const tls_1 = __importDefault(require("tls"));
 const api_1 = require("./api");
+const cached_1 = require("./utils/cached");
 const decoder_1 = require("./utils/decoder");
 const encoder_1 = require("./utils/encoder");
 const error_1 = require("./utils/error");
@@ -62,6 +63,7 @@ class Connection {
     queue = {};
     lastCorrelationId = 0;
     chunks = [];
+    versions;
     constructor(options) {
         this.options = options;
     }
@@ -105,7 +107,26 @@ class Connection {
             this.socket.end(resolve);
         });
     }
-    async sendRequest(api, body) {
+    setVersions(versions) {
+        this.versions = versions;
+        this.validateVersionCached.clear();
+    }
+    validateVersion(api) {
+        if (!this.versions)
+            return api;
+        const versionInfo = this.versions[api.apiKey];
+        if (!versionInfo)
+            throw new Error(`Broker does not support API ${(0, api_1.getApiName)(api)}`);
+        if (api.apiVersion < versionInfo.minVersion || api.apiVersion > versionInfo.maxVersion) {
+            if (api.fallback)
+                return this.validateVersion(api.fallback);
+            throw new Error(`Broker does not support API ${(0, api_1.getApiName)(api)} version ${api.apiVersion} (minVersion=${versionInfo.minVersion}, maxVersion=${versionInfo.maxVersion})`);
+        }
+        return api;
+    }
+    validateVersionCached = (0, cached_1.cached)(this.validateVersion, (api) => api.apiKey.toString());
+    async sendRequest(apiLatest, body) {
+        const api = this.validateVersionCached(apiLatest);
         const correlationId = this.nextCorrelationId();
         const apiName = (0, api_1.getApiName)(api);
         const encoder = new encoder_1.Encoder()
@@ -113,6 +134,8 @@ class Connection {
             .writeInt16(api.apiVersion)
             .writeInt32(correlationId)
             .writeString(this.options.clientId);
+        if (api.requestHeaderVersion === 2)
+            encoder.writeTagBuffer();
         const request = api.request(encoder, body);
         const requestEncoder = new encoder_1.Encoder().writeInt32(request.getBufferLength()).writeEncoder(request);
         const { stack } = new Error();
@@ -123,7 +146,7 @@ class Connection {
                 reject(new error_1.ConnectionError(`${apiName} timed out`, stack));
             }, this.options.requestTimeout);
             try {
-                this.queue[correlationId] = { resolve, reject };
+                this.queue[correlationId] = { api, resolve, reject };
                 await this.write(requestEncoder.value());
             }
             catch (error) {
@@ -160,6 +183,8 @@ class Connection {
         const responseDecoder = new decoder_1.Decoder(decoder.read(responseSize));
         const correlationId = responseDecoder.readInt32();
         const context = this.queue[correlationId];
+        if (context?.api.responseHeaderVersion === 1)
+            responseDecoder.readTagBuffer();
         if (context) {
             delete this.queue[correlationId];
             context.resolve({ responseDecoder, responseSize });

package/dist/consumer/consumer.js

@@ -173,10 +173,10 @@ class Consumer extends events_1.default {
         const { retrier } = options;
         this.consumerGroup?.handleLastHeartbeat();
         const topicPartitions = {};
-        const messages = response.responses.flatMap(({ topicId, partitions }) => {
-            const topic = this.metadata.getTopicNameById(topicId);
+        const messages = response.responses.flatMap((response) => {
+            const topic = 'topicName' in response ? response.topicName : this.metadata.getTopicNameById(response.topicId);
             topicPartitions[topic] ??= new Set();
-            return partitions.flatMap(({ partitionIndex, records }) => {
+            return response.partitions.flatMap(({ partitionIndex, records }) => {
                 topicPartitions[topic].add(partitionIndex);
                 return records.flatMap(({ baseTimestamp, baseOffset, records }) => records.flatMap((message) => ({
                     topic,
@@ -211,10 +211,13 @@ class Consumer extends events_1.default {
             this.off('rebalanceInProgress', onRebalance);
         }
         if (!abortController.signal.aborted) {
-            response.responses.forEach(({ topicId, partitions }) => {
-                partitions.forEach(({ partitionIndex, records }) => {
+            response.responses.forEach((response) => {
+                response.partitions.forEach(({ partitionIndex, records }) => {
                     records.forEach(({ baseOffset, lastOffsetDelta }) => {
-                        this.offsetManager.resolve(this.metadata.getTopicNameById(topicId), partitionIndex, baseOffset + BigInt(lastOffsetDelta) + 1n);
+                        const topic = 'topicName' in response
+                            ? response.topicName
+                            : this.metadata.getTopicNameById(response.topicId);
+                        this.offsetManager.resolve(topic, partitionIndex, baseOffset + BigInt(lastOffsetDelta) + 1n);
                     });
                 });
             });
@@ -232,12 +235,13 @@ class Consumer extends events_1.default {
                 isolationLevel,
                 sessionId: 0,
                 sessionEpoch: -1,
-                topics: Object.entries(assignment).map(([topic, partitions]) => ({
-                    topicId: this.metadata.getTopicIdByName(topic),
+                topics: Object.entries(assignment).map(([topicName, partitions]) => ({
+                    topicId: this.metadata.getTopicIdByName(topicName),
+                    topicName,
                     partitions: partitions.map((partition) => ({
                         partition,
                         currentLeaderEpoch: -1,
-                        fetchOffset: this.offsetManager.getCurrentOffset(topic, partition),
+                        fetchOffset: this.offsetManager.getCurrentOffset(topicName, partition),
                         lastFetchedEpoch: -1,
                         logStartOffset: -1n,
                         partitionMaxBytes,

package/dist/consumer/metadata.d.ts

@@ -0,0 +1,24 @@
+import { IsolationLevel } from "../api/fetch";
+import { Assignment } from "../api/sync-group";
+import { Cluster } from "../cluster";
+import { OffsetManager } from "./offset-manager";
+export type Metadata = ReturnType<typeof createMetadata>;
+type MetadataOptions = {
+    cluster: Cluster;
+    topics?: string[];
+    isolationLevel?: IsolationLevel;
+    allowTopicAutoCreation?: boolean;
+    fromBeginning?: boolean;
+    offsetManager?: OffsetManager;
+};
+export declare const createMetadata: ({ cluster, topics, isolationLevel, allowTopicAutoCreation, fromBeginning, offsetManager, }: MetadataOptions) => {
+    init: () => Promise<void>;
+    getTopicPartitions: () => Record<string, number[]>;
+    getTopicIdByName: (name: string) => string;
+    getTopicNameById: (id: string) => string;
+    getAssignment: () => Assignment;
+    setAssignment: (newAssignment: Assignment) => void;
+    getLeaderIdByTopicPartition: (topic: string, partition: number) => number;
+    getIsrNodeIdsByTopicPartition: (topic: string, partition: number) => number[];
+};
+export {};

package/dist/consumer/metadata.js

@@ -0,0 +1,64 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createMetadata = void 0;
+const api_1 = require("../api");
+const createMetadata = ({ cluster, topics, isolationLevel = 0 /* IsolationLevel.READ_UNCOMMITTED */, allowTopicAutoCreation = true, fromBeginning = false, offsetManager, }) => {
+    let topicPartitions = {};
+    let topicNameById = {};
+    let topicIdByName = {};
+    let leaderIdByTopicPartition = {};
+    let isrNodesByTopicPartition;
+    let assignment = {};
+    const fetchMetadata = async () => {
+        const response = await cluster.sendRequest(api_1.API.METADATA, {
+            allowTopicAutoCreation,
+            includeTopicAuthorizedOperations: false,
+            topics: topics?.map((name) => ({ id: null, name })) ?? null,
+        });
+        topicPartitions = Object.fromEntries(response.topics.map((topic) => [topic.name, topic.partitions.map((partition) => partition.partitionIndex)]));
+        topicNameById = Object.fromEntries(response.topics.map((topic) => [topic.topicId, topic.name]));
+        topicIdByName = Object.fromEntries(response.topics.map((topic) => [topic.name, topic.topicId]));
+        leaderIdByTopicPartition = Object.fromEntries(response.topics.map((topic) => [
+            topic.name,
+            Object.fromEntries(topic.partitions.map((partition) => [partition.partitionIndex, partition.leaderId])),
+        ]));
+        isrNodesByTopicPartition = Object.fromEntries(response.topics.map((topic) => [
+            topic.name,
+            Object.fromEntries(topic.partitions.map((partition) => [partition.partitionIndex, partition.isrNodes])),
+        ]));
+        assignment = topicPartitions;
+    };
+    const listOffsets = async () => {
+        const offsets = await cluster.sendRequest(api_1.API.LIST_OFFSETS, {
+            replicaId: -1,
+            isolationLevel,
+            topics: Object.entries(assignment)
+                .flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition })))
+                .map(({ topic, partition }) => ({
+                name: topic,
+                partitions: [{ partitionIndex: partition, currentLeaderEpoch: -1, timestamp: -1n }],
+            })),
+        });
+        offsets.topics.forEach(({ name, partitions }) => {
+            partitions.forEach(({ partitionIndex, offset }) => {
+                offsetManager?.resolve(name, partitionIndex, fromBeginning ? 0n : offset);
+            });
+        });
+    };
+    return {
+        init: async () => {
+            await fetchMetadata();
+            await listOffsets();
+        },
+        getTopicPartitions: () => topicPartitions,
+        getTopicIdByName: (name) => topicIdByName[name],
+        getTopicNameById: (id) => topicNameById[id],
+        getAssignment: () => assignment,
+        setAssignment: (newAssignment) => {
+            assignment = newAssignment;
+        },
+        getLeaderIdByTopicPartition: (topic, partition) => leaderIdByTopicPartition[topic][partition],
+        getIsrNodeIdsByTopicPartition: (topic, partition) => isrNodesByTopicPartition[topic][partition],
+    };
+};
+exports.createMetadata = createMetadata;

package/dist/distributors/messages-to-topic-partition-leaders.d.ts

@@ -0,0 +1,17 @@
+type TopicPartitionLeader = {
+    [topicName: string]: {
+        [partitionId: number]: number;
+    };
+};
+type MessagesByNodeTopicPartition<T> = {
+    [nodeId: number]: {
+        [topicName: string]: {
+            [partitionId: number]: T[];
+        };
+    };
+};
+export declare const distributeMessagesToTopicPartitionLeaders: <T extends {
+    topic: string;
+    partition: number;
+}>(messages: T[], topicPartitionLeader: TopicPartitionLeader) => MessagesByNodeTopicPartition<T>;
+export {};

package/dist/distributors/messages-to-topic-partition-leaders.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.distributeMessagesToTopicPartitionLeaders = void 0;
+const distributeMessagesToTopicPartitionLeaders = (messages, topicPartitionLeader) => {
+    const result = {};
+    messages.forEach((message) => {
+        const leaderId = topicPartitionLeader[message.topic][message.partition];
+        result[leaderId] ??= {};
+        result[leaderId][message.topic] ??= {};
+        result[leaderId][message.topic][message.partition] ??= [];
+        result[leaderId][message.topic][message.partition].push(message);
+    });
+    return result;
+};
+exports.distributeMessagesToTopicPartitionLeaders = distributeMessagesToTopicPartitionLeaders;

package/dist/distributors/messages-to-topic-partition-leaders.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/distributors/messages-to-topic-partition-leaders.test.js

@@ -0,0 +1,30 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const vitest_1 = require("vitest");
+const messages_to_topic_partition_leaders_1 = require("./messages-to-topic-partition-leaders");
+(0, vitest_1.describe)('Distribute messages to partition leader ids', () => {
+    (0, vitest_1.describe)('distributeMessagesToTopicPartitionLeaders', () => {
+        (0, vitest_1.it)('snoke', () => {
+            const result = (0, messages_to_topic_partition_leaders_1.distributeMessagesToTopicPartitionLeaders)([{ topic: 'topic', partition: 0, key: null, value: null, offset: 0n, timestamp: 0n, headers: {} }], { topic: { 0: 1 } });
+            (0, vitest_1.expect)(result).toMatchInlineSnapshot(`
+              {
+                "1": {
+                  "topic": {
+                    "0": [
+                      {
+                        "headers": {},
+                        "key": null,
+                        "offset": 0n,
+                        "partition": 0,
+                        "timestamp": 0n,
+                        "topic": "topic",
+                        "value": null,
+                      },
+                    ],
+                  },
+                },
+              }
+            `);
+        });
+    });
+});

package/dist/examples/src/replicator.js

@@ -0,0 +1,34 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const kafkats_1 = require("kafkats");
+const json_1 = require("./utils/json");
+(async () => {
+    const brokers = [{ host: "localhost", port: 9092 }];
+    const topic = "playground-topic";
+    // const producer = createProducer({ brokers });
+    // const producerInterval = setInterval(async () => {
+    //     await producer.send([
+    //         {
+    //             topic,
+    //             partition: 0,
+    //             offset: 1n,
+    //             timestamp: BigInt(Date.now()),
+    //             key: null,
+    //             value: `PING ${Math.random()}`,
+    //             headers: { timestamp: Date.now().toString() }
+    //         }
+    //     ])
+    // }, 5000);
+    const consumer = await (0, kafkats_1.startConsumer)({
+        topics: [topic],
+        brokers,
+        onBatch: (messages) => {
+            console.log(JSON.stringify(messages, json_1.serializer, 2));
+        },
+    });
+    process.on("SIGINT", async () => {
+        await consumer.close();
+        // clearInterval(producerInterval);
+        // await producer.close();
+    });
+})();

package/dist/examples/src/utils/json.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.serializer = void 0;
+const serializer = (_, value) => (typeof value === "bigint" ? value.toString() : value);
+exports.serializer = serializer;

package/dist/request-handler.d.ts

@@ -0,0 +1,16 @@
+import { Connection } from "./connection";
+import { Api } from "./utils/api";
+type RequestHandlerOptions = {
+    clientId: string | null;
+};
+export declare class RequestHandler {
+    private connection;
+    private options;
+    private queue;
+    private currentBuffer;
+    constructor(connection: Connection, options: RequestHandlerOptions);
+    private handleData;
+    sendRequest<Request, Response>(api: Api<Request, Response>, args: Request): Promise<Response>;
+}
+export type SendRequest = typeof RequestHandler.prototype.sendRequest;
+export {};

package/dist/request-handler.js

@@ -0,0 +1,67 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RequestHandler = void 0;
+const node_assert_1 = __importDefault(require("node:assert"));
+const decoder_1 = require("./utils/decoder");
+const encoder_1 = require("./utils/encoder");
+class RequestHandler {
+    connection;
+    options;
+    queue = {};
+    currentBuffer = null;
+    constructor(connection, options) {
+        this.connection = connection;
+        this.options = options;
+        this.connection.on("data", this.handleData);
+    }
+    handleData(buffer) {
+        this.currentBuffer = this.currentBuffer ? Buffer.concat([this.currentBuffer, buffer]) : buffer;
+        if (this.currentBuffer.length < 4) {
+            return;
+        }
+        const decoder = (0, decoder_1.createDecoder)({ buffer: this.currentBuffer });
+        const size = decoder.readInt32();
+        if (size > decoder.buffer.length) {
+            return;
+        }
+        const correlationId = decoder.readInt32();
+        const request = this.queue[correlationId];
+        delete this.queue[correlationId];
+        request.callback(decoder);
+        // debug(handleData.name, 'Response offsets', {
+        //     offset: decoder.offset,
+        //     length: decoder.buffer.length,
+        //     rest: decoder.buffer.subarray(decoder.offset, decoder.buffer.length)?.toString(),
+        // });
+        (0, node_assert_1.default)(decoder.offset - 4 === size, `Buffer not correctly consumed: ${decoder.offset - 4} !== ${buffer.length}`);
+        this.currentBuffer = null;
+    }
+    async sendRequest(api, args) {
+        const correlationId = Math.floor(Math.random() * 1000000);
+        const encoder = (0, encoder_1.createEncoder)()
+            .writeInt16(api.apiKey)
+            .writeInt16(api.apiVersion)
+            .writeInt32(correlationId)
+            .writeString(this.options.clientId);
+        const request = api.request(encoder, args).value();
+        const buffer = (0, encoder_1.createEncoder)().writeInt32(request.length).write(request).value();
+        return new Promise(async (resolve, reject) => {
+            await this.connection.write(buffer);
+            this.queue[correlationId] = {
+                callback: (decoder) => {
+                    try {
+                        const response = api.response(decoder);
+                        resolve(response);
+                    }
+                    catch (error) {
+                        reject(error);
+                    }
+                },
+            };
+        });
+    }
+}
+exports.RequestHandler = RequestHandler;

package/dist/request-handler.test.d.ts

@@ -0,0 +1 @@
+export declare const kafka: import("./client").Client;