kafka-ts 1.1.7 → 1.1.8

package/dist/src/connection.js

@@ -0,0 +1,21 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createConnection = void 0;
+const net_1 = require("net");
+const createConnection = (options) => {
+    const socket = new net_1.Socket();
+    const connect = () => {
+        return new Promise((resolve, reject) => {
+            socket.connect(options);
+            socket.once('connect', resolve);
+            socket.once('error', reject);
+        });
+    };
+    const disconnect = () => {
+        return new Promise((resolve) => {
+            socket.end(resolve);
+        });
+    };
+    return { connect, disconnect, socket };
+};
+exports.createConnection = createConnection;

package/dist/src/consumer/consumer-group.js

@@ -0,0 +1,131 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createConsumerGroup = void 0;
+const api_1 = require("../api");
+const find_coordinator_1 = require("../api/find-coordinator");
+const broker_1 = require("../broker");
+const createConsumerGroup = ({ topics, groupId, groupInstanceId, sessionTimeoutMs, rebalanceTimeoutMs, coordinator, metadata, offsetManager, }) => {
+    let memberId = "";
+    let generationId = -1;
+    let leaderId = "";
+    let memberIds = [];
+    const findCoordinator = async () => {
+        const { coordinators } = await coordinator.sendRequest(api_1.API.FIND_COORDINATOR, {
+            keyType: find_coordinator_1.KEY_TYPE.GROUP,
+            keys: [groupId],
+        });
+        if (coordinators[0].host !== coordinator.host || coordinators[0].port !== coordinator.port) {
+            await coordinator.disconnect();
+            coordinator = await (0, broker_1.connectBroker)({
+                clientId: coordinator.clientId,
+                options: { host: coordinators[0].host, port: coordinators[0].port },
+            });
+        }
+    };
+    const joinGroup = async () => {
+        const response = await coordinator.sendRequest(api_1.API.JOIN_GROUP, {
+            groupId,
+            groupInstanceId,
+            memberId,
+            sessionTimeoutMs,
+            rebalanceTimeoutMs,
+            protocolType: "consumer",
+            protocols: [{ name: "RoundRobinAssigner", metadata: { version: 0, topics } }],
+            reason: null,
+        });
+        memberId = response.memberId;
+        generationId = response.generationId;
+        leaderId = response.leader;
+        memberIds = response.members.map((member) => member.memberId);
+        if (response.errorCode === api_1.API_ERROR.MEMBER_ID_REQUIRED) {
+            return joinGroup();
+        }
+    };
+    const syncGroup = async () => {
+        let assignments = [];
+        if (memberId === leaderId) {
+            const memberAssignments = Object.entries(metadata.getTopicPartitions())
+                .flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition })))
+                .reduce((acc, { topic, partition }, index) => {
+                const memberId = memberIds[index % memberIds.length];
+                acc[memberId] ??= {};
+                acc[memberId][topic] ??= [];
+                acc[memberId][topic].push(partition);
+                return acc;
+            }, {});
+            assignments = Object.entries(memberAssignments).map(([memberId, assignment]) => ({ memberId, assignment }));
+        }
+        const response = await coordinator.sendRequest(api_1.API.SYNC_GROUP, {
+            groupId,
+            groupInstanceId,
+            memberId,
+            generationId,
+            protocolType: "consumer",
+            protocolName: "RoundRobinAssigner",
+            assignments,
+        });
+        metadata.setAssignment(JSON.parse(response.assignments));
+    };
+    const offsetFetch = async () => {
+        const response = await coordinator.sendRequest(api_1.API.OFFSET_FETCH, {
+            groups: [
+                {
+                    groupId,
+                    memberId,
+                    memberEpoch: -1,
+                    topics: topics.map((topic) => ({ name: topic, partitionIndexes: metadata.getAssignment()[topic] })),
+                },
+            ],
+            requireStable: true,
+        });
+        response.groups.forEach((group) => {
+            group.topics.forEach((topic) => {
+                topic.partitions
+                    .filter(({ committedOffset }) => committedOffset >= 0)
+                    .forEach(({ partitionIndex, committedOffset }) => offsetManager.resolve(topic.name, partitionIndex, committedOffset));
+            });
+        });
+    };
+    const offsetCommit = async () => {
+        await coordinator.sendRequest(api_1.API.OFFSET_COMMIT, {
+            groupId,
+            groupInstanceId,
+            memberId,
+            generationIdOrMemberEpoch: generationId,
+            topics: Object.entries(offsetManager.getPendingOffsets()).map(([topic, partitions]) => ({
+                name: topic,
+                partitions: Object.entries(partitions).map(([partition, offset]) => ({
+                    partitionIndex: parseInt(partition),
+                    committedOffset: offset,
+                    committedLeaderEpoch: -1,
+                    committedMetadata: null,
+                })),
+            })),
+        });
+        offsetManager.flush();
+    };
+    const leaveGroup = async () => {
+        if (!groupId) {
+            return;
+        }
+        await coordinator.sendRequest(api_1.API.LEAVE_GROUP, {
+            groupId,
+            members: [{ memberId, groupInstanceId, reason: null }],
+        });
+    };
+    return {
+        join: async () => {
+            await findCoordinator();
+            await joinGroup();
+            await syncGroup();
+            await offsetFetch();
+        },
+        commit: async () => {
+            await offsetCommit();
+        },
+        leave: async () => {
+            await leaveGroup();
+        },
+    };
+};
+exports.createConsumerGroup = createConsumerGroup;

package/dist/src/consumer/consumer.js

@@ -0,0 +1,103 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.startConsumer = void 0;
+const fetch_1 = require("../api/fetch");
+const broker_1 = require("../broker");
+const consumer_group_1 = require("./consumer-group");
+const metadata_1 = require("./metadata");
+const offset_manager_1 = require("./offset-manager");
+const startConsumer = async ({ brokers, clientId = null, groupId, topics, groupInstanceId = null, rackId = "", isolationLevel = 0 /* IsolationLevel.READ_UNCOMMITTED */, sessionTimeoutMs = 30_000, rebalanceTimeoutMs = 60_000, maxWaitMs = 5000, minBytes = 1, maxBytes = 1_000_000, partitionMaxBytes = 1_000_000, allowTopicAutoCreation = true, fromBeginning = false, ...options }) => {
+    let coordinator = await (0, broker_1.connectBroker)({ clientId, options: brokers[0] });
+    let stopHook;
+    const offsetManager = (0, offset_manager_1.createOffsetManager)();
+    const metadata = (0, metadata_1.createMetadata)({
+        topics,
+        isolationLevel,
+        allowTopicAutoCreation,
+        fromBeginning,
+        coordinator,
+        offsetManager,
+    });
+    const consumerGroup = groupId
+        ? (0, consumer_group_1.createConsumerGroup)({
+            topics,
+            groupId,
+            groupInstanceId,
+            sessionTimeoutMs,
+            rebalanceTimeoutMs,
+            coordinator,
+            metadata,
+            offsetManager,
+        })
+        : undefined;
+    const fetch = async () => {
+        const response = await coordinator.sendRequest(fetch_1.FETCH, {
+            maxWaitMs,
+            minBytes,
+            maxBytes,
+            isolationLevel,
+            sessionId: 0,
+            sessionEpoch: -1,
+            topics: Object.entries(metadata.getAssignment())
+                .flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition })))
+                .map(({ topic, partition }) => ({
+                topicId: metadata.getTopicIdByName(topic),
+                partitions: [
+                    {
+                        partition,
+                        currentLeaderEpoch: -1,
+                        fetchOffset: offsetManager.getCurrentOffset(topic, partition),
+                        lastFetchedEpoch: -1,
+                        logStartOffset: 0n,
+                        partitionMaxBytes,
+                    },
+                ],
+            })),
+            forgottenTopicsData: [],
+            rackId,
+        });
+        return response;
+    };
+    const fetchLoop = async () => {
+        while (!stopHook) {
+            const batch = await fetch();
+            const messages = batch.responses.flatMap(({ topicId, partitions }) => partitions.flatMap(({ partitionIndex, records }) => records.flatMap(({ baseTimestamp, baseOffset, records }) => records.map((message) => ({
+                topic: metadata.getTopicNameById(topicId),
+                partition: partitionIndex,
+                key: message.key ?? null,
+                value: message.value ?? null,
+                headers: Object.fromEntries(message.headers.map(({ key, value }) => [key, value])),
+                timestamp: baseTimestamp + BigInt(message.timestampDelta),
+                offset: baseOffset + BigInt(message.offsetDelta),
+            })))));
+            // TODO: Implement exponential backoff
+            try {
+                if ("onBatch" in options) {
+                    await options.onBatch(messages);
+                }
+                else if ("onMessage" in options) {
+                    for (const message of messages) {
+                        await options.onMessage(message);
+                    }
+                }
+            }
+            catch (error) {
+                console.error(error);
+                continue;
+            }
+            messages.forEach(({ topic, partition, offset }) => offsetManager.resolve(topic, partition, offset));
+            await consumerGroup?.commit();
+        }
+        stopHook();
+    };
+    const close = async () => {
+        await new Promise((resolve) => (stopHook = resolve));
+        await consumerGroup?.leave();
+        await coordinator.disconnect();
+    };
+    await metadata.init();
+    await consumerGroup?.join();
+    fetchLoop();
+    return { close };
+};
+exports.startConsumer = startConsumer;

package/dist/src/consumer/metadata.js

@@ -0,0 +1,52 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createMetadata = void 0;
+const api_1 = require("../api");
+const createMetadata = ({ topics, isolationLevel, allowTopicAutoCreation, fromBeginning, coordinator, offsetManager, }) => {
+    let topicPartitions = {};
+    let topicNameById = {};
+    let topicIdByName = {};
+    let assignment = {};
+    const fetchMetadata = async () => {
+        const response = await coordinator.sendRequest(api_1.API.METADATA, {
+            allowTopicAutoCreation,
+            includeTopicAuthorizedOperations: false,
+            topics: topics.map((name) => ({ id: null, name })),
+        });
+        topicPartitions = Object.fromEntries(response.topics.map((topic) => [topic.name, topic.partitions.map((partition) => partition.partitionIndex)]));
+        topicNameById = Object.fromEntries(response.topics.map((topic) => [topic.topicId, topic.name]));
+        topicIdByName = Object.fromEntries(response.topics.map((topic) => [topic.name, topic.topicId]));
+        assignment = topicPartitions;
+    };
+    const listOffsets = async () => {
+        const offsets = await coordinator.sendRequest(api_1.API.LIST_OFFSETS, {
+            replicaId: -1,
+            isolationLevel,
+            topics: Object.entries(assignment)
+                .flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition })))
+                .map(({ topic, partition }) => ({
+                name: topic,
+                partitions: [{ partitionIndex: partition, currentLeaderEpoch: -1, timestamp: -1n }],
+            })),
+        });
+        offsets.topics.forEach(({ name, partitions }) => {
+            partitions.forEach(({ partitionIndex, offset }) => {
+                offsetManager.resolve(name, partitionIndex, fromBeginning ? 0n : offset - 1n);
+            });
+        });
+    };
+    return {
+        init: async () => {
+            await fetchMetadata();
+            await listOffsets();
+        },
+        getTopicPartitions: () => topicPartitions,
+        getTopicIdByName: (name) => topicIdByName[name],
+        getTopicNameById: (id) => topicNameById[id],
+        getAssignment: () => assignment,
+        setAssignment: (newAssignment) => {
+            assignment = newAssignment;
+        },
+    };
+};
+exports.createMetadata = createMetadata;

package/dist/src/consumer/offset-manager.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createOffsetManager = void 0;
+const createOffsetManager = () => {
+    let currentOffsets = {};
+    let pendingOffsets = {};
+    const resolve = (topic, partition, offset) => {
+        pendingOffsets[topic] ??= {};
+        pendingOffsets[topic][partition] = offset + 1n;
+        currentOffsets[topic] ??= {};
+        currentOffsets[topic][partition] = offset + 1n;
+    };
+    const flush = () => {
+        pendingOffsets = {};
+    };
+    return {
+        getCurrentOffset: (topic, partition) => currentOffsets[topic]?.[partition] ?? 0n,
+        resolve,
+        flush,
+        getPendingOffsets: () => pendingOffsets,
+    };
+};
+exports.createOffsetManager = createOffsetManager;

package/dist/src/index.js

@@ -0,0 +1,19 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./consumer/consumer"), exports);
+__exportStar(require("./producer/producer"), exports);
+__exportStar(require("./types"), exports);

package/dist/src/producer/producer.js

@@ -0,0 +1,84 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createProducer = void 0;
+const api_1 = require("../api");
+const broker_1 = require("../broker");
+const createProducer = ({ clientId = null, brokers }) => {
+    let producerId = 0n;
+    let producerEpoch = 0;
+    let sequences = {};
+    let connect = (async () => {
+        const broker = await (0, broker_1.connectBroker)({ clientId, options: brokers[0] });
+        const result = await broker.sendRequest(api_1.API.INIT_PRODUCER_ID, {
+            transactionalId: null,
+            transactionTimeoutMs: 0,
+            producerId,
+            producerEpoch,
+        });
+        producerId = result.producerId;
+        producerEpoch = result.producerEpoch;
+        sequences = {};
+        return broker;
+    })();
+    const getSequence = (topic, partition) => {
+        sequences[topic] ??= {};
+        sequences[topic][partition] ??= 0;
+        return sequences[topic][partition]++;
+    };
+    const send = async (messages) => {
+        const { sendRequest } = await connect;
+        const topicPartitionMessages = {};
+        messages.forEach((message) => {
+            topicPartitionMessages[message.topic] ??= {};
+            topicPartitionMessages[message.topic][message.partition] ??= [];
+            topicPartitionMessages[message.topic][message.partition].push(message);
+        });
+        await sendRequest(api_1.API.PRODUCE, {
+            transactionalId: null,
+            acks: 1,
+            timeoutMs: 5000,
+            topicData: Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
+                name: topic,
+                partitionData: Object.entries(partitionMessages).map(([partition, messages]) => {
+                    let baseTimestamp;
+                    let maxTimestamp;
+                    messages.forEach(message => {
+                        if (!baseTimestamp || message.timestamp < baseTimestamp) {
+                            baseTimestamp = message.timestamp;
+                        }
+                        if (!maxTimestamp || message.timestamp > maxTimestamp) {
+                            maxTimestamp = message.timestamp;
+                        }
+                    });
+                    return ({
+                        index: parseInt(partition),
+                        baseOffset: 0n,
+                        partitionLeaderEpoch: -1,
+                        attributes: 0,
+                        lastOffsetDelta: messages.length - 1,
+                        baseTimestamp: baseTimestamp ?? 0n,
+                        maxTimestamp: maxTimestamp ?? 0n,
+                        producerId,
+                        producerEpoch: 0,
+                        baseSequence: getSequence(topic, parseInt(partition)),
+                        records: messages.map((message, index) => ({
+                            attributes: 0,
+                            timestampDelta: message.timestamp - (baseTimestamp ?? 0n),
+                            offsetDelta: index,
+                            key: message.key,
+                            value: message.value,
+                            headers: Object.entries(message.headers).map(([key, value]) => ({ key, value })),
+                        })),
+                    });
+                }),
+            })),
+        });
+    };
+    const close = async () => {
+        const { disconnect } = await connect;
+        // TODO: wait for inflight requests to complete
+        await disconnect();
+    };
+    return { send, close };
+};
+exports.createProducer = createProducer;

package/dist/src/request-handler.js

@@ -0,0 +1,57 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createRequestHandler = void 0;
+const node_assert_1 = __importDefault(require("node:assert"));
+const api_1 = require("./api");
+const decoder_1 = require("./utils/decoder");
+const encoder_1 = require("./utils/encoder");
+const createRequestHandler = ({ clientId, connection }) => {
+    const requestQueue = {};
+    let currentBuffer = null;
+    const handleData = (buffer) => {
+        currentBuffer = currentBuffer ? Buffer.concat([currentBuffer, buffer]) : buffer;
+        const decoder = (0, decoder_1.createDecoder)({ buffer: currentBuffer });
+        const size = decoder.readInt32();
+        if (size > decoder.buffer.length) {
+            return;
+        }
+        const correlationId = decoder.readInt32();
+        const request = requestQueue[correlationId];
+        delete requestQueue[correlationId];
+        request.callback(decoder);
+        // console.log({
+        //     offset: decoder.offset,
+        //     length: decoder.buffer.length,
+        //     rest: decoder.buffer.subarray(decoder.offset, decoder.buffer.length)?.toString(),
+        // });
+        (0, node_assert_1.default)(decoder.offset - 4 === size, `Buffer not correctly consumed: ${decoder.offset - 4} !== ${buffer.length}`);
+        currentBuffer = null;
+    };
+    const sendRequest = (api, args) => {
+        const [apiName] = Object.entries(api_1.API).find(([, value]) => value === api) ?? ["UNKNOWN"];
+        console.log(`[sendRequest] ${apiName}`);
+        const correlationId = Math.floor(Math.random() * 1000000);
+        const encoder = (0, encoder_1.createEncoder)()
+            .writeInt16(api.apiKey)
+            .writeInt16(api.apiVersion)
+            .writeInt32(correlationId)
+            .writeString(clientId);
+        const request = api.request(encoder, args);
+        const buffer = (0, encoder_1.createEncoder)()
+            .writeInt32(request.length)
+            .write(request)
+            .value();
+        return new Promise((resolve, reject) => {
+            requestQueue[correlationId] = {
+                callback: (decoder) => resolve(api.response(decoder)),
+            };
+            connection.socket.write(buffer, (err) => (err ? reject(err) : undefined));
+        });
+    };
+    connection.socket.on("data", handleData);
+    return { sendRequest };
+};
+exports.createRequestHandler = createRequestHandler;