kafka-ts 1.1.8 → 1.2.0

package/dist/src/broker.js

@@ -1,35 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.connectBroker = void 0;
-const api_1 = require("./api");
-const api_versions_1 = require("./api/api-versions");
-const connection_1 = require("./connection");
-const request_handler_1 = require("./request-handler");
-const error_1 = require("./utils/error");
-const connectBroker = async ({ clientId, options }) => {
-    const connection = (0, connection_1.createConnection)(options);
-    await connection.connect();
-    const { sendRequest } = (0, request_handler_1.createRequestHandler)({ clientId, connection });
-    await validateApiVersions(sendRequest);
-    return {
-        clientId,
-        sendRequest,
-        disconnect: connection.disconnect,
-        host: options.host,
-        port: options.port,
-    };
-};
-exports.connectBroker = connectBroker;
-const validateApiVersions = async (sendRequest) => {
-    const { versions } = await sendRequest(api_versions_1.API_VERSIONS, {});
-    const apiByKey = Object.fromEntries(Object.values(api_1.API).map((api) => [api.apiKey, api]));
-    versions.forEach(({ apiKey, minVersion, maxVersion }) => {
-        if (!apiByKey[apiKey]) {
-            return;
-        }
-        const { apiVersion } = apiByKey[apiKey];
-        if (apiVersion < minVersion || apiVersion > maxVersion) {
-            throw new error_1.KafkaTSError(`API ${apiKey} version ${apiVersion} is not supported by the broker`);
-        }
-    });
-};

package/dist/src/connection.js

@@ -1,21 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createConnection = void 0;
-const net_1 = require("net");
-const createConnection = (options) => {
-    const socket = new net_1.Socket();
-    const connect = () => {
-        return new Promise((resolve, reject) => {
-            socket.connect(options);
-            socket.once('connect', resolve);
-            socket.once('error', reject);
-        });
-    };
-    const disconnect = () => {
-        return new Promise((resolve) => {
-            socket.end(resolve);
-        });
-    };
-    return { connect, disconnect, socket };
-};
-exports.createConnection = createConnection;

package/dist/src/consumer/consumer-group.js

@@ -1,131 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createConsumerGroup = void 0;
-const api_1 = require("../api");
-const find_coordinator_1 = require("../api/find-coordinator");
-const broker_1 = require("../broker");
-const createConsumerGroup = ({ topics, groupId, groupInstanceId, sessionTimeoutMs, rebalanceTimeoutMs, coordinator, metadata, offsetManager, }) => {
-    let memberId = "";
-    let generationId = -1;
-    let leaderId = "";
-    let memberIds = [];
-    const findCoordinator = async () => {
-        const { coordinators } = await coordinator.sendRequest(api_1.API.FIND_COORDINATOR, {
-            keyType: find_coordinator_1.KEY_TYPE.GROUP,
-            keys: [groupId],
-        });
-        if (coordinators[0].host !== coordinator.host || coordinators[0].port !== coordinator.port) {
-            await coordinator.disconnect();
-            coordinator = await (0, broker_1.connectBroker)({
-                clientId: coordinator.clientId,
-                options: { host: coordinators[0].host, port: coordinators[0].port },
-            });
-        }
-    };
-    const joinGroup = async () => {
-        const response = await coordinator.sendRequest(api_1.API.JOIN_GROUP, {
-            groupId,
-            groupInstanceId,
-            memberId,
-            sessionTimeoutMs,
-            rebalanceTimeoutMs,
-            protocolType: "consumer",
-            protocols: [{ name: "RoundRobinAssigner", metadata: { version: 0, topics } }],
-            reason: null,
-        });
-        memberId = response.memberId;
-        generationId = response.generationId;
-        leaderId = response.leader;
-        memberIds = response.members.map((member) => member.memberId);
-        if (response.errorCode === api_1.API_ERROR.MEMBER_ID_REQUIRED) {
-            return joinGroup();
-        }
-    };
-    const syncGroup = async () => {
-        let assignments = [];
-        if (memberId === leaderId) {
-            const memberAssignments = Object.entries(metadata.getTopicPartitions())
-                .flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition })))
-                .reduce((acc, { topic, partition }, index) => {
-                const memberId = memberIds[index % memberIds.length];
-                acc[memberId] ??= {};
-                acc[memberId][topic] ??= [];
-                acc[memberId][topic].push(partition);
-                return acc;
-            }, {});
-            assignments = Object.entries(memberAssignments).map(([memberId, assignment]) => ({ memberId, assignment }));
-        }
-        const response = await coordinator.sendRequest(api_1.API.SYNC_GROUP, {
-            groupId,
-            groupInstanceId,
-            memberId,
-            generationId,
-            protocolType: "consumer",
-            protocolName: "RoundRobinAssigner",
-            assignments,
-        });
-        metadata.setAssignment(JSON.parse(response.assignments));
-    };
-    const offsetFetch = async () => {
-        const response = await coordinator.sendRequest(api_1.API.OFFSET_FETCH, {
-            groups: [
-                {
-                    groupId,
-                    memberId,
-                    memberEpoch: -1,
-                    topics: topics.map((topic) => ({ name: topic, partitionIndexes: metadata.getAssignment()[topic] })),
-                },
-            ],
-            requireStable: true,
-        });
-        response.groups.forEach((group) => {
-            group.topics.forEach((topic) => {
-                topic.partitions
-                    .filter(({ committedOffset }) => committedOffset >= 0)
-                    .forEach(({ partitionIndex, committedOffset }) => offsetManager.resolve(topic.name, partitionIndex, committedOffset));
-            });
-        });
-    };
-    const offsetCommit = async () => {
-        await coordinator.sendRequest(api_1.API.OFFSET_COMMIT, {
-            groupId,
-            groupInstanceId,
-            memberId,
-            generationIdOrMemberEpoch: generationId,
-            topics: Object.entries(offsetManager.getPendingOffsets()).map(([topic, partitions]) => ({
-                name: topic,
-                partitions: Object.entries(partitions).map(([partition, offset]) => ({
-                    partitionIndex: parseInt(partition),
-                    committedOffset: offset,
-                    committedLeaderEpoch: -1,
-                    committedMetadata: null,
-                })),
-            })),
-        });
-        offsetManager.flush();
-    };
-    const leaveGroup = async () => {
-        if (!groupId) {
-            return;
-        }
-        await coordinator.sendRequest(api_1.API.LEAVE_GROUP, {
-            groupId,
-            members: [{ memberId, groupInstanceId, reason: null }],
-        });
-    };
-    return {
-        join: async () => {
-            await findCoordinator();
-            await joinGroup();
-            await syncGroup();
-            await offsetFetch();
-        },
-        commit: async () => {
-            await offsetCommit();
-        },
-        leave: async () => {
-            await leaveGroup();
-        },
-    };
-};
-exports.createConsumerGroup = createConsumerGroup;

package/dist/src/consumer/consumer.js

@@ -1,103 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.startConsumer = void 0;
-const fetch_1 = require("../api/fetch");
-const broker_1 = require("../broker");
-const consumer_group_1 = require("./consumer-group");
-const metadata_1 = require("./metadata");
-const offset_manager_1 = require("./offset-manager");
-const startConsumer = async ({ brokers, clientId = null, groupId, topics, groupInstanceId = null, rackId = "", isolationLevel = 0 /* IsolationLevel.READ_UNCOMMITTED */, sessionTimeoutMs = 30_000, rebalanceTimeoutMs = 60_000, maxWaitMs = 5000, minBytes = 1, maxBytes = 1_000_000, partitionMaxBytes = 1_000_000, allowTopicAutoCreation = true, fromBeginning = false, ...options }) => {
-    let coordinator = await (0, broker_1.connectBroker)({ clientId, options: brokers[0] });
-    let stopHook;
-    const offsetManager = (0, offset_manager_1.createOffsetManager)();
-    const metadata = (0, metadata_1.createMetadata)({
-        topics,
-        isolationLevel,
-        allowTopicAutoCreation,
-        fromBeginning,
-        coordinator,
-        offsetManager,
-    });
-    const consumerGroup = groupId
-        ? (0, consumer_group_1.createConsumerGroup)({
-            topics,
-            groupId,
-            groupInstanceId,
-            sessionTimeoutMs,
-            rebalanceTimeoutMs,
-            coordinator,
-            metadata,
-            offsetManager,
-        })
-        : undefined;
-    const fetch = async () => {
-        const response = await coordinator.sendRequest(fetch_1.FETCH, {
-            maxWaitMs,
-            minBytes,
-            maxBytes,
-            isolationLevel,
-            sessionId: 0,
-            sessionEpoch: -1,
-            topics: Object.entries(metadata.getAssignment())
-                .flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition })))
-                .map(({ topic, partition }) => ({
-                topicId: metadata.getTopicIdByName(topic),
-                partitions: [
-                    {
-                        partition,
-                        currentLeaderEpoch: -1,
-                        fetchOffset: offsetManager.getCurrentOffset(topic, partition),
-                        lastFetchedEpoch: -1,
-                        logStartOffset: 0n,
-                        partitionMaxBytes,
-                    },
-                ],
-            })),
-            forgottenTopicsData: [],
-            rackId,
-        });
-        return response;
-    };
-    const fetchLoop = async () => {
-        while (!stopHook) {
-            const batch = await fetch();
-            const messages = batch.responses.flatMap(({ topicId, partitions }) => partitions.flatMap(({ partitionIndex, records }) => records.flatMap(({ baseTimestamp, baseOffset, records }) => records.map((message) => ({
-                topic: metadata.getTopicNameById(topicId),
-                partition: partitionIndex,
-                key: message.key ?? null,
-                value: message.value ?? null,
-                headers: Object.fromEntries(message.headers.map(({ key, value }) => [key, value])),
-                timestamp: baseTimestamp + BigInt(message.timestampDelta),
-                offset: baseOffset + BigInt(message.offsetDelta),
-            })))));
-            // TODO: Implement exponential backoff
-            try {
-                if ("onBatch" in options) {
-                    await options.onBatch(messages);
-                }
-                else if ("onMessage" in options) {
-                    for (const message of messages) {
-                        await options.onMessage(message);
-                    }
-                }
-            }
-            catch (error) {
-                console.error(error);
-                continue;
-            }
-            messages.forEach(({ topic, partition, offset }) => offsetManager.resolve(topic, partition, offset));
-            await consumerGroup?.commit();
-        }
-        stopHook();
-    };
-    const close = async () => {
-        await new Promise((resolve) => (stopHook = resolve));
-        await consumerGroup?.leave();
-        await coordinator.disconnect();
-    };
-    await metadata.init();
-    await consumerGroup?.join();
-    fetchLoop();
-    return { close };
-};
-exports.startConsumer = startConsumer;

package/dist/src/consumer/metadata.js

@@ -1,52 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createMetadata = void 0;
-const api_1 = require("../api");
-const createMetadata = ({ topics, isolationLevel, allowTopicAutoCreation, fromBeginning, coordinator, offsetManager, }) => {
-    let topicPartitions = {};
-    let topicNameById = {};
-    let topicIdByName = {};
-    let assignment = {};
-    const fetchMetadata = async () => {
-        const response = await coordinator.sendRequest(api_1.API.METADATA, {
-            allowTopicAutoCreation,
-            includeTopicAuthorizedOperations: false,
-            topics: topics.map((name) => ({ id: null, name })),
-        });
-        topicPartitions = Object.fromEntries(response.topics.map((topic) => [topic.name, topic.partitions.map((partition) => partition.partitionIndex)]));
-        topicNameById = Object.fromEntries(response.topics.map((topic) => [topic.topicId, topic.name]));
-        topicIdByName = Object.fromEntries(response.topics.map((topic) => [topic.name, topic.topicId]));
-        assignment = topicPartitions;
-    };
-    const listOffsets = async () => {
-        const offsets = await coordinator.sendRequest(api_1.API.LIST_OFFSETS, {
-            replicaId: -1,
-            isolationLevel,
-            topics: Object.entries(assignment)
-                .flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition })))
-                .map(({ topic, partition }) => ({
-                name: topic,
-                partitions: [{ partitionIndex: partition, currentLeaderEpoch: -1, timestamp: -1n }],
-            })),
-        });
-        offsets.topics.forEach(({ name, partitions }) => {
-            partitions.forEach(({ partitionIndex, offset }) => {
-                offsetManager.resolve(name, partitionIndex, fromBeginning ? 0n : offset - 1n);
-            });
-        });
-    };
-    return {
-        init: async () => {
-            await fetchMetadata();
-            await listOffsets();
-        },
-        getTopicPartitions: () => topicPartitions,
-        getTopicIdByName: (name) => topicIdByName[name],
-        getTopicNameById: (id) => topicNameById[id],
-        getAssignment: () => assignment,
-        setAssignment: (newAssignment) => {
-            assignment = newAssignment;
-        },
-    };
-};
-exports.createMetadata = createMetadata;

package/dist/src/consumer/offset-manager.js

@@ -1,23 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createOffsetManager = void 0;
-const createOffsetManager = () => {
-    let currentOffsets = {};
-    let pendingOffsets = {};
-    const resolve = (topic, partition, offset) => {
-        pendingOffsets[topic] ??= {};
-        pendingOffsets[topic][partition] = offset + 1n;
-        currentOffsets[topic] ??= {};
-        currentOffsets[topic][partition] = offset + 1n;
-    };
-    const flush = () => {
-        pendingOffsets = {};
-    };
-    return {
-        getCurrentOffset: (topic, partition) => currentOffsets[topic]?.[partition] ?? 0n,
-        resolve,
-        flush,
-        getPendingOffsets: () => pendingOffsets,
-    };
-};
-exports.createOffsetManager = createOffsetManager;

package/dist/src/index.js

@@ -1,19 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./consumer/consumer"), exports);
-__exportStar(require("./producer/producer"), exports);
-__exportStar(require("./types"), exports);

package/dist/src/producer/producer.js

@@ -1,84 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createProducer = void 0;
-const api_1 = require("../api");
-const broker_1 = require("../broker");
-const createProducer = ({ clientId = null, brokers }) => {
-    let producerId = 0n;
-    let producerEpoch = 0;
-    let sequences = {};
-    let connect = (async () => {
-        const broker = await (0, broker_1.connectBroker)({ clientId, options: brokers[0] });
-        const result = await broker.sendRequest(api_1.API.INIT_PRODUCER_ID, {
-            transactionalId: null,
-            transactionTimeoutMs: 0,
-            producerId,
-            producerEpoch,
-        });
-        producerId = result.producerId;
-        producerEpoch = result.producerEpoch;
-        sequences = {};
-        return broker;
-    })();
-    const getSequence = (topic, partition) => {
-        sequences[topic] ??= {};
-        sequences[topic][partition] ??= 0;
-        return sequences[topic][partition]++;
-    };
-    const send = async (messages) => {
-        const { sendRequest } = await connect;
-        const topicPartitionMessages = {};
-        messages.forEach((message) => {
-            topicPartitionMessages[message.topic] ??= {};
-            topicPartitionMessages[message.topic][message.partition] ??= [];
-            topicPartitionMessages[message.topic][message.partition].push(message);
-        });
-        await sendRequest(api_1.API.PRODUCE, {
-            transactionalId: null,
-            acks: 1,
-            timeoutMs: 5000,
-            topicData: Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
-                name: topic,
-                partitionData: Object.entries(partitionMessages).map(([partition, messages]) => {
-                    let baseTimestamp;
-                    let maxTimestamp;
-                    messages.forEach(message => {
-                        if (!baseTimestamp || message.timestamp < baseTimestamp) {
-                            baseTimestamp = message.timestamp;
-                        }
-                        if (!maxTimestamp || message.timestamp > maxTimestamp) {
-                            maxTimestamp = message.timestamp;
-                        }
-                    });
-                    return ({
-                        index: parseInt(partition),
-                        baseOffset: 0n,
-                        partitionLeaderEpoch: -1,
-                        attributes: 0,
-                        lastOffsetDelta: messages.length - 1,
-                        baseTimestamp: baseTimestamp ?? 0n,
-                        maxTimestamp: maxTimestamp ?? 0n,
-                        producerId,
-                        producerEpoch: 0,
-                        baseSequence: getSequence(topic, parseInt(partition)),
-                        records: messages.map((message, index) => ({
-                            attributes: 0,
-                            timestampDelta: message.timestamp - (baseTimestamp ?? 0n),
-                            offsetDelta: index,
-                            key: message.key,
-                            value: message.value,
-                            headers: Object.entries(message.headers).map(([key, value]) => ({ key, value })),
-                        })),
-                    });
-                }),
-            })),
-        });
-    };
-    const close = async () => {
-        const { disconnect } = await connect;
-        // TODO: wait for inflight requests to complete
-        await disconnect();
-    };
-    return { send, close };
-};
-exports.createProducer = createProducer;

package/dist/src/request-handler.js

@@ -1,57 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createRequestHandler = void 0;
-const node_assert_1 = __importDefault(require("node:assert"));
-const api_1 = require("./api");
-const decoder_1 = require("./utils/decoder");
-const encoder_1 = require("./utils/encoder");
-const createRequestHandler = ({ clientId, connection }) => {
-    const requestQueue = {};
-    let currentBuffer = null;
-    const handleData = (buffer) => {
-        currentBuffer = currentBuffer ? Buffer.concat([currentBuffer, buffer]) : buffer;
-        const decoder = (0, decoder_1.createDecoder)({ buffer: currentBuffer });
-        const size = decoder.readInt32();
-        if (size > decoder.buffer.length) {
-            return;
-        }
-        const correlationId = decoder.readInt32();
-        const request = requestQueue[correlationId];
-        delete requestQueue[correlationId];
-        request.callback(decoder);
-        // console.log({
-        //     offset: decoder.offset,
-        //     length: decoder.buffer.length,
-        //     rest: decoder.buffer.subarray(decoder.offset, decoder.buffer.length)?.toString(),
-        // });
-        (0, node_assert_1.default)(decoder.offset - 4 === size, `Buffer not correctly consumed: ${decoder.offset - 4} !== ${buffer.length}`);
-        currentBuffer = null;
-    };
-    const sendRequest = (api, args) => {
-        const [apiName] = Object.entries(api_1.API).find(([, value]) => value === api) ?? ["UNKNOWN"];
-        console.log(`[sendRequest] ${apiName}`);
-        const correlationId = Math.floor(Math.random() * 1000000);
-        const encoder = (0, encoder_1.createEncoder)()
-            .writeInt16(api.apiKey)
-            .writeInt16(api.apiVersion)
-            .writeInt32(correlationId)
-            .writeString(clientId);
-        const request = api.request(encoder, args);
-        const buffer = (0, encoder_1.createEncoder)()
-            .writeInt32(request.length)
-            .write(request)
-            .value();
-        return new Promise((resolve, reject) => {
-            requestQueue[correlationId] = {
-                callback: (decoder) => resolve(api.response(decoder)),
-            };
-            connection.socket.write(buffer, (err) => (err ? reject(err) : undefined));
-        });
-    };
-    connection.socket.on("data", handleData);
-    return { sendRequest };
-};
-exports.createRequestHandler = createRequestHandler;