kafka-ts 0.0.2-beta → 0.0.2

package/dist/metadata.d.ts

@@ -0,0 +1,24 @@
+import { Cluster } from './cluster';
+type MetadataOptions = {
+    cluster: Cluster;
+};
+export declare class Metadata {
+    private options;
+    private topicPartitions;
+    private topicNameById;
+    private topicIdByName;
+    private leaderIdByTopicPartition;
+    private isrNodesByTopicPartition;
+    constructor(options: MetadataOptions);
+    getTopicPartitionLeaderIds(): Record<string, Record<number, number>>;
+    getTopicPartitionReplicaIds(): Record<string, Record<number, number[]>>;
+    getTopicPartitions(): Record<string, number[]>;
+    getTopicIdByName(name: string): string;
+    getTopicNameById(id: string): string;
+    fetchMetadataIfNecessary({ topics, allowTopicAutoCreation, }: {
+        topics: string[];
+        allowTopicAutoCreation: boolean;
+    }): Promise<void>;
+    private fetchMetadata;
+}
+export {};

package/dist/metadata.js

@@ -0,0 +1,106 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Metadata = void 0;
+const api_1 = require("./api");
+const delay_1 = require("./utils/delay");
+const error_1 = require("./utils/error");
+const tracer_1 = require("./utils/tracer");
+const trace = (0, tracer_1.createTracer)('Metadata');
+class Metadata {
+    options;
+    topicPartitions = {};
+    topicNameById = {};
+    topicIdByName = {};
+    leaderIdByTopicPartition = {};
+    isrNodesByTopicPartition = {};
+    constructor(options) {
+        this.options = options;
+    }
+    getTopicPartitionLeaderIds() {
+        return this.leaderIdByTopicPartition;
+    }
+    getTopicPartitionReplicaIds() {
+        return this.isrNodesByTopicPartition;
+    }
+    getTopicPartitions() {
+        return this.topicPartitions;
+    }
+    getTopicIdByName(name) {
+        return this.topicIdByName[name];
+    }
+    getTopicNameById(id) {
+        return this.topicNameById[id];
+    }
+    async fetchMetadataIfNecessary({ topics, allowTopicAutoCreation, }) {
+        const missingTopics = topics.filter((topic) => !this.topicPartitions[topic]);
+        if (!missingTopics.length) {
+            return;
+        }
+        try {
+            return await this.fetchMetadata({ topics: missingTopics, allowTopicAutoCreation });
+        }
+        catch (error) {
+            if (error instanceof error_1.KafkaTSApiError &&
+                error.errorCode === api_1.API_ERROR.UNKNOWN_TOPIC_OR_PARTITION &&
+                allowTopicAutoCreation) {
+                // TODO: investigate if we can avoid the delay
+                await (0, delay_1.delay)(1000);
+                return await this.fetchMetadata({ topics: missingTopics, allowTopicAutoCreation });
+            }
+            throw error;
+        }
+    }
+    async fetchMetadata({ topics, allowTopicAutoCreation, }) {
+        const { cluster } = this.options;
+        const response = await cluster.sendRequest(api_1.API.METADATA, {
+            allowTopicAutoCreation,
+            includeTopicAuthorizedOperations: false,
+            topics: topics?.map((name) => ({ id: null, name })) ?? null,
+        });
+        this.topicPartitions = {
+            ...this.topicPartitions,
+            ...Object.fromEntries(response.topics.map((topic) => [
+                topic.name,
+                topic.partitions.map((partition) => partition.partitionIndex),
+            ])),
+        };
+        this.topicNameById = {
+            ...this.topicNameById,
+            ...Object.fromEntries(response.topics.map((topic) => [topic.topicId, topic.name])),
+        };
+        this.topicIdByName = {
+            ...this.topicIdByName,
+            ...Object.fromEntries(response.topics.map((topic) => [topic.name, topic.topicId])),
+        };
+        this.leaderIdByTopicPartition = {
+            ...this.leaderIdByTopicPartition,
+            ...Object.fromEntries(response.topics.map((topic) => [
+                topic.name,
+                Object.fromEntries(topic.partitions.map((partition) => [partition.partitionIndex, partition.leaderId])),
+            ])),
+        };
+        this.isrNodesByTopicPartition = {
+            ...this.isrNodesByTopicPartition,
+            ...Object.fromEntries(response.topics.map((topic) => [
+                topic.name,
+                Object.fromEntries(topic.partitions.map((partition) => [partition.partitionIndex, partition.isrNodes])),
+            ])),
+        };
+    }
+}
+exports.Metadata = Metadata;
+__decorate([
+    trace(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Metadata.prototype, "fetchMetadataIfNecessary", null);

package/dist/producer/producer.d.ts

@@ -0,0 +1,24 @@
+import { Cluster } from '../cluster';
+import { Partitioner } from '../distributors/partitioner';
+import { Message } from '../types';
+export type ProducerOptions = {
+    allowTopicAutoCreation?: boolean;
+    partitioner?: Partitioner;
+};
+export declare class Producer {
+    private cluster;
+    private options;
+    private metadata;
+    private producerId;
+    private producerEpoch;
+    private sequences;
+    private partition;
+    constructor(cluster: Cluster, options: ProducerOptions);
+    send(messages: Message[], { acks }?: {
+        acks?: -1 | 1;
+    }): Promise<void>;
+    close(): Promise<void>;
+    private ensureConnected;
+    private initProducerId;
+    private nextSequence;
+}

package/dist/producer/producer.js

@@ -0,0 +1,131 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Producer = void 0;
+const api_1 = require("../api");
+const messages_to_topic_partition_leaders_1 = require("../distributors/messages-to-topic-partition-leaders");
+const partitioner_1 = require("../distributors/partitioner");
+const metadata_1 = require("../metadata");
+const delay_1 = require("../utils/delay");
+const memo_1 = require("../utils/memo");
+const tracer_1 = require("../utils/tracer");
+const trace = (0, tracer_1.createTracer)('Producer');
+class Producer {
+    cluster;
+    options;
+    metadata;
+    producerId = 0n;
+    producerEpoch = 0;
+    sequences = {};
+    partition;
+    constructor(cluster, options) {
+        this.cluster = cluster;
+        this.options = {
+            ...options,
+            allowTopicAutoCreation: options.allowTopicAutoCreation ?? false,
+            partitioner: options.partitioner ?? partitioner_1.defaultPartitioner,
+        };
+        this.metadata = new metadata_1.Metadata({ cluster });
+        this.partition = this.options.partitioner({ metadata: this.metadata });
+    }
+    async send(messages, { acks = -1 } = {}) {
+        await this.ensureConnected();
+        const { allowTopicAutoCreation } = this.options;
+        const defaultTimestamp = BigInt(Date.now());
+        const topics = Array.from(new Set(messages.map((message) => message.topic)));
+        await this.metadata.fetchMetadataIfNecessary({ topics, allowTopicAutoCreation });
+        const nodeTopicPartitionMessages = (0, messages_to_topic_partition_leaders_1.distributeMessagesToTopicPartitionLeaders)(messages.map((message) => ({ ...message, partition: this.partition(message) })), this.metadata.getTopicPartitionLeaderIds());
+        await Promise.all(Object.entries(nodeTopicPartitionMessages).map(([nodeId, topicPartitionMessages]) => this.cluster.sendRequestToNode(parseInt(nodeId))(api_1.API.PRODUCE, {
+            transactionalId: null,
+            acks,
+            timeoutMs: 5000,
+            topicData: Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
+                name: topic,
+                partitionData: Object.entries(partitionMessages).map(([partition, messages]) => {
+                    const partitionIndex = parseInt(partition);
+                    let baseTimestamp;
+                    let maxTimestamp;
+                    messages.forEach(({ timestamp = defaultTimestamp }) => {
+                        if (!baseTimestamp || timestamp < baseTimestamp) {
+                            baseTimestamp = timestamp;
+                        }
+                        if (!maxTimestamp || timestamp > maxTimestamp) {
+                            maxTimestamp = timestamp;
+                        }
+                    });
+                    const baseSequence = this.nextSequence(topic, partitionIndex, messages.length);
+                    return {
+                        index: partitionIndex,
+                        baseOffset: 0n,
+                        partitionLeaderEpoch: -1,
+                        attributes: 0,
+                        lastOffsetDelta: messages.length - 1,
+                        baseTimestamp: baseTimestamp ?? 0n,
+                        maxTimestamp: maxTimestamp ?? 0n,
+                        producerId: this.producerId,
+                        producerEpoch: 0,
+                        baseSequence,
+                        records: messages.map((message, index) => ({
+                            attributes: 0,
+                            timestampDelta: (message.timestamp ?? defaultTimestamp) - (baseTimestamp ?? 0n),
+                            offsetDelta: index,
+                            key: message.key ?? null,
+                            value: message.value,
+                            headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
+                                key: Buffer.from(key),
+                                value: Buffer.from(value),
+                            })),
+                        })),
+                    };
+                }),
+            })),
+        })));
+    }
+    async close() {
+        await this.cluster.disconnect();
+    }
+    ensureConnected = (0, memo_1.memo)(async () => {
+        await this.cluster.connect();
+        await this.initProducerId();
+    });
+    async initProducerId() {
+        try {
+            const result = await this.cluster.sendRequest(api_1.API.INIT_PRODUCER_ID, {
+                transactionalId: null,
+                transactionTimeoutMs: 0,
+                producerId: this.producerId,
+                producerEpoch: this.producerEpoch,
+            });
+            this.producerId = result.producerId;
+            this.producerEpoch = result.producerEpoch;
+            this.sequences = {};
+        }
+        catch (error) {
+            if (error.errorCode === api_1.API_ERROR.COORDINATOR_LOAD_IN_PROGRESS) {
+                await (0, delay_1.delay)(100);
+                return this.initProducerId();
+            }
+            throw error;
+        }
+    }
+    nextSequence(topic, partition, messagesCount) {
+        this.sequences[topic] ??= {};
+        this.sequences[topic][partition] ??= 0;
+        return (this.sequences[topic][partition] += messagesCount || 1);
+    }
+}
+exports.Producer = Producer;
+__decorate([
+    trace(() => ({ root: true })),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Array, Object]),
+    __metadata("design:returntype", Promise)
+], Producer.prototype, "send", null);

package/dist/types.d.ts

@@ -0,0 +1,11 @@
+/// <reference types="node" />
+export type Message = {
+    topic: string;
+    partition?: number;
+    offset?: bigint;
+    timestamp?: bigint;
+    key?: Buffer | null;
+    value: Buffer | null;
+    headers?: Record<string, string>;
+};
+export type Batch = Required<Message>[];

package/dist/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/utils/api.d.ts

@@ -0,0 +1,9 @@
+import { Decoder } from './decoder';
+import { Encoder } from './encoder';
+export type Api<Request, Response> = {
+    apiKey: number;
+    apiVersion: number;
+    request: (encoder: Encoder, body: Request) => Encoder;
+    response: (buffer: Decoder) => Promise<Response> | Response;
+};
+export declare const createApi: <Request, Response>(api: Api<Request, Response>) => Api<Request, Response>;

package/dist/utils/api.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createApi = void 0;
+const createApi = (api) => api;
+exports.createApi = createApi;

package/dist/utils/crypto.d.ts

@@ -0,0 +1,8 @@
+/// <reference types="node" />
+export declare const generateNonce: () => string;
+export declare const saltPassword: (password: string, salt: string, iterations: number, keyLength: number, digest: string) => Promise<Buffer>;
+export declare const base64Encode: (input: Buffer | string) => string;
+export declare const base64Decode: (input: string) => string;
+export declare const hash: (data: Buffer, digest: string) => Buffer;
+export declare const hmac: (key: Buffer, data: Buffer | string, digest: string) => Buffer;
+export declare const xor: (a: Buffer, b: Buffer) => Buffer;

package/dist/utils/crypto.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.xor = exports.hmac = exports.hash = exports.base64Decode = exports.base64Encode = exports.saltPassword = exports.generateNonce = void 0;
+const crypto_1 = require("crypto");
+const generateNonce = () => (0, crypto_1.randomBytes)(16).toString('base64').replace(/[\/=]/g, '');
+exports.generateNonce = generateNonce;
+const saltPassword = (password, salt, iterations, keyLength, digest) => new Promise((resolve, reject) => (0, crypto_1.pbkdf2)(password, salt, iterations, keyLength, digest, (err, key) => (err ? reject(err) : resolve(key))));
+exports.saltPassword = saltPassword;
+const base64Encode = (input) => Buffer.from(input).toString('base64');
+exports.base64Encode = base64Encode;
+const base64Decode = (input) => Buffer.from(input, 'base64').toString();
+exports.base64Decode = base64Decode;
+const hash = (data, digest) => (0, crypto_1.createHash)(digest).update(data).digest();
+exports.hash = hash;
+const hmac = (key, data, digest) => (0, crypto_1.createHmac)(digest, key).update(data).digest();
+exports.hmac = hmac;
+const xor = (a, b) => Buffer.from(a.map((byte, i) => byte ^ b[i]));
+exports.xor = xor;

package/dist/utils/decoder.d.ts

@@ -0,0 +1,30 @@
+/// <reference types="node" />
+export declare class Decoder {
+    private buffer;
+    private offset;
+    constructor(buffer: Buffer);
+    getOffset(): number;
+    getBufferLength(): number;
+    readInt8(): number;
+    readInt16(): number;
+    readInt32(): number;
+    readUInt32(): number;
+    readInt64(): bigint;
+    readUVarInt(): number;
+    readVarInt(): number;
+    readUVarLong(): bigint;
+    readVarLong(): bigint;
+    readString(): string | null;
+    readCompactString(): string | null;
+    readVarIntBuffer(): Buffer | null;
+    readUUID(): string;
+    readBoolean(): boolean;
+    readArray<T>(callback: (opts: Decoder) => T): T[];
+    readCompactArray<T>(callback: (opts: Decoder) => T): T[];
+    readVarIntArray<T>(callback: (opts: Decoder) => T): T[];
+    readRecords<T>(callback: (opts: Decoder) => T): T[];
+    read(length?: number): Buffer;
+    readBytes(): Buffer;
+    readCompactBytes(): Buffer | null;
+    readTagBuffer(): void;
+}

package/dist/utils/decoder.js

@@ -0,0 +1,152 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Decoder = void 0;
+class Decoder {
+    buffer;
+    offset = 0;
+    constructor(buffer) {
+        this.buffer = buffer;
+    }
+    getOffset() {
+        return this.offset;
+    }
+    getBufferLength() {
+        return this.buffer.length;
+    }
+    readInt8() {
+        const value = this.buffer.readInt8(this.offset);
+        this.offset += 1;
+        return value;
+    }
+    readInt16() {
+        const value = this.buffer.readInt16BE(this.offset);
+        this.offset += 2;
+        return value;
+    }
+    readInt32() {
+        const value = this.buffer.readInt32BE(this.offset);
+        this.offset += 4;
+        return value;
+    }
+    readUInt32() {
+        const value = this.buffer.readUInt32BE(this.offset);
+        this.offset += 4;
+        return value;
+    }
+    readInt64() {
+        const value = this.buffer.readBigInt64BE(this.offset);
+        this.offset += 8;
+        return value;
+    }
+    readUVarInt() {
+        let result = 0;
+        let shift = 0;
+        let currentByte;
+        do {
+            currentByte = this.buffer[this.offset++];
+            result |= (currentByte & 0x7f) << shift;
+            shift += 7;
+        } while ((currentByte & 0x80) !== 0);
+        return result;
+    }
+    readVarInt() {
+        const decodedValue = this.readUVarInt();
+        return (decodedValue >>> 1) ^ -(decodedValue & 1);
+    }
+    readUVarLong() {
+        let result = BigInt(0);
+        let shift = BigInt(0);
+        let currentByte;
+        do {
+            currentByte = BigInt(this.buffer[this.offset++]);
+            result |= (currentByte & BigInt(0x7f)) << shift;
+            shift += BigInt(7);
+        } while ((currentByte & BigInt(0x80)) !== BigInt(0));
+        return result;
+    }
+    readVarLong() {
+        const decodedValue = this.readUVarLong();
+        return (decodedValue >> BigInt(1)) ^ -(decodedValue & BigInt(1));
+    }
+    readString() {
+        const length = this.readInt16();
+        if (length < 0) {
+            return null;
+        }
+        const value = this.buffer.toString('utf-8', this.offset, this.offset + length);
+        this.offset += length;
+        return value;
+    }
+    readCompactString() {
+        const length = this.readUVarInt() - 1;
+        if (length < 0) {
+            return null;
+        }
+        const value = this.buffer.toString('utf-8', this.offset, this.offset + length);
+        this.offset += length;
+        return value;
+    }
+    readVarIntBuffer() {
+        const length = this.readVarInt();
+        if (length < 0) {
+            return null;
+        }
+        const value = this.buffer.subarray(this.offset, this.offset + length);
+        this.offset += length;
+        return value;
+    }
+    readUUID() {
+        const value = this.buffer.toString('hex', this.offset, this.offset + 16);
+        this.offset += 16;
+        return value;
+    }
+    readBoolean() {
+        const value = this.buffer.readInt8(this.offset) === 1;
+        this.offset += 1;
+        return value;
+    }
+    readArray(callback) {
+        const length = this.readInt32();
+        const results = Array.from({ length }).map(() => callback(this));
+        return results;
+    }
+    readCompactArray(callback) {
+        const length = this.readUVarInt() - 1;
+        const results = Array.from({ length }).map(() => callback(this));
+        return results;
+    }
+    readVarIntArray(callback) {
+        const length = this.readVarInt();
+        const results = Array.from({ length }).map(() => callback(this));
+        return results;
+    }
+    readRecords(callback) {
+        const length = this.readInt32();
+        return Array.from({ length }).map(() => {
+            const size = this.readVarInt();
+            const child = new Decoder(this.buffer.subarray(this.offset, this.offset + size));
+            this.offset += size;
+            return callback(child);
+        });
+    }
+    read(length) {
+        const value = this.buffer.subarray(this.offset, length !== undefined ? this.offset + length : undefined);
+        this.offset += Buffer.byteLength(value);
+        return value;
+    }
+    readBytes() {
+        const length = this.readInt32();
+        return this.read(length);
+    }
+    readCompactBytes() {
+        const length = this.readUVarInt() - 1;
+        if (length < 0) {
+            return null;
+        }
+        return this.read(length);
+    }
+    readTagBuffer() {
+        this.readUVarInt();
+    }
+}
+exports.Decoder = Decoder;

package/dist/utils/delay.d.ts

@@ -0,0 +1 @@
+export declare const delay: (delayMs: number) => Promise<void>;

package/dist/utils/delay.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.delay = void 0;
+const delay = (delayMs) => new Promise((resolve) => setTimeout(resolve, delayMs));
+exports.delay = delay;

package/dist/utils/encoder.d.ts

@@ -0,0 +1,28 @@
+/// <reference types="node" />
+export declare class Encoder {
+    private chunks;
+    getChunks(): Buffer[];
+    getByteLength(): number;
+    write(...buffers: Buffer[]): this;
+    writeEncoder(encoder: Encoder): this;
+    writeInt8(value: number): this;
+    writeInt16(value: number): this;
+    writeInt32(value: number): this;
+    writeUInt32(value: number): this;
+    writeInt64(value: bigint): this;
+    writeUVarInt(value: number): this;
+    writeVarInt(value: number): this;
+    writeUVarLong(value: bigint): this;
+    writeVarLong(value: bigint): this;
+    writeString(value: string | null): this;
+    writeCompactString(value: string | null): this;
+    writeVarIntBuffer(buffer: Buffer | null): this;
+    writeUUID(value: string | null): this;
+    writeBoolean(value: boolean): this;
+    writeArray<T>(arr: T[], callback: (encoder: Encoder, item: T) => Encoder): this;
+    writeCompactArray<T>(arr: T[] | null, callback: (encoder: Encoder, item: T) => Encoder): this;
+    writeVarIntArray<T>(arr: T[], callback: (encoder: Encoder, item: T) => Encoder): this;
+    writeBytes(value: Buffer): this;
+    writeCompactBytes(value: Buffer): this;
+    value(): Buffer;
+}