kafka-ts 0.0.3 → 0.0.4

package/src/metadata.ts

@@ -1,126 +0,0 @@
-import { API, API_ERROR } from './api';
-import { Cluster } from './cluster';
-import { delay } from './utils/delay';
-import { KafkaTSApiError } from './utils/error';
-import { createTracer } from './utils/tracer';
-
-const trace = createTracer('Metadata');
-
-type MetadataOptions = {
-    cluster: Cluster;
-};
-
-export class Metadata {
-    private topicPartitions: Record<string, number[]> = {};
-    private topicNameById: Record<string, string> = {};
-    private topicIdByName: Record<string, string> = {};
-    private leaderIdByTopicPartition: Record<string, Record<number, number>> = {};
-    private isrNodesByTopicPartition: Record<string, Record<number, number[]>> = {};
-
-    constructor(private options: MetadataOptions) {}
-
-    public getTopicPartitionLeaderIds() {
-        return this.leaderIdByTopicPartition;
-    }
-
-    public getTopicPartitionReplicaIds() {
-        return this.isrNodesByTopicPartition;
-    }
-
-    public getTopicPartitions() {
-        return this.topicPartitions;
-    }
-
-    public getTopicIdByName(name: string) {
-        return this.topicIdByName[name];
-    }
-
-    public getTopicNameById(id: string) {
-        return this.topicNameById[id];
-    }
-
-    @trace()
-    public async fetchMetadataIfNecessary({
-        topics,
-        allowTopicAutoCreation,
-    }: {
-        topics: string[];
-        allowTopicAutoCreation: boolean;
-    }) {
-        const missingTopics = topics.filter((topic) => !this.topicPartitions[topic]);
-        if (!missingTopics.length) {
-            return;
-        }
-
-        try {
-            return await this.fetchMetadata({ topics: missingTopics, allowTopicAutoCreation });
-        } catch (error) {
-            if (
-                error instanceof KafkaTSApiError &&
-                error.errorCode === API_ERROR.UNKNOWN_TOPIC_OR_PARTITION &&
-                allowTopicAutoCreation
-            ) {
-                // TODO: investigate if we can avoid the delay
-                await delay(1000);
-                return await this.fetchMetadata({ topics: missingTopics, allowTopicAutoCreation });
-            }
-            throw error;
-        }
-    }
-
-    private async fetchMetadata({
-        topics,
-        allowTopicAutoCreation,
-    }: {
-        topics: string[] | null;
-        allowTopicAutoCreation: boolean;
-    }) {
-        const { cluster } = this.options;
-
-        const response = await cluster.sendRequest(API.METADATA, {
-            allowTopicAutoCreation,
-            includeTopicAuthorizedOperations: false,
-            topics: topics?.map((name) => ({ id: null, name })) ?? null,
-        });
-
-        this.topicPartitions = {
-            ...this.topicPartitions,
-            ...Object.fromEntries(
-                response.topics.map((topic) => [
-                    topic.name,
-                    topic.partitions.map((partition) => partition.partitionIndex),
-                ]),
-            ),
-        };
-        this.topicNameById = {
-            ...this.topicNameById,
-            ...Object.fromEntries(response.topics.map((topic) => [topic.topicId, topic.name])),
-        };
-        this.topicIdByName = {
-            ...this.topicIdByName,
-            ...Object.fromEntries(response.topics.map((topic) => [topic.name, topic.topicId])),
-        };
-        this.leaderIdByTopicPartition = {
-            ...this.leaderIdByTopicPartition,
-            ...Object.fromEntries(
-                response.topics.map((topic) => [
-                    topic.name,
-                    Object.fromEntries(
-                        topic.partitions.map((partition) => [partition.partitionIndex, partition.leaderId]),
-                    ),
-                ]),
-            ),
-        };
-        this.isrNodesByTopicPartition = {
-            ...this.isrNodesByTopicPartition,
-            ...Object.fromEntries(
-                response.topics.map((topic) => [
-                    topic.name,
-                    Object.fromEntries(
-                        topic.partitions.map((partition) => [partition.partitionIndex, partition.isrNodes]),
-                    ),
-                ]),
-            ),
-        };
-    }
-}

package/src/producer/producer.ts

@@ -1,142 +0,0 @@
-import { API, API_ERROR } from '../api';
-import { Cluster } from '../cluster';
-import { distributeMessagesToTopicPartitionLeaders } from '../distributors/messages-to-topic-partition-leaders';
-import { defaultPartitioner, Partition, Partitioner } from '../distributors/partitioner';
-import { Metadata } from '../metadata';
-import { Message } from '../types';
-import { delay } from '../utils/delay';
-import { KafkaTSApiError } from '../utils/error';
-import { memo } from '../utils/memo';
-import { createTracer } from '../utils/tracer';
-
-const trace = createTracer('Producer');
-
-export type ProducerOptions = {
-    allowTopicAutoCreation?: boolean;
-    partitioner?: Partitioner;
-};
-
-export class Producer {
-    private options: Required<ProducerOptions>;
-    private metadata: Metadata;
-    private producerId = 0n;
-    private producerEpoch = 0;
-    private sequences: Record<string, Record<number, number>> = {};
-    private partition: Partition;
-
-    constructor(
-        private cluster: Cluster,
-        options: ProducerOptions,
-    ) {
-        this.options = {
-            ...options,
-            allowTopicAutoCreation: options.allowTopicAutoCreation ?? false,
-            partitioner: options.partitioner ?? defaultPartitioner,
-        };
-        this.metadata = new Metadata({ cluster });
-        this.partition = this.options.partitioner({ metadata: this.metadata });
-    }
-
-    @trace(() => ({ root: true }))
-    public async send(messages: Message[], { acks = -1 }: { acks?: -1 | 1 } = {}) {
-        await this.ensureConnected();
-
-        const { allowTopicAutoCreation } = this.options;
-        const defaultTimestamp = BigInt(Date.now());
-
-        const topics = Array.from(new Set(messages.map((message) => message.topic)));
-        await this.metadata.fetchMetadataIfNecessary({ topics, allowTopicAutoCreation });
-
-        const nodeTopicPartitionMessages = distributeMessagesToTopicPartitionLeaders(
-            messages.map((message) => ({ ...message, partition: this.partition(message) })),
-            this.metadata.getTopicPartitionLeaderIds(),
-        );
-
-        await Promise.all(
-            Object.entries(nodeTopicPartitionMessages).map(([nodeId, topicPartitionMessages]) =>
-                this.cluster.sendRequestToNode(parseInt(nodeId))(API.PRODUCE, {
-                    transactionalId: null,
-                    acks,
-                    timeoutMs: 5000,
-                    topicData: Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
-                        name: topic,
-                        partitionData: Object.entries(partitionMessages).map(([partition, messages]) => {
-                            const partitionIndex = parseInt(partition);
-                            let baseTimestamp: bigint | undefined;
-                            let maxTimestamp: bigint | undefined;
-
-                            messages.forEach(({ timestamp = defaultTimestamp }) => {
-                                if (!baseTimestamp || timestamp < baseTimestamp) {
-                                    baseTimestamp = timestamp;
-                                }
-                                if (!maxTimestamp || timestamp > maxTimestamp) {
-                                    maxTimestamp = timestamp;
-                                }
-                            });
-
-                            const baseSequence = this.nextSequence(topic, partitionIndex, messages.length);
-                            return {
-                                index: partitionIndex,
-                                baseOffset: 0n,
-                                partitionLeaderEpoch: -1,
-                                attributes: 0,
-                                lastOffsetDelta: messages.length - 1,
-                                baseTimestamp: baseTimestamp ?? 0n,
-                                maxTimestamp: maxTimestamp ?? 0n,
-                                producerId: this.producerId,
-                                producerEpoch: 0,
-                                baseSequence,
-                                records: messages.map((message, index) => ({
-                                    attributes: 0,
-                                    timestampDelta: (message.timestamp ?? defaultTimestamp) - (baseTimestamp ?? 0n),
-                                    offsetDelta: index,
-                                    key: message.key ?? null,
-                                    value: message.value,
-                                    headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
-                                        key: Buffer.from(key),
-                                        value: Buffer.from(value),
-                                    })),
-                                })),
-                            };
-                        }),
-                    })),
-                }),
-            ),
-        );
-    }
-
-    public async close() {
-        await this.cluster.disconnect();
-    }
-
-    private ensureConnected = memo(async () => {
-        await this.cluster.connect();
-        await this.initProducerId();
-    });
-
-    private async initProducerId(): Promise<void> {
-        try {
-            const result = await this.cluster.sendRequest(API.INIT_PRODUCER_ID, {
-                transactionalId: null,
-                transactionTimeoutMs: 0,
-                producerId: this.producerId,
-                producerEpoch: this.producerEpoch,
-            });
-            this.producerId = result.producerId;
-            this.producerEpoch = result.producerEpoch;
-            this.sequences = {};
-        } catch (error) {
-            if ((error as KafkaTSApiError).errorCode === API_ERROR.COORDINATOR_LOAD_IN_PROGRESS) {
-                await delay(100);
-                return this.initProducerId();
-            }
-            throw error;
-        }
-    }
-
-    private nextSequence(topic: string, partition: number, messagesCount: number) {
-        this.sequences[topic] ??= {};
-        this.sequences[topic][partition] ??= 0;
-        return (this.sequences[topic][partition] += messagesCount || 1);
-    }
-}

package/src/types.ts

@@ -1,11 +0,0 @@
-export type Message = {
-    topic: string;
-    partition?: number;
-    offset?: bigint;
-    timestamp?: bigint;
-    key?: Buffer | null;
-    value: Buffer | null;
-    headers?: Record<string, string>;
-};
-
-export type Batch = Required<Message>[];

package/src/utils/api.ts

@@ -1,11 +0,0 @@
-import { Decoder } from './decoder';
-import { Encoder } from './encoder';
-
-export type Api<Request, Response> = {
-    apiKey: number;
-    apiVersion: number;
-    request: (encoder: Encoder, body: Request) => Encoder;
-    response: (buffer: Decoder) => Promise<Response> | Response;
-};
-
-export const createApi = <Request, Response>(api: Api<Request, Response>) => api;

package/src/utils/crypto.ts

@@ -1,15 +0,0 @@
-import { createHash, createHmac, pbkdf2, randomBytes } from 'crypto';
-
-export const generateNonce = () => randomBytes(16).toString('base64').replace(/[\/=]/g, '');
-
-export const saltPassword = (password: string, salt: string, iterations: number, keyLength: number, digest: string) =>
-    new Promise<Buffer>((resolve, reject) =>
-        pbkdf2(password, salt, iterations, keyLength, digest, (err, key) => (err ? reject(err) : resolve(key))),
-    );
-
-export const base64Encode = (input: Buffer | string) => Buffer.from(input).toString('base64');
-export const base64Decode = (input: string) => Buffer.from(input, 'base64').toString();
-export const hash = (data: Buffer, digest: string) => createHash(digest).update(data).digest();
-export const hmac = (key: Buffer, data: Buffer | string, digest: string) =>
-    createHmac(digest, key).update(data).digest();
-export const xor = (a: Buffer, b: Buffer) => Buffer.from(a.map((byte, i) => byte ^ b[i]));

package/src/utils/decoder.ts

@@ -1,174 +0,0 @@
-export class Decoder {
-    private offset = 0;
-
-    constructor(private buffer: Buffer) {}
-
-    public getOffset() {
-        return this.offset;
-    }
-
-    public getBufferLength() {
-        return this.buffer.length;
-    }
-
-    public readInt8() {
-        const value = this.buffer.readInt8(this.offset);
-        this.offset += 1;
-        return value;
-    }
-
-    public readInt16() {
-        const value = this.buffer.readInt16BE(this.offset);
-        this.offset += 2;
-        return value;
-    }
-
-    public readInt32() {
-        const value = this.buffer.readInt32BE(this.offset);
-        this.offset += 4;
-        return value;
-    }
-
-    public readUInt32() {
-        const value = this.buffer.readUInt32BE(this.offset);
-        this.offset += 4;
-        return value;
-    }
-
-    public readInt64() {
-        const value = this.buffer.readBigInt64BE(this.offset);
-        this.offset += 8;
-        return value;
-    }
-
-    public readUVarInt() {
-        let result = 0;
-        let shift = 0;
-        let currentByte;
-        do {
-            currentByte = this.buffer[this.offset++];
-            result |= (currentByte & 0x7f) << shift;
-            shift += 7;
-        } while ((currentByte & 0x80) !== 0);
-        return result;
-    }
-
-    public readVarInt() {
-        const decodedValue = this.readUVarInt();
-        return (decodedValue >>> 1) ^ -(decodedValue & 1);
-    }
-
-    public readUVarLong() {
-        let result = BigInt(0);
-        let shift = BigInt(0);
-        let currentByte;
-        do {
-            currentByte = BigInt(this.buffer[this.offset++]);
-            result |= (currentByte & BigInt(0x7f)) << shift;
-            shift += BigInt(7);
-        } while ((currentByte & BigInt(0x80)) !== BigInt(0));
-        return result;
-    }
-
-    public readVarLong() {
-        const decodedValue = this.readUVarLong();
-        return (decodedValue >> BigInt(1)) ^ -(decodedValue & BigInt(1));
-    }
-
-    public readString() {
-        const length = this.readInt16();
-        if (length < 0) {
-            return null;
-        }
-
-        const value = this.buffer.toString('utf-8', this.offset, this.offset + length);
-        this.offset += length;
-        return value;
-    }
-
-    public readCompactString() {
-        const length = this.readUVarInt() - 1;
-        if (length < 0) {
-            return null;
-        }
-
-        const value = this.buffer.toString('utf-8', this.offset, this.offset + length);
-        this.offset += length;
-        return value;
-    }
-
-    public readVarIntBuffer() {
-        const length = this.readVarInt();
-        if (length < 0) {
-            return null;
-        }
-
-        const value = this.buffer.subarray(this.offset, this.offset + length);
-        this.offset += length;
-        return value;
-    }
-
-    public readUUID() {
-        const value = this.buffer.toString('hex', this.offset, this.offset + 16);
-        this.offset += 16;
-        return value;
-    }
-
-    public readBoolean() {
-        const value = this.buffer.readInt8(this.offset) === 1;
-        this.offset += 1;
-        return value;
-    }
-
-    public readArray<T>(callback: (opts: Decoder) => T): T[] {
-        const length = this.readInt32();
-        const results = Array.from({ length }).map(() => callback(this));
-        return results;
-    }
-
-    public readCompactArray<T>(callback: (opts: Decoder) => T): T[] {
-        const length = this.readUVarInt() - 1;
-        const results = Array.from({ length }).map(() => callback(this));
-        return results;
-    }
-
-    public readVarIntArray<T>(callback: (opts: Decoder) => T): T[] {
-        const length = this.readVarInt();
-        const results = Array.from({ length }).map(() => callback(this));
-        return results;
-    }
-
-    public readRecords<T>(callback: (opts: Decoder) => T): T[] {
-        const length = this.readInt32();
-
-        return Array.from({ length }).map(() => {
-            const size = this.readVarInt();
-            const child = new Decoder(this.buffer.subarray(this.offset, this.offset + size));
-            this.offset += size;
-            return callback(child);
-        });
-    }
-
-    public read(length?: number) {
-        const value = this.buffer.subarray(this.offset, length !== undefined ? this.offset + length : undefined);
-        this.offset += Buffer.byteLength(value);
-        return value;
-    }
-
-    public readBytes() {
-        const length = this.readInt32();
-        return this.read(length);
-    }
-
-    public readCompactBytes() {
-        const length = this.readUVarInt() - 1;
-        if (length < 0) {
-            return null;
-        }
-        return this.read(length);
-    }
-
-    public readTagBuffer() {
-        this.readUVarInt();
-    }
-}

package/src/utils/delay.ts

@@ -1 +0,0 @@
-export const delay = (delayMs: number) => new Promise<void>((resolve) => setTimeout(resolve, delayMs));

package/src/utils/encoder.ts

@@ -1,148 +0,0 @@
-export class Encoder {
-    private chunks: Buffer[] = [];
-
-    public getChunks() {
-        return this.chunks;
-    }
-
-    public getByteLength() {
-        return this.chunks.reduce((acc, chunk) => acc + chunk.byteLength, 0);
-    }
-
-    public write(...buffers: Buffer[]) {
-        this.chunks.push(...buffers);
-        return this;
-    }
-
-    public writeEncoder(encoder: Encoder) {
-        return this.write(...encoder.getChunks());
-    }
-
-    public writeInt8(value: number) {
-        const buffer = Buffer.allocUnsafe(1);
-        buffer.writeInt8(value);
-        return this.write(buffer);
-    }
-
-    public writeInt16(value: number) {
-        const buffer = Buffer.allocUnsafe(2);
-        buffer.writeInt16BE(value);
-        return this.write(buffer);
-    }
-
-    public writeInt32(value: number) {
-        const buffer = Buffer.allocUnsafe(4);
-        buffer.writeInt32BE(value);
-        return this.write(buffer);
-    }
-
-    public writeUInt32(value: number) {
-        const buffer = Buffer.allocUnsafe(4);
-        buffer.writeUInt32BE(value);
-        return this.write(buffer);
-    }
-
-    public writeInt64(value: bigint) {
-        const buffer = Buffer.allocUnsafe(8);
-        buffer.writeBigInt64BE(value);
-        return this.write(buffer);
-    }
-
-    public writeUVarInt(value: number) {
-        const byteArray = [];
-        while ((value & 0xffffffff) !== 0) {
-            byteArray.push((value & 0x7f) | 0x80);
-            value >>>= 7;
-        }
-        byteArray.push(value & 0x7f);
-        return this.write(Buffer.from(byteArray));
-    }
-
-    public writeVarInt(value: number) {
-        const encodedValue = (value << 1) ^ (value >> 31);
-        return this.writeUVarInt(encodedValue);
-    }
-
-    public writeUVarLong(value: bigint) {
-        const byteArray = [];
-        while ((value & BigInt(0xffffffffffffffff)) !== BigInt(0)) {
-            byteArray.push(Number((value & BigInt(0x7f)) | BigInt(0x80)));
-            value = value >> BigInt(7);
-        }
-        byteArray.push(Number(value));
-        return this.write(Buffer.from(byteArray));
-    }
-
-    public writeVarLong(value: bigint) {
-        const encodedValue = (value << BigInt(1)) ^ (value >> BigInt(63));
-        return this.writeUVarLong(encodedValue);
-    }
-
-    public writeString(value: string | null) {
-        if (value === null) {
-            return this.writeInt16(-1);
-        }
-        const byteLength = Buffer.byteLength(value, 'utf-8');
-        const buffer = Buffer.allocUnsafe(byteLength);
-        buffer.write(value, 0, byteLength, 'utf-8');
-        return this.writeInt16(byteLength).write(buffer);
-    }
-
-    public writeCompactString(value: string | null) {
-        if (value === null) {
-            return this.writeUVarInt(0);
-        }
-
-        const byteLength = Buffer.byteLength(value, 'utf-8');
-        const buffer = Buffer.allocUnsafe(byteLength);
-        buffer.write(value, 0, byteLength, 'utf-8');
-        return this.writeUVarInt(byteLength + 1).write(buffer);
-    }
-
-    public writeVarIntBuffer(buffer: Buffer | null) {
-        if (buffer === null) {
-            return this.writeVarInt(-1);
-        }
-        return this.writeVarInt(buffer.byteLength).write(buffer);
-    }
-
-    public writeUUID(value: string | null) {
-        if (value === null) {
-            return this.write(Buffer.alloc(16));
-        }
-        return this.write(Buffer.from(value, 'hex'));
-    }
-
-    public writeBoolean(value: boolean) {
-        return this.writeInt8(value ? 1 : 0);
-    }
-
-    public writeArray<T>(arr: T[], callback: (encoder: Encoder, item: T) => Encoder) {
-        return this.writeInt32(arr.length).write(...arr.flatMap((item) => callback(new Encoder(), item).getChunks()));
-    }
-
-    public writeCompactArray<T>(arr: T[] | null, callback: (encoder: Encoder, item: T) => Encoder) {
-        if (arr === null) {
-            return this.writeUVarInt(0);
-        }
-        return this.writeUVarInt(arr.length + 1).write(
-            ...arr.flatMap((item) => callback(new Encoder(), item).getChunks()),
-        );
-    }
-
-    public writeVarIntArray<T>(arr: T[], callback: (encoder: Encoder, item: T) => Encoder) {
-        return this.writeVarInt(arr.length).write(...arr.flatMap((item) => callback(new Encoder(), item).getChunks()));
-    }
-
-    public writeBytes(value: Buffer) {
-        return this.writeInt32(value.length).write(value);
-    }
-
-    public writeCompactBytes(value: Buffer) {
-        return this.writeUVarInt(value.length + 1).write(value);
-    }
-
-    public value() {
-        return Buffer.concat(this.chunks);
-    }
-}

package/src/utils/error.ts

@@ -1,21 +0,0 @@
-import { API_ERROR } from '../api';
-
-export class KafkaTSError extends Error {
-    constructor(message: string) {
-        super(message);
-        this.name = this.constructor.name;
-    }
-}
-
-export class KafkaTSApiError<T = any> extends KafkaTSError {
-    constructor(
-        public errorCode: number,
-        public errorMessage: string | null,
-        public response: T,
-    ) {
-        const [errorName] = Object.entries(API_ERROR).find(([, value]) => value === errorCode) ?? ['UNKNOWN'];
-        super(`${errorName}${errorMessage ? `: ${errorMessage}` : ''}`);
-    }
-}
-
-export class ConnectionError extends KafkaTSError {}

package/src/utils/logger.ts

@@ -1,37 +0,0 @@
-export interface Logger {
-    debug: (message: string, metadata?: unknown) => void;
-    info: (message: string, metadata?: unknown) => void;
-    warn: (message: string, metadata?: unknown) => void;
-    error: (message: string, metadata?: unknown) => void;
-}
-
-export const jsonSerializer = (_: unknown, v: unknown) => {
-    if (v instanceof Error) {
-        return { name: v.name, message: v.message, stack: v.stack, cause: v.cause };
-    }
-    if (typeof v === 'bigint') {
-        return v.toString();
-    }
-    return v;
-};
-
-class JsonLogger implements Logger {
-    debug(message: string, metadata?: unknown) {
-        console.debug(JSON.stringify({ message, metadata, level: 'debug' }, jsonSerializer));
-    }
-    info(message: string, metadata?: unknown) {
-        console.info(JSON.stringify({ message, metadata, level: 'info' }, jsonSerializer));
-    }
-    warn(message: string, metadata?: unknown) {
-        console.warn(JSON.stringify({ message, metadata, level: 'warning' }, jsonSerializer));
-    }
-    error(message: string, metadata?: unknown) {
-        console.error(JSON.stringify({ message, metadata, level: 'error' }, jsonSerializer));
-    }
-}
-
-export let log: Logger = new JsonLogger();
-
-export const setLogger = (newLogger: Logger) => {
-    log = newLogger;
-};