kafka-ts 0.0.3-beta → 0.0.4

package/src/distributors/messages-to-topic-partition-leaders.ts

@@ -1,19 +0,0 @@
-type TopicPartitionLeader = { [topicName: string]: { [partitionId: number]: number } };
-type MessagesByNodeTopicPartition<T> = {
-    [nodeId: number]: { [topicName: string]: { [partitionId: number]: T[] } };
-};
-
-export const distributeMessagesToTopicPartitionLeaders = <T extends { topic: string; partition: number }>(
-    messages: T[],
-    topicPartitionLeader: TopicPartitionLeader,
-) => {
-    const result: MessagesByNodeTopicPartition<T> = {};
-    messages.forEach((message) => {
-        const leaderId = topicPartitionLeader[message.topic][message.partition];
-        result[leaderId] ??= {};
-        result[leaderId][message.topic] ??= {};
-        result[leaderId][message.topic][message.partition] ??= [];
-        result[leaderId][message.topic][message.partition].push(message);
-    });
-    return result;
-};

package/src/index.ts

@@ -1,4 +0,0 @@
-export * from './utils/error';
-export * from './client';
-export * from './api';
-export * from './types';

package/src/metadata.ts

@@ -1,122 +0,0 @@
-import { API, API_ERROR } from './api';
-import { Cluster } from './cluster';
-import { delay } from './utils/delay';
-import { KafkaTSApiError } from './utils/error';
-
-type MetadataOptions = {
-    cluster: Cluster;
-};
-
-export class Metadata {
-    private topicPartitions: Record<string, number[]> = {};
-    private topicNameById: Record<string, string> = {};
-    private topicIdByName: Record<string, string> = {};
-    private leaderIdByTopicPartition: Record<string, Record<number, number>> = {};
-    private isrNodesByTopicPartition: Record<string, Record<number, number[]>> = {};
-
-    constructor(private options: MetadataOptions) {}
-
-    public getTopicPartitionLeaderIds() {
-        return this.leaderIdByTopicPartition;
-    }
-
-    public getTopicPartitionReplicaIds() {
-        return this.isrNodesByTopicPartition;
-    }
-
-    public getTopicPartitions() {
-        return this.topicPartitions;
-    }
-
-    public getTopicIdByName(name: string) {
-        return this.topicIdByName[name];
-    }
-
-    public getTopicNameById(id: string) {
-        return this.topicNameById[id];
-    }
-
-    public async fetchMetadataIfNecessary({
-        topics,
-        allowTopicAutoCreation,
-    }: {
-        topics: string[];
-        allowTopicAutoCreation: boolean;
-    }) {
-        const missingTopics = topics.filter((topic) => !this.topicPartitions[topic]);
-        if (!missingTopics.length) {
-            return;
-        }
-
-        try {
-            return await this.fetchMetadata({ topics: missingTopics, allowTopicAutoCreation });
-        } catch (error) {
-            if (
-                error instanceof KafkaTSApiError &&
-                error.errorCode === API_ERROR.UNKNOWN_TOPIC_OR_PARTITION &&
-                allowTopicAutoCreation
-            ) {
-                // TODO: investigate if we can avoid the delay
-                await delay(1000);
-                return await this.fetchMetadata({ topics: missingTopics, allowTopicAutoCreation });
-            }
-            throw error;
-        }
-    }
-
-    private async fetchMetadata({
-        topics,
-        allowTopicAutoCreation,
-    }: {
-        topics: string[] | null;
-        allowTopicAutoCreation: boolean;
-    }) {
-        const { cluster } = this.options;
-
-        const response = await cluster.sendRequest(API.METADATA, {
-            allowTopicAutoCreation,
-            includeTopicAuthorizedOperations: false,
-            topics: topics?.map((name) => ({ id: null, name })) ?? null,
-        });
-
-        this.topicPartitions = {
-            ...this.topicPartitions,
-            ...Object.fromEntries(
-                response.topics.map((topic) => [
-                    topic.name,
-                    topic.partitions.map((partition) => partition.partitionIndex),
-                ]),
-            ),
-        };
-        this.topicNameById = {
-            ...this.topicNameById,
-            ...Object.fromEntries(response.topics.map((topic) => [topic.topicId, topic.name])),
-        };
-        this.topicIdByName = {
-            ...this.topicIdByName,
-            ...Object.fromEntries(response.topics.map((topic) => [topic.name, topic.topicId])),
-        };
-        this.leaderIdByTopicPartition = {
-            ...this.leaderIdByTopicPartition,
-            ...Object.fromEntries(
-                response.topics.map((topic) => [
-                    topic.name,
-                    Object.fromEntries(
-                        topic.partitions.map((partition) => [partition.partitionIndex, partition.leaderId]),
-                    ),
-                ]),
-            ),
-        };
-        this.isrNodesByTopicPartition = {
-            ...this.isrNodesByTopicPartition,
-            ...Object.fromEntries(
-                response.topics.map((topic) => [
-                    topic.name,
-                    Object.fromEntries(
-                        topic.partitions.map((partition) => [partition.partitionIndex, partition.isrNodes]),
-                    ),
-                ]),
-            ),
-        };
-    }
-}

package/src/producer/producer.ts

@@ -1,132 +0,0 @@
-import { API, API_ERROR } from '../api';
-import { Cluster } from '../cluster';
-import { distributeMessagesToTopicPartitionLeaders } from '../distributors/messages-to-topic-partition-leaders';
-import { Metadata } from '../metadata';
-import { Message } from '../types';
-import { delay } from '../utils/delay';
-import { KafkaTSApiError } from '../utils/error';
-import { memo } from '../utils/memo';
-
-export type ProducerOptions = {
-    allowTopicAutoCreation?: boolean;
-};
-
-export class Producer {
-    private options: Required<ProducerOptions>;
-    private metadata: Metadata;
-    private producerId = 0n;
-    private producerEpoch = 0;
-    private sequences: Record<string, Record<number, number>> = {};
-
-    constructor(
-        private cluster: Cluster,
-        options: ProducerOptions,
-    ) {
-        this.options = {
-            ...options,
-            allowTopicAutoCreation: options.allowTopicAutoCreation ?? false,
-        };
-        this.metadata = new Metadata({ cluster });
-    }
-
-    public async send(messages: Message[]) {
-        await this.ensureConnected();
-
-        const { allowTopicAutoCreation } = this.options;
-        const defaultTimestamp = BigInt(Date.now());
-
-        const topics = Array.from(new Set(messages.map((message) => message.topic)));
-        await this.metadata.fetchMetadataIfNecessary({ topics, allowTopicAutoCreation });
-
-        const nodeTopicPartitionMessages = distributeMessagesToTopicPartitionLeaders(
-            messages,
-            this.metadata.getTopicPartitionLeaderIds(),
-        );
-
-        await Promise.all(
-            Object.entries(nodeTopicPartitionMessages).map(async ([nodeId, topicPartitionMessages]) => {
-                await this.cluster.sendRequestToNode(parseInt(nodeId))(API.PRODUCE, {
-                    transactionalId: null,
-                    acks: 1,
-                    timeoutMs: 5000,
-                    topicData: Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
-                        name: topic,
-                        partitionData: Object.entries(partitionMessages).map(([partition, messages]) => {
-                            let baseTimestamp: bigint | undefined;
-                            let maxTimestamp: bigint | undefined;
-
-                            messages.forEach(({ timestamp = defaultTimestamp }) => {
-                                if (!baseTimestamp || timestamp < baseTimestamp) {
-                                    baseTimestamp = timestamp;
-                                }
-                                if (!maxTimestamp || timestamp > maxTimestamp) {
-                                    maxTimestamp = timestamp;
-                                }
-                            });
-
-                            const baseSequence = this.nextSequence(topic, parseInt(partition), messages.length);
-                            return {
-                                index: parseInt(partition),
-                                baseOffset: 0n,
-                                partitionLeaderEpoch: -1,
-                                attributes: 0,
-                                lastOffsetDelta: messages.length - 1,
-                                baseTimestamp: baseTimestamp ?? 0n,
-                                maxTimestamp: maxTimestamp ?? 0n,
-                                producerId: this.producerId,
-                                producerEpoch: 0,
-                                baseSequence,
-                                records: messages.map((message, index) => ({
-                                    attributes: 0,
-                                    timestampDelta: (message.timestamp ?? defaultTimestamp) - (baseTimestamp ?? 0n),
-                                    offsetDelta: index,
-                                    key: message.key,
-                                    value: message.value,
-                                    headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
-                                        key,
-                                        value,
-                                    })),
-                                })),
-                            };
-                        }),
-                    })),
-                });
-            }),
-        );
-    }
-
-    public async close() {
-        await this.cluster.disconnect();
-    }
-
-    private ensureConnected = memo(async () => {
-        await this.cluster.connect();
-        await this.initProducerId();
-    });
-
-    private async initProducerId(): Promise<void> {
-        try {
-            const result = await this.cluster.sendRequest(API.INIT_PRODUCER_ID, {
-                transactionalId: null,
-                transactionTimeoutMs: 0,
-                producerId: this.producerId,
-                producerEpoch: this.producerEpoch,
-            });
-            this.producerId = result.producerId;
-            this.producerEpoch = result.producerEpoch;
-            this.sequences = {};
-        } catch (error) {
-            if ((error as KafkaTSApiError).errorCode === API_ERROR.COORDINATOR_LOAD_IN_PROGRESS) {
-                await delay(100);
-                return this.initProducerId();
-            }
-            throw error;
-        }
-    }
-
-    private nextSequence(topic: string, partition: number, messagesCount: number) {
-        this.sequences[topic] ??= {};
-        this.sequences[topic][partition] ??= 0;
-        return (this.sequences[topic][partition] += messagesCount || 1);
-    }
-}

package/src/utils/debug.ts

@@ -1,9 +0,0 @@
-export const serializer = (_: string, value: unknown) => (typeof value === 'bigint' ? value.toString() : value);
-
-export const createDebugger = (module: string) => (func: string, message: string, data?: unknown) => {
-    if (!process.env.DEBUG?.includes('kafka-ts')) return;
-    console.debug(
-        `[${module}] ${func}: ${message}`,
-        data && `(${data instanceof Error ? data : JSON.stringify(data, serializer, 4)})`,
-    );
-};

package/src/utils/delay.ts

@@ -1 +0,0 @@
-export const delay = (delayMs: number) => new Promise<void>((resolve) => setTimeout(resolve, delayMs));

package/src/utils/error.ts

@@ -1,21 +0,0 @@
-import { API_ERROR } from '../api';
-
-export class KafkaTSError extends Error {
-    constructor(message: string) {
-        super(message);
-        this.name = this.constructor.name;
-    }
-}
-
-export class KafkaTSApiError<T = any> extends KafkaTSError {
-    constructor(
-        public errorCode: number,
-        public errorMessage: string | null,
-        public response: T,
-    ) {
-        const [errorName] = Object.entries(API_ERROR).find(([, value]) => value === errorCode) ?? ['UNKNOWN'];
-        super(`${errorName}${errorMessage ? `: ${errorMessage}` : ''}`);
-    }
-}
-
-export class ConnectionError extends KafkaTSError {}

package/src/utils/retrier.ts

@@ -1,39 +0,0 @@
-import { delay } from './delay';
-
-export type Retrier = (func: () => unknown) => Promise<void>;
-
-export const createExponentialBackoffRetrier =
-    (options: {
-        onFailure?: (error: unknown) => Promise<void>;
-        maxRetries?: number;
-        initialDelayMs?: number;
-        maxDelayMs?: number;
-        multiplier?: number;
-        retry?: number;
-    }): Retrier =>
-    async (func) => {
-        try {
-            await func();
-        } catch (error) {
-            const {
-                retry = 0,
-                maxRetries = 3,
-                onFailure = (error) => {
-                    throw error;
-                },
-                initialDelayMs = 100,
-                maxDelayMs = 3000,
-                multiplier = 2,
-            } = options;
-
-            const isMaxRetriesExceeded = retry > maxRetries;
-            if (isMaxRetriesExceeded) return onFailure(error);
-
-            const delayMs = Math.min(maxDelayMs, initialDelayMs * multiplier ** retry);
-            await delay(delayMs);
-
-            return createExponentialBackoffRetrier({ ...options, retry: retry + 1 })(func);
-        }
-    };
-
-export const defaultRetrier = createExponentialBackoffRetrier({});

package/src/utils/tracer.ts

@@ -1,31 +0,0 @@
-import { serializer } from './debug';
-
-export const createTracer =
-    (module: string, attributes?: Record<string, unknown>) =>
-    (fn?: (...args: any[]) => Record<string, unknown> | undefined) =>
-    (target: any, propertyKey: string, descriptor: PropertyDescriptor) => {
-        if (!process.env.DEBUG?.includes('kafka-ts')) return;
-
-        const original = descriptor.value;
-        descriptor.value = function (...args: any[]) {
-            const startTime = Date.now();
-            const metadata = fn?.(...args);
-
-            const onEnd = <T>(result: T): T => {
-                console.log(
-                    `[${module}.${propertyKey}] +${Date.now() - startTime}ms ${JSON.stringify({ ...attributes, ...metadata, result }, serializer)}`,
-                );
-                return result;
-            };
-
-            const result = original.apply(this, args);
-            if (result instanceof Promise) {
-                return result.then(onEnd);
-            } else {
-                onEnd(result);
-                return result;
-            }
-        };
-    };
-
-export const trace = createTracer('GLOBAL');

package/tsconfig.json

@@ -1,17 +0,0 @@
-{
-    "compilerOptions": {
-        "lib": ["es2023"],
-        "target": "es2022",
-        "module": "node16",
-        "strict": true,
-        "forceConsistentCasingInFileNames": true,
-        "esModuleInterop": true,
-        "skipLibCheck": true,
-        "moduleResolution": "node16",
-        "outDir": "dist",
-        "declaration": true,
-        "emitDecoratorMetadata": true,
-        "experimentalDecorators": true
-    },
-    "include": ["src/**/*"]
-}