@platformatic/kafka 0.1.0

package/dist/clients/consumer/topics-map.js

@@ -0,0 +1,48 @@
+export class TopicsMap extends Map {
+    #current = [];
+    get current() {
+        return this.#current;
+    }
+    track(topic) {
+        let updated = false;
+        let existing = this.get(topic);
+        if (typeof existing === 'undefined') {
+            existing = 0;
+            updated = true;
+        }
+        this.set(topic, existing + 1);
+        if (updated) {
+            this.#updateCurrentList();
+        }
+        return updated;
+    }
+    trackAll(...topics) {
+        const updated = [];
+        for (const topic of topics.flat()) {
+            updated.push(this.track(topic));
+        }
+        return updated;
+    }
+    untrack(topic) {
+        const existing = this.get(topic);
+        if (existing === 1) {
+            this.delete(topic);
+            this.#updateCurrentList();
+            return true;
+        }
+        else if (typeof existing === 'number') {
+            this.set(topic, existing - 1);
+        }
+        return false;
+    }
+    untrackAll(...topics) {
+        const updated = [];
+        for (const topic of topics.flat()) {
+            updated.push(this.untrack(topic));
+        }
+        return updated;
+    }
+    #updateCurrentList() {
+        this.#current = Array.from(this.keys());
+    }
+}

package/dist/clients/consumer/types.d.ts

@@ -0,0 +1,74 @@
+import { type FetchRequestTopic } from '../../apis/consumer/fetch.ts';
+import { type FetchIsolationLevel } from '../../apis/enumerations.ts';
+import { type BaseOptions, type TopicWithPartitionAndOffset } from '../base/types.ts';
+import { type Deserializers } from '../serde.ts';
+export interface GroupProtocolSubscription {
+    name: string;
+    version: number;
+    metadata?: Buffer | string;
+}
+export interface GroupAssignment {
+    topic: string;
+    partitions: number[];
+}
+export interface ExtendedGroupProtocolSubscription extends Omit<GroupProtocolSubscription, 'name'> {
+    topics?: string[];
+    memberId: string;
+}
+export type Offsets = Map<string, bigint[]>;
+export declare const MessagesStreamModes: {
+    readonly LATEST: "latest";
+    readonly EARLIEST: "earliest";
+    readonly COMMITTED: "committed";
+    readonly MANUAL: "manual";
+};
+export type MessagesStreamMode = keyof typeof MessagesStreamModes;
+export declare const MessagesStreamFallbackModes: {
+    readonly LATEST: "latest";
+    readonly EARLIEST: "earliest";
+    readonly FAIL: "fail";
+};
+export type MessagesStreamFallbackMode = keyof typeof MessagesStreamFallbackModes;
+export interface GroupOptions {
+    sessionTimeout?: number;
+    rebalanceTimeout?: number;
+    heartbeatInterval?: number;
+    protocols?: GroupProtocolSubscription[];
+}
+export interface ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> {
+    autocommit?: boolean | number;
+    minBytes?: number;
+    maxBytes?: number;
+    maxWaitTime?: number;
+    isolationLevel?: FetchIsolationLevel;
+    deserializers?: Partial<Deserializers<Key, Value, HeaderKey, HeaderValue>>;
+    highWaterMark?: number;
+}
+export interface StreamOptions {
+    topics: string[];
+    mode?: string;
+    fallbackMode?: string;
+    offsets?: TopicWithPartitionAndOffset[];
+}
+export type ConsumeOptions<Key, Value, HeaderKey, HeaderValue> = StreamOptions & ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> & GroupOptions;
+export type ConsumerOptions<Key, Value, HeaderKey, HeaderValue> = BaseOptions & {
+    groupId: string;
+} & GroupOptions & ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue>;
+export type FetchOptions<Key, Value, HeaderKey, HeaderValue> = Omit<ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue>, 'deserializers'> & GroupOptions & {
+    node: number;
+    topics: FetchRequestTopic[];
+};
+export interface CommitOptionsPartition extends TopicWithPartitionAndOffset {
+    leaderEpoch: number;
+}
+export interface CommitOptions {
+    offsets: CommitOptionsPartition[];
+}
+export interface ListCommitsOptions {
+    topics: GroupAssignment[];
+}
+export interface ListOffsetsOptions {
+    topics: string[];
+    timestamp?: bigint;
+    isolationLevel?: FetchIsolationLevel;
+}

package/dist/clients/consumer/types.js

@@ -0,0 +1,11 @@
+export const MessagesStreamModes = {
+    LATEST: 'latest',
+    EARLIEST: 'earliest',
+    COMMITTED: 'committed',
+    MANUAL: 'manual'
+};
+export const MessagesStreamFallbackModes = {
+    LATEST: 'latest',
+    EARLIEST: 'earliest',
+    FAIL: 'fail'
+};

package/dist/clients/index.d.ts

@@ -0,0 +1,6 @@
+export * from './callbacks.ts';
+export * from './serde.ts';
+export * from './admin/index.ts';
+export * from './base/index.ts';
+export * from './consumer/index.ts';
+export * from './producer/index.ts';

package/dist/clients/index.js

@@ -0,0 +1,6 @@
+export * from "./callbacks.js";
+export * from "./serde.js";
+export * from "./admin/index.js";
+export * from "./base/index.js";
+export * from "./consumer/index.js";
+export * from "./producer/index.js";

package/dist/clients/producer/index.d.ts

@@ -0,0 +1,3 @@
+export * from './options.ts';
+export * from './producer.ts';
+export * from './types.ts';

package/dist/clients/producer/index.js

@@ -0,0 +1,3 @@
+export * from "./options.js";
+export * from "./producer.js";
+export * from "./types.js";

package/dist/clients/producer/options.d.ts

@@ -0,0 +1,122 @@
+export declare const produceOptionsSchema: {
+    type: string;
+    properties: {
+        producerId: {
+            bigint: boolean;
+        };
+        producerEpoch: {
+            type: string;
+        };
+        idempotent: {
+            type: string;
+        };
+        acks: {
+            type: string;
+            enum: (0 | 1 | -1)[];
+            errorMessage: string;
+        };
+        compression: {
+            type: string;
+            enum: string[];
+            errorMessage: string;
+        };
+        partitioner: {
+            function: boolean;
+        };
+        autocreateTopics: {
+            type: string;
+        };
+        repeatOnStaleMetadata: {
+            type: string;
+        };
+    };
+    additionalProperties: boolean;
+};
+export declare const produceOptionsValidator: import("ajv").ValidateFunction<unknown>;
+export declare const producerOptionsValidator: import("ajv").ValidateFunction<unknown>;
+export declare const sendOptionsSchema: {
+    type: string;
+    properties: {
+        producerId: {
+            bigint: boolean;
+        };
+        producerEpoch: {
+            type: string;
+        };
+        idempotent: {
+            type: string;
+        };
+        acks: {
+            type: string;
+            enum: (0 | 1 | -1)[];
+            errorMessage: string;
+        };
+        compression: {
+            type: string;
+            enum: string[];
+            errorMessage: string;
+        };
+        partitioner: {
+            function: boolean;
+        };
+        autocreateTopics: {
+            type: string;
+        };
+        repeatOnStaleMetadata: {
+            type: string;
+        };
+        messages: {
+            type: string;
+            items: {
+                type: string;
+                properties: {
+                    key: {
+                        oneOf: ({
+                            type: string;
+                            buffer?: undefined;
+                        } | {
+                            buffer: boolean;
+                            type?: undefined;
+                        })[];
+                    };
+                    value: {
+                        oneOf: ({
+                            type: string;
+                            buffer?: undefined;
+                        } | {
+                            buffer: boolean;
+                            type?: undefined;
+                        })[];
+                    };
+                    headers: {
+                        anyOf: ({
+                            map: boolean;
+                            type?: undefined;
+                            additionalProperties?: undefined;
+                        } | {
+                            type: string;
+                            additionalProperties: boolean;
+                            map?: undefined;
+                        })[];
+                    };
+                    topic: {
+                        type: string;
+                    };
+                    partition: {
+                        type: string;
+                    };
+                    timestamp: {
+                        bigint: boolean;
+                    };
+                };
+                required: string[];
+                additionalProperties: boolean;
+            };
+        };
+    };
+    required: string[];
+    additionalProperties: boolean;
+};
+export declare const sendOptionsValidator: import("ajv").ValidateFunction<{
+    [x: string]: {};
+}>;

package/dist/clients/producer/options.js

@@ -0,0 +1,47 @@
+import { ProduceAcks } from "../../apis/enumerations.js";
+import { compressionsAlgorithms } from "../../protocol/compression.js";
+import { messageSchema } from "../../protocol/records.js";
+import { ajv, enumErrorMessage } from "../../utils.js";
+import { serdeProperties } from "../serde.js";
+const produceOptionsProperties = {
+    producerId: { bigint: true },
+    producerEpoch: { type: 'number' },
+    idempotent: { type: 'boolean' },
+    acks: {
+        type: 'number',
+        enum: Object.values(ProduceAcks),
+        errorMessage: enumErrorMessage(ProduceAcks)
+    },
+    compression: {
+        type: 'string',
+        enum: Object.keys(compressionsAlgorithms),
+        errorMessage: enumErrorMessage(compressionsAlgorithms, true)
+    },
+    partitioner: { function: true },
+    autocreateTopics: { type: 'boolean' },
+    repeatOnStaleMetadata: { type: 'boolean' }
+};
+export const produceOptionsSchema = {
+    type: 'object',
+    properties: produceOptionsProperties,
+    additionalProperties: false
+};
+export const produceOptionsValidator = ajv.compile(produceOptionsSchema);
+export const producerOptionsValidator = ajv.compile({
+    type: 'object',
+    properties: {
+        ...produceOptionsProperties,
+        serializers: serdeProperties
+    },
+    additionalProperties: true
+});
+export const sendOptionsSchema = {
+    type: 'object',
+    properties: {
+        messages: { type: 'array', items: messageSchema },
+        ...produceOptionsProperties
+    },
+    required: ['messages'],
+    additionalProperties: false
+};
+export const sendOptionsValidator = ajv.compile(sendOptionsSchema);

package/dist/clients/producer/producer.d.ts

@@ -0,0 +1,13 @@
+import { Base } from '../base/base.ts';
+import { type CallbackWithPromise } from '../callbacks.ts';
+import { type ProduceOptions, type ProduceResult, type ProducerInfo, type ProducerOptions, type SendOptions } from './types.ts';
+export declare class Producer<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends Base<ProducerOptions<Key, Value, HeaderKey, HeaderValue>> {
+    #private;
+    constructor(options: ProducerOptions<Key, Value, HeaderKey, HeaderValue>);
+    get producerId(): bigint | undefined;
+    get producerEpoch(): number | undefined;
+    initIdempotentProducer(options: ProduceOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<ProducerInfo>): void;
+    initIdempotentProducer(options: ProduceOptions<Key, Value, HeaderKey, HeaderValue>): Promise<ProducerInfo>;
+    send(options: SendOptions<Key, Value, HeaderKey, HeaderValue>, callback: CallbackWithPromise<ProduceResult>): void;
+    send(options: SendOptions<Key, Value, HeaderKey, HeaderValue>): Promise<ProduceResult>;
+}

package/dist/clients/producer/producer.js

@@ -0,0 +1,272 @@
+import { ProduceAcks } from "../../apis/enumerations.js";
+import { api as initProducerIdV5 } from "../../apis/producer/init-producer-id.js";
+import { api as produceV11 } from "../../apis/producer/produce.js";
+import { UserError } from "../../errors.js";
+import { murmur2 } from "../../protocol/murmur2.js";
+import { NumericMap } from "../../utils.js";
+import { Base, kBootstrapBrokers, kCheckNotClosed, kClearMetadata, kConnections, kMetadata, kOptions, kPerformDeduplicated, kPerformWithRetry, kValidateOptions } from "../base/base.js";
+import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../callbacks.js";
+import { produceOptionsValidator, producerOptionsValidator, sendOptionsValidator } from "./options.js";
+// Don't move this function as being in the same file will enable V8 to remove.
+// For futher info, ask Matteo.
+function noopSerializer(data) {
+    return data;
+}
+export class Producer extends Base {
+    #partitionsRoundRobin;
+    // These two values should be serializable and loadable in the constructor in order to restore
+    // the idempotent producer status.
+    #producerInfo;
+    #sequences;
+    #keySerializer;
+    #valueSerializer;
+    #headerKeySerializer;
+    #headerValueSerializer;
+    constructor(options) {
+        if (options.idempotent) {
+            options.maxInflights = 1;
+            options.acks = ProduceAcks.ALL;
+            options.retries = Number.MAX_SAFE_INTEGER;
+        }
+        options.repeatOnStaleMetadata ??= true;
+        super(options);
+        this.#partitionsRoundRobin = new NumericMap();
+        this.#sequences = new NumericMap();
+        this.#keySerializer = options.serializers?.key ?? noopSerializer;
+        this.#valueSerializer = options.serializers?.value ?? noopSerializer;
+        this.#headerKeySerializer = options.serializers?.headerKey ?? noopSerializer;
+        this.#headerValueSerializer = options.serializers?.headerValue ?? noopSerializer;
+        this[kValidateOptions](options, producerOptionsValidator, '/options');
+    }
+    get producerId() {
+        return this.#producerInfo?.producerId;
+    }
+    get producerEpoch() {
+        return this.#producerInfo?.producerEpoch;
+    }
+    initIdempotentProducer(options, callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        if (this[kCheckNotClosed](callback)) {
+            return callback[kCallbackPromise];
+        }
+        const validationError = this[kValidateOptions](options, produceOptionsValidator, '/options', false);
+        if (validationError) {
+            callback(validationError, undefined);
+            return callback[kCallbackPromise];
+        }
+        return this[kPerformDeduplicated]('initProducerId', deduplicateCallback => {
+            this[kPerformWithRetry]('initProducerId', retryCallback => {
+                this[kConnections].getFirstAvailable(this[kBootstrapBrokers], (error, connection) => {
+                    if (error) {
+                        callback(error, undefined);
+                        return;
+                    }
+                    initProducerIdV5(connection, null, this[kOptions].timeout, options.producerId ?? this[kOptions].producerId ?? 0n, options.producerEpoch ?? this[kOptions].producerEpoch ?? 0, retryCallback);
+                });
+            }, (error, response) => {
+                if (error) {
+                    callback(error, undefined);
+                    return;
+                }
+                this.#producerInfo = { producerId: response.producerId, producerEpoch: response.producerEpoch };
+                deduplicateCallback(null, this.#producerInfo);
+            }, 0);
+        }, callback);
+    }
+    send(options, callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        if (this[kCheckNotClosed](callback)) {
+            return callback[kCallbackPromise];
+        }
+        const validationError = this[kValidateOptions](options, sendOptionsValidator, '/options', false);
+        if (validationError) {
+            callback(validationError, undefined);
+            return callback[kCallbackPromise];
+        }
+        options.idempotent ??= this[kOptions].idempotent ?? false;
+        /* c8 ignore next */
+        options.repeatOnStaleMetadata ??= this[kOptions].repeatOnStaleMetadata ?? true;
+        options.partitioner ??= this[kOptions].partitioner;
+        const { idempotent, partitioner } = options;
+        if (idempotent) {
+            options.acks ??= ProduceAcks.ALL;
+        }
+        else {
+            options.acks ??= ProduceAcks.LEADER;
+        }
+        // We still need to initialize the producerId
+        if (idempotent) {
+            if (!this.#producerInfo) {
+                const { messages, ...initOptions } = options;
+                this.initIdempotentProducer(initOptions, error => {
+                    if (error) {
+                        callback(error, undefined);
+                        return;
+                    }
+                    this.send(options, callback);
+                });
+                return callback[kCallbackPromise];
+            }
+            if (typeof options.producerId !== 'undefined' || typeof options.producerEpoch !== 'undefined') {
+                callback(new UserError('Cannot specify producerId or producerEpoch when using idempotent producer.'), undefined);
+                return callback[kCallbackPromise];
+            }
+            if (options.acks !== ProduceAcks.ALL) {
+                callback(new UserError('Idempotent producer requires acks to be ALL (-1).'), undefined);
+                return callback[kCallbackPromise];
+            }
+        }
+        const produceOptions = {
+            compression: options.compression ?? this[kOptions].compression,
+            producerId: idempotent ? this.#producerInfo.producerId : options.producerId,
+            producerEpoch: idempotent ? this.#producerInfo.producerEpoch : options.producerEpoch,
+            sequences: idempotent ? this.#sequences : undefined
+        };
+        // Build messages records out of messages
+        const topics = new Set();
+        const messages = [];
+        for (const message of options.messages) {
+            const topic = message.topic;
+            const key = this.#keySerializer(message.key);
+            const headers = new Map();
+            let partition = 0;
+            if (typeof message.partition !== 'number') {
+                if (partitioner) {
+                    partition = partitioner(message);
+                }
+                else if (key) {
+                    partition = murmur2(key) >>> 0;
+                }
+                else {
+                    // Use the roundrobin
+                    partition = this.#partitionsRoundRobin.postIncrement(topic, 1, -1);
+                }
+            }
+            else {
+                partition = message.partition;
+            }
+            if (message.headers) {
+                const entries = message.headers instanceof Map ? message.headers : Object.entries(message.headers);
+                for (const [key, value] of entries) {
+                    headers.set(this.#headerKeySerializer(key), this.#headerValueSerializer(value));
+                }
+            }
+            topics.add(topic);
+            messages.push({
+                key,
+                value: this.#valueSerializer(message.value),
+                headers,
+                topic,
+                partition,
+                timestamp: message.timestamp
+            });
+        }
+        this.#performSend(Array.from(topics), messages, options, produceOptions, callback);
+        return callback[kCallbackPromise];
+    }
+    #performSend(topics, messages, sendOptions, produceOptions, callback) {
+        // Get the metadata with the topic/partitions informations
+        this[kMetadata]({ topics, autocreateTopics: sendOptions.autocreateTopics }, (error, metadata) => {
+            if (error) {
+                callback(error, undefined);
+                return;
+            }
+            const messagesByDestination = new Map();
+            // Track the number of messages per partition so we can update the sequence number
+            const messagesPerPartition = new NumericMap();
+            // Normalize the partition of all messages, then enqueue them to their destination
+            for (const message of messages) {
+                message.partition %= metadata.topics.get(message.topic).partitionsCount;
+                const { topic, partition } = message;
+                const leader = metadata.topics.get(topic).partitions[partition].leader;
+                let destination = messagesByDestination.get(leader);
+                if (!destination) {
+                    destination = [];
+                    messagesByDestination.set(leader, destination);
+                }
+                const messagePartitionKey = `${message.topic}:${partition}`;
+                messagesPerPartition.postIncrement(messagePartitionKey, 1, 0);
+                destination.push(message);
+            }
+            // Track nodes so that we can get their ID for delayed write reporting
+            const nodes = [];
+            runConcurrentCallbacks('Producing messages failed.', messagesByDestination, ([destination, messages], concurrentCallback) => {
+                nodes.push(destination);
+                this.#performSingleDestinationSend(topics, messages, this[kOptions].timeout, sendOptions.acks, sendOptions.autocreateTopics, sendOptions.repeatOnStaleMetadata, produceOptions, concurrentCallback);
+            }, (error, apiResults) => {
+                if (error) {
+                    callback(error, undefined);
+                    return;
+                }
+                const results = {};
+                if (sendOptions.acks === ProduceAcks.NO_RESPONSE) {
+                    const unwritableNodes = [];
+                    for (let i = 0; i < apiResults.length; i++) {
+                        if (apiResults[i] === false) {
+                            unwritableNodes.push(nodes[i]);
+                        }
+                    }
+                    results.unwritableNodes = unwritableNodes;
+                }
+                else {
+                    const topics = [];
+                    for (const result of apiResults) {
+                        for (const { name, partitionResponses } of result.responses) {
+                            for (const partitionResponse of partitionResponses) {
+                                topics.push({
+                                    topic: name,
+                                    partition: partitionResponse.index,
+                                    offset: partitionResponse.baseOffset
+                                });
+                                const partitionKey = `${name}:${partitionResponse.index}`;
+                                this.#sequences.postIncrement(partitionKey, messagesPerPartition.get(partitionKey), 0);
+                            }
+                        }
+                        results.offsets = topics;
+                    }
+                }
+                callback(null, results);
+            });
+        });
+    }
+    #performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, repeatOnStaleMetadata, produceOptions, callback) {
+        // Get the metadata with the topic/partitions informations
+        this[kMetadata]({ topics, autocreateTopics }, (error, metadata) => {
+            if (error) {
+                callback(error, undefined);
+                return;
+            }
+            const { topic, partition } = messages[0];
+            const leader = metadata.topics.get(topic).partitions[partition].leader;
+            this[kPerformWithRetry]('produce', retryCallback => {
+                this[kConnections].get(metadata.brokers.get(leader), (error, connection) => {
+                    if (error) {
+                        retryCallback(error, undefined);
+                        return;
+                    }
+                    produceV11(connection, acks, timeout, messages, produceOptions, retryCallback);
+                });
+            }, (error, results) => {
+                if (error) {
+                    // If the last error was due to stale metadata, we retry the operation with this set of messages
+                    // since the partition is already set, it should attempt on the new destination
+                    const hasStaleMetadata = error.findBy('hasStaleMetadata', true);
+                    if (hasStaleMetadata && repeatOnStaleMetadata) {
+                        this[kClearMetadata]();
+                        this.#performSingleDestinationSend(topics, messages, timeout, acks, autocreateTopics, false, produceOptions, callback);
+                        return;
+                    }
+                    callback(error, undefined);
+                    return;
+                }
+                callback(error, results);
+            }, 0, [], (error) => {
+                return repeatOnStaleMetadata && !!error.findBy('hasStaleMetadata', true);
+            });
+        });
+    }
+}

package/dist/clients/producer/types.d.ts

@@ -0,0 +1,29 @@
+import { type CompressionAlgorithms } from '../../protocol/compression.ts';
+import { type MessageToProduce } from '../../protocol/records.ts';
+import { type BaseOptions, type TopicWithPartitionAndOffset } from '../base/types.ts';
+import { type Serializers } from '../serde.ts';
+export interface ProducerInfo {
+    producerId: bigint;
+    producerEpoch: number;
+}
+export interface ProduceResult {
+    offsets?: TopicWithPartitionAndOffset[];
+    unwritableNodes?: number[];
+}
+export type Partitioner<Key, Value, HeaderKey, HeaderValue> = (message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>) => number;
+export interface ProduceOptions<Key, Value, HeaderKey, HeaderValue> {
+    producerId?: bigint;
+    producerEpoch?: number;
+    idempotent?: boolean;
+    acks?: number;
+    compression?: CompressionAlgorithms;
+    partitioner?: Partitioner<Key, Value, HeaderKey, HeaderValue>;
+    autocreateTopics?: boolean;
+    repeatOnStaleMetadata?: boolean;
+}
+export type ProducerOptions<Key, Value, HeaderKey, HeaderValue> = BaseOptions & ProduceOptions<Key, Value, HeaderKey, HeaderValue> & {
+    serializers?: Partial<Serializers<Key, Value, HeaderKey, HeaderValue>>;
+};
+export type SendOptions<Key, Value, HeaderKey, HeaderValue> = {
+    messages: MessageToProduce<Key, Value, HeaderKey, HeaderValue>[];
+} & ProduceOptions<Key, Value, HeaderKey, HeaderValue>;

package/dist/clients/producer/types.js

@@ -0,0 +1 @@
+export {};