@platformatic/kafka 0.1.0

package/dist/clients/admin/options.js

@@ -0,0 +1,81 @@
+import { ConsumerGroupStates } from "../../apis/enumerations.js";
+import { ajv, listErrorMessage } from "../../utils.js";
+import { idProperty } from "../base/options.js";
+export const groupsProperties = {
+    groups: {
+        type: 'array',
+        items: idProperty,
+        minItems: 1
+    }
+};
+export const createTopicOptionsSchema = {
+    type: 'object',
+    properties: {
+        topics: { type: 'array', items: idProperty },
+        partitions: { type: 'number' },
+        replicas: { type: 'number' },
+        assignments: {
+            type: 'array',
+            items: {
+                type: 'object',
+                properties: {
+                    partition: { type: 'number', minimum: 0 },
+                    brokers: { type: 'array', items: { type: 'number' }, minItems: 1 }
+                },
+                required: ['partition', 'brokers'],
+                additionalProperties: false
+            },
+            minItems: 1
+        }
+    },
+    required: ['topics'],
+    additionalProperties: false
+};
+export const deleteTopicOptionsSchema = {
+    type: 'object',
+    properties: {
+        topics: { type: 'array', items: idProperty }
+    },
+    required: ['topics'],
+    additionalProperties: false
+};
+export const listGroupsOptionsSchema = {
+    type: 'object',
+    properties: {
+        states: {
+            type: 'array',
+            items: {
+                type: 'string',
+                enum: ConsumerGroupStates,
+                errorMessage: listErrorMessage(ConsumerGroupStates)
+            },
+            minItems: 0
+        },
+        types: {
+            type: 'array',
+            items: idProperty,
+            minItems: 0
+        }
+    },
+    additionalProperties: false
+};
+export const describeGroupsOptionsSchema = {
+    type: 'object',
+    properties: {
+        ...groupsProperties,
+        includeAuthorizedOperations: { type: 'boolean' }
+    },
+    required: ['groups'],
+    additionalProperties: false
+};
+export const deleteGroupsOptionsSchema = {
+    type: 'object',
+    properties: groupsProperties,
+    required: ['groups'],
+    additionalProperties: false
+};
+export const createTopicsOptionsValidator = ajv.compile(createTopicOptionsSchema);
+export const deleteTopicsOptionsValidator = ajv.compile(deleteTopicOptionsSchema);
+export const listGroupsOptionsValidator = ajv.compile(listGroupsOptionsSchema);
+export const describeGroupsOptionsValidator = ajv.compile(describeGroupsOptionsSchema);
+export const deleteGroupsOptionsValidator = ajv.compile(deleteGroupsOptionsSchema);

package/dist/clients/admin/types.d.ts

@@ -0,0 +1,56 @@
+import { type ConsumerGroupState } from '../../apis/enumerations.ts';
+import { type NullableString } from '../../protocol/definitions.ts';
+import { type BaseOptions } from '../base/types.ts';
+import { type ExtendedGroupProtocolSubscription, type GroupAssignment } from '../consumer/types.ts';
+export interface BrokerAssignment {
+    partition: number;
+    brokers: number[];
+}
+export interface CreatedTopic {
+    id: string;
+    name: string;
+    partitions: number;
+    replicas: number;
+    configuration: Record<string, NullableString>;
+}
+export interface GroupMember {
+    id: string;
+    groupInstanceId: NullableString;
+    clientId: string;
+    clientHost: string;
+    metadata: Omit<ExtendedGroupProtocolSubscription, 'memberId'>;
+    assignments: Map<string, GroupAssignment>;
+}
+export interface GroupBase {
+    id: string;
+    state: ConsumerGroupState;
+    groupType: string;
+    protocolType: string;
+}
+export interface Group extends Omit<GroupBase, 'groupType'> {
+    protocol: string;
+    members: Map<string, GroupMember>;
+    authorizedOperations: number;
+}
+export interface AdminOptions extends BaseOptions {
+}
+export interface CreateTopicsOptions {
+    topics: string[];
+    partitions?: number;
+    replicas?: number;
+    assignments?: BrokerAssignment[];
+}
+export interface DeleteTopicsOptions {
+    topics: string[];
+}
+export interface ListGroupsOptions {
+    states?: ConsumerGroupState[];
+    types?: string[];
+}
+export interface DescribeGroupsOptions {
+    groups: string[];
+    includeAuthorizedOperations?: boolean;
+}
+export interface DeleteGroupsOptions {
+    groups: string[];
+}

package/dist/clients/admin/types.js

@@ -0,0 +1 @@
+export {};

package/dist/clients/base/base.d.ts

@@ -0,0 +1,48 @@
+import { type ValidateFunction } from 'ajv';
+import { EventEmitter } from 'node:events';
+import { type Callback } from '../../apis/definitions.ts';
+import { ConnectionPool } from '../../network/connection-pool.ts';
+import { type Broker } from '../../network/connection.ts';
+import { type CallbackWithPromise } from '../callbacks.ts';
+import { type BaseOptions, type ClusterMetadata, type MetadataOptions } from './types.ts';
+export declare const kClientId: unique symbol;
+export declare const kBootstrapBrokers: unique symbol;
+export declare const kOptions: unique symbol;
+export declare const kConnections: unique symbol;
+export declare const kCreateConnectionPool: unique symbol;
+export declare const kClosed: unique symbol;
+export declare const kMetadata: unique symbol;
+export declare const kCheckNotClosed: unique symbol;
+export declare const kClearMetadata: unique symbol;
+export declare const kParseBroker: unique symbol;
+export declare const kPerformWithRetry: unique symbol;
+export declare const kPerformDeduplicated: unique symbol;
+export declare const kValidateOptions: unique symbol;
+export declare const kInspect: unique symbol;
+export declare const kInstance: unique symbol;
+export declare class Base<OptionsType extends BaseOptions> extends EventEmitter {
+    #private;
+    [kInstance]: number;
+    [kClientId]: string;
+    [kBootstrapBrokers]: Broker[];
+    [kOptions]: OptionsType;
+    [kConnections]: ConnectionPool;
+    [kClosed]: boolean;
+    constructor(options: OptionsType);
+    get clientId(): string;
+    get closed(): boolean;
+    emitWithDebug(section: string | null, name: string, ...args: any[]): boolean;
+    close(callback: CallbackWithPromise<void>): void;
+    close(): Promise<void>;
+    metadata(options: MetadataOptions, callback: CallbackWithPromise<ClusterMetadata>): void;
+    metadata(options: MetadataOptions): Promise<ClusterMetadata>;
+    [kCreateConnectionPool](): ConnectionPool;
+    [kMetadata](options: MetadataOptions, callback: CallbackWithPromise<ClusterMetadata>): void;
+    [kCheckNotClosed](callback: CallbackWithPromise<any>): boolean;
+    [kClearMetadata](): void;
+    [kParseBroker](broker: Broker | string): Broker;
+    [kPerformWithRetry]<ReturnType>(operationId: string, operation: (callback: Callback<ReturnType>) => void, callback: CallbackWithPromise<ReturnType>, attempt?: number, errors?: Error[], shouldSkipRetry?: (e: Error) => boolean): void | Promise<ReturnType>;
+    [kPerformDeduplicated]<ReturnType>(operationId: string, operation: (callback: CallbackWithPromise<ReturnType>) => void, callback: CallbackWithPromise<ReturnType>): void | Promise<ReturnType>;
+    [kValidateOptions](target: unknown, validator: ValidateFunction<unknown>, targetName: string, throwOnErrors?: boolean): Error | null;
+    [kInspect](...args: unknown[]): void;
+}

package/dist/clients/base/base.js

@@ -0,0 +1,242 @@
+import { EventEmitter } from 'node:events';
+import { api as metadataV12 } from "../../apis/metadata/metadata.js";
+import { MultipleErrors, NetworkError, UserError } from "../../errors.js";
+import { ConnectionPool } from "../../network/connection-pool.js";
+import { ajv, debugDump, loggers } from "../../utils.js";
+import { createPromisifiedCallback, kCallbackPromise } from "../callbacks.js";
+import { baseOptionsValidator, defaultBaseOptions, defaultPort, metadataOptionsValidator } from "./options.js";
+export const kClientId = Symbol('plt.kafka.base.clientId');
+export const kBootstrapBrokers = Symbol('plt.kafka.base.bootstrapBrokers');
+export const kOptions = Symbol('plt.kafka.base.options');
+export const kConnections = Symbol('plt.kafka.base.kConnections');
+export const kCreateConnectionPool = Symbol('plt.kafka.base.kCreateConnectionPool');
+export const kClosed = Symbol('plt.kafka.base.kClosed');
+export const kMetadata = Symbol('plt.kafka.base.metadata');
+export const kCheckNotClosed = Symbol('plt.kafka.base.checkNotClosed');
+export const kClearMetadata = Symbol('plt.kafka.base.clearMetadata');
+export const kParseBroker = Symbol('plt.kafka.base.parseBroker');
+export const kPerformWithRetry = Symbol('plt.kafka.base.performWithRetry');
+export const kPerformDeduplicated = Symbol('plt.kafka.base.performDeduplicated');
+export const kValidateOptions = Symbol('plt.kafka.base.validateOptions');
+export const kInspect = Symbol('plt.kafka.base.inspect');
+export const kInstance = Symbol('plt.kafka.base.instance');
+let currentInstance = 0;
+export class Base extends EventEmitter {
+    // This is just used for debugging
+    [kInstance];
+    // General status - Use symbols rather than JS private property to make them "protected" as in C++
+    [kClientId];
+    [kBootstrapBrokers];
+    [kOptions];
+    [kConnections];
+    [kClosed];
+    #metadata;
+    #inflightDeduplications;
+    constructor(options) {
+        super();
+        this[kInstance] = currentInstance++;
+        // Validate options
+        this[kOptions] = Object.assign({}, defaultBaseOptions, options);
+        this[kValidateOptions](this[kOptions], baseOptionsValidator, '/options');
+        this[kClientId] = options.clientId;
+        // Initialize bootstrap brokers
+        this[kBootstrapBrokers] = [];
+        for (const broker of options.bootstrapBrokers) {
+            this[kBootstrapBrokers].push(this[kParseBroker](broker));
+        }
+        // Initialize main connection pool
+        this[kConnections] = this[kCreateConnectionPool]();
+        this[kClosed] = false;
+        this.#inflightDeduplications = new Map();
+    }
+    /* c8 ignore next 3 */
+    get clientId() {
+        return this[kClientId];
+    }
+    /* c8 ignore next 3 */
+    get closed() {
+        return this[kClosed] === true;
+    }
+    emitWithDebug(section, name, ...args) {
+        if (!section) {
+            return this.emit(name, ...args);
+        }
+        /* c8 ignore next */
+        loggers[section]?.({ event: name, payload: args });
+        return this.emit(`${section}:${name}`, ...args);
+    }
+    close(callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        this[kClosed] = true;
+        this[kConnections].close(callback);
+        return callback[kCallbackPromise];
+    }
+    metadata(options, callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        const validationError = this[kValidateOptions](options, metadataOptionsValidator, '/options', false);
+        if (validationError) {
+            callback(validationError, undefined);
+            return callback[kCallbackPromise];
+        }
+        this[kMetadata](options, callback);
+        return callback[kCallbackPromise];
+    }
+    [kCreateConnectionPool]() {
+        const pool = new ConnectionPool(this[kClientId], {
+            ownerId: this[kInstance],
+            ...this[kOptions]
+        });
+        for (const event of ['connect', 'disconnect', 'failed', 'drain']) {
+            pool.on(event, payload => this.emitWithDebug('client', `broker:${event}`, payload));
+        }
+        return pool;
+    }
+    [kMetadata](options, callback) {
+        const metadataMaxAge = options.metadataMaxAge ?? this[kOptions].metadataMaxAge;
+        const isStale = options.forceUpdate ||
+            !this.#metadata ||
+            Date.now() > this.#metadata.lastUpdate + metadataMaxAge ||
+            options.topics.some(topic => !this.#metadata?.topics.has(topic));
+        if (!isStale) {
+            callback(null, this.#metadata);
+            return;
+        }
+        const autocreateTopics = options.autocreateTopics ?? this[kOptions].autocreateTopics;
+        this[kPerformDeduplicated]('metadata', deduplicateCallback => {
+            this[kPerformWithRetry]('metadata', retryCallback => {
+                this[kConnections].getFirstAvailable(this[kBootstrapBrokers], (error, connection) => {
+                    if (error) {
+                        retryCallback(error, undefined);
+                        return;
+                    }
+                    metadataV12(connection, options.topics, autocreateTopics, true, retryCallback);
+                });
+            }, (error, metadata) => {
+                if (error) {
+                    deduplicateCallback(error, undefined);
+                    return;
+                }
+                const brokers = new Map();
+                const topics = new Map();
+                for (const broker of metadata.brokers) {
+                    const { host, port } = broker;
+                    brokers.set(broker.nodeId, { host, port });
+                }
+                for (const { name, topicId: id, partitions: rawPartitions, isInternal } of metadata.topics) {
+                    /* c8 ignore next 3 */
+                    if (isInternal) {
+                        continue;
+                    }
+                    const partitions = [];
+                    for (const rawPartition of rawPartitions.sort((a, b) => a.partitionIndex - b.partitionIndex)) {
+                        partitions[rawPartition.partitionIndex] = {
+                            leader: rawPartition.leaderId,
+                            leaderEpoch: rawPartition.leaderEpoch,
+                            replicas: rawPartition.replicaNodes
+                        };
+                    }
+                    topics.set(name, { id, partitions, partitionsCount: rawPartitions.length });
+                }
+                this.#metadata = {
+                    id: metadata.clusterId,
+                    brokers,
+                    topics,
+                    lastUpdate: Date.now()
+                };
+                this.emitWithDebug('client', 'metadata', this.#metadata);
+                deduplicateCallback(null, this.#metadata);
+            }, 0);
+        }, callback);
+    }
+    [kCheckNotClosed](callback) {
+        if (this[kClosed]) {
+            const error = new NetworkError('Client is closed.', { closed: true, instance: this[kInstance] });
+            callback(error, undefined);
+            return true;
+        }
+        return false;
+    }
+    [kClearMetadata]() {
+        this.#metadata = undefined;
+    }
+    [kParseBroker](broker) {
+        if (typeof broker === 'string') {
+            if (broker.includes(':')) {
+                const [host, port] = broker.split(':');
+                return { host, port: Number(port) };
+            }
+            else {
+                return { host: broker, port: defaultPort };
+            }
+        }
+        return broker;
+    }
+    [kPerformWithRetry](operationId, operation, callback, attempt = 0, errors = [], shouldSkipRetry) {
+        const retries = this[kOptions].retries;
+        this.emitWithDebug('client', 'performWithRetry', operationId, attempt, retries);
+        operation((error, result) => {
+            if (error) {
+                const genericError = error;
+                const retriable = genericError.findBy?.('code', NetworkError.code) || genericError.findBy?.('canRetry', true);
+                errors.push(error);
+                if (attempt < retries && retriable && !shouldSkipRetry?.(error)) {
+                    setTimeout(() => {
+                        this[kPerformWithRetry](operationId, operation, callback, attempt + 1, errors, shouldSkipRetry);
+                    }, this[kOptions].retryDelay);
+                }
+                else {
+                    if (attempt === 0) {
+                        callback(error, undefined);
+                        return;
+                    }
+                    callback(new MultipleErrors(`${operationId} failed ${attempt + 1} times.`, errors), undefined);
+                }
+                return;
+            }
+            callback(null, result);
+        });
+        return callback[kCallbackPromise];
+    }
+    [kPerformDeduplicated](operationId, operation, callback) {
+        let inflights = this.#inflightDeduplications.get(operationId);
+        if (!inflights) {
+            inflights = [];
+            this.#inflightDeduplications.set(operationId, inflights);
+        }
+        inflights.push(callback);
+        if (inflights.length === 1) {
+            this.emitWithDebug('client', 'performDeduplicated', operationId);
+            operation((error, result) => {
+                this.#inflightDeduplications.set(operationId, []);
+                for (const cb of inflights) {
+                    cb(error, result);
+                }
+                inflights = [];
+            });
+        }
+        return callback[kCallbackPromise];
+    }
+    [kValidateOptions](target, validator, targetName, throwOnErrors = true) {
+        if (!this[kOptions].strict) {
+            return null;
+        }
+        const valid = validator(target);
+        if (!valid) {
+            const error = new UserError(ajv.errorsText(validator.errors, { dataVar: targetName }) + '.');
+            if (throwOnErrors) {
+                throw error;
+            }
+            return error;
+        }
+        return null;
+    }
+    // This is a private API used to debug during development
+    /* c8 ignore next 3 */
+    [kInspect](...args) {
+        debugDump(`client:${this[kInstance]}`, ...args);
+    }
+}

package/dist/clients/base/index.d.ts

@@ -0,0 +1,3 @@
+export { Base } from './base.ts';
+export * from './options.ts';
+export * from './types.ts';

package/dist/clients/base/index.js

@@ -0,0 +1,3 @@
+export { Base } from "./base.js";
+export * from "./options.js";
+export * from "./types.js";

package/dist/clients/base/options.d.ts

@@ -0,0 +1,115 @@
+import { type BaseOptions } from './types.ts';
+export declare const idProperty: {
+    type: string;
+    pattern: string;
+};
+export declare const topicWithPartitionAndOffsetProperties: {
+    topic: {
+        type: string;
+        pattern: string;
+    };
+    partition: {
+        type: string;
+        minimum: number;
+    };
+    offset: {
+        bigint: boolean;
+    };
+};
+export declare const baseOptionsSchema: {
+    type: string;
+    properties: {
+        clientId: {
+            type: string;
+            pattern: string;
+        };
+        bootstrapBrokers: {
+            oneOf: ({
+                type: string;
+                items: {
+                    type: string;
+                    properties?: undefined;
+                };
+            } | {
+                type: string;
+                items: {
+                    type: string;
+                    properties: {
+                        host: {
+                            type: string;
+                        };
+                        port: {
+                            type: string;
+                            minimum: number;
+                            maximum: number;
+                        };
+                    };
+                };
+            })[];
+        };
+        timeout: {
+            type: string;
+            minimum: number;
+        };
+        connectTimeout: {
+            type: string;
+            minimum: number;
+        };
+        retries: {
+            type: string;
+            minimum: number;
+        };
+        retryDelay: {
+            type: string;
+            minimum: number;
+        };
+        maxInflights: {
+            type: string;
+            minimum: number;
+        };
+        metadataMaxAge: {
+            type: string;
+            minimum: number;
+        };
+        autocreateTopics: {
+            type: string;
+        };
+        strict: {
+            type: string;
+        };
+    };
+    required: string[];
+    additionalProperties: boolean;
+};
+export declare const metadataOptionsSchema: {
+    type: string;
+    properties: {
+        topics: {
+            type: string;
+            items: {
+                type: string;
+                pattern: string;
+            };
+        };
+        autocreateTopics: {
+            type: string;
+        };
+        forceUpdate: {
+            type: string;
+        };
+        metadataMaxAge: {
+            type: string;
+            minimum: number;
+        };
+    };
+    required: string[];
+    additionalProperties: boolean;
+};
+export declare const baseOptionsValidator: import("ajv").ValidateFunction<{
+    [x: string]: {};
+}>;
+export declare const metadataOptionsValidator: import("ajv").ValidateFunction<{
+    [x: string]: {};
+}>;
+export declare const defaultPort = 9092;
+export declare const defaultBaseOptions: Partial<BaseOptions>;

package/dist/clients/base/options.js

@@ -0,0 +1,59 @@
+import { ajv } from "../../utils.js";
+export const idProperty = { type: 'string', pattern: '^\\S+$' };
+export const topicWithPartitionAndOffsetProperties = {
+    topic: idProperty,
+    partition: { type: 'number', minimum: 0 },
+    offset: { bigint: true }
+};
+export const baseOptionsSchema = {
+    type: 'object',
+    properties: {
+        clientId: idProperty,
+        bootstrapBrokers: {
+            oneOf: [
+                { type: 'array', items: { type: 'string' } },
+                {
+                    type: 'array',
+                    items: {
+                        type: 'object',
+                        properties: { host: { type: 'string' }, port: { type: 'number', minimum: 0, maximum: 65535 } }
+                    }
+                }
+            ]
+        },
+        timeout: { type: 'number', minimum: 0 },
+        connectTimeout: { type: 'number', minimum: 0 },
+        retries: { type: 'number', minimum: 0 },
+        retryDelay: { type: 'number', minimum: 0 },
+        maxInflights: { type: 'number', minimum: 0 },
+        metadataMaxAge: { type: 'number', minimum: 0 },
+        autocreateTopics: { type: 'boolean' },
+        strict: { type: 'boolean' }
+    },
+    required: ['clientId', 'bootstrapBrokers'],
+    additionalProperties: true
+};
+export const metadataOptionsSchema = {
+    type: 'object',
+    properties: {
+        topics: { type: 'array', items: idProperty },
+        autocreateTopics: { type: 'boolean' },
+        forceUpdate: { type: 'boolean' },
+        metadataMaxAge: { type: 'number', minimum: 0 }
+    },
+    required: ['topics'],
+    additionalProperties: false
+};
+export const baseOptionsValidator = ajv.compile(baseOptionsSchema);
+export const metadataOptionsValidator = ajv.compile(metadataOptionsSchema);
+export const defaultPort = 9092;
+export const defaultBaseOptions = {
+    connectTimeout: 5000,
+    maxInflights: 5,
+    timeout: 5000,
+    retries: 3,
+    retryDelay: 1000,
+    metadataMaxAge: 5000, // 5 minutes
+    autocreateTopics: false,
+    strict: false
+};

package/dist/clients/base/types.d.ts

@@ -0,0 +1,38 @@
+import { type Broker, type ConnectionOptions } from '../../network/connection.ts';
+export interface TopicWithPartitionAndOffset {
+    topic: string;
+    partition: number;
+    offset: bigint;
+}
+export interface ClusterPartitionMetadata {
+    leader: number;
+    leaderEpoch: number;
+    replicas: number[];
+}
+export interface ClusterTopicMetadata {
+    id: string;
+    partitions: ClusterPartitionMetadata[];
+    partitionsCount: number;
+}
+export interface ClusterMetadata {
+    id: string;
+    brokers: Map<number, Broker>;
+    topics: Map<string, ClusterTopicMetadata>;
+    lastUpdate: number;
+}
+export interface BaseOptions extends ConnectionOptions {
+    clientId: string;
+    bootstrapBrokers: Broker[] | string[];
+    timeout?: number;
+    retries?: number;
+    retryDelay?: number;
+    metadataMaxAge?: number;
+    autocreateTopics?: boolean;
+    strict?: boolean;
+}
+export interface MetadataOptions {
+    topics: string[];
+    autocreateTopics?: boolean;
+    forceUpdate?: boolean;
+    metadataMaxAge?: number;
+}

package/dist/clients/base/types.js

@@ -0,0 +1 @@
+export {};

package/dist/clients/callbacks.d.ts

@@ -0,0 +1,8 @@
+import { type Callback } from '../apis/definitions.ts';
+export declare const kCallbackPromise: unique symbol;
+export declare const noopCallback: CallbackWithPromise<any>;
+export type CallbackWithPromise<ReturnType> = Callback<ReturnType> & {
+    [kCallbackPromise]?: Promise<ReturnType>;
+};
+export declare function createPromisifiedCallback<ReturnType>(): CallbackWithPromise<ReturnType>;
+export declare function runConcurrentCallbacks<ReturnType>(errorMessage: string, collection: unknown[] | Set<unknown> | Map<unknown, unknown>, operation: (item: any, cb: Callback<ReturnType>) => void, callback: Callback<ReturnType[]>): void;