@platformatic/kafka 0.1.0

package/dist/clients/serde.d.ts

@@ -0,0 +1,40 @@
+export type Serializer<InputType = unknown> = (data?: InputType) => Buffer | undefined;
+export type Deserializer<OutputType = unknown> = (data?: Buffer) => OutputType | undefined;
+export interface Serializers<Key, Value, HeaderKey, HeaderValue> {
+    key: Serializer<Key>;
+    value: Serializer<Value>;
+    headerKey: Serializer<HeaderKey>;
+    headerValue: Serializer<HeaderValue>;
+}
+export interface Deserializers<Key, Value, HeaderKey, HeaderValue> {
+    key: Deserializer<Key>;
+    value: Deserializer<Value>;
+    headerKey: Deserializer<HeaderKey>;
+    headerValue: Deserializer<HeaderValue>;
+}
+export declare function stringSerializer(data?: string): Buffer | undefined;
+export declare function stringDeserializer(data?: string | Buffer): string | undefined;
+export declare function jsonSerializer<T = Record<string, any>>(data?: T): Buffer | undefined;
+export declare function jsonDeserializer<T = Record<string, any>>(data?: string | Buffer): T | undefined;
+export declare function serializersFrom<T>(serializer: Serializer<T>): Serializers<T, T, T, T>;
+export declare function deserializersFrom<T>(deserializer: Deserializer<T>): Deserializers<T, T, T, T>;
+export declare const serdeProperties: {
+    type: string;
+    properties: {
+        key: {
+            function: boolean;
+        };
+        value: {
+            function: boolean;
+        };
+        headerKey: {
+            function: boolean;
+        };
+        headerValue: {
+            function: boolean;
+        };
+    };
+    additionalProperties: boolean;
+};
+export declare const stringSerializers: Serializers<string, string, string, string>;
+export declare const stringDeserializers: Deserializers<string, string, string, string>;

package/dist/clients/serde.js

@@ -0,0 +1,49 @@
+export function stringSerializer(data) {
+    if (typeof data !== 'string') {
+        return undefined;
+    }
+    return Buffer.from(data, 'utf-8');
+}
+export function stringDeserializer(data) {
+    if (!Buffer.isBuffer(data)) {
+        return undefined;
+    }
+    return data.toString('utf-8');
+}
+export function jsonSerializer(data) {
+    return Buffer.from(JSON.stringify(data), 'utf-8');
+}
+export function jsonDeserializer(data) {
+    if (!Buffer.isBuffer(data)) {
+        return undefined;
+    }
+    return JSON.parse(data.toString('utf-8'));
+}
+export function serializersFrom(serializer) {
+    return {
+        key: serializer,
+        value: serializer,
+        headerKey: serializer,
+        headerValue: serializer
+    };
+}
+export function deserializersFrom(deserializer) {
+    return {
+        key: deserializer,
+        value: deserializer,
+        headerKey: deserializer,
+        headerValue: deserializer
+    };
+}
+export const serdeProperties = {
+    type: 'object',
+    properties: {
+        key: { function: true },
+        value: { function: true },
+        headerKey: { function: true },
+        headerValue: { function: true }
+    },
+    additionalProperties: false
+};
+export const stringSerializers = serializersFrom(stringSerializer);
+export const stringDeserializers = deserializersFrom(stringDeserializer);

package/dist/errors.d.ts

@@ -0,0 +1,67 @@
+declare const kGenericError: unique symbol;
+declare const kMultipleErrors: unique symbol;
+export declare const ERROR_PREFIX = "PLT_KFK_";
+export declare const errorCodes: readonly ["PLT_KFK_AUTHENTICATION", "PLT_KFK_MULTIPLE", "PLT_KFK_NETWORK", "PLT_KFK_PROTOCOL", "PLT_KFK_RESPONSE", "PLT_KFK_TIMEOUT", "PLT_KFK_UNEXPECTED_CORRELATION_ID", "PLT_KFK_UNFINISHED_WRITE_BUFFER", "PLT_KFK_UNSUPPORTED_COMPRESSION", "PLT_KFK_UNSUPPORTED", "PLT_KFK_USER"];
+export type ErrorCode = (typeof errorCodes)[number];
+export type ErrorProperties = {
+    cause?: Error;
+} & Record<string, any>;
+export declare class GenericError extends Error {
+    code: string;
+    [index: string]: any;
+    [kGenericError]: true;
+    static isGenericError(error: Error): boolean;
+    constructor(code: ErrorCode, message: string, { cause, ...rest }?: ErrorProperties);
+    findBy<ErrorType extends GenericError = GenericError>(property: string, value: unknown): ErrorType | null;
+}
+export declare class MultipleErrors extends AggregateError {
+    code: string;
+    [index: string]: any;
+    [kGenericError]: true;
+    [kMultipleErrors]: true;
+    static code: ErrorCode;
+    static isGenericError(error: Error): boolean;
+    static isMultipleErrors(error: Error): boolean;
+    constructor(message: string, errors: Error[], { cause, ...rest }?: ErrorProperties);
+    findBy<ErrorType extends GenericError | MultipleErrors = MultipleErrors>(property: string, value: unknown): ErrorType | null;
+}
+export * from './protocol/errors.ts';
+export declare class AuthenticationError extends GenericError {
+    static code: ErrorCode;
+    constructor(message: string, properties?: ErrorProperties);
+}
+export declare class NetworkError extends GenericError {
+    static code: ErrorCode;
+    constructor(message: string, properties?: ErrorProperties);
+}
+export declare class ProtocolError extends GenericError {
+    constructor(codeOrId: string | number, properties?: ErrorProperties, response?: unknown);
+}
+export declare class ResponseError extends MultipleErrors {
+    static code: ErrorCode;
+    constructor(apiName: number, apiVersion: number, errors: Record<string, number>, response: unknown, properties?: ErrorProperties);
+}
+export declare class TimeoutError extends GenericError {
+    static code: ErrorCode;
+    constructor(message: string, properties?: ErrorProperties);
+}
+export declare class UnexpectedCorrelationIdError extends GenericError {
+    static code: ErrorCode;
+    constructor(message: string, properties?: ErrorProperties);
+}
+export declare class UnfinishedWriteBufferError extends GenericError {
+    static code: ErrorCode;
+    constructor(message: string, properties?: ErrorProperties);
+}
+export declare class UnsupportedCompressionError extends GenericError {
+    static code: ErrorCode;
+    constructor(message: string, properties?: ErrorProperties);
+}
+export declare class UnsupportedError extends GenericError {
+    static code: ErrorCode;
+    constructor(message: string, properties?: ErrorProperties);
+}
+export declare class UserError extends GenericError {
+    static code: ErrorCode;
+    constructor(message: string, properties?: ErrorProperties);
+}

package/dist/errors.js

@@ -0,0 +1,160 @@
+import { protocolAPIsById } from "./protocol/apis.js";
+import { protocolErrors, protocolErrorsCodesById } from "./protocol/errors.js";
+const kGenericError = Symbol('plt.kafka.genericError');
+const kMultipleErrors = Symbol('plt.kafka.multipleErrors');
+export const ERROR_PREFIX = 'PLT_KFK_';
+export const errorCodes = [
+    'PLT_KFK_AUTHENTICATION',
+    'PLT_KFK_MULTIPLE',
+    'PLT_KFK_NETWORK',
+    'PLT_KFK_PROTOCOL',
+    'PLT_KFK_RESPONSE',
+    'PLT_KFK_TIMEOUT',
+    'PLT_KFK_UNEXPECTED_CORRELATION_ID',
+    'PLT_KFK_UNFINISHED_WRITE_BUFFER',
+    'PLT_KFK_UNSUPPORTED_COMPRESSION',
+    'PLT_KFK_UNSUPPORTED',
+    'PLT_KFK_USER'
+];
+export class GenericError extends Error {
+    code;
+    [kGenericError];
+    static isGenericError(error) {
+        return error[kGenericError] === true;
+    }
+    constructor(code, message, { cause, ...rest } = {}) {
+        super(message, cause ? { cause } : {});
+        this.code = code;
+        this[kGenericError] = true;
+        Reflect.defineProperty(this, 'message', { enumerable: true });
+        Reflect.defineProperty(this, 'code', { enumerable: true });
+        if ('stack' in this) {
+            Reflect.defineProperty(this, 'stack', { enumerable: true });
+        }
+        for (const [key, value] of Object.entries(rest)) {
+            Reflect.defineProperty(this, key, { value, enumerable: true });
+        }
+        Reflect.defineProperty(this, kGenericError, { value: true, enumerable: false });
+    }
+    findBy(property, value) {
+        if (this[property] === value) {
+            return this;
+        }
+        return null;
+    }
+}
+export class MultipleErrors extends AggregateError {
+    code;
+    [kGenericError];
+    [kMultipleErrors];
+    static code = 'PLT_KFK_MULTIPLE';
+    static isGenericError(error) {
+        return error[kGenericError] === true;
+    }
+    static isMultipleErrors(error) {
+        return error[kMultipleErrors] === true;
+    }
+    constructor(message, errors, { cause, ...rest } = {}) {
+        super(errors, message, cause ? { cause } : {});
+        this.code = MultipleErrors.code;
+        this[kGenericError] = true;
+        this[kMultipleErrors] = true;
+        Reflect.defineProperty(this, 'message', { enumerable: true });
+        Reflect.defineProperty(this, 'code', { enumerable: true });
+        if ('stack' in this) {
+            Reflect.defineProperty(this, 'stack', { enumerable: true });
+        }
+        for (const [key, value] of Object.entries(rest)) {
+            Reflect.defineProperty(this, key, { value, enumerable: true });
+        }
+        Reflect.defineProperty(this, kGenericError, { value: true, enumerable: false });
+        Reflect.defineProperty(this, kMultipleErrors, { value: true, enumerable: false });
+    }
+    findBy(property, value) {
+        if (this[property] === value) {
+            return this;
+        }
+        for (const error of this.errors) {
+            /* c8 ignore next */
+            if (error[kGenericError] ? error.findBy(property, value) : error[property] === value) {
+                return error;
+            }
+        }
+        return null;
+    }
+}
+export * from "./protocol/errors.js";
+export class AuthenticationError extends GenericError {
+    static code = 'PLT_KFK_AUTHENTICATION';
+    constructor(message, properties = {}) {
+        super(AuthenticationError.code, message, properties);
+    }
+}
+export class NetworkError extends GenericError {
+    static code = 'PLT_KFK_NETWORK';
+    constructor(message, properties = {}) {
+        super(NetworkError.code, message, properties);
+    }
+}
+export class ProtocolError extends GenericError {
+    constructor(codeOrId, properties = {}, response = undefined) {
+        const { id, code, message, canRetry } = protocolErrors[typeof codeOrId === 'number' ? protocolErrorsCodesById[codeOrId] : codeOrId];
+        super('PLT_KFK_PROTOCOL', message, {
+            apiId: id,
+            apiCode: code,
+            canRetry,
+            hasStaleMetadata: ['UNKNOWN_TOPIC_OR_PARTITION', 'LEADER_NOT_AVAILABLE', 'NOT_LEADER_OR_FOLLOWER'].includes(id),
+            needsRejoin: ['MEMBER_ID_REQUIRED', 'UNKNOWN_MEMBER_ID', 'REBALANCE_IN_PROGRESS'].includes(id),
+            rebalanceInProgress: id === 'REBALANCE_IN_PROGRESS',
+            unknownMemberId: id === 'UNKNOWN_MEMBER_ID',
+            memberId: response?.memberId,
+            ...properties
+        });
+    }
+}
+export class ResponseError extends MultipleErrors {
+    static code = 'PLT_KFK_RESPONSE';
+    constructor(apiName, apiVersion, errors, response, properties = {}) {
+        super(`Received response with error while executing API ${protocolAPIsById[apiName]}(v${apiVersion})`, Object.entries(errors).map(([path, errorCode]) => new ProtocolError(errorCode, { path }, response)), {
+            ...properties,
+            response
+        });
+        this.code = ResponseError.code;
+    }
+}
+export class TimeoutError extends GenericError {
+    static code = 'PLT_KFK_TIMEOUT';
+    constructor(message, properties = {}) {
+        super(NetworkError.code, message, properties);
+    }
+}
+export class UnexpectedCorrelationIdError extends GenericError {
+    static code = 'PLT_KFK_UNEXPECTED_CORRELATION_ID';
+    constructor(message, properties = {}) {
+        super(UnexpectedCorrelationIdError.code, message, properties);
+    }
+}
+export class UnfinishedWriteBufferError extends GenericError {
+    static code = 'PLT_KFK_UNFINISHED_WRITE_BUFFER';
+    constructor(message, properties = {}) {
+        super(UnfinishedWriteBufferError.code, message, properties);
+    }
+}
+export class UnsupportedCompressionError extends GenericError {
+    static code = 'PLT_KFK_UNSUPPORTED_COMPRESSION';
+    constructor(message, properties = {}) {
+        super(UnsupportedCompressionError.code, message, properties);
+    }
+}
+export class UnsupportedError extends GenericError {
+    static code = 'PLT_KFK_UNSUPPORTED';
+    constructor(message, properties = {}) {
+        super(UnsupportedError.code, message, properties);
+    }
+}
+export class UserError extends GenericError {
+    static code = 'PLT_KFK_USER';
+    constructor(message, properties = {}) {
+        super(UserError.code, message, properties);
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,6 @@
+export * from './errors.ts';
+export * from './utils.ts';
+export * from './network/index.ts';
+export * from './protocol/index.ts';
+export * from './apis/index.ts';
+export * from './clients/index.ts';

package/dist/index.js

@@ -0,0 +1,10 @@
+// General
+export * from "./errors.js";
+export * from "./utils.js";
+// Networking
+export * from "./network/index.js";
+export * from "./protocol/index.js";
+// APIs
+export * from "./apis/index.js";
+// Clients
+export * from "./clients/index.js";

package/dist/network/connection-pool.d.ts

@@ -0,0 +1,11 @@
+import EventEmitter from 'node:events';
+import { type CallbackWithPromise } from '../clients/callbacks.ts';
+import { Connection, type Broker, type ConnectionOptions } from './connection.ts';
+export declare class ConnectionPool extends EventEmitter {
+    #private;
+    constructor(clientId: string, connectionOptions?: ConnectionOptions);
+    get(broker: Broker, callback: CallbackWithPromise<Connection>): void;
+    get(broker: Broker): Promise<Connection>;
+    getFirstAvailable(brokers: Broker[], callback?: CallbackWithPromise<Connection>, current?: number, errors?: Error[]): void | Promise<Connection>;
+    close(callback?: CallbackWithPromise<void>): void | Promise<void>;
+}

package/dist/network/connection-pool.js

@@ -0,0 +1,101 @@
+import EventEmitter from 'node:events';
+import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../clients/callbacks.js";
+import { MultipleErrors } from "../errors.js";
+import { Connection, ConnectionStatuses } from "./connection.js";
+export class ConnectionPool extends EventEmitter {
+    #clientId;
+    // @ts-ignore This is used just for debugging
+    #ownerId;
+    #connections;
+    #connectionOptions;
+    constructor(clientId, connectionOptions = {}) {
+        super();
+        this.#clientId = clientId;
+        this.#ownerId = connectionOptions.ownerId;
+        this.#connections = new Map();
+        this.#connectionOptions = connectionOptions;
+    }
+    get(broker, callback) {
+        const key = `${broker.host}:${broker.port}`;
+        const existing = this.#connections.get(key);
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        if (existing) {
+            if (existing.status !== ConnectionStatuses.CONNECTED) {
+                existing.ready(error => {
+                    if (error) {
+                        callback(error, undefined);
+                        return;
+                    }
+                    callback(null, existing);
+                });
+            }
+            else {
+                callback(null, existing);
+            }
+            return callback[kCallbackPromise];
+        }
+        const connection = new Connection(this.#clientId, this.#connectionOptions);
+        this.#connections.set(key, connection);
+        const eventPayload = { broker, connection };
+        this.emit('connecting', eventPayload);
+        connection.connect(broker.host, broker.port, error => {
+            if (error) {
+                this.#connections.delete(key);
+                this.emit('failed', eventPayload);
+                callback(error, undefined);
+                return;
+            }
+            this.emit('connect', eventPayload);
+            callback(null, connection);
+        });
+        // Remove stale connections from the pool
+        connection.once('close', () => {
+            this.emit('disconnect', eventPayload);
+            this.#connections.delete(key);
+        });
+        connection.once('error', () => {
+            this.#connections.delete(key);
+        });
+        connection.on('drain', () => {
+            this.emit('drain', eventPayload);
+        });
+        return callback[kCallbackPromise];
+    }
+    getFirstAvailable(brokers, callback, current = 0, errors = []) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        this.#getFirstAvailable(brokers, current, errors, callback);
+        return callback[kCallbackPromise];
+    }
+    close(callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        if (this.#connections.size === 0) {
+            callback(null);
+            return callback[kCallbackPromise];
+        }
+        runConcurrentCallbacks('Closing connections failed.', this.#connections, ([key, connection], cb) => {
+            connection.close(cb);
+            this.#connections.delete(key);
+        }, error => callback(error));
+        return callback[kCallbackPromise];
+    }
+    #getFirstAvailable(brokers, current = 0, errors = [], callback) {
+        this.get(brokers[current], (error, connection) => {
+            if (error) {
+                errors.push(error);
+                if (current === brokers.length - 1) {
+                    callback(new MultipleErrors('Cannot connect to any broker.', errors), undefined);
+                    return;
+                }
+                this.#getFirstAvailable(brokers, current + 1, errors, callback);
+                return;
+            }
+            callback(null, connection);
+        });
+    }
+}

package/dist/network/connection.d.ts

@@ -0,0 +1,49 @@
+import EventEmitter from 'node:events';
+import { type Socket } from 'node:net';
+import { type ConnectionOptions as TLSConnectionOptions } from 'node:tls';
+import { type Callback, type ResponseParser } from '../apis/definitions.ts';
+import { type CallbackWithPromise } from '../clients/callbacks.ts';
+import { Writer } from '../protocol/writer.ts';
+export interface Broker {
+    host: string;
+    port: number;
+}
+export interface ConnectionOptions {
+    connectTimeout?: number;
+    maxInflights?: number;
+    tls?: TLSConnectionOptions;
+    ownerId?: number;
+}
+export interface Request {
+    correlationId: number;
+    apiKey: number;
+    apiVersion: number;
+    hasRequestHeaderTaggedFields: boolean;
+    hasResponseHeaderTaggedFields: boolean;
+    payload: () => Writer;
+    parser: ResponseParser<unknown>;
+    callback: Callback<any>;
+}
+export declare const ConnectionStatuses: {
+    readonly NONE: "none";
+    readonly CONNECTING: "connecting";
+    readonly CONNECTED: "connected";
+    readonly CLOSED: "closed";
+    readonly CLOSING: "closing";
+    readonly ERROR: "error";
+};
+export type ConnectionStatus = keyof typeof ConnectionStatuses;
+export type ConnectionStatusValue = (typeof ConnectionStatuses)[keyof typeof ConnectionStatuses];
+export declare const defaultOptions: ConnectionOptions;
+export declare function noResponseCallback(..._: any[]): void;
+export declare namespace noResponseCallback { }
+export declare class Connection extends EventEmitter {
+    #private;
+    constructor(clientId?: string, options?: ConnectionOptions);
+    get status(): ConnectionStatusValue;
+    get socket(): Socket;
+    connect(host: string, port: number, callback?: CallbackWithPromise<void>): void | Promise<void>;
+    ready(callback?: CallbackWithPromise<void>): void | Promise<void>;
+    close(callback?: CallbackWithPromise<void>): void | Promise<void>;
+    send<ReturnType>(apiKey: number, apiVersion: number, payload: () => Writer, responseParser: ResponseParser<ReturnType>, hasRequestHeaderTaggedFields: boolean, hasResponseHeaderTaggedFields: boolean, callback: Callback<ReturnType>): void;
+}