@platformatic/kafka 0.1.0

package/dist/network/connection.js

@@ -0,0 +1,319 @@
+import fastq from 'fastq';
+import EventEmitter from 'node:events';
+import { createConnection } from 'node:net';
+import { connect as createTLSConnection } from 'node:tls';
+import { createPromisifiedCallback, kCallbackPromise } from "../clients/callbacks.js";
+import { NetworkError, TimeoutError, UnexpectedCorrelationIdError } from "../errors.js";
+import { protocolAPIsById } from "../protocol/apis.js";
+import { EMPTY_OR_SINGLE_COMPACT_LENGTH_SIZE, INT32_SIZE } from "../protocol/definitions.js";
+import { DynamicBuffer } from "../protocol/dynamic-buffer.js";
+import { Reader } from "../protocol/reader.js";
+import { Writer } from "../protocol/writer.js";
+import { loggers } from "../utils.js";
+export const ConnectionStatuses = {
+    NONE: 'none',
+    CONNECTING: 'connecting',
+    CONNECTED: 'connected',
+    CLOSED: 'closed',
+    CLOSING: 'closing',
+    ERROR: 'error'
+};
+export const defaultOptions = {
+    connectTimeout: 5000,
+    maxInflights: 5
+};
+const kNoResponse = Symbol('plt.kafka.noResponse');
+/* c8 ignore next */
+export function noResponseCallback(..._) { }
+noResponseCallback[kNoResponse] = true;
+export class Connection extends EventEmitter {
+    #options;
+    #status;
+    #clientId;
+    // @ts-ignore This is used just for debugging
+    #ownerId;
+    #correlationId;
+    #nextMessage;
+    #afterDrainRequests;
+    #requestsQueue;
+    #inflightRequests;
+    #responseBuffer;
+    #responseReader;
+    #socket;
+    #socketMustBeDrained;
+    constructor(clientId, options = {}) {
+        super();
+        this.setMaxListeners(0);
+        this.#options = Object.assign({}, defaultOptions, options);
+        this.#status = ConnectionStatuses.NONE;
+        this.#clientId = clientId;
+        this.#ownerId = options.ownerId;
+        this.#correlationId = 0;
+        this.#nextMessage = 0;
+        this.#afterDrainRequests = [];
+        this.#requestsQueue = fastq((op, cb) => op(cb), this.#options.maxInflights);
+        this.#inflightRequests = new Map();
+        this.#responseBuffer = new DynamicBuffer();
+        this.#responseReader = new Reader(this.#responseBuffer);
+        this.#socketMustBeDrained = false;
+    }
+    get status() {
+        return this.#status;
+    }
+    get socket() {
+        return this.#socket;
+    }
+    connect(host, port, callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        if (this.#status === ConnectionStatuses.CONNECTED) {
+            callback(null);
+            return callback[kCallbackPromise];
+        }
+        this.ready(callback);
+        if (this.#status === ConnectionStatuses.CONNECTING) {
+            return callback[kCallbackPromise];
+        }
+        this.#status = ConnectionStatuses.CONNECTING;
+        const connectionOptions = {
+            timeout: this.#options.connectTimeout
+        };
+        const connectionTimeoutHandler = () => {
+            const error = new TimeoutError(`Connection to ${host}:${port} timed out.`);
+            this.#socket.destroy();
+            this.#status = ConnectionStatuses.ERROR;
+            this.emit('timeout', error);
+            this.emit('error', error);
+        };
+        const connectionErrorHandler = (e) => {
+            const error = new NetworkError(`Connection to ${host}:${port} failed.`, { cause: e });
+            this.#status = ConnectionStatuses.ERROR;
+            this.emit('error', error);
+        };
+        this.emit('connecting');
+        /* c8 ignore next 3 */
+        this.#socket = this.#options.tls
+            ? createTLSConnection(port, host, { ...this.#options.tls, ...connectionOptions })
+            : createConnection({ ...connectionOptions, port, host });
+        this.#socket.setNoDelay(true);
+        this.#socket.once('connect', () => {
+            this.#socket.removeListener('timeout', connectionTimeoutHandler);
+            this.#socket.removeListener('error', connectionErrorHandler);
+            this.#socket.on('error', this.#onError.bind(this));
+            this.#socket.on('data', this.#onData.bind(this));
+            this.#socket.on('drain', this.#onDrain.bind(this));
+            this.#socket.on('close', this.#onClose.bind(this));
+            this.#socket.setTimeout(0);
+            this.#status = ConnectionStatuses.CONNECTED;
+            this.emit('connect');
+        });
+        this.#socket.once('timeout', connectionTimeoutHandler);
+        this.#socket.once('error', connectionErrorHandler);
+        return callback[kCallbackPromise];
+    }
+    ready(callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        const onConnect = () => {
+            this.removeListener('error', onError);
+            callback(null);
+        };
+        const onError = (error) => {
+            this.removeListener('connect', onConnect);
+            callback(error);
+        };
+        this.once('connect', onConnect);
+        this.once('error', onError);
+        this.emit('ready');
+        return callback[kCallbackPromise];
+    }
+    close(callback) {
+        if (!callback) {
+            callback = createPromisifiedCallback();
+        }
+        if (this.#status === ConnectionStatuses.CLOSED ||
+            this.#status === ConnectionStatuses.ERROR ||
+            this.#status === ConnectionStatuses.NONE) {
+            callback(null);
+            return callback[kCallbackPromise];
+        }
+        else if (this.#status === ConnectionStatuses.CLOSING) {
+            this.once('close', () => {
+                callback(null);
+            });
+            return callback[kCallbackPromise];
+        }
+        // Ignore all disconnection errors
+        this.#socket.removeAllListeners('error');
+        this.#socket.once('error', () => { });
+        this.#socket.once('close', () => {
+            this.#status = ConnectionStatuses.CLOSED;
+            this.emit('close');
+            callback(null);
+        });
+        this.#status = ConnectionStatuses.CLOSING;
+        this.emit('closing');
+        this.#socket.end();
+        return callback[kCallbackPromise];
+    }
+    send(apiKey, apiVersion, payload, responseParser, hasRequestHeaderTaggedFields, hasResponseHeaderTaggedFields, callback) {
+        this.#requestsQueue.push(fastQueueCallback => {
+            const correlationId = ++this.#correlationId;
+            const request = {
+                correlationId,
+                apiKey,
+                apiVersion,
+                hasRequestHeaderTaggedFields,
+                hasResponseHeaderTaggedFields,
+                parser: responseParser,
+                payload,
+                callback: fastQueueCallback
+            };
+            if (this.#socketMustBeDrained) {
+                this.#afterDrainRequests.push(request);
+                return false;
+            }
+            return this.#sendRequest(request);
+        }, callback);
+    }
+    /*
+      Request => Size [Request Header v2] [payload]
+      Request Header v2 => request_api_key request_api_version correlation_id client_id TAG_BUFFER
+        request_api_key => INT16
+        request_api_version => INT16
+        correlation_id => INT32
+        client_id => NULLABLE_STRING
+    */
+    #sendRequest(request) {
+        if (this.#status !== ConnectionStatuses.CONNECTED) {
+            request.callback(new NetworkError('Connection closed'), undefined);
+            return false;
+        }
+        let canWrite = true;
+        const { correlationId, apiKey, apiVersion, payload: payloadFn, hasRequestHeaderTaggedFields } = request;
+        const writer = Writer.create()
+            .appendInt16(apiKey)
+            .appendInt16(apiVersion)
+            .appendInt32(correlationId)
+            .appendString(this.#clientId, false);
+        if (hasRequestHeaderTaggedFields) {
+            writer.appendTaggedFields();
+        }
+        const payload = payloadFn();
+        writer.appendFrom(payload);
+        writer.prependLength();
+        // Write the header
+        this.#socket.cork();
+        if (!payload.context.noResponse) {
+            this.#inflightRequests.set(correlationId, request);
+        }
+        /* c8 ignore next */
+        loggers.protocol({ apiKey: protocolAPIsById[apiKey], correlationId, request }, 'Sending request.');
+        for (const buf of writer.buffers) {
+            if (!this.#socket.write(buf)) {
+                canWrite = false;
+            }
+        }
+        if (!canWrite) {
+            this.#socketMustBeDrained = true;
+        }
+        this.#socket.uncork();
+        if (payload.context.noResponse) {
+            request.callback(null, canWrite);
+        }
+        // debugDump(Date.now() % 100000, 'send', { owner: this.#ownerId, apiKey: protocolAPIsById[apiKey], correlationId })
+        return canWrite;
+    }
+    /*
+      Response Header v1 => correlation_id TAG_BUFFER
+        correlation_id => INT32
+    */
+    #onData(chunk) {
+        this.#responseBuffer.append(chunk);
+        // There is at least one message size to add
+        // Note that here the initial position is always 0
+        while (this.#responseBuffer.length > INT32_SIZE) {
+            if (this.#nextMessage < 1) {
+                this.#nextMessage = this.#responseReader.readInt32();
+            }
+            // Less data than the message size, wait for more data
+            if (this.#nextMessage > this.#responseBuffer.length - INT32_SIZE) {
+                break;
+            }
+            // Read the correlationId and get the handler
+            const correlationId = this.#responseReader.readInt32();
+            const request = this.#inflightRequests.get(correlationId);
+            if (!request) {
+                this.emit('error', new UnexpectedCorrelationIdError(`Received unexpected response with correlation_id=${correlationId}`, {
+                    raw: this.#responseReader.buffer.slice(0, this.#nextMessage + INT32_SIZE)
+                }));
+                return;
+            }
+            this.#inflightRequests.delete(correlationId);
+            const { apiKey, apiVersion, hasResponseHeaderTaggedFields, parser, callback } = request;
+            let deserialized;
+            let responseError = null;
+            try {
+                // Due to inconsistency in the wire protocol, the tag buffer in the header might have to be handled by the APIs
+                // For example: https://github.com/apache/kafka/blob/84caaa6e9da06435411510a81fa321d4f99c351f/clients/src/main/resources/common/message/ApiVersionsResponse.json#L24
+                if (hasResponseHeaderTaggedFields) {
+                    this.#responseReader.skip(EMPTY_OR_SINGLE_COMPACT_LENGTH_SIZE);
+                }
+                deserialized = parser(correlationId, apiKey, apiVersion, new Reader(this.#responseReader.buffer.subarray(this.#responseReader.position, this.#nextMessage + INT32_SIZE)));
+            }
+            catch (error) {
+                responseError = error;
+                // debugDump(Date.now() % 100000, 'received error', {
+                //   owner: this.#ownerId,
+                //   apiKey: protocolAPIsById[apiKey],
+                //   error
+                // })
+            }
+            finally {
+                this.#responseBuffer.consume(this.#nextMessage + INT32_SIZE);
+                this.#responseReader.position = 0;
+                this.#nextMessage = -1;
+            }
+            // debugDump(Date.now() % 100000, 'receive', {
+            //   owner: this.#ownerId,
+            //   apiKey: protocolAPIsById[apiKey],
+            //   correlationId
+            // })
+            /* c8 ignore next */
+            loggers.protocol({ apiKey: protocolAPIsById[apiKey], correlationId, request }, 'Received response.');
+            callback(responseError, deserialized);
+        }
+    }
+    #onDrain() {
+        // First of all, send all the requests that were waiting for the socket to drain
+        while (this.#afterDrainRequests.length) {
+            const request = this.#afterDrainRequests.shift();
+            // If no more request or  after sending the request the socket is blocked again, abort
+            if (!request || !this.#sendRequest(request)) {
+                return;
+            }
+        }
+        // Start getting requests again
+        this.#socketMustBeDrained = false;
+        this.emit('drain');
+    }
+    #onClose() {
+        this.#status = ConnectionStatuses.CLOSED;
+        this.emit('close');
+        const error = new NetworkError('Connection closed');
+        for (const request of this.#afterDrainRequests) {
+            const payload = request.payload();
+            if (!payload.context.noResponse) {
+                request.callback(error, undefined);
+            }
+        }
+        for (const inflight of this.#inflightRequests.values()) {
+            inflight.callback(error, undefined);
+        }
+    }
+    #onError(error) {
+        this.emit('error', new NetworkError('Connection error', { cause: error }));
+    }
+}

package/dist/network/index.d.ts

@@ -0,0 +1,2 @@
+export * from './connection-pool.ts';
+export * from './connection.ts';

package/dist/network/index.js

@@ -0,0 +1,2 @@
+export * from "./connection-pool.js";
+export * from "./connection.js";

package/dist/protocol/apis.d.ts

@@ -0,0 +1,2 @@
+export declare const protocolAPIsByName: Record<string, number>;
+export declare const protocolAPIsById: Record<number, string>;

package/dist/protocol/apis.js

@@ -0,0 +1,191 @@
+// This is autogenerated from the generate:apis script, do not edit manually.
+export const protocolAPIsByName = Object.freeze({
+    AddOffsetsToTxn: 25,
+    AddPartitionsToTxn: 24,
+    AddRaftVoter: 80,
+    AllocateProducerIds: 67,
+    AlterClientQuotas: 49,
+    AlterConfigs: 33,
+    AlterPartitionReassignments: 45,
+    AlterPartition: 56,
+    AlterReplicaLogDirs: 34,
+    AlterShareGroupOffsets: 91,
+    AlterUserScramCredentials: 51,
+    ApiVersions: 18,
+    AssignReplicasToDirs: 73,
+    BeginQuorumEpoch: 53,
+    BrokerHeartbeat: 63,
+    BrokerRegistration: 62,
+    ConsumerGroupDescribe: 69,
+    ConsumerGroupHeartbeat: 68,
+    ControlledShutdown: 7,
+    ControllerRegistration: 70,
+    CreateAcls: 30,
+    CreateDelegationToken: 38,
+    CreatePartitions: 37,
+    CreateTopics: 19,
+    DeleteAcls: 31,
+    DeleteGroups: 42,
+    DeleteRecords: 21,
+    DeleteShareGroupOffsets: 92,
+    DeleteShareGroupState: 86,
+    DeleteTopics: 20,
+    DescribeAcls: 29,
+    DescribeClientQuotas: 48,
+    DescribeCluster: 60,
+    DescribeConfigs: 32,
+    DescribeDelegationToken: 41,
+    DescribeGroups: 15,
+    DescribeLogDirs: 35,
+    DescribeProducers: 61,
+    DescribeQuorum: 55,
+    DescribeShareGroupOffsets: 90,
+    DescribeTopicPartitions: 75,
+    DescribeTransactions: 65,
+    DescribeUserScramCredentials: 50,
+    ElectLeaders: 43,
+    EndQuorumEpoch: 54,
+    EndTxn: 26,
+    Envelope: 58,
+    ExpireDelegationToken: 40,
+    Fetch: 1,
+    FetchSnapshot: 59,
+    FindCoordinator: 10,
+    GetTelemetrySubscriptions: 71,
+    Heartbeat: 12,
+    IncrementalAlterConfigs: 44,
+    InitProducerId: 22,
+    InitializeShareGroupState: 83,
+    JoinGroup: 11,
+    LeaderAndIsr: 4,
+    LeaveGroup: 13,
+    ListClientMetricsResources: 74,
+    ListGroups: 16,
+    ListOffsets: 2,
+    ListPartitionReassignments: 46,
+    ListTransactions: 66,
+    Metadata: 3,
+    OffsetCommit: 8,
+    OffsetDelete: 47,
+    OffsetFetch: 9,
+    OffsetForLeaderEpoch: 23,
+    Produce: 0,
+    PushTelemetry: 72,
+    ReadShareGroupState: 84,
+    ReadShareGroupStateSummary: 87,
+    RemoveRaftVoter: 81,
+    RenewDelegationToken: 39,
+    SaslAuthenticate: 36,
+    SaslHandshake: 17,
+    ShareAcknowledge: 79,
+    ShareFetch: 78,
+    ShareGroupDescribe: 77,
+    ShareGroupHeartbeat: 76,
+    StopReplica: 5,
+    StreamsGroupDescribe: 89,
+    StreamsGroupHeartbeat: 88,
+    SyncGroup: 14,
+    TxnOffsetCommit: 28,
+    UnregisterBroker: 64,
+    UpdateFeatures: 57,
+    UpdateMetadata: 6,
+    UpdateRaftVoter: 82,
+    Vote: 52,
+    WriteShareGroupState: 85,
+    WriteTxnMarkers: 27
+});
+export const protocolAPIsById = Object.freeze({
+    0: 'Produce',
+    1: 'Fetch',
+    2: 'ListOffsets',
+    3: 'Metadata',
+    4: 'LeaderAndIsr',
+    5: 'StopReplica',
+    6: 'UpdateMetadata',
+    7: 'ControlledShutdown',
+    8: 'OffsetCommit',
+    9: 'OffsetFetch',
+    10: 'FindCoordinator',
+    11: 'JoinGroup',
+    12: 'Heartbeat',
+    13: 'LeaveGroup',
+    14: 'SyncGroup',
+    15: 'DescribeGroups',
+    16: 'ListGroups',
+    17: 'SaslHandshake',
+    18: 'ApiVersions',
+    19: 'CreateTopics',
+    20: 'DeleteTopics',
+    21: 'DeleteRecords',
+    22: 'InitProducerId',
+    23: 'OffsetForLeaderEpoch',
+    24: 'AddPartitionsToTxn',
+    25: 'AddOffsetsToTxn',
+    26: 'EndTxn',
+    27: 'WriteTxnMarkers',
+    28: 'TxnOffsetCommit',
+    29: 'DescribeAcls',
+    30: 'CreateAcls',
+    31: 'DeleteAcls',
+    32: 'DescribeConfigs',
+    33: 'AlterConfigs',
+    34: 'AlterReplicaLogDirs',
+    35: 'DescribeLogDirs',
+    36: 'SaslAuthenticate',
+    37: 'CreatePartitions',
+    38: 'CreateDelegationToken',
+    39: 'RenewDelegationToken',
+    40: 'ExpireDelegationToken',
+    41: 'DescribeDelegationToken',
+    42: 'DeleteGroups',
+    43: 'ElectLeaders',
+    44: 'IncrementalAlterConfigs',
+    45: 'AlterPartitionReassignments',
+    46: 'ListPartitionReassignments',
+    47: 'OffsetDelete',
+    48: 'DescribeClientQuotas',
+    49: 'AlterClientQuotas',
+    50: 'DescribeUserScramCredentials',
+    51: 'AlterUserScramCredentials',
+    52: 'Vote',
+    53: 'BeginQuorumEpoch',
+    54: 'EndQuorumEpoch',
+    55: 'DescribeQuorum',
+    56: 'AlterPartition',
+    57: 'UpdateFeatures',
+    58: 'Envelope',
+    59: 'FetchSnapshot',
+    60: 'DescribeCluster',
+    61: 'DescribeProducers',
+    62: 'BrokerRegistration',
+    63: 'BrokerHeartbeat',
+    64: 'UnregisterBroker',
+    65: 'DescribeTransactions',
+    66: 'ListTransactions',
+    67: 'AllocateProducerIds',
+    68: 'ConsumerGroupHeartbeat',
+    69: 'ConsumerGroupDescribe',
+    70: 'ControllerRegistration',
+    71: 'GetTelemetrySubscriptions',
+    72: 'PushTelemetry',
+    73: 'AssignReplicasToDirs',
+    74: 'ListClientMetricsResources',
+    75: 'DescribeTopicPartitions',
+    76: 'ShareGroupHeartbeat',
+    77: 'ShareGroupDescribe',
+    78: 'ShareFetch',
+    79: 'ShareAcknowledge',
+    80: 'AddRaftVoter',
+    81: 'RemoveRaftVoter',
+    82: 'UpdateRaftVoter',
+    83: 'InitializeShareGroupState',
+    84: 'ReadShareGroupState',
+    85: 'WriteShareGroupState',
+    86: 'DeleteShareGroupState',
+    87: 'ReadShareGroupStateSummary',
+    88: 'StreamsGroupHeartbeat',
+    89: 'StreamsGroupDescribe',
+    90: 'DescribeShareGroupOffsets',
+    91: 'AlterShareGroupOffsets',
+    92: 'DeleteShareGroupOffsets'
+});

package/dist/protocol/compression.d.ts

@@ -0,0 +1,70 @@
+import { DynamicBuffer } from './dynamic-buffer.ts';
+export type SyncCompressionPhase = (data: Buffer | DynamicBuffer) => Buffer;
+export type CompressionOperation = (data: Buffer) => Buffer;
+export interface CompressionAlgorithm {
+    compressSync: SyncCompressionPhase;
+    decompressSync: SyncCompressionPhase;
+    bitmask: number;
+    available?: boolean;
+}
+export declare const compressionsAlgorithms: {
+    readonly none: {
+        readonly compressSync: (data: Buffer | DynamicBuffer) => Buffer;
+        readonly decompressSync: (data: Buffer | DynamicBuffer) => Buffer;
+        readonly bitmask: 0;
+        readonly available: true;
+    };
+    readonly gzip: {
+        readonly compressSync: (data: Buffer | DynamicBuffer) => Buffer;
+        readonly decompressSync: (data: Buffer | DynamicBuffer) => Buffer;
+        readonly bitmask: 1;
+        readonly available: true;
+    };
+    readonly snappy: {
+        readonly compressSync: (data: Buffer | DynamicBuffer) => Buffer;
+        readonly decompressSync: (data: Buffer | DynamicBuffer) => Buffer;
+        readonly bitmask: 2;
+        readonly available: true;
+    };
+    readonly lz4: {
+        readonly compressSync: (data: Buffer | DynamicBuffer) => Buffer;
+        readonly decompressSync: (data: Buffer | DynamicBuffer) => Buffer;
+        readonly bitmask: 3;
+        readonly available: true;
+    };
+    readonly zstd: {
+        readonly compressSync: (data: Buffer | DynamicBuffer) => Buffer;
+        readonly decompressSync: (data: Buffer | DynamicBuffer) => Buffer;
+        readonly bitmask: 4;
+        readonly available: boolean;
+    };
+};
+export declare const compressionsAlgorithmsByBitmask: {
+    [k: string]: {
+        readonly compressSync: (data: Buffer | DynamicBuffer) => Buffer;
+        readonly decompressSync: (data: Buffer | DynamicBuffer) => Buffer;
+        readonly bitmask: 0;
+        readonly available: true;
+    } | {
+        readonly compressSync: (data: Buffer | DynamicBuffer) => Buffer;
+        readonly decompressSync: (data: Buffer | DynamicBuffer) => Buffer;
+        readonly bitmask: 1;
+        readonly available: true;
+    } | {
+        readonly compressSync: (data: Buffer | DynamicBuffer) => Buffer;
+        readonly decompressSync: (data: Buffer | DynamicBuffer) => Buffer;
+        readonly bitmask: 2;
+        readonly available: true;
+    } | {
+        readonly compressSync: (data: Buffer | DynamicBuffer) => Buffer;
+        readonly decompressSync: (data: Buffer | DynamicBuffer) => Buffer;
+        readonly bitmask: 3;
+        readonly available: true;
+    } | {
+        readonly compressSync: (data: Buffer | DynamicBuffer) => Buffer;
+        readonly decompressSync: (data: Buffer | DynamicBuffer) => Buffer;
+        readonly bitmask: 4;
+        readonly available: boolean;
+    };
+};
+export type CompressionAlgorithms = keyof typeof compressionsAlgorithms;