@platformatic/kafka 0.1.0

package/dist/protocol/index.d.ts

@@ -0,0 +1,14 @@
+export * from './apis.ts';
+export * from './compression.ts';
+export * from './crc32c.ts';
+export * from './definitions.ts';
+export * from './dynamic-buffer.ts';
+export * from './errors.ts';
+export * from './index.ts';
+export * from './murmur2.ts';
+export * from './reader.ts';
+export * from './records.ts';
+export * as saslPlain from './sasl/plain.ts';
+export * as saslScramSha from './sasl/scram-sha.ts';
+export * from './varint.ts';
+export * from './writer.ts';

package/dist/protocol/index.js

@@ -0,0 +1,14 @@
+export * from "./apis.js";
+export * from "./compression.js";
+export * from "./crc32c.js";
+export * from "./definitions.js";
+export * from "./dynamic-buffer.js";
+export * from "./errors.js";
+export * from "./index.js";
+export * from "./murmur2.js";
+export * from "./reader.js";
+export * from "./records.js";
+export * as saslPlain from "./sasl/plain.js";
+export * as saslScramSha from "./sasl/scram-sha.js";
+export * from "./varint.js";
+export * from "./writer.js";

package/dist/protocol/murmur2.d.ts

@@ -0,0 +1 @@
+export declare function murmur2(data: string | Buffer, useCache?: boolean): number;

package/dist/protocol/murmur2.js

@@ -0,0 +1,55 @@
+import { LRUCache } from 'mnemonist';
+// This is a Javascript port of
+// https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/utils/Utils.java#L492
+// 'm' and 'r' are mixing constants generated offline. They're not really 'magic', they just happen to work well.
+const seed = 0x9747b28cn;
+const m = 0x5bd1e995n;
+const r = 24;
+const cache = new LRUCache(1000);
+const tail = Buffer.alloc(4);
+function tripleRightShift(num, bits) {
+    return BigInt(Number(BigInt.asIntN(32, num)) >>> bits);
+}
+export function murmur2(data, useCache = false) {
+    let key;
+    if (typeof data === 'string') {
+        if (useCache) {
+            key = data;
+            const existing = key ? cache.get(key) : undefined;
+            if (existing) {
+                return existing;
+            }
+        }
+        data = Buffer.from(data);
+    }
+    const length = data.length;
+    // Initialize the hash to a random value
+    let h = seed ^ BigInt(length);
+    let i = 0;
+    while (i < length - 3) {
+        let k = BigInt(data.readInt32LE(i));
+        i += 4;
+        k *= m;
+        k ^= tripleRightShift(k, r);
+        k *= m;
+        h *= m;
+        h ^= k;
+    }
+    // Read the last few bytes of the input array
+    // If this becomes an hot-path, consider to optimize it further and avoid copying
+    if (length % 4 > 0) {
+        data.copy(tail, 0, length - (length % 4));
+        h ^= BigInt(tail.readInt32LE(0));
+        tail.fill(0);
+        h *= m;
+    }
+    // Perform the final mixing
+    h ^= tripleRightShift(h, 13);
+    h *= m;
+    h ^= tripleRightShift(h, 15);
+    const hash = Number(BigInt.asIntN(32, h));
+    if (key !== undefined) {
+        cache.set(key, hash);
+    }
+    return hash;
+}

package/dist/protocol/reader.d.ts

@@ -0,0 +1,58 @@
+import { DynamicBuffer } from './dynamic-buffer.ts';
+import { Writer } from './writer.ts';
+export type EntryReader<OutputType> = (reader: Reader, index: number) => OutputType;
+declare const instanceIdentifier: unique symbol;
+export declare class Reader {
+    buffer: DynamicBuffer;
+    position: number;
+    [instanceIdentifier]: boolean;
+    static isReader(target: any): boolean;
+    static from(buffer: Buffer | DynamicBuffer | Writer): Reader;
+    constructor(buffer: DynamicBuffer);
+    reset(buffer?: Buffer | DynamicBuffer): void;
+    inspect(): string;
+    skip(length: number): this;
+    peekUnsignedInt8(): number;
+    peekUnsignedInt16(): number;
+    peekUnsignedInt32(): number;
+    peekUnsignedInt64(): bigint;
+    peekUnsignedVarInt(): number;
+    peekUnsignedVarInt64(): bigint;
+    peekInt8(): number;
+    peekInt16(): number;
+    peekInt32(): number;
+    peekInt64(): bigint;
+    peekFloat64(): number;
+    peekVarInt(): number;
+    peekVarInt64(): bigint;
+    peekBoolean(): boolean;
+    peekUUID(): string;
+    readUnsignedInt8(): number;
+    readUnsignedInt16(): number;
+    readUnsignedInt32(): number;
+    readUnsignedInt64(): bigint;
+    readUnsignedVarInt(): number;
+    readUnsignedVarInt64(): bigint;
+    readInt8(): number;
+    readInt16(): number;
+    readInt32(): number;
+    readInt64(): bigint;
+    readFloat64(): number;
+    readVarInt(): number;
+    readVarInt64(): bigint;
+    readBoolean(): boolean;
+    readNullableString(compact?: boolean, encoding?: BufferEncoding): string | null;
+    readString(compact?: boolean, encoding?: BufferEncoding): string;
+    readUUID(): string;
+    readNullableBytes(compact?: boolean): Buffer | null;
+    readBytes(compact?: boolean): Buffer;
+    readVarIntBytes(): Buffer;
+    readNullableArray<OutputType>(reader: EntryReader<OutputType>, compact?: boolean, discardTrailingTaggedFields?: boolean): OutputType[] | null;
+    readNullableMap<Key, Value>(reader: EntryReader<[Key, Value]>, compact?: boolean, discardTrailingTaggedFields?: boolean): Map<Key, Value> | null;
+    readArray<OutputType>(reader: EntryReader<OutputType>, compact?: boolean, discardTrailingTaggedFields?: boolean): OutputType[];
+    readMap<Key, Value>(reader: EntryReader<[Key, Value]>, compact?: boolean, discardTrailingTaggedFields?: boolean): Map<Key, Value>;
+    readVarIntArray<OutputType>(reader: EntryReader<OutputType>): OutputType[];
+    readVarIntMap<Key, Value>(reader: EntryReader<[Key, Value]>): Map<Key, Value>;
+    readTaggedFields(): void;
+}
+export {};

package/dist/protocol/reader.js

@@ -0,0 +1,296 @@
+import { EMPTY_BUFFER, INT16_SIZE, INT32_SIZE, INT64_SIZE, INT8_SIZE, UUID_SIZE } from "./definitions.js";
+import { DynamicBuffer } from "./dynamic-buffer.js";
+import { Writer } from "./writer.js";
+const instanceIdentifier = Symbol('plt.kafka.reader.instanceIdentifier');
+export class Reader {
+    buffer;
+    position;
+    [instanceIdentifier];
+    static isReader(target) {
+        return target?.[instanceIdentifier] === true;
+    }
+    static from(buffer) {
+        if (Writer.isWriter(buffer)) {
+            return new Reader(buffer.dynamicBuffer);
+        }
+        else if (Buffer.isBuffer(buffer)) {
+            buffer = new DynamicBuffer(buffer);
+        }
+        return new Reader(buffer);
+    }
+    constructor(buffer) {
+        this.buffer = buffer;
+        this.position = 0;
+        this[instanceIdentifier] = true;
+    }
+    reset(buffer) {
+        if (buffer) {
+            if (Buffer.isBuffer(buffer)) {
+                buffer = new DynamicBuffer(buffer);
+            }
+            this.buffer = buffer;
+        }
+        this.position = 0;
+    }
+    inspect() {
+        return this.buffer
+            .subarray(this.position)
+            .toString('hex')
+            .replaceAll(/(.{4})/g, '$1 ')
+            .trim();
+    }
+    skip(length) {
+        this.position += length;
+        return this;
+    }
+    peekUnsignedInt8() {
+        return this.buffer.readUInt8(this.position);
+    }
+    peekUnsignedInt16() {
+        return this.buffer.readUInt16BE(this.position);
+    }
+    peekUnsignedInt32() {
+        return this.buffer.readUInt32BE(this.position);
+    }
+    peekUnsignedInt64() {
+        return this.buffer.readBigUInt64BE(this.position);
+    }
+    peekUnsignedVarInt() {
+        return this.buffer.readUnsignedVarInt(this.position)[0];
+    }
+    peekUnsignedVarInt64() {
+        return this.buffer.readUnsignedVarInt64(this.position)[0];
+    }
+    peekInt8() {
+        return this.buffer.readInt8(this.position);
+    }
+    peekInt16() {
+        return this.buffer.readInt16BE(this.position);
+    }
+    peekInt32() {
+        return this.buffer.readInt32BE(this.position);
+    }
+    peekInt64() {
+        return this.buffer.readBigInt64BE(this.position);
+    }
+    peekFloat64() {
+        return this.buffer.readDoubleBE(this.position);
+    }
+    peekVarInt() {
+        return this.buffer.readVarInt(this.position)[0];
+    }
+    peekVarInt64() {
+        return this.buffer.readVarInt64(this.position)[0];
+    }
+    peekBoolean() {
+        return this.buffer.readInt8(this.position) === 1;
+    }
+    peekUUID() {
+        return this.buffer.toString('hex', this.position, this.position + UUID_SIZE);
+    }
+    readUnsignedInt8() {
+        const value = this.peekUnsignedInt8();
+        this.position += INT8_SIZE;
+        return value;
+    }
+    readUnsignedInt16() {
+        const value = this.peekUnsignedInt16();
+        this.position += INT16_SIZE;
+        return value;
+    }
+    readUnsignedInt32() {
+        const value = this.peekUnsignedInt32();
+        this.position += INT32_SIZE;
+        return value;
+    }
+    readUnsignedInt64() {
+        const value = this.peekUnsignedInt64();
+        this.position += INT64_SIZE;
+        return value;
+    }
+    readUnsignedVarInt() {
+        const [value, read] = this.buffer.readUnsignedVarInt(this.position);
+        this.position += read;
+        return value;
+    }
+    readUnsignedVarInt64() {
+        const [value, read] = this.buffer.readUnsignedVarInt64(this.position);
+        this.position += read;
+        return value;
+    }
+    readInt8() {
+        const value = this.peekInt8();
+        this.position += INT8_SIZE;
+        return value;
+    }
+    readInt16() {
+        const value = this.peekInt16();
+        this.position += INT16_SIZE;
+        return value;
+    }
+    readInt32() {
+        const value = this.peekInt32();
+        this.position += INT32_SIZE;
+        return value;
+    }
+    readInt64() {
+        const value = this.peekInt64();
+        this.position += INT64_SIZE;
+        return value;
+    }
+    readFloat64() {
+        const value = this.peekFloat64();
+        this.position += INT64_SIZE;
+        return value;
+    }
+    readVarInt() {
+        const [value, read] = this.buffer.readVarInt(this.position);
+        this.position += read;
+        return value;
+    }
+    readVarInt64() {
+        const [value, read] = this.buffer.readVarInt64(this.position);
+        this.position += read;
+        return value;
+    }
+    readBoolean() {
+        const value = this.peekUnsignedInt8();
+        this.position += INT8_SIZE;
+        return value === 1;
+    }
+    readNullableString(compact = true, encoding = 'utf-8') {
+        let length;
+        if (compact) {
+            length = this.readUnsignedVarInt();
+            if (length === 0) {
+                return null;
+            }
+            length--;
+        }
+        else {
+            length = this.readInt16();
+            if (length === -1) {
+                return null;
+            }
+        }
+        const value = this.buffer.toString(encoding, this.position, this.position + length);
+        this.position += length;
+        return value;
+    }
+    readString(compact = true, encoding = 'utf-8') {
+        return this.readNullableString(compact, encoding) || '';
+    }
+    readUUID() {
+        const value = this.peekUUID();
+        this.position += UUID_SIZE;
+        return value.replace(/(.{8})(.{4})(.{4})(.{4})(.{12})/, '$1-$2-$3-$4-$5');
+    }
+    readNullableBytes(compact = true) {
+        let length;
+        if (compact) {
+            length = this.readUnsignedVarInt();
+            if (length === 0) {
+                return null;
+            }
+            length--;
+        }
+        else {
+            length = this.readInt32();
+            if (length === -1) {
+                return null;
+            }
+        }
+        const value = this.buffer.slice(this.position, this.position + length);
+        this.position += length;
+        return value;
+    }
+    readBytes(compact = true) {
+        /* c8 ignore next */
+        return this.readNullableBytes(compact) || EMPTY_BUFFER;
+    }
+    readVarIntBytes() {
+        const length = this.readVarInt();
+        const value = this.buffer.slice(this.position, this.position + length);
+        this.position += length;
+        return value;
+    }
+    readNullableArray(reader, compact = true, discardTrailingTaggedFields = true) {
+        let length;
+        if (compact) {
+            length = this.readUnsignedVarInt();
+            if (length === 0) {
+                return null;
+            }
+            length--;
+        }
+        else {
+            length = this.readInt32();
+            if (length === -1) {
+                return null;
+            }
+        }
+        const value = [];
+        for (let i = 0; i < length; i++) {
+            value.push(reader(this, i));
+            if (discardTrailingTaggedFields) {
+                this.readTaggedFields();
+            }
+        }
+        return value;
+    }
+    readNullableMap(reader, compact = true, discardTrailingTaggedFields = true) {
+        let length;
+        if (compact) {
+            length = this.readUnsignedVarInt();
+            if (length === 0) {
+                return null;
+            }
+            length--;
+        }
+        else {
+            length = this.readInt32();
+            if (length === -1) {
+                return null;
+            }
+        }
+        const map = new Map();
+        for (let i = 0; i < length; i++) {
+            const [key, value] = reader(this, i);
+            map.set(key, value);
+            if (discardTrailingTaggedFields) {
+                this.readTaggedFields();
+            }
+        }
+        return map;
+    }
+    readArray(reader, compact = true, discardTrailingTaggedFields = true) {
+        return this.readNullableArray(reader, compact, discardTrailingTaggedFields) || [];
+    }
+    readMap(reader, compact = true, discardTrailingTaggedFields = true) {
+        return this.readNullableMap(reader, compact, discardTrailingTaggedFields) ?? new Map();
+    }
+    readVarIntArray(reader) {
+        const length = this.readVarInt();
+        const value = [];
+        for (let i = 0; i < length; i++) {
+            value.push(reader(this, i));
+        }
+        return value;
+    }
+    readVarIntMap(reader) {
+        const length = this.readVarInt();
+        const map = new Map();
+        for (let i = 0; i < length; i++) {
+            const [key, value] = reader(this, i);
+            map.set(key, value);
+        }
+        return map;
+    }
+    // TODO(ShogunPanda): Tagged fields are not supported yet
+    readTaggedFields() {
+        const length = this.readVarInt();
+        if (length > 0) {
+            this.skip(length);
+        }
+    }
+}

package/dist/protocol/records.d.ts

@@ -0,0 +1,110 @@
+import { type NumericMap } from '../utils.ts';
+import { type CompressionAlgorithms } from './compression.ts';
+import { type NullableString } from './definitions.ts';
+import { Reader } from './reader.ts';
+import { Writer } from './writer.ts';
+export interface MessageBase<Key = Buffer, Value = Buffer> {
+    key?: Key;
+    value?: Value;
+    topic: string;
+    partition?: number;
+    timestamp?: bigint;
+}
+export interface MessageToProduce<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends MessageBase<Key, Value> {
+    headers?: Map<HeaderKey, HeaderValue> | Record<string, HeaderValue>;
+}
+export interface Message<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends Required<MessageBase<Key, Value>> {
+    headers: Map<HeaderKey, HeaderValue>;
+    offset: bigint;
+    commit(callback?: (error?: Error) => void): void | Promise<void>;
+}
+export interface MessageRecord {
+    key?: Buffer;
+    value: Buffer;
+    headers?: Map<Buffer, Buffer>;
+    topic: string;
+    partition?: number;
+    timestamp?: bigint;
+}
+export interface CreateRecordsBatchOptions {
+    transactionalId?: NullableString;
+    compression: CompressionAlgorithms;
+    firstSequence?: number;
+    producerId: bigint;
+    producerEpoch: number;
+    sequences: NumericMap;
+    partitionLeaderEpoch: number;
+}
+export interface KafkaRecord {
+    length: number;
+    attributes: number;
+    timestampDelta: bigint;
+    offsetDelta: number;
+    key: Buffer;
+    value: Buffer;
+    headers: [Buffer, Buffer][];
+}
+export interface RecordsBatch {
+    firstOffset: bigint;
+    length: number;
+    partitionLeaderEpoch: number;
+    magic: number;
+    crc: number;
+    attributes: number;
+    lastOffsetDelta: number;
+    firstTimestamp: bigint;
+    maxTimestamp: bigint;
+    producerId: bigint;
+    producerEpoch: number;
+    firstSequence: number;
+    records: KafkaRecord[];
+}
+export declare const messageSchema: {
+    type: string;
+    properties: {
+        key: {
+            oneOf: ({
+                type: string;
+                buffer?: undefined;
+            } | {
+                buffer: boolean;
+                type?: undefined;
+            })[];
+        };
+        value: {
+            oneOf: ({
+                type: string;
+                buffer?: undefined;
+            } | {
+                buffer: boolean;
+                type?: undefined;
+            })[];
+        };
+        headers: {
+            anyOf: ({
+                map: boolean;
+                type?: undefined;
+                additionalProperties?: undefined;
+            } | {
+                type: string;
+                additionalProperties: boolean;
+                map?: undefined;
+            })[];
+        };
+        topic: {
+            type: string;
+        };
+        partition: {
+            type: string;
+        };
+        timestamp: {
+            bigint: boolean;
+        };
+    };
+    required: string[];
+    additionalProperties: boolean;
+};
+export declare function createRecord(message: MessageRecord, offsetDelta: number, firstTimestamp: bigint): Writer;
+export declare function readRecord(reader: Reader): KafkaRecord;
+export declare function createRecordsBatch(messages: MessageRecord[], options?: Partial<CreateRecordsBatchOptions>): Writer;
+export declare function readRecordsBatch(reader: Reader): RecordsBatch;

package/dist/protocol/records.js

@@ -0,0 +1,149 @@
+import { UnsupportedCompressionError } from "../errors.js";
+import { compressionsAlgorithms, compressionsAlgorithmsByBitmask } from "./compression.js";
+import { crc32c } from "./crc32c.js";
+import { DynamicBuffer } from "./dynamic-buffer.js";
+import { Reader } from "./reader.js";
+import { Writer } from "./writer.js";
+const CURRENT_RECORD_VERSION = 2;
+const IS_TRANSACTIONAL = 0b10000; // Bit 4 set
+const IS_COMPRESSED = 0b111; // Bits 0, 1 and/or 2 set
+export const messageSchema = {
+    type: 'object',
+    properties: {
+        key: {
+            oneOf: [{ type: 'string' }, { buffer: true }]
+        },
+        value: {
+            oneOf: [{ type: 'string' }, { buffer: true }]
+        },
+        headers: {
+            // Note: we can't use oneOf here since a Map is also a 'object'. Thanks JS.
+            anyOf: [
+                {
+                    map: true
+                },
+                {
+                    type: 'object',
+                    additionalProperties: true
+                }
+            ]
+        },
+        topic: { type: 'string' },
+        partition: { type: 'integer' },
+        timestamp: { bigint: true }
+    },
+    required: ['value', 'topic'],
+    additionalProperties: true
+};
+export function createRecord(message, offsetDelta, firstTimestamp) {
+    return Writer.create()
+        .appendInt8(0) // Attributes are unused for now
+        .appendVarInt64((message.timestamp ?? BigInt(Date.now())) - firstTimestamp)
+        .appendVarInt(offsetDelta)
+        .appendVarIntBytes(message.key)
+        .appendVarIntBytes(message.value)
+        .appendVarIntMap(message.headers, (w, [key, value]) => {
+        w.appendVarIntBytes(key).appendVarIntBytes(value);
+    })
+        .prependVarIntLength();
+}
+export function readRecord(reader) {
+    return {
+        length: reader.readVarInt(),
+        attributes: reader.readInt8(),
+        timestampDelta: reader.readVarInt64(),
+        offsetDelta: reader.readVarInt(),
+        key: reader.readVarIntBytes(),
+        value: reader.readVarIntBytes(),
+        headers: reader.readVarIntArray(r => [r.readVarIntBytes(), r.readVarIntBytes()])
+    };
+}
+export function createRecordsBatch(messages, options = {}) {
+    const now = BigInt(Date.now());
+    const timestamps = [];
+    for (let i = 0; i < messages.length; i++) {
+        timestamps.push(messages[i].timestamp ?? now);
+    }
+    messages.sort();
+    const firstTimestamp = timestamps[0];
+    const maxTimestamp = timestamps[timestamps.length - 1];
+    let buffer = new DynamicBuffer();
+    for (let i = 0; i < messages.length; i++) {
+        const record = createRecord(messages[i], i, firstTimestamp);
+        buffer.appendFrom(record.dynamicBuffer);
+    }
+    let attributes = 0;
+    let firstSequence = 0;
+    if (options.sequences) {
+        const firstMessage = messages[0];
+        firstSequence = options.sequences.getWithDefault(`${firstMessage.topic}:${firstMessage.partition}`, 0);
+    }
+    // Set the transaction
+    if (options.transactionalId) {
+        attributes |= IS_TRANSACTIONAL;
+    }
+    // Set the compression, if any
+    /* c8 ignore next */
+    if ((options.compression ?? 'none') !== 'none') {
+        const algorithm = compressionsAlgorithms[options.compression];
+        if (!algorithm) {
+            throw new UnsupportedCompressionError(`Unsupported compression algorithm ${options.compression}`);
+        }
+        attributes |= algorithm.bitmask;
+        const compressed = algorithm.compressSync(buffer.buffer);
+        buffer = new DynamicBuffer(compressed);
+    }
+    const writer = Writer.create()
+        // Phase 1: Prepare the message from Attributes (included) to the end
+        .appendInt16(attributes)
+        // LastOffsetDelta, FirstTimestamp and MaxTimestamp are extracted from the messages
+        .appendInt32(messages.length - 1)
+        .appendInt64(BigInt(firstTimestamp))
+        .appendInt64(BigInt(maxTimestamp))
+        /* c8 ignore next */
+        .appendInt64(options.producerId ?? -1n)
+        /* c8 ignore next */
+        .appendInt16(options.producerEpoch ?? 0)
+        .appendInt32(firstSequence)
+        .appendInt32(messages.length) // Number of records
+        .appendFrom(buffer);
+    // Phase 2: Prepend the PartitionLeaderEpoch, Magic and CRC, then the Length and firstOffset, in reverse order
+    return (writer
+        .appendUnsignedInt32(crc32c(writer.dynamicBuffer), false)
+        .appendInt8(CURRENT_RECORD_VERSION, false)
+        .appendInt32(options.partitionLeaderEpoch ?? 0, false)
+        .prependLength()
+        // FirstOffset is 0
+        .appendInt64(0n, false));
+}
+export function readRecordsBatch(reader) {
+    const batch = {
+        firstOffset: reader.readInt64(),
+        length: reader.readInt32(),
+        partitionLeaderEpoch: reader.readInt32(),
+        magic: reader.readInt8(),
+        crc: reader.readUnsignedInt32(),
+        attributes: reader.readInt16(),
+        lastOffsetDelta: reader.readInt32(),
+        firstTimestamp: reader.readInt64(),
+        maxTimestamp: reader.readInt64(),
+        producerId: reader.readInt64(),
+        producerEpoch: reader.readInt16(),
+        firstSequence: reader.readInt32(),
+        records: []
+    };
+    const recordsLength = reader.readInt32();
+    const compression = batch.attributes & IS_COMPRESSED;
+    if (compression !== 0) {
+        const algorithm = compressionsAlgorithmsByBitmask[compression];
+        if (!algorithm) {
+            throw new UnsupportedCompressionError(`Unsupported compression algorithm with bitmask ${compression}`);
+        }
+        const buffer = algorithm.decompressSync(reader.buffer.slice(reader.position, reader.buffer.length));
+        reader = Reader.from(buffer);
+    }
+    for (let i = 0; i < recordsLength; i++) {
+        batch.records.push(readRecord(reader));
+    }
+    return batch;
+}