kafka-ts 0.0.1-beta

package/dist/src/utils/decoder.js

@@ -0,0 +1,161 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createDecoder = void 0;
+const createDecoder = ({ buffer, offset = 0 }) => {
+    const readInt8 = () => {
+        const value = buffer.readInt8(offset);
+        offset += 1;
+        return value;
+    };
+    const readInt16 = () => {
+        const value = buffer.readInt16BE(offset);
+        offset += 2;
+        return value;
+    };
+    const readInt32 = () => {
+        const value = buffer.readInt32BE(offset);
+        offset += 4;
+        return value;
+    };
+    const readUInt32 = () => {
+        const value = buffer.readUInt32BE(offset);
+        offset += 4;
+        return value;
+    };
+    const readInt64 = () => {
+        const value = buffer.readBigInt64BE(offset);
+        offset += 8;
+        return value;
+    };
+    const readString = () => {
+        const length = readInt16();
+        if (length === -1) {
+            return null;
+        }
+        const value = buffer.toString('utf-8', offset, offset + length);
+        offset += length;
+        return value;
+    };
+    const readCompactString = () => {
+        const length = readUVarInt() - 1;
+        if (length < 0) {
+            return null;
+        }
+        const value = buffer.toString('utf-8', offset, offset + length);
+        offset += length;
+        return value;
+    };
+    const readVarInt = () => {
+        let currentByte;
+        let result = 0;
+        let i = 0;
+        while (((currentByte = buffer[offset++]) & 0x80) !== 0) {
+            result |= (currentByte & 0x7f) << i;
+            i += 7;
+            if (i > 28) {
+                throw new Error("Invalid VarInt, must contain 5 bytes or less");
+            }
+        }
+        result |= currentByte << i;
+        return result;
+    };
+    const readUVarInt = () => {
+        const result = readVarInt();
+        return result >>> 0;
+    };
+    const readVarLong = () => {
+        let currentByte;
+        let result = BigInt(0);
+        let i = 0;
+        while (((currentByte = buffer[offset++]) & 0x80) !== 0) {
+            result |= BigInt(currentByte & 0x7f) << BigInt(i);
+            i += 7;
+            if (i > 63) {
+                throw new Error("Invalid VarLong, must contain 9 bytes or less");
+            }
+        }
+        result |= BigInt(currentByte) << BigInt(i);
+        return result;
+    };
+    const readUUID = () => {
+        const value = buffer.toString("hex", offset, offset + 16);
+        offset += 16;
+        return value;
+    };
+    const readBoolean = () => {
+        const value = buffer.readInt8(offset) === 1;
+        offset += 1;
+        return value;
+    };
+    const readArray = (callback) => {
+        const length = readInt32();
+        const decoder = (0, exports.createDecoder)({ buffer, offset });
+        const results = Array.from({ length }).map(() => callback(decoder));
+        offset = decoder.offset;
+        return results;
+    };
+    const readCompactArray = (callback) => {
+        const length = readUVarInt() - 1;
+        const decoder = (0, exports.createDecoder)({ buffer, offset });
+        const results = Array.from({ length }).map(() => callback(decoder));
+        offset = decoder.offset;
+        return results;
+    };
+    const readRecords = (callback) => {
+        const length = readInt32();
+        return Array.from({ length }).map(() => {
+            const size = Math.ceil((readVarInt() - 1) / 2);
+            const child = (0, exports.createDecoder)({ buffer: buffer.subarray(offset, offset + size) });
+            offset += size;
+            return callback(child);
+        });
+    };
+    const read = (length) => {
+        const value = buffer.subarray(offset, offset + length);
+        offset += length;
+        return value;
+    };
+    const readBytes = () => {
+        const length = readInt32();
+        return read(length);
+    };
+    const readCompactBytes = (divide = false) => {
+        let length = readUVarInt() - 1;
+        if (length < 0) {
+            return null;
+        }
+        if (divide) {
+            length = Math.round(length / 2);
+        }
+        return read(length);
+    };
+    const readTagBuffer = () => {
+        readUVarInt();
+    };
+    return {
+        readInt8,
+        readInt16,
+        readInt32,
+        readInt64,
+        readUInt32,
+        readVarInt,
+        readUVarInt,
+        readVarLong,
+        readString,
+        readBytes,
+        readCompactString,
+        readUUID,
+        readBoolean,
+        readArray,
+        readCompactArray,
+        read,
+        readCompactBytes,
+        readTagBuffer,
+        readRecords,
+        buffer,
+        get offset() {
+            return offset;
+        },
+    };
+};
+exports.createDecoder = createDecoder;

package/dist/src/utils/encoder.js

@@ -0,0 +1,137 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createEncoder = void 0;
+const createEncoder = ({ buffer = Buffer.alloc(0) } = {}) => {
+    const write = (rightBuffer) => (0, exports.createEncoder)({ buffer: Buffer.concat([buffer, rightBuffer]) });
+    const writeInt8 = (value) => {
+        const buffer = Buffer.alloc(1);
+        buffer.writeInt8(value);
+        return write(buffer);
+    };
+    const writeInt16 = (value) => {
+        const buffer = Buffer.alloc(2);
+        buffer.writeInt16BE(value);
+        return write(buffer);
+    };
+    const writeInt32 = (value) => {
+        const buffer = Buffer.alloc(4);
+        buffer.writeInt32BE(value);
+        return write(buffer);
+    };
+    const writeUInt32 = (value) => {
+        const buffer = Buffer.alloc(4);
+        buffer.writeUInt32BE(value);
+        return write(buffer);
+    };
+    const writeInt64 = (value) => {
+        const buffer = Buffer.alloc(8);
+        buffer.writeBigInt64BE(value);
+        return write(buffer);
+    };
+    const writeString = (value) => {
+        if (value === null) {
+            return writeInt16(-1);
+        }
+        const byteLength = Buffer.byteLength(value, "utf-8");
+        const buffer = Buffer.alloc(2 + byteLength);
+        buffer.writeInt16BE(byteLength);
+        buffer.write(value, 2, byteLength, "utf-8");
+        return write(buffer);
+    };
+    const writeUVarInt = (value) => {
+        const byteArray = [];
+        while ((value & 0xffffffff) !== 0) {
+            byteArray.push((value & 0x7f) | 0x80);
+            value >>>= 7;
+        }
+        byteArray.push(value & 0x7f);
+        return write(Buffer.from(byteArray));
+    };
+    const writeVarInt = (value) => {
+        const encodedValue = (value << 1) ^ (value >> 31);
+        return writeUVarInt(encodedValue);
+    };
+    const writeUVarLong = (value) => {
+        const byteArray = [];
+        while ((value & BigInt(0xffffffffffffffff)) !== BigInt(0)) {
+            byteArray.push(Number(value & BigInt(0x7f) | BigInt(0x80)));
+            value = value >> BigInt(7);
+        }
+        byteArray.push(Number(value));
+        return write(Buffer.from(byteArray));
+    };
+    const writeVarLong = (value) => {
+        const encodedValue = (value << BigInt(1)) ^ (value >> BigInt(63));
+        return writeUVarLong(encodedValue);
+    };
+    // const writeVarLong = (value: bigint) => {
+    //     const encodedValue = (value << BigInt(1)) ^ (value >> BigInt(63));
+    //     return writeUVarInt(Number(encodedValue));
+    // };
+    const writeCompactString = (value) => {
+        if (value === null) {
+            return writeUVarInt(0);
+        }
+        const byteLength = Buffer.byteLength(value, "utf-8");
+        const buffer = Buffer.alloc(byteLength);
+        buffer.write(value, 0, byteLength, "utf-8");
+        return writeUVarInt(byteLength + 1).write(buffer);
+    };
+    const writeVarIntString = (value) => {
+        if (value === null) {
+            return writeVarInt(-1);
+        }
+        return writeVarInt(Buffer.byteLength(value, 'utf-8')).write(Buffer.from(value, 'utf-8'));
+    };
+    const writeUUID = (value) => {
+        if (value === null) {
+            return write(Buffer.alloc(16));
+        }
+        return write(Buffer.from(value, "hex"));
+    };
+    const writeCompactArray = (arr, callback) => {
+        if (arr === null) {
+            return writeUVarInt(0);
+        }
+        const buffers = arr.map((item) => callback((0, exports.createEncoder)(), item).value());
+        return writeUVarInt(buffers.length + 1).write(Buffer.concat(buffers));
+    };
+    const writeArray = (arr, callback) => {
+        const buffers = arr.map((item) => callback((0, exports.createEncoder)(), item).value());
+        return writeInt32(arr.length).write(Buffer.concat(buffers));
+    };
+    const writeBoolean = (value) => {
+        const buffer = Buffer.alloc(1);
+        buffer.writeInt8(value ? 1 : 0);
+        return write(buffer);
+    };
+    const writeBytes = (value) => {
+        return writeInt32(value.length).write(value);
+    };
+    const writeCompactBytes = (value) => {
+        return writeUVarInt(value.length + 1).write(value);
+    };
+    const value = () => buffer;
+    return {
+        write,
+        writeInt8,
+        writeInt16,
+        writeInt32,
+        writeUInt32,
+        writeInt64,
+        writeCompactString,
+        writeVarIntString,
+        writeString,
+        writeUVarInt,
+        writeVarInt,
+        writeVarLong,
+        writeUUID,
+        writeCompactArray,
+        writeArray,
+        writeBoolean,
+        writeBytes,
+        writeCompactBytes,
+        value,
+    };
+};
+exports.createEncoder = createEncoder;

package/dist/src/utils/error.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KafkaTSError = void 0;
+class KafkaTSError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "KafkaTSError";
+    }
+}
+exports.KafkaTSError = KafkaTSError;

package/dist/types.d.ts

@@ -0,0 +1,9 @@
+export type Message = {
+    topic: string;
+    partition: number;
+    offset?: bigint;
+    timestamp?: bigint;
+    key: string | null;
+    value: string | null;
+    headers?: Record<string, string>;
+};

package/dist/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/utils/api.d.ts

@@ -0,0 +1,9 @@
+import { Decoder } from "./decoder";
+import { Encoder } from "./encoder";
+export type Api<Request, Response> = {
+    apiKey: number;
+    apiVersion: number;
+    request: (encoder: Encoder, body: Request) => Encoder;
+    response: (buffer: Decoder) => Response;
+};
+export declare const createApi: <Request, Response>(api: Api<Request, Response>) => Api<Request, Response>;

package/dist/utils/api.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createApi = void 0;
+const createApi = (api) => api;
+exports.createApi = createApi;

package/dist/utils/debug.d.ts

@@ -0,0 +1,2 @@
+export declare const serializer: (_: string, value: unknown) => unknown;
+export declare const createDebugger: (module: string) => (func: string, message: string, data?: unknown) => void;

package/dist/utils/debug.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createDebugger = exports.serializer = void 0;
+const serializer = (_, value) => (typeof value === "bigint" ? value.toString() : value);
+exports.serializer = serializer;
+const createDebugger = (module) => (func, message, data) => {
+    if (!process.env.DEBUG?.includes("kafkats"))
+        return;
+    console.debug(`[${module}] ${func}: ${message}`, data && `(${data instanceof Error ? data : JSON.stringify(data, exports.serializer, 4)})`);
+};
+exports.createDebugger = createDebugger;

package/dist/utils/decoder.d.ts

@@ -0,0 +1,29 @@
+/// <reference types="node" />
+export declare class Decoder {
+    private buffer;
+    private offset;
+    constructor(buffer: Buffer);
+    getOffset(): number;
+    getBufferLength(): number;
+    readInt8(): number;
+    readInt16(): number;
+    readInt32(): number;
+    readUInt32(): number;
+    readInt64(): bigint;
+    readUVarInt(): number;
+    readVarInt(): number;
+    readUVarLong(): bigint;
+    readVarLong(): bigint;
+    readString(): string | null;
+    readCompactString(): string | null;
+    readVarIntString(): string | null;
+    readUUID(): string;
+    readBoolean(): boolean;
+    readArray<T>(callback: (opts: Decoder) => T): T[];
+    readCompactArray<T>(callback: (opts: Decoder) => T): T[];
+    readRecords<T>(callback: (opts: Decoder) => T): T[];
+    read(length: number): Buffer;
+    readBytes(): Buffer;
+    readCompactBytes(): Buffer | null;
+    readTagBuffer(): void;
+}

package/dist/utils/decoder.js

@@ -0,0 +1,147 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Decoder = void 0;
+class Decoder {
+    buffer;
+    offset = 0;
+    constructor(buffer) {
+        this.buffer = buffer;
+    }
+    getOffset() {
+        return this.offset;
+    }
+    getBufferLength() {
+        return this.buffer.length;
+    }
+    readInt8() {
+        const value = this.buffer.readInt8(this.offset);
+        this.offset += 1;
+        return value;
+    }
+    readInt16() {
+        const value = this.buffer.readInt16BE(this.offset);
+        this.offset += 2;
+        return value;
+    }
+    readInt32() {
+        const value = this.buffer.readInt32BE(this.offset);
+        this.offset += 4;
+        return value;
+    }
+    readUInt32() {
+        const value = this.buffer.readUInt32BE(this.offset);
+        this.offset += 4;
+        return value;
+    }
+    readInt64() {
+        const value = this.buffer.readBigInt64BE(this.offset);
+        this.offset += 8;
+        return value;
+    }
+    readUVarInt() {
+        let result = 0;
+        let shift = 0;
+        let currentByte;
+        do {
+            currentByte = this.buffer[this.offset++];
+            result |= (currentByte & 0x7f) << shift;
+            shift += 7;
+        } while ((currentByte & 0x80) !== 0);
+        return result;
+    }
+    readVarInt() {
+        const decodedValue = this.readUVarInt();
+        return (decodedValue >>> 1) ^ -(decodedValue & 1);
+    }
+    readUVarLong() {
+        let result = BigInt(0);
+        let shift = BigInt(0);
+        let currentByte;
+        do {
+            currentByte = BigInt(this.buffer[this.offset++]);
+            result |= (currentByte & BigInt(0x7f)) << shift;
+            shift += BigInt(7);
+        } while ((currentByte & BigInt(0x80)) !== BigInt(0));
+        return result;
+    }
+    readVarLong() {
+        const decodedValue = this.readUVarLong();
+        return (decodedValue >> BigInt(1)) ^ -(decodedValue & BigInt(1));
+    }
+    readString() {
+        const length = this.readInt16();
+        if (length < 0) {
+            return null;
+        }
+        const value = this.buffer.toString("utf-8", this.offset, this.offset + length);
+        this.offset += length;
+        return value;
+    }
+    readCompactString() {
+        const length = this.readUVarInt() - 1;
+        if (length < 0) {
+            return null;
+        }
+        const value = this.buffer.toString("utf-8", this.offset, this.offset + length);
+        this.offset += length;
+        return value;
+    }
+    readVarIntString() {
+        const length = this.readVarInt();
+        if (length < 0) {
+            return null;
+        }
+        const value = this.buffer.toString("utf-8", this.offset, this.offset + length);
+        this.offset += length;
+        return value;
+    }
+    readUUID() {
+        const value = this.buffer.toString("hex", this.offset, this.offset + 16);
+        this.offset += 16;
+        return value;
+    }
+    readBoolean() {
+        const value = this.buffer.readInt8(this.offset) === 1;
+        this.offset += 1;
+        return value;
+    }
+    readArray(callback) {
+        const length = this.readInt32();
+        const results = Array.from({ length }).map(() => callback(this));
+        return results;
+    }
+    readCompactArray(callback) {
+        const length = this.readUVarInt() - 1;
+        const results = Array.from({ length }).map(() => callback(this));
+        return results;
+    }
+    readRecords(callback) {
+        const length = this.readInt32();
+        return Array.from({ length }).map(() => {
+            const size = this.readVarInt();
+            const child = new Decoder(this.buffer.subarray(this.offset, this.offset + size));
+            this.offset += size;
+            return callback(child);
+        });
+    }
+    read(length) {
+        const value = this.buffer.subarray(this.offset, this.offset + length);
+        this.offset += length;
+        return value;
+    }
+    readBytes() {
+        const length = this.readInt32();
+        return this.read(length);
+    }
+    readCompactBytes() {
+        const length = this.readUVarInt() - 1;
+        if (length < 0) {
+            return null;
+        }
+        return this.read(length);
+    }
+    readTagBuffer() {
+        this.readUVarInt();
+    }
+}
+exports.Decoder = Decoder;

package/dist/utils/delay.d.ts

@@ -0,0 +1 @@
+export declare const delay: (delayMs: number) => Promise<void>;

package/dist/utils/delay.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.delay = void 0;
+const delay = (delayMs) => new Promise((resolve) => setTimeout(resolve, delayMs));
+exports.delay = delay;

package/dist/utils/encoder.d.ts

@@ -0,0 +1,28 @@
+/// <reference types="node" />
+export declare class Encoder {
+    private buffer;
+    constructor({ buffer }?: {
+        buffer?: Buffer;
+    });
+    write(rightBuffer: Buffer): this;
+    writeInt8(value: number): this;
+    writeInt16(value: number): this;
+    writeInt32(value: number): this;
+    writeUInt32(value: number): this;
+    writeInt64(value: bigint): this;
+    writeUVarInt(value: number): this;
+    writeVarInt(value: number): this;
+    writeUVarLong(value: bigint): this;
+    writeVarLong(value: bigint): this;
+    writeString(value: string | null): this;
+    writeCompactString(value: string | null): this;
+    writeVarIntString(value: string | null): this;
+    writeUUID(value: string | null): this;
+    writeBoolean(value: boolean): this;
+    writeArray<T>(arr: T[], callback: (encoder: Encoder, item: T) => Encoder): this;
+    writeCompactArray<T>(arr: T[] | null, callback: (encoder: Encoder, item: T) => Encoder): this;
+    writeVarIntArray<T>(arr: T[], callback: (encoder: Encoder, item: T) => Encoder): this;
+    writeBytes(value: Buffer): this;
+    writeCompactBytes(value: Buffer): this;
+    value(): Buffer;
+}

package/dist/utils/encoder.js

@@ -0,0 +1,122 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Encoder = void 0;
+class Encoder {
+    buffer;
+    constructor({ buffer = Buffer.alloc(0) } = {}) {
+        this.buffer = buffer;
+    }
+    write(rightBuffer) {
+        this.buffer = Buffer.concat([this.buffer, rightBuffer]);
+        return this;
+    }
+    writeInt8(value) {
+        const buffer = Buffer.alloc(1);
+        buffer.writeInt8(value);
+        return this.write(buffer);
+    }
+    writeInt16(value) {
+        const buffer = Buffer.alloc(2);
+        buffer.writeInt16BE(value);
+        return this.write(buffer);
+    }
+    writeInt32(value) {
+        const buffer = Buffer.alloc(4);
+        buffer.writeInt32BE(value);
+        return this.write(buffer);
+    }
+    writeUInt32(value) {
+        const buffer = Buffer.alloc(4);
+        buffer.writeUInt32BE(value);
+        return this.write(buffer);
+    }
+    writeInt64(value) {
+        const buffer = Buffer.alloc(8);
+        buffer.writeBigInt64BE(value);
+        return this.write(buffer);
+    }
+    writeUVarInt(value) {
+        const byteArray = [];
+        while ((value & 0xffffffff) !== 0) {
+            byteArray.push((value & 0x7f) | 0x80);
+            value >>>= 7;
+        }
+        byteArray.push(value & 0x7f);
+        return this.write(Buffer.from(byteArray));
+    }
+    writeVarInt(value) {
+        const encodedValue = (value << 1) ^ (value >> 31);
+        return this.writeUVarInt(encodedValue);
+    }
+    writeUVarLong(value) {
+        const byteArray = [];
+        while ((value & BigInt(0xffffffffffffffff)) !== BigInt(0)) {
+            byteArray.push(Number((value & BigInt(0x7f)) | BigInt(0x80)));
+            value = value >> BigInt(7);
+        }
+        byteArray.push(Number(value));
+        return this.write(Buffer.from(byteArray));
+    }
+    writeVarLong(value) {
+        const encodedValue = (value << BigInt(1)) ^ (value >> BigInt(63));
+        return this.writeUVarLong(encodedValue);
+    }
+    writeString(value) {
+        if (value === null) {
+            return this.writeInt16(-1);
+        }
+        const byteLength = Buffer.byteLength(value, "utf-8");
+        const buffer = Buffer.alloc(byteLength);
+        buffer.write(value, 0, byteLength, "utf-8");
+        return this.writeInt16(byteLength).write(buffer);
+    }
+    writeCompactString(value) {
+        if (value === null) {
+            return this.writeUVarInt(0);
+        }
+        const byteLength = Buffer.byteLength(value, "utf-8");
+        const buffer = Buffer.alloc(byteLength);
+        buffer.write(value, 0, byteLength, "utf-8");
+        return this.writeUVarInt(byteLength + 1).write(buffer);
+    }
+    writeVarIntString(value) {
+        if (value === null) {
+            return this.writeVarInt(-1);
+        }
+        return this.writeVarInt(Buffer.byteLength(value, "utf-8")).write(Buffer.from(value, "utf-8"));
+    }
+    writeUUID(value) {
+        if (value === null) {
+            return this.write(Buffer.alloc(16));
+        }
+        return this.write(Buffer.from(value, "hex"));
+    }
+    writeBoolean(value) {
+        return this.writeInt8(value ? 1 : 0);
+    }
+    writeArray(arr, callback) {
+        const buffers = arr.map((item) => callback(new Encoder(), item).value());
+        return this.writeInt32(arr.length).write(Buffer.concat(buffers));
+    }
+    writeCompactArray(arr, callback) {
+        if (arr === null) {
+            return this.writeUVarInt(0);
+        }
+        const buffers = arr.map((item) => callback(new Encoder(), item).value());
+        return this.writeUVarInt(buffers.length + 1).write(Buffer.concat(buffers));
+    }
+    writeVarIntArray(arr, callback) {
+        const buffers = arr.map((item) => callback(new Encoder(), item).value());
+        return this.writeVarInt(buffers.length).write(Buffer.concat(buffers));
+    }
+    writeBytes(value) {
+        return this.writeInt32(value.length).write(value);
+    }
+    writeCompactBytes(value) {
+        return this.writeUVarInt(value.length + 1).write(value);
+    }
+    value() {
+        return this.buffer;
+    }
+}
+exports.Encoder = Encoder;