kafka-ts 1.3.1-beta.1 → 1.3.1-beta.2

package/dist/src/utils/decoder.js

@@ -0,0 +1,161 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createDecoder = void 0;
+const createDecoder = ({ buffer, offset = 0 }) => {
+    const readInt8 = () => {
+        const value = buffer.readInt8(offset);
+        offset += 1;
+        return value;
+    };
+    const readInt16 = () => {
+        const value = buffer.readInt16BE(offset);
+        offset += 2;
+        return value;
+    };
+    const readInt32 = () => {
+        const value = buffer.readInt32BE(offset);
+        offset += 4;
+        return value;
+    };
+    const readUInt32 = () => {
+        const value = buffer.readUInt32BE(offset);
+        offset += 4;
+        return value;
+    };
+    const readInt64 = () => {
+        const value = buffer.readBigInt64BE(offset);
+        offset += 8;
+        return value;
+    };
+    const readString = () => {
+        const length = readInt16();
+        if (length === -1) {
+            return null;
+        }
+        const value = buffer.toString('utf-8', offset, offset + length);
+        offset += length;
+        return value;
+    };
+    const readCompactString = () => {
+        const length = readUVarInt() - 1;
+        if (length < 0) {
+            return null;
+        }
+        const value = buffer.toString('utf-8', offset, offset + length);
+        offset += length;
+        return value;
+    };
+    const readVarInt = () => {
+        let currentByte;
+        let result = 0;
+        let i = 0;
+        while (((currentByte = buffer[offset++]) & 0x80) !== 0) {
+            result |= (currentByte & 0x7f) << i;
+            i += 7;
+            if (i > 28) {
+                throw new Error("Invalid VarInt, must contain 5 bytes or less");
+            }
+        }
+        result |= currentByte << i;
+        return result;
+    };
+    const readUVarInt = () => {
+        const result = readVarInt();
+        return result >>> 0;
+    };
+    const readVarLong = () => {
+        let currentByte;
+        let result = BigInt(0);
+        let i = 0;
+        while (((currentByte = buffer[offset++]) & 0x80) !== 0) {
+            result |= BigInt(currentByte & 0x7f) << BigInt(i);
+            i += 7;
+            if (i > 63) {
+                throw new Error("Invalid VarLong, must contain 9 bytes or less");
+            }
+        }
+        result |= BigInt(currentByte) << BigInt(i);
+        return result;
+    };
+    const readUUID = () => {
+        const value = buffer.toString("hex", offset, offset + 16);
+        offset += 16;
+        return value;
+    };
+    const readBoolean = () => {
+        const value = buffer.readInt8(offset) === 1;
+        offset += 1;
+        return value;
+    };
+    const readArray = (callback) => {
+        const length = readInt32();
+        const decoder = (0, exports.createDecoder)({ buffer, offset });
+        const results = Array.from({ length }).map(() => callback(decoder));
+        offset = decoder.offset;
+        return results;
+    };
+    const readCompactArray = (callback) => {
+        const length = readUVarInt() - 1;
+        const decoder = (0, exports.createDecoder)({ buffer, offset });
+        const results = Array.from({ length }).map(() => callback(decoder));
+        offset = decoder.offset;
+        return results;
+    };
+    const readRecords = (callback) => {
+        const length = readInt32();
+        return Array.from({ length }).map(() => {
+            const size = Math.ceil((readVarInt() - 1) / 2);
+            const child = (0, exports.createDecoder)({ buffer: buffer.subarray(offset, offset + size) });
+            offset += size;
+            return callback(child);
+        });
+    };
+    const read = (length) => {
+        const value = buffer.subarray(offset, offset + length);
+        offset += length;
+        return value;
+    };
+    const readBytes = () => {
+        const length = readInt32();
+        return read(length);
+    };
+    const readCompactBytes = (divide = false) => {
+        let length = readUVarInt() - 1;
+        if (length < 0) {
+            return null;
+        }
+        if (divide) {
+            length = Math.round(length / 2);
+        }
+        return read(length);
+    };
+    const readTagBuffer = () => {
+        readUVarInt();
+    };
+    return {
+        readInt8,
+        readInt16,
+        readInt32,
+        readInt64,
+        readUInt32,
+        readVarInt,
+        readUVarInt,
+        readVarLong,
+        readString,
+        readBytes,
+        readCompactString,
+        readUUID,
+        readBoolean,
+        readArray,
+        readCompactArray,
+        read,
+        readCompactBytes,
+        readTagBuffer,
+        readRecords,
+        buffer,
+        get offset() {
+            return offset;
+        },
+    };
+};
+exports.createDecoder = createDecoder;

package/dist/src/utils/encoder.js

@@ -0,0 +1,137 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createEncoder = void 0;
+const createEncoder = ({ buffer = Buffer.alloc(0) } = {}) => {
+    const write = (rightBuffer) => (0, exports.createEncoder)({ buffer: Buffer.concat([buffer, rightBuffer]) });
+    const writeInt8 = (value) => {
+        const buffer = Buffer.alloc(1);
+        buffer.writeInt8(value);
+        return write(buffer);
+    };
+    const writeInt16 = (value) => {
+        const buffer = Buffer.alloc(2);
+        buffer.writeInt16BE(value);
+        return write(buffer);
+    };
+    const writeInt32 = (value) => {
+        const buffer = Buffer.alloc(4);
+        buffer.writeInt32BE(value);
+        return write(buffer);
+    };
+    const writeUInt32 = (value) => {
+        const buffer = Buffer.alloc(4);
+        buffer.writeUInt32BE(value);
+        return write(buffer);
+    };
+    const writeInt64 = (value) => {
+        const buffer = Buffer.alloc(8);
+        buffer.writeBigInt64BE(value);
+        return write(buffer);
+    };
+    const writeString = (value) => {
+        if (value === null) {
+            return writeInt16(-1);
+        }
+        const byteLength = Buffer.byteLength(value, "utf-8");
+        const buffer = Buffer.alloc(2 + byteLength);
+        buffer.writeInt16BE(byteLength);
+        buffer.write(value, 2, byteLength, "utf-8");
+        return write(buffer);
+    };
+    const writeUVarInt = (value) => {
+        const byteArray = [];
+        while ((value & 0xffffffff) !== 0) {
+            byteArray.push((value & 0x7f) | 0x80);
+            value >>>= 7;
+        }
+        byteArray.push(value & 0x7f);
+        return write(Buffer.from(byteArray));
+    };
+    const writeVarInt = (value) => {
+        const encodedValue = (value << 1) ^ (value >> 31);
+        return writeUVarInt(encodedValue);
+    };
+    const writeUVarLong = (value) => {
+        const byteArray = [];
+        while ((value & BigInt(0xffffffffffffffff)) !== BigInt(0)) {
+            byteArray.push(Number(value & BigInt(0x7f) | BigInt(0x80)));
+            value = value >> BigInt(7);
+        }
+        byteArray.push(Number(value));
+        return write(Buffer.from(byteArray));
+    };
+    const writeVarLong = (value) => {
+        const encodedValue = (value << BigInt(1)) ^ (value >> BigInt(63));
+        return writeUVarLong(encodedValue);
+    };
+    // const writeVarLong = (value: bigint) => {
+    //     const encodedValue = (value << BigInt(1)) ^ (value >> BigInt(63));
+    //     return writeUVarInt(Number(encodedValue));
+    // };
+    const writeCompactString = (value) => {
+        if (value === null) {
+            return writeUVarInt(0);
+        }
+        const byteLength = Buffer.byteLength(value, "utf-8");
+        const buffer = Buffer.alloc(byteLength);
+        buffer.write(value, 0, byteLength, "utf-8");
+        return writeUVarInt(byteLength + 1).write(buffer);
+    };
+    const writeVarIntString = (value) => {
+        if (value === null) {
+            return writeVarInt(-1);
+        }
+        return writeVarInt(Buffer.byteLength(value, 'utf-8')).write(Buffer.from(value, 'utf-8'));
+    };
+    const writeUUID = (value) => {
+        if (value === null) {
+            return write(Buffer.alloc(16));
+        }
+        return write(Buffer.from(value, "hex"));
+    };
+    const writeCompactArray = (arr, callback) => {
+        if (arr === null) {
+            return writeUVarInt(0);
+        }
+        const buffers = arr.map((item) => callback((0, exports.createEncoder)(), item).value());
+        return writeUVarInt(buffers.length + 1).write(Buffer.concat(buffers));
+    };
+    const writeArray = (arr, callback) => {
+        const buffers = arr.map((item) => callback((0, exports.createEncoder)(), item).value());
+        return writeInt32(arr.length).write(Buffer.concat(buffers));
+    };
+    const writeBoolean = (value) => {
+        const buffer = Buffer.alloc(1);
+        buffer.writeInt8(value ? 1 : 0);
+        return write(buffer);
+    };
+    const writeBytes = (value) => {
+        return writeInt32(value.length).write(value);
+    };
+    const writeCompactBytes = (value) => {
+        return writeUVarInt(value.length + 1).write(value);
+    };
+    const value = () => buffer;
+    return {
+        write,
+        writeInt8,
+        writeInt16,
+        writeInt32,
+        writeUInt32,
+        writeInt64,
+        writeCompactString,
+        writeVarIntString,
+        writeString,
+        writeUVarInt,
+        writeVarInt,
+        writeVarLong,
+        writeUUID,
+        writeCompactArray,
+        writeArray,
+        writeBoolean,
+        writeBytes,
+        writeCompactBytes,
+        value,
+    };
+};
+exports.createEncoder = createEncoder;

package/dist/src/utils/error.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KafkaTSError = void 0;
+class KafkaTSError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "KafkaTSError";
+    }
+}
+exports.KafkaTSError = KafkaTSError;

package/dist/utils/api.d.ts

@@ -3,7 +3,10 @@ import { Encoder } from './encoder';
 export type Api<Request, Response> = {
     apiKey: number;
     apiVersion: number;
-    request: (encoder: Encoder, body: Request) => Encoder;
+    requestHeaderVersion: 1 | 2;
+    responseHeaderVersion: 0 | 1;
+    fallback?: Api<Request, Response>;
+    request: (encoder: Encoder, data: Request) => Encoder;
     response: (buffer: Decoder) => Promise<Response> | Response;
 };
 export declare const createApi: <Request, Response>(api: Api<Request, Response>) => Api<Request, Response>;

package/dist/utils/cached.d.ts

@@ -0,0 +1,3 @@
+export declare const cached: <F extends (...args: any[]) => any>(func: F, createKey: (...args: Parameters<F>) => string) => F & {
+    clear: () => void;
+};

package/dist/utils/cached.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.cached = void 0;
+const cached = (func, createKey) => {
+    const cache = {};
+    const cachedFunc = ((...args) => {
+        const key = createKey(...args);
+        if (!(key in cache)) {
+            cache[key] = func(...args);
+        }
+        return cache[key];
+    });
+    cachedFunc.clear = () => {
+        for (const key in cache)
+            delete cache[key];
+    };
+    return cachedFunc;
+};
+exports.cached = cached;

package/dist/utils/debug.d.ts

@@ -0,0 +1,2 @@
+export declare const serializer: (_: string, value: unknown) => unknown;
+export declare const createDebugger: (module: string) => (func: string, message: string, data?: unknown) => void;

package/dist/utils/debug.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createDebugger = exports.serializer = void 0;
+const serializer = (_, value) => (typeof value === 'bigint' ? value.toString() : value);
+exports.serializer = serializer;
+const createDebugger = (module) => (func, message, data) => {
+    if (!process.env.DEBUG?.includes('kafka-ts'))
+        return;
+    console.debug(`[${module}] ${func}: ${message}`, data && `(${data instanceof Error ? data : JSON.stringify(data, exports.serializer, 4)})`);
+};
+exports.createDebugger = createDebugger;

package/dist/utils/decoder.d.ts

@@ -24,7 +24,7 @@ export declare class Decoder {
     readVarIntArray<T>(callback: (opts: Decoder) => T): T[];
     readRecords<T>(callback: (opts: Decoder) => T): T[];
     read(length?: number): Buffer<ArrayBufferLike>;
-    readBytes(): Buffer<ArrayBufferLike>;
+    readBytes(): Buffer<ArrayBufferLike> | null;
     readCompactBytes(): Buffer<ArrayBufferLike> | null;
-    readTagBuffer(): void;
+    readTagBuffer(): Record<number, Buffer>;
 }

package/dist/utils/decoder.js

@@ -149,6 +149,9 @@ class Decoder {
     }
     readBytes() {
         const length = this.readInt32();
+        if (length < 0) {
+            return null;
+        }
         return this.read(length);
     }
     readCompactBytes() {
@@ -159,7 +162,17 @@ class Decoder {
         return this.read(length);
     }
     readTagBuffer() {
-        this.readUVarInt();
+        const numTags = this.readUVarInt();
+        if (!numTags)
+            return {};
+        const tags = {};
+        for (let i = 0; i < numTags; i++) {
+            const tagId = this.readUVarInt();
+            const size = this.readUVarInt();
+            const value = this.read(size);
+            tags[tagId] = value;
+        }
+        return tags;
     }
 }
 exports.Decoder = Decoder;

package/dist/utils/encoder.d.ts

@@ -25,5 +25,6 @@ export declare class Encoder {
     writeVarIntArray<T>(arr: T[], callback: (encoder: Encoder, item: T) => void): this;
     writeBytes(value: Buffer): this;
     writeCompactBytes(value: Buffer): this;
+    writeTagBuffer(tags?: Record<number, Buffer>): this;
     value(): Buffer<ArrayBufferLike>;
 }

package/dist/utils/encoder.js

@@ -152,6 +152,20 @@ class Encoder {
         this.write(value);
         return this;
     }
+    writeTagBuffer(tags) {
+        if (!tags)
+            return this.writeUVarInt(0);
+        const entries = Object.entries(tags)
+            .map(([k, v]) => [Number(k), v])
+            .sort(([a], [b]) => a - b);
+        this.writeUVarInt(entries.length);
+        for (const [tagId, tagBuffer] of entries) {
+            this.writeUVarInt(tagId);
+            this.writeUVarInt(tagBuffer.length);
+            this.write(tagBuffer);
+        }
+        return this;
+    }
     value() {
         return this.buffer.subarray(0, this.offset);
     }

package/dist/utils/lock.d.ts

@@ -0,0 +1,8 @@
+import EventEmitter from 'events';
+export declare class Lock extends EventEmitter {
+    private locks;
+    constructor();
+    acquire(keys: string[], callback: () => Promise<void>): Promise<void>;
+    private acquireKey;
+    private releaseKey;
+}

package/dist/utils/lock.js

@@ -0,0 +1,44 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Lock = void 0;
+const events_1 = __importDefault(require("events"));
+const logger_1 = require("./logger");
+class Lock extends events_1.default {
+    locks = {};
+    constructor() {
+        super();
+        this.setMaxListeners(Infinity);
+    }
+    async acquire(keys, callback) {
+        await Promise.all(keys.map((key) => this.acquireKey(key)));
+        try {
+            await callback();
+        }
+        finally {
+            keys.forEach((key) => this.releaseKey(key));
+        }
+    }
+    async acquireKey(key) {
+        while (this.locks[key]) {
+            await new Promise((resolve) => {
+                const timeout = setTimeout(() => {
+                    logger_1.log.warn(`Lock timed out`, { key });
+                    this.releaseKey(key);
+                }, 60_000);
+                this.once(`release:${key}`, () => {
+                    clearTimeout(timeout);
+                    resolve();
+                });
+            });
+        }
+        this.locks[key] = true;
+    }
+    releaseKey(key) {
+        this.locks[key] = false;
+        this.emit(`release:${key}`);
+    }
+}
+exports.Lock = Lock;

package/dist/utils/memo.d.ts

@@ -0,0 +1 @@
+export declare const memo: <T extends (...args: any[]) => any>(fn: T) => (...args: Parameters<T>) => ReturnType<T>;

package/dist/utils/memo.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.memo = void 0;
+const memo = (fn) => {
+    const cache = {};
+    return (...args) => {
+        const key = JSON.stringify(args);
+        if (cache[key]) {
+            return cache[key];
+        }
+        const result = fn(...args);
+        cache[key] = result;
+        return result;
+    };
+};
+exports.memo = memo;

package/dist/utils/mutex.d.ts

@@ -0,0 +1,3 @@
+export declare const createMutex: () => {
+    exclusive: (fn: () => Promise<void>) => Promise<void>;
+};

package/dist/utils/mutex.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createMutex = void 0;
+const createMutex = () => {
+    const queue = [];
+    let isLocked = false;
+    const acquire = () => {
+        return new Promise((resolve) => {
+            if (!isLocked) {
+                isLocked = true;
+                return resolve();
+            }
+            queue.push(resolve);
+        });
+    };
+    const release = () => {
+        isLocked = false;
+        const next = queue.shift();
+        next && next();
+    };
+    const exclusive = async (fn) => {
+        await acquire();
+        try {
+            await fn();
+        }
+        finally {
+            release();
+        }
+    };
+    return { exclusive };
+};
+exports.createMutex = createMutex;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kafka-ts",
-    "version": "1.3.1-beta.1",
+    "version": "1.3.1-beta.2",
     "main": "dist/index.js",
     "author": "Priit Käärd",
     "license": "MIT",