kafka-ts 0.0.1-beta → 0.0.1-beta.0

package/src/utils/crypto.ts

@@ -0,0 +1,14 @@
+import { createHash, createHmac, pbkdf2, randomBytes } from 'crypto';
+
+export const generateNonce = () => randomBytes(16).toString('base64').replace(/[\/=]/g, '');
+
+export const saltPassword = (password: string, salt: string, iterations: number, keyLength: number, digest: string) =>
+    new Promise<Buffer>((resolve, reject) =>
+        pbkdf2(password, salt, iterations, keyLength, digest, (err, key) => (err ? reject(err) : resolve(key))),
+    );
+
+export const base64Encode = (input: Buffer | string) => Buffer.from(input).toString('base64');
+export const base64Decode = (input: string) => Buffer.from(input, 'base64').toString();
+export const hash = (data: Buffer) => createHash('sha256').update(data).digest();
+export const hmac = (key: Buffer, data: Buffer | string) => createHmac('sha256', key).update(data).digest();
+export const xor = (a: Buffer, b: Buffer) => Buffer.from(a.map((byte, i) => byte ^ b[i]));

package/src/utils/debug.ts

@@ -1,7 +1,7 @@
-export const serializer = (_: string, value: unknown) => (typeof value === "bigint" ? value.toString() : value);
+export const serializer = (_: string, value: unknown) => (typeof value === 'bigint' ? value.toString() : value);
 
 export const createDebugger = (module: string) => (func: string, message: string, data?: unknown) => {
-    if (!process.env.DEBUG?.includes("kafkats")) return;
+    if (!process.env.DEBUG?.includes('kafka-ts')) return;
     console.debug(
         `[${module}] ${func}: ${message}`,
         data && `(${data instanceof Error ? data : JSON.stringify(data, serializer, 4)})`,

package/src/utils/decoder.ts

@@ -81,7 +81,7 @@ export class Decoder {
             return null;
         }
 
-        const value = this.buffer.toString("utf-8", this.offset, this.offset + length);
+        const value = this.buffer.toString('utf-8', this.offset, this.offset + length);
         this.offset += length;
         return value;
     }
@@ -92,7 +92,7 @@ export class Decoder {
             return null;
         }
 
-        const value = this.buffer.toString("utf-8", this.offset, this.offset + length);
+        const value = this.buffer.toString('utf-8', this.offset, this.offset + length);
         this.offset += length;
         return value;
     }
@@ -103,13 +103,13 @@ export class Decoder {
             return null;
         }
 
-        const value = this.buffer.toString("utf-8", this.offset, this.offset + length);
+        const value = this.buffer.toString('utf-8', this.offset, this.offset + length);
         this.offset += length;
         return value;
     }
 
     public readUUID() {
-        const value = this.buffer.toString("hex", this.offset, this.offset + 16);
+        const value = this.buffer.toString('hex', this.offset, this.offset + 16);
         this.offset += 16;
         return value;
     }

package/src/utils/encoder.ts

@@ -74,9 +74,9 @@ export class Encoder {
         if (value === null) {
             return this.writeInt16(-1);
         }
-        const byteLength = Buffer.byteLength(value, "utf-8");
+        const byteLength = Buffer.byteLength(value, 'utf-8');
         const buffer = Buffer.alloc(byteLength);
-        buffer.write(value, 0, byteLength, "utf-8");
+        buffer.write(value, 0, byteLength, 'utf-8');
         return this.writeInt16(byteLength).write(buffer);
     }
 
@@ -85,9 +85,9 @@ export class Encoder {
             return this.writeUVarInt(0);
         }
 
-        const byteLength = Buffer.byteLength(value, "utf-8");
+        const byteLength = Buffer.byteLength(value, 'utf-8');
         const buffer = Buffer.alloc(byteLength);
-        buffer.write(value, 0, byteLength, "utf-8");
+        buffer.write(value, 0, byteLength, 'utf-8');
         return this.writeUVarInt(byteLength + 1).write(buffer);
     }
 
@@ -95,14 +95,14 @@ export class Encoder {
         if (value === null) {
             return this.writeVarInt(-1);
         }
-        return this.writeVarInt(Buffer.byteLength(value, "utf-8")).write(Buffer.from(value, "utf-8"));
+        return this.writeVarInt(Buffer.byteLength(value, 'utf-8')).write(Buffer.from(value, 'utf-8'));
     }
 
     public writeUUID(value: string | null) {
         if (value === null) {
             return this.write(Buffer.alloc(16));
         }
-        return this.write(Buffer.from(value, "hex"));
+        return this.write(Buffer.from(value, 'hex'));
     }
 
     public writeBoolean(value: boolean) {

package/src/utils/error.ts

@@ -1,4 +1,4 @@
-import { API_ERROR } from "../api";
+import { API_ERROR } from '../api';
 
 export class KafkaTSError extends Error {
     constructor(message: string) {
@@ -13,8 +13,8 @@ export class KafkaTSApiError<T = any> extends KafkaTSError {
         public errorMessage: string | null,
         public response: T,
     ) {
-        const [errorName] = Object.entries(API_ERROR).find(([, value]) => value === errorCode) ?? ["UNKNOWN"];
-        super(`${errorName}${errorMessage ? `: ${errorMessage}` : ""}`);
+        const [errorName] = Object.entries(API_ERROR).find(([, value]) => value === errorCode) ?? ['UNKNOWN'];
+        super(`${errorName}${errorMessage ? `: ${errorMessage}` : ''}`);
     }
 }
 

package/src/utils/retrier.ts

@@ -1,4 +1,4 @@
-import { delay } from "./delay";
+import { delay } from './delay';
 
 export type Retrier = (func: () => unknown) => Promise<void>;
 

package/src/utils/tracer.ts

@@ -1,9 +1,10 @@
-import { serializer } from "./debug";
+import { serializer } from './debug';
 
-export const trace =
+export const createTracer =
+    (module: string, attributes?: Record<string, unknown>) =>
     (fn?: (...args: any[]) => Record<string, unknown> | undefined) =>
     (target: any, propertyKey: string, descriptor: PropertyDescriptor) => {
-        if (!process.env.DEBUG?.includes("kafkats")) return;
+        if (!process.env.DEBUG?.includes('kafka-ts')) return;
 
         const original = descriptor.value;
         descriptor.value = function (...args: any[]) {
@@ -12,7 +13,7 @@ export const trace =
 
             const onEnd = <T>(result: T): T => {
                 console.log(
-                    `[${propertyKey}] +${Date.now() - startTime}ms ${JSON.stringify({ ...metadata, result }, serializer)}`,
+                    `[${module}.${propertyKey}] +${Date.now() - startTime}ms ${JSON.stringify({ ...attributes, ...metadata, result }, serializer)}`,
                 );
                 return result;
             };
@@ -26,3 +27,5 @@ export const trace =
             }
         };
     };
+
+export const trace = createTracer('GLOBAL');

package/tsconfig.json

@@ -1,17 +1,17 @@
 {
-  "compilerOptions": {
-    "lib": ["es2023"],
-    "target": "es2022",
-    "module": "node16",
-    "strict": true,
-    "forceConsistentCasingInFileNames": true,
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "moduleResolution": "node16",
-    "outDir": "dist",
-    "declaration": true,
-    "emitDecoratorMetadata": true,
-    "experimentalDecorators": true
-  },
-  "include": ["src/**/*"],
-}
+    "compilerOptions": {
+        "lib": ["es2023"],
+        "target": "es2022",
+        "module": "node16",
+        "strict": true,
+        "forceConsistentCasingInFileNames": true,
+        "esModuleInterop": true,
+        "skipLibCheck": true,
+        "moduleResolution": "node16",
+        "outDir": "dist",
+        "declaration": true,
+        "emitDecoratorMetadata": true,
+        "experimentalDecorators": true
+    },
+    "include": ["src/**/*"]
+}

package/dist/api/api-versions.d.ts

@@ -1,9 +0,0 @@
-export declare const API_VERSIONS: import("../utils/api.js").Api<unknown, {
-    errorCode: number;
-    versions: {
-        apiKey: number;
-        minVersion: number;
-        maxVersion: number;
-    }[];
-    throttleTimeMs: number;
-}>;

package/dist/api/api-versions.js

@@ -1,24 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.API_VERSIONS = void 0;
-const api_js_1 = require("../utils/api.js");
-const error_js_1 = require("../utils/error.js");
-exports.API_VERSIONS = (0, api_js_1.createApi)({
-    apiKey: 18,
-    apiVersion: 2,
-    request: (encoder) => encoder,
-    response: (decoder) => {
-        const result = {
-            errorCode: decoder.readInt16(),
-            versions: decoder.readArray((version) => ({
-                apiKey: version.readInt16(),
-                minVersion: version.readInt16(),
-                maxVersion: version.readInt16(),
-            })),
-            throttleTimeMs: decoder.readInt32(),
-        };
-        if (result.errorCode)
-            throw new error_js_1.KafkaTSApiError(result.errorCode, null, result);
-        return result;
-    },
-});

package/dist/api/create-topics.d.ts

@@ -1,38 +0,0 @@
-export declare const CREATE_TOPICS: import("../utils/api").Api<{
-    topics: {
-        name: string;
-        numPartitions: number;
-        replicationFactor: number;
-        assignments: {
-            partitionIndex: number;
-            brokerIds: number[];
-        }[];
-        configs: {
-            name: string;
-            value: string | null;
-        }[];
-    }[];
-    timeoutMs: number;
-    validateOnly: boolean;
-}, {
-    _tag: void;
-    throttleTimeMs: number;
-    topics: {
-        name: string | null;
-        topicId: string;
-        errorCode: number;
-        errorMessage: string | null;
-        numPartitions: number;
-        replicationFactor: number;
-        configs: {
-            name: string | null;
-            value: string | null;
-            readOnly: boolean;
-            configSource: number;
-            isSensitive: boolean;
-            _tag: void;
-        }[];
-        _tag: void;
-    }[];
-    _tag2: void;
-}>;

package/dist/api/create-topics.js

@@ -1,53 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CREATE_TOPICS = void 0;
-const api_1 = require("../utils/api");
-const error_1 = require("../utils/error");
-exports.CREATE_TOPICS = (0, api_1.createApi)({
-    apiKey: 19,
-    apiVersion: 7,
-    request: (encoder, data) => encoder
-        .writeUVarInt(0)
-        .writeCompactArray(data.topics, (encoder, topic) => encoder
-        .writeCompactString(topic.name)
-        .writeInt32(topic.numPartitions)
-        .writeInt16(topic.replicationFactor)
-        .writeCompactArray(topic.assignments, (encoder, assignment) => encoder
-        .writeInt32(assignment.partitionIndex)
-        .writeCompactArray(assignment.brokerIds, (encoder, brokerId) => encoder.writeInt32(brokerId))
-        .writeUVarInt(0))
-        .writeCompactArray(topic.configs, (encoder, config) => encoder.writeCompactString(config.name).writeCompactString(config.value).writeUVarInt(0))
-        .writeUVarInt(0))
-        .writeInt32(data.timeoutMs)
-        .writeBoolean(data.validateOnly)
-        .writeUVarInt(0),
-    response: (decoder) => {
-        const result = {
-            _tag: decoder.readTagBuffer(),
-            throttleTimeMs: decoder.readInt32(),
-            topics: decoder.readCompactArray((topic) => ({
-                name: topic.readCompactString(),
-                topicId: topic.readUUID(),
-                errorCode: topic.readInt16(),
-                errorMessage: topic.readCompactString(),
-                numPartitions: topic.readInt32(),
-                replicationFactor: topic.readInt16(),
-                configs: topic.readCompactArray((config) => ({
-                    name: config.readCompactString(),
-                    value: config.readCompactString(),
-                    readOnly: config.readBoolean(),
-                    configSource: config.readInt8(),
-                    isSensitive: config.readBoolean(),
-                    _tag: config.readTagBuffer(),
-                })),
-                _tag: topic.readTagBuffer(),
-            })),
-            _tag2: decoder.readTagBuffer(),
-        };
-        result.topics.forEach((topic) => {
-            if (topic.errorCode)
-                throw new error_1.KafkaTSApiError(topic.errorCode, topic.errorMessage, result);
-        });
-        return result;
-    },
-});

package/dist/api/delete-topics.d.ts

@@ -1,18 +0,0 @@
-export declare const DELETE_TOPICS: import("../utils/api").Api<{
-    topics: {
-        name: string | null;
-        topicId: string | null;
-    }[];
-    timeoutMs: number;
-}, {
-    _tag: void;
-    throttleTimeMs: number;
-    responses: {
-        name: string | null;
-        topicId: string;
-        errorCode: number;
-        errorMessage: string | null;
-        _tag: void;
-    }[];
-    _tag2: void;
-}>;

package/dist/api/delete-topics.js

@@ -1,33 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DELETE_TOPICS = void 0;
-const api_1 = require("../utils/api");
-const error_1 = require("../utils/error");
-exports.DELETE_TOPICS = (0, api_1.createApi)({
-    apiKey: 20,
-    apiVersion: 6,
-    request: (encoder, data) => encoder
-        .writeUVarInt(0)
-        .writeCompactArray(data.topics, (encoder, topic) => encoder.writeCompactString(topic.name).writeUUID(topic.topicId).writeUVarInt(0))
-        .writeInt32(data.timeoutMs)
-        .writeUVarInt(0),
-    response: (decoder) => {
-        const result = {
-            _tag: decoder.readTagBuffer(),
-            throttleTimeMs: decoder.readInt32(),
-            responses: decoder.readCompactArray((decoder) => ({
-                name: decoder.readCompactString(),
-                topicId: decoder.readUUID(),
-                errorCode: decoder.readInt16(),
-                errorMessage: decoder.readCompactString(),
-                _tag: decoder.readTagBuffer(),
-            })),
-            _tag2: decoder.readTagBuffer(),
-        };
-        result.responses.forEach((response) => {
-            if (response.errorCode)
-                throw new error_1.KafkaTSApiError(response.errorCode, response.errorMessage, result);
-        });
-        return result;
-    },
-});

package/dist/api/fetch.d.ts

@@ -1,77 +0,0 @@
-export declare const enum IsolationLevel {
-    READ_UNCOMMITTED = 0,
-    READ_COMMITTED = 1
-}
-export declare const FETCH: import("../utils/api").Api<{
-    maxWaitMs: number;
-    minBytes: number;
-    maxBytes: number;
-    isolationLevel: IsolationLevel;
-    sessionId: number;
-    sessionEpoch: number;
-    topics: {
-        topicId: string;
-        partitions: {
-            partition: number;
-            currentLeaderEpoch: number;
-            fetchOffset: bigint;
-            lastFetchedEpoch: number;
-            logStartOffset: bigint;
-            partitionMaxBytes: number;
-        }[];
-    }[];
-    forgottenTopicsData: {
-        topicId: string;
-        partitions: number[];
-    }[];
-    rackId: string;
-}, {
-    _tag: void;
-    throttleTimeMs: number;
-    errorCode: number;
-    sessionId: number;
-    responses: {
-        topicId: string;
-        partitions: {
-            partitionIndex: number;
-            errorCode: number;
-            highWatermark: bigint;
-            lastStableOffset: bigint;
-            logStartOffset: bigint;
-            abortedTransactions: {
-                producerId: bigint;
-                firstOffset: bigint;
-                _tag: void;
-            }[];
-            preferredReadReplica: number;
-            records: {
-                baseOffset: bigint;
-                batchLength: number;
-                partitionLeaderEpoch: number;
-                magic: number;
-                crc: number;
-                attributes: number;
-                lastOffsetDelta: number;
-                baseTimestamp: bigint;
-                maxTimestamp: bigint;
-                producerId: bigint;
-                producerEpoch: number;
-                baseSequence: number;
-                records: {
-                    attributes: number;
-                    timestampDelta: bigint;
-                    offsetDelta: number;
-                    key: string | null;
-                    value: string | null;
-                    headers: {
-                        key: string | null;
-                        value: string | null;
-                    }[];
-                }[];
-            }[];
-            _tag: void;
-        }[];
-        _tag: void;
-    }[];
-    _tag2: void;
-}>;

package/dist/api/fetch.js

@@ -1,106 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.FETCH = void 0;
-const api_1 = require("../utils/api");
-const error_1 = require("../utils/error");
-exports.FETCH = (0, api_1.createApi)({
-    apiKey: 1,
-    apiVersion: 16,
-    request: (encoder, data) => encoder
-        .writeUVarInt(0)
-        .writeInt32(data.maxWaitMs)
-        .writeInt32(data.minBytes)
-        .writeInt32(data.maxBytes)
-        .writeInt8(data.isolationLevel)
-        .writeInt32(data.sessionId)
-        .writeInt32(data.sessionEpoch)
-        .writeCompactArray(data.topics, (encoder, topic) => encoder
-        .writeUUID(topic.topicId)
-        .writeCompactArray(topic.partitions, (encoder, partition) => encoder
-        .writeInt32(partition.partition)
-        .writeInt32(partition.currentLeaderEpoch)
-        .writeInt64(partition.fetchOffset)
-        .writeInt32(partition.lastFetchedEpoch)
-        .writeInt64(partition.logStartOffset)
-        .writeInt32(partition.partitionMaxBytes)
-        .writeUVarInt(0))
-        .writeUVarInt(0))
-        .writeCompactArray(data.forgottenTopicsData, (encoder, forgottenTopic) => encoder
-        .writeUUID(forgottenTopic.topicId)
-        .writeCompactArray(forgottenTopic.partitions, (encoder, partition) => encoder.writeInt32(partition))
-        .writeUVarInt(0))
-        .writeCompactString(data.rackId)
-        .writeUVarInt(0),
-    response: (decoder) => {
-        const result = {
-            _tag: decoder.readTagBuffer(),
-            throttleTimeMs: decoder.readInt32(),
-            errorCode: decoder.readInt16(),
-            sessionId: decoder.readInt32(),
-            responses: decoder.readCompactArray((response) => ({
-                topicId: response.readUUID(),
-                partitions: response.readCompactArray((partition) => ({
-                    partitionIndex: partition.readInt32(),
-                    errorCode: partition.readInt16(),
-                    highWatermark: partition.readInt64(),
-                    lastStableOffset: partition.readInt64(),
-                    logStartOffset: partition.readInt64(),
-                    abortedTransactions: partition.readCompactArray((abortedTransaction) => ({
-                        producerId: abortedTransaction.readInt64(),
-                        firstOffset: abortedTransaction.readInt64(),
-                        _tag: abortedTransaction.readTagBuffer(),
-                    })),
-                    preferredReadReplica: partition.readInt32(),
-                    records: decodeRecords(partition),
-                    _tag: partition.readTagBuffer(),
-                })),
-                _tag: response.readTagBuffer(),
-            })),
-            _tag2: decoder.readTagBuffer(),
-        };
-        if (result.errorCode)
-            throw new error_1.KafkaTSApiError(result.errorCode, null, result);
-        result.responses.forEach((response) => {
-            response.partitions.forEach((partition) => {
-                if (partition.errorCode)
-                    throw new error_1.KafkaTSApiError(partition.errorCode, null, result);
-            });
-        });
-        return result;
-    },
-});
-const decodeRecords = (decoder) => {
-    const size = decoder.readUVarInt() - 1;
-    if (size <= 0) {
-        return [];
-    }
-    const results = [];
-    while (decoder.getBufferLength() > decoder.getOffset() + 49) {
-        results.push({
-            baseOffset: decoder.readInt64(),
-            batchLength: decoder.readInt32(),
-            partitionLeaderEpoch: decoder.readInt32(),
-            magic: decoder.readInt8(),
-            crc: decoder.readUInt32(),
-            attributes: decoder.readInt16(),
-            lastOffsetDelta: decoder.readInt32(),
-            baseTimestamp: decoder.readInt64(),
-            maxTimestamp: decoder.readInt64(),
-            producerId: decoder.readInt64(),
-            producerEpoch: decoder.readInt16(),
-            baseSequence: decoder.readInt32(),
-            records: decoder.readRecords((record) => ({
-                attributes: record.readInt8(),
-                timestampDelta: record.readVarLong(),
-                offsetDelta: record.readVarInt(),
-                key: record.readVarIntString(),
-                value: record.readVarIntString(),
-                headers: record.readCompactArray((header) => ({
-                    key: header.readVarIntString(),
-                    value: header.readVarIntString(),
-                })),
-            })),
-        });
-    }
-    return results;
-};

package/dist/api/find-coordinator.d.ts

@@ -1,21 +0,0 @@
-export declare const KEY_TYPE: {
-    GROUP: number;
-    TRANSACTION: number;
-};
-export declare const FIND_COORDINATOR: import("../utils/api").Api<{
-    keyType: number;
-    keys: string[];
-}, {
-    _tag: void;
-    throttleTimeMs: number;
-    coordinators: {
-        key: string | null;
-        nodeId: number;
-        host: string;
-        port: number;
-        errorCode: number;
-        errorMessage: string | null;
-        _tag: void;
-    }[];
-    _tag2: void;
-}>;

package/dist/api/find-coordinator.js

@@ -1,39 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.FIND_COORDINATOR = exports.KEY_TYPE = void 0;
-const api_1 = require("../utils/api");
-const error_1 = require("../utils/error");
-exports.KEY_TYPE = {
-    GROUP: 0,
-    TRANSACTION: 1,
-};
-exports.FIND_COORDINATOR = (0, api_1.createApi)({
-    apiKey: 10,
-    apiVersion: 4,
-    request: (encoder, data) => encoder
-        .writeUVarInt(0)
-        .writeInt8(data.keyType)
-        .writeCompactArray(data.keys, (encoder, key) => encoder.writeCompactString(key))
-        .writeUVarInt(0),
-    response: (decoder) => {
-        const result = {
-            _tag: decoder.readTagBuffer(),
-            throttleTimeMs: decoder.readInt32(),
-            coordinators: decoder.readCompactArray((decoder) => ({
-                key: decoder.readCompactString(),
-                nodeId: decoder.readInt32(),
-                host: decoder.readCompactString(),
-                port: decoder.readInt32(),
-                errorCode: decoder.readInt16(),
-                errorMessage: decoder.readCompactString(),
-                _tag: decoder.readTagBuffer(),
-            })),
-            _tag2: decoder.readTagBuffer(),
-        };
-        result.coordinators.forEach((coordinator) => {
-            if (coordinator.errorCode)
-                throw new error_1.KafkaTSApiError(coordinator.errorCode, coordinator.errorMessage, result);
-        });
-        return result;
-    },
-});

package/dist/api/heartbeat.d.ts

@@ -1,11 +0,0 @@
-export declare const HEARTBEAT: import("../utils/api").Api<{
-    groupId: string;
-    generationId: number;
-    memberId: string;
-    groupInstanceId: string | null;
-}, {
-    _tag: void;
-    throttleTimeMs: number;
-    errorCode: number;
-    _tag2: void;
-}>;

package/dist/api/heartbeat.js

@@ -1,27 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.HEARTBEAT = void 0;
-const api_1 = require("../utils/api");
-const error_1 = require("../utils/error");
-exports.HEARTBEAT = (0, api_1.createApi)({
-    apiKey: 12,
-    apiVersion: 4,
-    request: (encoder, data) => encoder
-        .writeUVarInt(0)
-        .writeCompactString(data.groupId)
-        .writeInt32(data.generationId)
-        .writeCompactString(data.memberId)
-        .writeCompactString(data.groupInstanceId)
-        .writeUVarInt(0),
-    response: (decoder) => {
-        const result = {
-            _tag: decoder.readTagBuffer(),
-            throttleTimeMs: decoder.readInt32(),
-            errorCode: decoder.readInt16(),
-            _tag2: decoder.readTagBuffer(),
-        };
-        if (result.errorCode)
-            throw new error_1.KafkaTSApiError(result.errorCode, null, result);
-        return result;
-    },
-});