kafka-ts 0.0.1-beta → 0.0.3-beta

package/src/consumer/processor.ts

@@ -0,0 +1,47 @@
+import { EventEmitter } from 'stream';
+import { Batch } from '../types';
+import { createTracer } from '../utils/tracer';
+
+const trace = createTracer('Processor');
+
+type ProcessorOptions = {
+    poll: () => Promise<Batch>;
+    process: (batch: Batch) => Promise<void>;
+};
+
+export class Processor extends EventEmitter<{ stop: []; stopped: [] }> {
+    private isRunning = false;
+
+    constructor(private options: ProcessorOptions) {
+        super();
+    }
+
+    public async loop() {
+        const { poll, process } = this.options;
+
+        this.isRunning = true;
+        this.once('stop', () => (this.isRunning = false));
+
+        try {
+            while (this.isRunning) {
+                const batch = await poll();
+                await process(batch);
+            }
+        } finally {
+            this.isRunning = false;
+            this.emit('stopped');
+        }
+    }
+
+    @trace()
+    public async stop() {
+        if (!this.isRunning) {
+            return;
+        }
+
+        return new Promise<void>((resolve) => {
+            this.once('stopped', resolve);
+            this.emit('stop');
+        });
+    }
+}

package/src/distributors/assignments-to-replicas.test.ts

@@ -1,9 +1,9 @@
-import { describe, expect, it } from "vitest";
-import { distributeAssignmentsToNodesBalanced, distributeAssignmentsToNodesOptimized } from "./assignments-to-replicas";
+import { describe, expect, it } from 'vitest';
+import { distributeAssignmentsToNodesBalanced, distributeAssignmentsToNodesOptimized } from './assignments-to-replicas';
 
-describe("Distribute assignments to replica ids", () => {
-    describe("distributeAssignmentsToNodesBalanced", () => {
-        it("smoke", () => {
+describe('Distribute assignments to replica ids', () => {
+    describe('distributeAssignmentsToNodesBalanced', () => {
+        it('smoke', () => {
             const result = distributeAssignmentsToNodesBalanced({ topic: [0, 1] }, { topic: { 0: [0, 1], 1: [1, 2] } });
             expect(result).toMatchInlineSnapshot(`
               {
@@ -22,8 +22,8 @@ describe("Distribute assignments to replica ids", () => {
         });
     });
 
-    describe("distributeAssignmentsToNodesOptimized", () => {
-        it("smoke", () => {
+    describe('distributeAssignmentsToNodesOptimized', () => {
+        it('smoke', () => {
             const result = distributeAssignmentsToNodesOptimized(
                 { topic: [0, 1] },
                 { topic: { 0: [0, 1], 1: [1, 2] } },

package/src/distributors/assignments-to-replicas.ts

@@ -50,7 +50,7 @@ export const distributeAssignmentsToNodesOptimized = (
         }
 
         result[parseInt(replicaId)] = partitions.reduce((acc, partition) => {
-            const [topicName, partitionId] = partition.split(":");
+            const [topicName, partitionId] = partition.split(':');
             acc[topicName] ??= [];
             acc[topicName].push(parseInt(partitionId));
             return acc;

package/src/distributors/messages-to-topic-partition-leaders.test.ts

@@ -1,11 +1,11 @@
-import { describe, expect, it } from "vitest";
-import { distributeMessagesToTopicPartitionLeaders } from "./messages-to-topic-partition-leaders";
+import { describe, expect, it } from 'vitest';
+import { distributeMessagesToTopicPartitionLeaders } from './messages-to-topic-partition-leaders';
 
-describe("Distribute messages to partition leader ids", () => {
-    describe("distributeMessagesToTopicPartitionLeaders", () => {
-        it("snoke", () => {
+describe('Distribute messages to partition leader ids', () => {
+    describe('distributeMessagesToTopicPartitionLeaders', () => {
+        it('snoke', () => {
             const result = distributeMessagesToTopicPartitionLeaders(
-                [{ topic: "topic", partition: 0, key: null, value: null, offset: 0n, timestamp: 0n, headers: {} }],
+                [{ topic: 'topic', partition: 0, key: null, value: null, offset: 0n, timestamp: 0n, headers: {} }],
                 { topic: { 0: 1 } },
             );
             expect(result).toMatchInlineSnapshot(`

package/src/index.ts

@@ -1,3 +1,4 @@
-export * from "./client";
-export * from "./api";
-export * from "./types";
+export * from './utils/error';
+export * from './client';
+export * from './api';
+export * from './types';

package/src/metadata.ts

@@ -1,7 +1,7 @@
-import { API, API_ERROR } from "./api";
-import { Cluster } from "./cluster";
-import { delay } from "./utils/delay";
-import { KafkaTSApiError } from "./utils/error";
+import { API, API_ERROR } from './api';
+import { Cluster } from './cluster';
+import { delay } from './utils/delay';
+import { KafkaTSApiError } from './utils/error';
 
 type MetadataOptions = {
     cluster: Cluster;

package/src/producer/producer.ts

@@ -1,11 +1,11 @@
-import { API, API_ERROR } from "../api";
-import { Cluster } from "../cluster";
-import { distributeMessagesToTopicPartitionLeaders } from "../distributors/messages-to-topic-partition-leaders";
-import { Metadata } from "../metadata";
-import { Message } from "../types";
-import { delay } from "../utils/delay";
-import { KafkaTSApiError } from "../utils/error";
-import { memo } from "../utils/memo";
+import { API, API_ERROR } from '../api';
+import { Cluster } from '../cluster';
+import { distributeMessagesToTopicPartitionLeaders } from '../distributors/messages-to-topic-partition-leaders';
+import { Metadata } from '../metadata';
+import { Message } from '../types';
+import { delay } from '../utils/delay';
+import { KafkaTSApiError } from '../utils/error';
+import { memo } from '../utils/memo';
 
 export type ProducerOptions = {
     allowTopicAutoCreation?: boolean;

package/src/types.ts

@@ -7,3 +7,5 @@ export type Message = {
     value: string | null;
     headers?: Record<string, string>;
 };
+
+export type Batch = Required<Message>[];

package/src/utils/api.ts

@@ -1,11 +1,11 @@
-import { Decoder } from "./decoder";
-import { Encoder } from "./encoder";
+import { Decoder } from './decoder';
+import { Encoder } from './encoder';
 
 export type Api<Request, Response> = {
     apiKey: number;
     apiVersion: number;
     request: (encoder: Encoder, body: Request) => Encoder;
     response: (buffer: Decoder) => Response;
-}
+};
 
-export const createApi = <Request, Response>(api: Api<Request, Response>) => api;
+export const createApi = <Request, Response>(api: Api<Request, Response>) => api;

package/src/utils/debug.ts

@@ -1,7 +1,7 @@
-export const serializer = (_: string, value: unknown) => (typeof value === "bigint" ? value.toString() : value);
+export const serializer = (_: string, value: unknown) => (typeof value === 'bigint' ? value.toString() : value);
 
 export const createDebugger = (module: string) => (func: string, message: string, data?: unknown) => {
-    if (!process.env.DEBUG?.includes("kafkats")) return;
+    if (!process.env.DEBUG?.includes('kafka-ts')) return;
     console.debug(
         `[${module}] ${func}: ${message}`,
         data && `(${data instanceof Error ? data : JSON.stringify(data, serializer, 4)})`,

package/src/utils/decoder.ts

@@ -81,7 +81,7 @@ export class Decoder {
             return null;
         }
 
-        const value = this.buffer.toString("utf-8", this.offset, this.offset + length);
+        const value = this.buffer.toString('utf-8', this.offset, this.offset + length);
         this.offset += length;
         return value;
     }
@@ -92,7 +92,7 @@ export class Decoder {
             return null;
         }
 
-        const value = this.buffer.toString("utf-8", this.offset, this.offset + length);
+        const value = this.buffer.toString('utf-8', this.offset, this.offset + length);
         this.offset += length;
         return value;
     }
@@ -103,13 +103,13 @@ export class Decoder {
             return null;
         }
 
-        const value = this.buffer.toString("utf-8", this.offset, this.offset + length);
+        const value = this.buffer.toString('utf-8', this.offset, this.offset + length);
         this.offset += length;
         return value;
     }
 
     public readUUID() {
-        const value = this.buffer.toString("hex", this.offset, this.offset + 16);
+        const value = this.buffer.toString('hex', this.offset, this.offset + 16);
         this.offset += 16;
         return value;
     }

package/src/utils/encoder.ts

@@ -74,9 +74,9 @@ export class Encoder {
         if (value === null) {
             return this.writeInt16(-1);
         }
-        const byteLength = Buffer.byteLength(value, "utf-8");
+        const byteLength = Buffer.byteLength(value, 'utf-8');
         const buffer = Buffer.alloc(byteLength);
-        buffer.write(value, 0, byteLength, "utf-8");
+        buffer.write(value, 0, byteLength, 'utf-8');
         return this.writeInt16(byteLength).write(buffer);
     }
 
@@ -85,9 +85,9 @@ export class Encoder {
             return this.writeUVarInt(0);
         }
 
-        const byteLength = Buffer.byteLength(value, "utf-8");
+        const byteLength = Buffer.byteLength(value, 'utf-8');
         const buffer = Buffer.alloc(byteLength);
-        buffer.write(value, 0, byteLength, "utf-8");
+        buffer.write(value, 0, byteLength, 'utf-8');
         return this.writeUVarInt(byteLength + 1).write(buffer);
     }
 
@@ -95,14 +95,14 @@ export class Encoder {
         if (value === null) {
             return this.writeVarInt(-1);
         }
-        return this.writeVarInt(Buffer.byteLength(value, "utf-8")).write(Buffer.from(value, "utf-8"));
+        return this.writeVarInt(Buffer.byteLength(value, 'utf-8')).write(Buffer.from(value, 'utf-8'));
     }
 
     public writeUUID(value: string | null) {
         if (value === null) {
             return this.write(Buffer.alloc(16));
         }
-        return this.write(Buffer.from(value, "hex"));
+        return this.write(Buffer.from(value, 'hex'));
     }
 
     public writeBoolean(value: boolean) {

package/src/utils/error.ts

@@ -1,4 +1,4 @@
-import { API_ERROR } from "../api";
+import { API_ERROR } from '../api';
 
 export class KafkaTSError extends Error {
     constructor(message: string) {
@@ -13,8 +13,8 @@ export class KafkaTSApiError<T = any> extends KafkaTSError {
         public errorMessage: string | null,
         public response: T,
     ) {
-        const [errorName] = Object.entries(API_ERROR).find(([, value]) => value === errorCode) ?? ["UNKNOWN"];
-        super(`${errorName}${errorMessage ? `: ${errorMessage}` : ""}`);
+        const [errorName] = Object.entries(API_ERROR).find(([, value]) => value === errorCode) ?? ['UNKNOWN'];
+        super(`${errorName}${errorMessage ? `: ${errorMessage}` : ''}`);
     }
 }
 

package/src/utils/retrier.ts

@@ -1,4 +1,4 @@
-import { delay } from "./delay";
+import { delay } from './delay';
 
 export type Retrier = (func: () => unknown) => Promise<void>;
 

package/src/utils/tracer.ts

@@ -1,9 +1,10 @@
-import { serializer } from "./debug";
+import { serializer } from './debug';
 
-export const trace =
+export const createTracer =
+    (module: string, attributes?: Record<string, unknown>) =>
     (fn?: (...args: any[]) => Record<string, unknown> | undefined) =>
     (target: any, propertyKey: string, descriptor: PropertyDescriptor) => {
-        if (!process.env.DEBUG?.includes("kafkats")) return;
+        if (!process.env.DEBUG?.includes('kafka-ts')) return;
 
         const original = descriptor.value;
         descriptor.value = function (...args: any[]) {
@@ -12,7 +13,7 @@ export const trace =
 
             const onEnd = <T>(result: T): T => {
                 console.log(
-                    `[${propertyKey}] +${Date.now() - startTime}ms ${JSON.stringify({ ...metadata, result }, serializer)}`,
+                    `[${module}.${propertyKey}] +${Date.now() - startTime}ms ${JSON.stringify({ ...attributes, ...metadata, result }, serializer)}`,
                 );
                 return result;
             };
@@ -26,3 +27,5 @@ export const trace =
             }
         };
     };
+
+export const trace = createTracer('GLOBAL');

package/tsconfig.json

@@ -1,17 +1,17 @@
 {
-  "compilerOptions": {
-    "lib": ["es2023"],
-    "target": "es2022",
-    "module": "node16",
-    "strict": true,
-    "forceConsistentCasingInFileNames": true,
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "moduleResolution": "node16",
-    "outDir": "dist",
-    "declaration": true,
-    "emitDecoratorMetadata": true,
-    "experimentalDecorators": true
-  },
-  "include": ["src/**/*"],
-}
+    "compilerOptions": {
+        "lib": ["es2023"],
+        "target": "es2022",
+        "module": "node16",
+        "strict": true,
+        "forceConsistentCasingInFileNames": true,
+        "esModuleInterop": true,
+        "skipLibCheck": true,
+        "moduleResolution": "node16",
+        "outDir": "dist",
+        "declaration": true,
+        "emitDecoratorMetadata": true,
+        "experimentalDecorators": true
+    },
+    "include": ["src/**/*"]
+}

package/dist/api/api-versions.d.ts

@@ -1,9 +0,0 @@
-export declare const API_VERSIONS: import("../utils/api.js").Api<unknown, {
-    errorCode: number;
-    versions: {
-        apiKey: number;
-        minVersion: number;
-        maxVersion: number;
-    }[];
-    throttleTimeMs: number;
-}>;

package/dist/api/api-versions.js

@@ -1,24 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.API_VERSIONS = void 0;
-const api_js_1 = require("../utils/api.js");
-const error_js_1 = require("../utils/error.js");
-exports.API_VERSIONS = (0, api_js_1.createApi)({
-    apiKey: 18,
-    apiVersion: 2,
-    request: (encoder) => encoder,
-    response: (decoder) => {
-        const result = {
-            errorCode: decoder.readInt16(),
-            versions: decoder.readArray((version) => ({
-                apiKey: version.readInt16(),
-                minVersion: version.readInt16(),
-                maxVersion: version.readInt16(),
-            })),
-            throttleTimeMs: decoder.readInt32(),
-        };
-        if (result.errorCode)
-            throw new error_js_1.KafkaTSApiError(result.errorCode, null, result);
-        return result;
-    },
-});

package/dist/api/create-topics.d.ts

@@ -1,38 +0,0 @@
-export declare const CREATE_TOPICS: import("../utils/api").Api<{
-    topics: {
-        name: string;
-        numPartitions: number;
-        replicationFactor: number;
-        assignments: {
-            partitionIndex: number;
-            brokerIds: number[];
-        }[];
-        configs: {
-            name: string;
-            value: string | null;
-        }[];
-    }[];
-    timeoutMs: number;
-    validateOnly: boolean;
-}, {
-    _tag: void;
-    throttleTimeMs: number;
-    topics: {
-        name: string | null;
-        topicId: string;
-        errorCode: number;
-        errorMessage: string | null;
-        numPartitions: number;
-        replicationFactor: number;
-        configs: {
-            name: string | null;
-            value: string | null;
-            readOnly: boolean;
-            configSource: number;
-            isSensitive: boolean;
-            _tag: void;
-        }[];
-        _tag: void;
-    }[];
-    _tag2: void;
-}>;

package/dist/api/create-topics.js

@@ -1,53 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CREATE_TOPICS = void 0;
-const api_1 = require("../utils/api");
-const error_1 = require("../utils/error");
-exports.CREATE_TOPICS = (0, api_1.createApi)({
-    apiKey: 19,
-    apiVersion: 7,
-    request: (encoder, data) => encoder
-        .writeUVarInt(0)
-        .writeCompactArray(data.topics, (encoder, topic) => encoder
-        .writeCompactString(topic.name)
-        .writeInt32(topic.numPartitions)
-        .writeInt16(topic.replicationFactor)
-        .writeCompactArray(topic.assignments, (encoder, assignment) => encoder
-        .writeInt32(assignment.partitionIndex)
-        .writeCompactArray(assignment.brokerIds, (encoder, brokerId) => encoder.writeInt32(brokerId))
-        .writeUVarInt(0))
-        .writeCompactArray(topic.configs, (encoder, config) => encoder.writeCompactString(config.name).writeCompactString(config.value).writeUVarInt(0))
-        .writeUVarInt(0))
-        .writeInt32(data.timeoutMs)
-        .writeBoolean(data.validateOnly)
-        .writeUVarInt(0),
-    response: (decoder) => {
-        const result = {
-            _tag: decoder.readTagBuffer(),
-            throttleTimeMs: decoder.readInt32(),
-            topics: decoder.readCompactArray((topic) => ({
-                name: topic.readCompactString(),
-                topicId: topic.readUUID(),
-                errorCode: topic.readInt16(),
-                errorMessage: topic.readCompactString(),
-                numPartitions: topic.readInt32(),
-                replicationFactor: topic.readInt16(),
-                configs: topic.readCompactArray((config) => ({
-                    name: config.readCompactString(),
-                    value: config.readCompactString(),
-                    readOnly: config.readBoolean(),
-                    configSource: config.readInt8(),
-                    isSensitive: config.readBoolean(),
-                    _tag: config.readTagBuffer(),
-                })),
-                _tag: topic.readTagBuffer(),
-            })),
-            _tag2: decoder.readTagBuffer(),
-        };
-        result.topics.forEach((topic) => {
-            if (topic.errorCode)
-                throw new error_1.KafkaTSApiError(topic.errorCode, topic.errorMessage, result);
-        });
-        return result;
-    },
-});

package/dist/api/delete-topics.d.ts

@@ -1,18 +0,0 @@
-export declare const DELETE_TOPICS: import("../utils/api").Api<{
-    topics: {
-        name: string | null;
-        topicId: string | null;
-    }[];
-    timeoutMs: number;
-}, {
-    _tag: void;
-    throttleTimeMs: number;
-    responses: {
-        name: string | null;
-        topicId: string;
-        errorCode: number;
-        errorMessage: string | null;
-        _tag: void;
-    }[];
-    _tag2: void;
-}>;

package/dist/api/delete-topics.js

@@ -1,33 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DELETE_TOPICS = void 0;
-const api_1 = require("../utils/api");
-const error_1 = require("../utils/error");
-exports.DELETE_TOPICS = (0, api_1.createApi)({
-    apiKey: 20,
-    apiVersion: 6,
-    request: (encoder, data) => encoder
-        .writeUVarInt(0)
-        .writeCompactArray(data.topics, (encoder, topic) => encoder.writeCompactString(topic.name).writeUUID(topic.topicId).writeUVarInt(0))
-        .writeInt32(data.timeoutMs)
-        .writeUVarInt(0),
-    response: (decoder) => {
-        const result = {
-            _tag: decoder.readTagBuffer(),
-            throttleTimeMs: decoder.readInt32(),
-            responses: decoder.readCompactArray((decoder) => ({
-                name: decoder.readCompactString(),
-                topicId: decoder.readUUID(),
-                errorCode: decoder.readInt16(),
-                errorMessage: decoder.readCompactString(),
-                _tag: decoder.readTagBuffer(),
-            })),
-            _tag2: decoder.readTagBuffer(),
-        };
-        result.responses.forEach((response) => {
-            if (response.errorCode)
-                throw new error_1.KafkaTSApiError(response.errorCode, response.errorMessage, result);
-        });
-        return result;
-    },
-});

package/dist/api/fetch.d.ts

@@ -1,77 +0,0 @@
-export declare const enum IsolationLevel {
-    READ_UNCOMMITTED = 0,
-    READ_COMMITTED = 1
-}
-export declare const FETCH: import("../utils/api").Api<{
-    maxWaitMs: number;
-    minBytes: number;
-    maxBytes: number;
-    isolationLevel: IsolationLevel;
-    sessionId: number;
-    sessionEpoch: number;
-    topics: {
-        topicId: string;
-        partitions: {
-            partition: number;
-            currentLeaderEpoch: number;
-            fetchOffset: bigint;
-            lastFetchedEpoch: number;
-            logStartOffset: bigint;
-            partitionMaxBytes: number;
-        }[];
-    }[];
-    forgottenTopicsData: {
-        topicId: string;
-        partitions: number[];
-    }[];
-    rackId: string;
-}, {
-    _tag: void;
-    throttleTimeMs: number;
-    errorCode: number;
-    sessionId: number;
-    responses: {
-        topicId: string;
-        partitions: {
-            partitionIndex: number;
-            errorCode: number;
-            highWatermark: bigint;
-            lastStableOffset: bigint;
-            logStartOffset: bigint;
-            abortedTransactions: {
-                producerId: bigint;
-                firstOffset: bigint;
-                _tag: void;
-            }[];
-            preferredReadReplica: number;
-            records: {
-                baseOffset: bigint;
-                batchLength: number;
-                partitionLeaderEpoch: number;
-                magic: number;
-                crc: number;
-                attributes: number;
-                lastOffsetDelta: number;
-                baseTimestamp: bigint;
-                maxTimestamp: bigint;
-                producerId: bigint;
-                producerEpoch: number;
-                baseSequence: number;
-                records: {
-                    attributes: number;
-                    timestampDelta: bigint;
-                    offsetDelta: number;
-                    key: string | null;
-                    value: string | null;
-                    headers: {
-                        key: string | null;
-                        value: string | null;
-                    }[];
-                }[];
-            }[];
-            _tag: void;
-        }[];
-        _tag: void;
-    }[];
-    _tag2: void;
-}>;