@twt-494/nestjs-kafka 6.0.2 → 7.0.0

package/dist/abstract-consumer.d.ts

@@ -0,0 +1,6 @@
+import { OnModuleInit } from "@nestjs/common";
+export declare abstract class AbstractConsumer implements OnModuleInit {
+    protected abstract registerTopics(): void;
+    onModuleInit(): void;
+    protected addTopic(topic: string): void;
+}

package/dist/abstract-consumer.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AbstractConsumer = void 0;
+const kafka_decorator_1 = require("./kafka-decorator");
+class AbstractConsumer {
+    onModuleInit() {
+        this.registerTopics();
+    }
+    addTopic(topic) {
+        kafka_decorator_1.SUB_CONTEXT_MAP.set(topic, {
+            context: this,
+        });
+    }
+}
+exports.AbstractConsumer = AbstractConsumer;

package/dist/enums/index.d.ts

@@ -0,0 +1,5 @@
+export declare enum ConsumerEventEnum {
+    CONNECTED = "CONSUMER_CONNECTED",
+    DISCONNECTED = "CONSUMER_DISCONNECTED",
+    FINISHED = "CONSUMER_FINISHED"
+}

package/dist/enums/index.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConsumerEventEnum = void 0;
+var ConsumerEventEnum;
+(function (ConsumerEventEnum) {
+    ConsumerEventEnum["CONNECTED"] = "CONSUMER_CONNECTED";
+    ConsumerEventEnum["DISCONNECTED"] = "CONSUMER_DISCONNECTED";
+    ConsumerEventEnum["FINISHED"] = "CONSUMER_FINISHED";
+})(ConsumerEventEnum || (exports.ConsumerEventEnum = ConsumerEventEnum = {}));

package/dist/kafka-client.d.ts

@@ -0,0 +1,25 @@
+import { Kafka, Producer, RecordMetadata } from 'kafkajs';
+import { CreateKafkaConsumerOptions, KafkaClientOptions, KafkaConsumerHandler } from './types';
+declare class KafkaClient extends Kafka {
+    mainProducer: Producer;
+    private isConnected;
+    private groupedConsumers;
+    private startingCloseProcess;
+    private consumersIsHealthy;
+    constructor({ logger, ...options }: KafkaClientOptions);
+    init(isDev?: boolean): Promise<void>;
+    produceMessage(topic: string, payload: any, key?: string): Promise<RecordMetadata[]>;
+    produceBatchMessages(topic: string, events: {
+        payload: string;
+        key?: string;
+    }[]): Promise<RecordMetadata[]>;
+    private eventConsumerConnect;
+    private eventConsumerDisconnect;
+    connected(): boolean;
+    getHealth(): Map<string, boolean>;
+    disconnect(): Promise<void>;
+    private processMessage;
+    private processMessagesSingle;
+    createConsumer<T>({ topics, fromStart, sessionTimeout, ...options }: CreateKafkaConsumerOptions, handler: KafkaConsumerHandler<T>): Promise<void>;
+}
+export default KafkaClient;

package/dist/kafka-client.js

@@ -0,0 +1,141 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const kafkajs_1 = require("kafkajs");
+const create_logger_1 = require("./utlis/create-logger");
+const enums_1 = require("./enums");
+class KafkaClient extends kafkajs_1.Kafka {
+    constructor({ logger = console, ...options }) {
+        super({ ...options, logCreator: (0, create_logger_1.createLogger)(logger) });
+        this.isConnected = false;
+        this.groupedConsumers = new Map();
+        this.startingCloseProcess = false;
+        this.consumersIsHealthy = new Map();
+    }
+    async init(isDev = false) {
+        this.mainProducer = this.producer({ createPartitioner: kafkajs_1.Partitioners.LegacyPartitioner });
+        this.mainProducer.on(this.mainProducer.events.CONNECT, () => this.isConnected = true);
+        this.mainProducer.on(this.mainProducer.events.DISCONNECT, () => this.isConnected = false);
+        await this.mainProducer.connect();
+        ['SIGTERM', 'SIGINT'].map((signal) => {
+            process.once(signal, async () => {
+                try {
+                    await this.disconnect();
+                }
+                finally {
+                    process.kill(process.pid, signal);
+                }
+            });
+        });
+    }
+    async produceMessage(topic, payload, key) {
+        const data = typeof payload === 'string' ? payload : JSON.stringify(payload);
+        return this.produceBatchMessages(topic, [{ payload: data, key }]);
+    }
+    async produceBatchMessages(topic, events) {
+        const messages = await Promise.all(events.map(({ payload, key }) => {
+            const message = {
+                value: payload,
+            };
+            if (key) {
+                message.key = key;
+            }
+            return message;
+        }));
+        if (!this.isConnected) {
+            await this.mainProducer.connect();
+        }
+        return this.mainProducer.send({ topic, messages });
+    }
+    eventConsumerConnect(event, topics) {
+        const connected = event === enums_1.ConsumerEventEnum.CONNECTED;
+        for (const topic of topics) {
+            this.consumersIsHealthy.set(topic, connected);
+        }
+    }
+    eventConsumerDisconnect(topics, event) {
+        var _a;
+        const error = (_a = event === null || event === void 0 ? void 0 : event.payload) === null || _a === void 0 ? void 0 : _a.error;
+        const consumerEvent = this.startingCloseProcess ? enums_1.ConsumerEventEnum.FINISHED : enums_1.ConsumerEventEnum.DISCONNECTED;
+        if (error) {
+            this.logger().error(`Kafka client consumer disconnect error: ${error.message}`, {
+                topics,
+                eventType: event.type,
+                stack: error.stack,
+            });
+        }
+        this.eventConsumerConnect(consumerEvent, topics);
+    }
+    connected() {
+        return this.isConnected;
+    }
+    getHealth() {
+        return this.consumersIsHealthy;
+    }
+    async disconnect() {
+        try {
+            this.startingCloseProcess = true;
+            await Promise.all([
+                ...[...this.groupedConsumers.values()].map(consumer => consumer.disconnect()),
+            ]);
+            this.startingCloseProcess = false;
+        }
+        catch (err) {
+            // @ts-ignore
+            this.logger().error(`Kafka client disconnect error: ${err.message}`, { error: err });
+        }
+    }
+    async processMessage(value, topic, handler) {
+        await handler(topic);
+    }
+    async processMessagesSingle({ batch: { topic, messages }, isRunning, isStale, resolveOffset, heartbeat }, handler) {
+        for (let { value, key, offset } of messages) {
+            try {
+                if (!isRunning() || isStale()) {
+                    break;
+                }
+                await this.processMessage(value, topic, handler);
+            }
+            catch (err) {
+                this.logger().error(`Kafka client process message error: ${err.message}`, {
+                    stack: err.stack,
+                    extra: {
+                        topic,
+                        value,
+                    }
+                });
+            }
+            finally {
+                await resolveOffset(offset);
+                await heartbeat();
+            }
+        }
+    }
+    async createConsumer({ topics, fromStart = true, sessionTimeout = 30000, ...options }, handler) {
+        const consumer = this.consumer({ ...options, sessionTimeout });
+        consumer.on(consumer.events.GROUP_JOIN, () => {
+            this.groupedConsumers.set(options.groupId, consumer);
+        });
+        const handleDisconnect = this.eventConsumerDisconnect.bind(this, topics);
+        consumer.on(consumer.events.CONNECT, () => this.eventConsumerConnect(enums_1.ConsumerEventEnum.CONNECTED, topics));
+        consumer.on(consumer.events.DISCONNECT, handleDisconnect);
+        consumer.on(consumer.events.CRASH, handleDisconnect);
+        consumer.on(consumer.events.REQUEST_TIMEOUT, handleDisconnect);
+        try {
+            await consumer.connect();
+            await Promise.all(topics.map(topic => consumer.subscribe({ topic, fromBeginning: fromStart })));
+            await consumer.run({
+                eachBatchAutoResolve: false,
+                eachBatch: async (payload) => {
+                    const { batch: { topic }, heartbeat } = payload;
+                    const timer = setTimeout(() => heartbeat(), Math.ceil(sessionTimeout / 2));
+                    await this.processMessagesSingle(payload, handler);
+                    clearTimeout(timer);
+                }
+            });
+        }
+        catch (err) {
+            this.logger().error(`Kafka client consumer error: ${err.message}`, { stack: err.stack });
+        }
+    }
+}
+exports.default = KafkaClient;

package/dist/kafka-decorator.d.ts

@@ -0,0 +1,5 @@
+import { CreateKafkaConsumerOptions } from "./types";
+export declare const SUB_CONTEXT_MAP: Map<any, any>;
+export declare const SUB_HANDLER_MAP: Map<any, any>;
+export declare const SUB_CONFIG_MAP: Map<any, any>;
+export declare function listenTo(topic: string, config?: Omit<CreateKafkaConsumerOptions, 'topic' | 'topics' | 'groupId'>): (target: any, propertyKey: any, descriptor: any) => any;

package/dist/kafka-decorator.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SUB_CONFIG_MAP = exports.SUB_HANDLER_MAP = exports.SUB_CONTEXT_MAP = void 0;
+exports.listenTo = listenTo;
+exports.SUB_CONTEXT_MAP = new Map();
+exports.SUB_HANDLER_MAP = new Map();
+exports.SUB_CONFIG_MAP = new Map();
+function listenTo(topic, config) {
+    // @ts-ignore
+    return function (target, propertyKey, descriptor) {
+        const method = target[propertyKey];
+        exports.SUB_HANDLER_MAP.set(topic, method);
+        if (config) {
+            exports.SUB_HANDLER_MAP.set(topic, method);
+        }
+        return descriptor;
+    };
+}

package/dist/kafka.constants.d.ts

@@ -0,0 +1 @@
+export declare const KAFKA_MODULE_OPTIONS = "KAFKA_MODULE_OPTIONS";

package/dist/kafka.constants.js

@@ -0,0 +1,4 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KAFKA_MODULE_OPTIONS = void 0;
+exports.KAFKA_MODULE_OPTIONS = 'KAFKA_MODULE_OPTIONS';

package/dist/kafka.module.d.ts

@@ -1,16 +1,5 @@
-import { DynamicModule } from '@nestjs/common';
-export interface KafkaModuleOptions {
-    brokers: string[];
-    serviceName: string;
-    user: string;
-}
-export interface KafkaModuleAsyncOptions {
-    imports?: any[];
-    useFactory?: (...args: any[]) => Promise<KafkaModuleOptions> | KafkaModuleOptions;
-    inject?: any[];
-    isGlobal?: boolean;
-}
+import { KafkaModuleOptions } from "./types";
+import { DynamicModule } from "@nestjs/common";
 export declare class KafkaModule {
-    static register(options: KafkaModuleOptions): DynamicModule;
-    static registerAsync(options: KafkaModuleAsyncOptions): DynamicModule;
+    static registerAsync(options: KafkaModuleOptions): DynamicModule;
 }

package/dist/kafka.module.js

@@ -1,47 +1,24 @@
 "use strict";
-var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
-    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
-    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
-    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
-    return c > 3 && r && Object.defineProperty(target, key, r), r;
-};
-var KafkaModule_1;
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.KafkaModule = void 0;
-// kafka.module.ts
-const common_1 = require("@nestjs/common");
+const kafka_constants_1 = require("./kafka.constants");
 const kafka_service_1 = require("./kafka.service");
-let KafkaModule = KafkaModule_1 = class KafkaModule {
-    static register(options) {
+class KafkaModule {
+    static registerAsync(options) {
         return {
-            module: KafkaModule_1,
+            global: true,
+            module: KafkaModule,
             providers: [
                 kafka_service_1.KafkaService,
-                { provide: 'KAFKA_MODULE_OPTIONS', useValue: options },
+                {
+                    provide: kafka_constants_1.KAFKA_MODULE_OPTIONS,
+                    useFactory: options.useFactory,
+                    inject: options.inject || [],
+                }
             ],
-            exports: [kafka_service_1.KafkaService],
-        };
-    }
-    static registerAsync(options) {
-        var _a;
-        if (!options.useFactory) {
-            throw new Error('KafkaModule.registerAsync requires useFactory');
-        }
-        const kafkaOptionsProvider = {
-            provide: 'KAFKA_MODULE_OPTIONS',
-            useFactory: options.useFactory,
-            inject: options.inject || [],
-        };
-        return {
-            module: KafkaModule_1,
             imports: options.imports || [],
-            providers: [kafka_service_1.KafkaService, kafkaOptionsProvider],
             exports: [kafka_service_1.KafkaService],
-            global: (_a = options.isGlobal) !== null && _a !== void 0 ? _a : false,
         };
     }
-};
+}
 exports.KafkaModule = KafkaModule;
-exports.KafkaModule = KafkaModule = KafkaModule_1 = __decorate([
-    (0, common_1.Module)({})
-], KafkaModule);

package/dist/kafka.service.d.ts

@@ -1,23 +1,10 @@
-import { OnModuleInit } from '@nestjs/common';
-import { Consumer } from 'kafkajs';
-import { KafkaModuleOptions } from './kafka.module';
-export declare class KafkaService implements OnModuleInit {
+import { OnApplicationBootstrap } from '@nestjs/common';
+import KafkaClient from './kafka-client';
+export declare class KafkaService implements OnApplicationBootstrap {
     private readonly options;
     private kafka;
-    private producer;
-    constructor(options: KafkaModuleOptions);
-    onModuleInit(): Promise<void>;
-    shutdown(): Promise<void>;
-    createConsumer(topic: string, options?: {
-        sessionTimeout?: number;
-    }): Promise<Consumer>;
-    sendBatch(topic: string, data: {
-        key?: string;
-        value: string;
-    }[]): Promise<void>;
-    send(topic: string, data: any, key?: string): Promise<void>;
-    produceBatch(topic: string, messages: {
-        value: string;
-        key?: string;
-    }[]): Promise<void>;
+    constructor(options: any);
+    private getOptions;
+    onApplicationBootstrap(): Promise<void>;
+    getClient(): KafkaClient;
 }

package/dist/kafka.service.js

@@ -11,73 +11,49 @@ var __metadata = (this && this.__metadata) || function (k, v) {
 var __param = (this && this.__param) || function (paramIndex, decorator) {
     return function (target, key) { decorator(target, key, paramIndex); }
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.KafkaService = void 0;
-// kafka.service.ts
 const common_1 = require("@nestjs/common");
-const kafkajs_1 = require("kafkajs");
+const kafka_client_1 = __importDefault(require("./kafka-client"));
+const kafka_constants_1 = require("./kafka.constants");
+const kafka_decorator_1 = require("./kafka-decorator");
 let KafkaService = class KafkaService {
+    // @ts-ignore
     constructor(options) {
         this.options = options;
+        const { isDev, connectionTimeout = 5000, ...clientOptions } = this.options;
+        this.kafka = new kafka_client_1.default({ connectionTimeout, ...clientOptions });
+        this.kafka.init(isDev);
     }
-    async onModuleInit() {
-        if (!this.options.brokers || !this.options.serviceName || !this.options.user) {
-            throw new Error('Missing required options');
-        }
-        this.kafka = new kafkajs_1.Kafka({
-            clientId: `${this.options.serviceName}Client`,
-            brokers: this.options.brokers,
-        });
-        this.producer = this.kafka.producer({
-            createPartitioner: kafkajs_1.Partitioners.LegacyPartitioner,
-        });
-        await this.producer.connect();
-        process.on('SIGTERM', this.shutdown);
-        process.on('SIGINT', this.shutdown);
-    }
-    async shutdown() {
-        await this.producer.disconnect();
+    getOptions(topic) {
+        return {
+            topics: [`${this.options.topicPrefix}_${topic}`],
+            groupId: `${this.options.clientId} ${topic} ${this.options.user} Consumer`,
+        };
     }
-    async createConsumer(topic, options) {
-        const consumer = this.kafka.consumer({
-            groupId: `${this.options.serviceName}Service${this.options.user}ConsumerGroup`,
-            sessionTimeout: (options === null || options === void 0 ? void 0 : options.sessionTimeout) || 30000,
-        });
-        await consumer.connect();
-        await consumer.subscribe({ topic, fromBeginning: true });
-        return consumer;
-    }
-    async sendBatch(topic, data) {
-        await this.producer.send({
-            topic,
-            messages: data,
-        });
-    }
-    async send(topic, data, key) {
-        await this.producer.send({
-            topic,
-            messages: [
-                {
-                    key,
-                    value: JSON.stringify(data),
-                },
-            ],
+    async onApplicationBootstrap() {
+        if (!this.options.user && this.options.isDev) {
+            return;
+        }
+        kafka_decorator_1.SUB_HANDLER_MAP.forEach((handler, topic) => {
+            const { context } = kafka_decorator_1.SUB_CONTEXT_MAP.get(topic);
+            const consumerOptions = this.getOptions(topic);
+            if (kafka_decorator_1.SUB_CONFIG_MAP.has(topic)) {
+                Object.assign(consumerOptions, kafka_decorator_1.SUB_CONFIG_MAP.get(topic));
+            }
+            return this.kafka.createConsumer(consumerOptions, handler.bind(context));
         });
     }
-    async produceBatch(topic, messages) {
-        await this.producer.sendBatch({
-            topicMessages: [
-                {
-                    topic,
-                    messages,
-                },
-            ],
-        });
+    getClient() {
+        return this.kafka;
     }
 };
 exports.KafkaService = KafkaService;
 exports.KafkaService = KafkaService = __decorate([
     (0, common_1.Injectable)(),
-    __param(0, (0, common_1.Inject)('KAFKA_MODULE_OPTIONS')),
+    __param(0, (0, common_1.Inject)(kafka_constants_1.KAFKA_MODULE_OPTIONS)),
     __metadata("design:paramtypes", [Object])
 ], KafkaService);

package/dist/types/index.d.ts

@@ -0,0 +1,18 @@
+import { ConsumerConfig } from 'kafkajs';
+import { FactoryProvider, ModuleMetadata } from "@nestjs/common";
+import { LoggerInterface } from "../utlis/create-logger";
+export type KafkaConsumerHandler<T> = (topic: string) => void | Promise<void>;
+export type CreateKafkaConsumerOptions = {
+    topics: string[];
+    batchSize?: number;
+    fromStart?: boolean;
+} & ConsumerConfig;
+export type KafkaClientOptions = {
+    clientId: string;
+    brokers: string[];
+    logger?: LoggerInterface;
+    connectionTimeout?: number;
+};
+export type KafkaModuleOptions = Omit<FactoryProvider<KafkaClientOptions & {
+    isDev: boolean;
+}> & Pick<ModuleMetadata, 'imports'>, 'provide'>;

package/dist/types/index.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/utlis/create-logger.d.ts

@@ -0,0 +1,12 @@
+export interface LoggerInterface {
+    debug: (log: any) => void;
+    info: (log: any) => void;
+    warn: (log: any) => void;
+    error: (log: any) => void;
+}
+export declare const createLogger: (logger: LoggerInterface) => () => ({ level, log }: {
+    namespace: string;
+    level: number;
+    label: string;
+    log: Record<string, unknown>;
+}) => void;

package/dist/utlis/create-logger.js

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createLogger = void 0;
+const kafkajs_1 = require("kafkajs");
+const createLogger = (logger) => () => {
+    return ({ level, log }) => {
+        const { message, error, logger: loggerName, ...extra } = log;
+        if (level === kafkajs_1.logLevel.ERROR && error === 'string' && error.includes('rebalancing')) {
+            level = kafkajs_1.logLevel.WARN;
+        }
+        const logMessage = {
+            logger: loggerName,
+            message,
+            ...extra,
+        };
+        if (message instanceof Error) {
+            logMessage.stack = message.stack;
+        }
+        switch (level) {
+            case kafkajs_1.logLevel.ERROR:
+                logger.error(logMessage);
+                return;
+            case kafkajs_1.logLevel.WARN:
+                logger.warn(logMessage);
+                return;
+            case kafkajs_1.logLevel.DEBUG:
+                logger.debug(logMessage);
+                return;
+            case kafkajs_1.logLevel.INFO:
+                logger.info(logMessage);
+                return;
+            default:
+                logger.info(logMessage);
+        }
+    };
+};
+exports.createLogger = createLogger;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@twt-494/nestjs-kafka",
-  "version": "6.0.2",
+  "version": "7.0.0",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -8,7 +8,8 @@
   "author": "Oganes Mgerabyan",
   "license": "MIT",
   "dependencies": {
-    "kafkajs": "^2.2.4"
+    "kafkajs": "^2.2.4",
+    "lodash.camelcase": "^4.3.0"
   },
   "peerDependencies": {
     "@nestjs/common": "11.1.6"