@mondart/nestjs-common-module 1.0.3

package/common/interface/pagination-query.ts

@@ -0,0 +1,39 @@
+import { IsNumber, IsOptional } from 'class-validator';
+import { IsString } from '../decorators/validations/default';
+import { Transform } from 'class-transformer';
+import { PaginateQuery } from 'nestjs-paginate';
+
+export class PaginationQueryCustom implements PaginateQuery {
+  @IsNumber()
+  @IsOptional()
+  @Transform(({ value }) => (value ? parseInt(value, 10) : undefined))
+  page?: number;
+
+  @IsNumber()
+  @IsOptional()
+  @Transform(({ value }) => (value ? parseInt(value, 10) : undefined))
+  limit?: number;
+
+  @IsString({ each: true })
+  sortBy?: [string, string][];
+
+  @IsString({ each: true })
+  @IsOptional()
+  searchBy?: string[];
+
+  @IsString({ each: true })
+  @IsOptional()
+  search?: string;
+
+  @IsString({ each: true })
+  @IsOptional()
+  select?: string[];
+
+  @IsString()
+  @IsOptional()
+  path: string;
+
+  filter?: {
+    [column: string]: string | string[];
+  };
+}

package/common/lib/index.ts

@@ -0,0 +1 @@
+export * from './kafka';

package/common/lib/kafka/constant/consumer.const.ts

@@ -0,0 +1,13 @@
+export enum ConsumerManagerTask {
+  Bootstrap = 'bootstrap',
+  Start = 'start',
+  Stop = 'stop',
+  Pause = 'pause',
+  Resume = 'resume',
+  Check = 'check',
+}
+
+export enum TaskStatus {
+  SuccessFull = 'SuccessFull',
+  Failed = 'Failed',
+}

package/common/lib/kafka/constant/kafka.const.ts

@@ -0,0 +1,9 @@
+export enum KafkaHealthState {
+  Unknown = 'Unknown',
+  PreparingRebalance = 'PreparingRebalance',
+  CompletingRebalance = 'CompletingRebalance',
+  Stable = 'Stable',
+  Dead = 'Dead',
+  Empty = 'Empty',
+  Pause = 'Pause',
+}

package/common/lib/kafka/index.ts

@@ -0,0 +1,5 @@
+import { KafkaModule } from './kafka.module';
+import { KafkaModuleOptions } from './kafka.interface';
+import { KafkaService } from './kafka.service';
+
+export { KafkaModule, KafkaService, KafkaModuleOptions };

package/common/lib/kafka/kafka-admin.service.ts

@@ -0,0 +1,51 @@
+import { Admin } from 'kafkajs';
+import { Logger } from '@nestjs/common';
+import { kafka } from './kafka-instance.const';
+
+export default class KafkaAdminService {
+  private kafkaAdmin: Admin;
+
+  public constructor(clientId: string, brokers: string[]) {
+    this.kafkaAdmin = this.createKafkaAdmin(clientId, brokers);
+
+    const errorTypes = ['unhandledRejection', 'uncaughtException'];
+    const signalTraps = ['SIGTERM', 'SIGINT', 'SIGUSR2'];
+
+    errorTypes.forEach((type) => {
+      process.on(type, async (e) => {
+        try {
+          Logger.log(`process.on ${type}`);
+          Logger.error(e);
+          await this.shutdown();
+          process.exit(0);
+        } catch (_) {
+          process.exit(1);
+        }
+      });
+    });
+
+    signalTraps.forEach((type) => {
+      process.once(type, async () => {
+        try {
+          await this.shutdown();
+        } finally {
+          process.kill(process.pid, type);
+        }
+      });
+    });
+  }
+
+  async getTopics() {
+    let topics = await this.kafkaAdmin.listTopics();
+    topics = topics.filter((topic) => topic.slice(0, 1) !== '_');
+    return topics;
+  }
+
+  private async shutdown(): Promise<void> {
+    await this.kafkaAdmin.disconnect();
+  }
+
+  private createKafkaAdmin(clientId, brokers): Admin {
+    return kafka(clientId, brokers).admin();
+  }
+}

package/common/lib/kafka/kafka-consumer.service.ts

@@ -0,0 +1,59 @@
+import { Consumer } from 'kafkajs';
+import { Logger } from '@nestjs/common';
+import { kafka } from './kafka-instance.const';
+
+export default class KafkaConsumerService {
+  private kafkaConsumer: Consumer;
+
+  public constructor(clientId, brokers, groupId) {
+    this.kafkaConsumer = this.createKafkaConsumer(clientId, brokers, groupId);
+
+    const errorTypes = ['unhandledRejection', 'uncaughtException'];
+    const signalTraps = ['SIGTERM', 'SIGINT', 'SIGUSR2'];
+
+    errorTypes.forEach((type) => {
+      process.on(type, async (e) => {
+        try {
+          Logger.log(`process.on ${type}`);
+          Logger.error(e);
+          await this.shutdown();
+          process.exit(0);
+        } catch (_) {
+          process.exit(1);
+        }
+      });
+    });
+
+    signalTraps.forEach((type) => {
+      process.once(type, async () => {
+        try {
+          await this.shutdown();
+        } finally {
+          process.kill(process.pid, type);
+        }
+      });
+    });
+  }
+
+  public async connectConsumer(): Promise<Consumer> {
+    await this.kafkaConsumer.connect();
+    return this.kafkaConsumer;
+  }
+
+  public async healthCheck() {
+    const describe = await this.kafkaConsumer.describeGroup();
+    return describe.state;
+  }
+
+  public async shutdown(): Promise<void> {
+    await this.kafkaConsumer.disconnect();
+  }
+
+  private createKafkaConsumer(
+    clientId,
+    brokers,
+    groupId = 'consumer-group',
+  ): Consumer {
+    return kafka(clientId, brokers).consumer({ groupId });
+  }
+}

package/common/lib/kafka/kafka-instance.const.ts

@@ -0,0 +1,9 @@
+import { createKafkaLoggerAdapter } from './kafka-logger';
+import { Kafka } from 'kafkajs';
+
+export const kafka = (clientId: string, brokers: string[]) =>
+  new Kafka({
+    clientId,
+    brokers,
+    logCreator: createKafkaLoggerAdapter,
+  });

package/common/lib/kafka/kafka-logger.ts

@@ -0,0 +1,25 @@
+import { Logger } from '@nestjs/common';
+import { logLevel, LogEntry, logCreator } from 'kafkajs';
+
+export // This function returns a LogCreator that is compatible with KafkaJS requirements
+const createKafkaLoggerAdapter: logCreator = () => {
+  const nestLogger = new Logger('KafkaJS', { timestamp: true });
+  // Converts KafkaJS log levels to appropriate NestJS log methods
+  const logMethodMap = {
+    [logLevel.ERROR]: 'error',
+    [logLevel.WARN]: 'warn',
+    [logLevel.INFO]: 'log',
+    [logLevel.DEBUG]: 'debug',
+    [logLevel.NOTHING]: 'verbose', // Adjust according to your preference
+  };
+
+  return ({ namespace, level, label, log }: LogEntry) => {
+    const { message, ...extra } = log;
+    const method = logMethodMap[level] || 'log';
+
+    nestLogger[method](
+      `${label} [${namespace}] ${message}`,
+      JSON.stringify(extra),
+    );
+  };
+};

package/common/lib/kafka/kafka-producer.service.ts

@@ -0,0 +1,97 @@
+import { Message, Producer, ProducerBatch, TopicMessages } from 'kafkajs';
+import { Logger } from '@nestjs/common';
+import { kafka } from './kafka-instance.const';
+
+export default class KafkaProducerService {
+  private producer: Producer;
+
+  constructor(clientId: string, brokers: string[]) {
+    this.producer = this.createProducer(clientId, brokers);
+    this.setupGracefulShutdown();
+  }
+
+  private setupGracefulShutdown(): void {
+    const errorTypes = ['unhandledRejection', 'uncaughtException'];
+    const signalTraps = ['SIGTERM', 'SIGINT', 'SIGUSR2'];
+
+    errorTypes.forEach((type) => {
+      process.on(type, async (e) => {
+        try {
+          Logger.log(`process.on ${type}`);
+          Logger.error(e);
+          await this.shutdown();
+          process.exit(0);
+        } catch (_) {
+          process.exit(1);
+        }
+      });
+    });
+
+    signalTraps.forEach((type) => {
+      process.once(type, async () => {
+        try {
+          await this.shutdown();
+        } finally {
+          process.kill(process.pid, type);
+        }
+      });
+    });
+  }
+
+  public async start() {
+    await this.producer.connect();
+    return this;
+  }
+
+  public async shutdown(): Promise<void> {
+    await this.producer.disconnect();
+  }
+
+  public async sendBatch(
+    topic: string,
+    messages: Array<object>,
+  ): Promise<void> {
+    const kafkaMessages: Array<Message> = messages.map((message) => {
+      return {
+        value: JSON.stringify(message),
+      };
+    });
+
+    const topicMessages: TopicMessages = {
+      topic,
+      messages: kafkaMessages,
+    };
+
+    const batch: ProducerBatch = {
+      topicMessages: [topicMessages],
+    };
+
+    await this.producer.sendBatch(batch);
+  }
+
+  async sendEvent(
+    topic: string,
+    message: unknown,
+    partition: number = undefined,
+  ) {
+    const producer: Producer = this.producer;
+
+    const kafkaMessage: Message = {
+      value: typeof message == 'string' ? message : JSON.stringify(message),
+    };
+
+    // If partition is specified, set it in the send method
+    kafkaMessage.partition = partition;
+
+    const sendOptions: { topic: string; messages: Message[] } = {
+      topic,
+      messages: [kafkaMessage],
+    };
+
+    await producer.send(sendOptions);
+  }
+
+  private createProducer(clientId: string, brokers: string[]): Producer {
+    return kafka(clientId, brokers).producer();
+  }
+}

package/common/lib/kafka/kafka.interface.ts

@@ -0,0 +1,22 @@
+import { ModuleMetadata, Type } from '@nestjs/common';
+
+export const KAFKA_MODULE_OPTIONS = 'KAFKA_MODULE_OPTIONS';
+
+export interface KafkaModuleOptions {
+  clientId: string;
+  brokers: string[];
+}
+
+export interface KafkaModuleFactory {
+  createModuleOptions: () => Promise<KafkaModuleOptions> | KafkaModuleOptions;
+}
+
+export interface KafkaModuleAsyncOptions
+  extends Pick<ModuleMetadata, 'imports'> {
+  inject?: any[];
+  useClass?: Type<KafkaModuleFactory>;
+  useExisting?: Type<KafkaModuleFactory>;
+  useFactory?: (
+    ...args: any[]
+  ) => Promise<KafkaModuleOptions> | KafkaModuleOptions;
+}

package/common/lib/kafka/kafka.module.ts

@@ -0,0 +1,80 @@
+import { DynamicModule, Global, Module, Provider } from '@nestjs/common';
+
+import {
+  KAFKA_MODULE_OPTIONS,
+  KafkaModuleAsyncOptions,
+  KafkaModuleFactory,
+  KafkaModuleOptions,
+} from './kafka.interface';
+import { KafkaService } from './index';
+
+@Global()
+@Module({})
+export class KafkaModule {
+  static forRoot(options: KafkaModuleOptions): DynamicModule {
+    return {
+      module: KafkaModule,
+      providers: [
+        {
+          provide: KAFKA_MODULE_OPTIONS,
+          useValue: options,
+        },
+        KafkaService,
+      ],
+      exports: [KafkaService],
+    };
+  }
+
+  static forRootAsync(options: KafkaModuleAsyncOptions): DynamicModule {
+    return {
+      module: KafkaModule,
+      imports: [...options.imports],
+      providers: [...this.createAsyncProviders(options), KafkaService],
+      exports: [...this.createAsyncProviders(options), KafkaService],
+    };
+  }
+
+  private static createAsyncProviders(
+    options: KafkaModuleAsyncOptions,
+  ): Provider[] {
+    if (options.useExisting || options.useFactory) {
+      return [this.createAsyncOptionsProvider(options)];
+    }
+    return [
+      this.createAsyncOptionsProvider(options),
+      {
+        provide: options.useClass,
+        useClass: options.useClass,
+      },
+    ];
+  }
+
+  private static createProviders(options: KafkaModuleOptions): Provider[] {
+    return [this.createOptionsProvider(options)];
+  }
+  private static createAsyncOptionsProvider(
+    options: KafkaModuleAsyncOptions,
+  ): Provider {
+    if (options.useFactory) {
+      return {
+        provide: KAFKA_MODULE_OPTIONS,
+        useFactory: options.useFactory,
+        inject: options.inject || [],
+      };
+    }
+    return {
+      provide: KAFKA_MODULE_OPTIONS,
+      useFactory: async (optionsFactory: KafkaModuleFactory) =>
+        await optionsFactory.createModuleOptions(),
+      inject: [options.useExisting || options.useClass],
+    };
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  private static createOptionsProvider(options: KafkaModuleOptions): Provider {
+    return {
+      provide: KAFKA_MODULE_OPTIONS,
+      useFactory: async (optionsFactory: KafkaModuleFactory) =>
+        await optionsFactory.createModuleOptions(),
+    };
+  }
+}

package/common/lib/kafka/kafka.provider.ts

@@ -0,0 +1,10 @@
+import { KafkaService } from './kafka.service';
+import { KafkaModuleOptions } from './kafka.interface';
+
+export const KafkaProviders = [
+  {
+    provide: 'KAFKA_MODEL',
+    useFactory: (options: KafkaModuleOptions) => new KafkaService(options),
+    inject: ['DATABASE_CONNECTION'],
+  },
+];

package/common/lib/kafka/kafka.service.ts

@@ -0,0 +1,224 @@
+import { Inject, Injectable, Logger } from '@nestjs/common';
+import { Consumer } from 'kafkajs';
+import KafkaConsumerService from './kafka-consumer.service';
+import { KAFKA_MODULE_OPTIONS, KafkaModuleOptions } from './kafka.interface';
+import { env } from '../../helper/get-env.helper';
+import { KafkaHealthState } from './constant/kafka.const';
+import KafkaAdminService from './kafka-admin.service';
+import KafkaProducerService from './kafka-producer.service';
+
+@Injectable()
+export class KafkaService {
+  private static instance: KafkaService;
+  private consumers: { [groupId: string]: KafkaConsumerService };
+  private consumersInstance: { [groupId: string]: Consumer };
+  private producerInstance: KafkaProducerService;
+  private adminInstance: KafkaAdminService;
+  public topics: { [groupId: string]: string[] };
+  public groupIds: { [item: string]: string };
+
+  constructor(
+    @Inject(KAFKA_MODULE_OPTIONS)
+    private kafkaModuleOptions: KafkaModuleOptions,
+  ) {
+    this.adminInstance = new KafkaAdminService(
+      this.kafkaModuleOptions.clientId,
+      this.kafkaModuleOptions.brokers,
+    );
+  }
+
+  // Kafka Admin
+
+  /**
+   * Getting available topics using Kafka admin instance
+   */
+  async getAvailableTopics() {
+    return await this.adminInstance.getTopics();
+  }
+
+  // Kafka Producer
+
+  /**
+   * Make Producer instance and starting a Kafka producer service
+   */
+  async startProducer() {
+    this.producerInstance = new KafkaProducerService(
+      this.kafkaModuleOptions.clientId,
+      this.kafkaModuleOptions.brokers,
+    );
+    await this.producerInstance.start();
+    Logger.log('Producer is Connected', 'KafkaProducer');
+  }
+
+  /**
+   * Stopping Producer instance of Kafka producer service
+   */
+  async stopProducer() {
+    await this.producerInstance.shutdown();
+    Logger.log('Producer is Disconnected', 'KafkaProducer');
+  }
+
+  /**
+   * sending an event using Kafka producer
+   * @param topic
+   * @param message
+   * @param partition
+   */
+  async sendProducerEvent(
+    topic: string,
+    message: unknown,
+    partition: number = undefined,
+  ) {
+    await this.producerInstance.sendEvent(topic, message, partition);
+  }
+
+  // Kafka Consumer
+
+  /**
+   * This function sets up multiple Kafka consumers based on the provided information,
+   * initializing group IDs, topics, and consumer instances for each group.
+   */
+  async setBulkConsumer(
+    consumersInfo: {
+      groupIdKey: string;
+      topics: string[];
+      groupIdDefaultValue?: string;
+    }[],
+  ) {
+    await Promise.all(
+      consumersInfo.map((item) => {
+        this.groupIds = {
+          ...this.groupIds,
+          [item.groupIdKey]: env(item.groupIdKey, item?.groupIdDefaultValue),
+        };
+        this.topics = {
+          ...this.topics,
+          [this.groupIds[item.groupIdKey]]: item?.topics,
+        };
+        this.consumers = {
+          ...this.consumers,
+          [this.groupIds[item.groupIdKey]]: new KafkaConsumerService(
+            this.kafkaModuleOptions.clientId,
+            this.kafkaModuleOptions.brokers,
+            this.groupIds[item.groupIdKey],
+          ),
+        };
+      }),
+    );
+  }
+
+  /**
+   * method setConsumer which sets up a Kafka consumer with the provided:
+   * groupIdKey, topics, and optional groupIdDefaultValue value.
+   * @param consumersInfo
+   */
+  async setConsumer(consumersInfo: {
+    groupIdKey: string;
+    topics: string[];
+    groupIdDefaultValue?: string;
+  }) {
+    this.groupIds = {
+      ...this.groupIds,
+      [consumersInfo.groupIdKey]: env(
+        consumersInfo.groupIdKey,
+        consumersInfo?.groupIdDefaultValue,
+      ),
+    };
+    this.topics = {
+      ...this.topics,
+      [this.groupIds[consumersInfo.groupIdKey]]: consumersInfo?.topics,
+    };
+    this.consumers = {
+      ...this.consumers,
+      [this.groupIds[consumersInfo.groupIdKey]]: new KafkaConsumerService(
+        this.kafkaModuleOptions.clientId,
+        this.kafkaModuleOptions.brokers,
+        this.groupIds[consumersInfo.groupIdKey],
+      ),
+    };
+  }
+
+  /**
+   * Asynchronously starts all consumers, subscribes them to respective topics.
+   */
+  async startAllConsumers() {
+    const consumerKeys = Object.keys(this.consumers);
+    for await (const key of consumerKeys) {
+      this.consumersInstance = {
+        ...this.consumersInstance,
+        [key]: await this.consumers[key].connectConsumer(),
+      };
+      await this.consumersInstance[key].subscribe({ topics: this.topics[key] });
+    }
+    Logger.log(
+      `Consumer is Connected. consumer-list: ${consumerKeys.join(', ')}`,
+      'KafkaConsumer',
+    );
+  }
+
+  /**
+   * Stops all consumers by asynchronously shutting them down.
+   */
+  async stopAllConsumers() {
+    return await Promise.all(
+      Object.keys(this.consumers).map(async (key) => {
+        await this.consumers[key].shutdown();
+      }),
+    );
+  }
+
+  /**
+   * Method to start a consumer by group ID asynchronously.
+   * @param groupId
+   * @param fromBeginning
+   */
+  async startConsumerByGroupId(
+    groupId: string,
+    fromBeginning: boolean = false,
+  ) {
+    // Start the consumer for the given group ID and store the instance
+    this.consumersInstance = {
+      ...this.consumersInstance,
+      [groupId]: await this.consumers[groupId].connectConsumer(),
+    };
+    await this.consumersInstance[groupId].subscribe({
+      fromBeginning,
+      topics: this.topics[groupId],
+    });
+    Logger.log(`\"${groupId}\" is Connected.`, 'KafkaConsumer');
+    // Return the consumer instance
+    return this.consumersInstance[groupId];
+  }
+
+  /**
+   * Method to stop a consumer by group ID.
+   */
+  async stopConsumerByGroupId(groupId: string) {
+    return await this.consumers[groupId].shutdown();
+  }
+
+  /**
+   * This method performs a health check on a specific Kafka consumer group and determines its state.
+   * It checks the Kafka state and whether any topics are paused in the consumer group.
+   * Returns the KafkaHealthState based on the conditions met.
+   */
+  async healthCheck(groupId: string) {
+    const kafkaState =
+      (await this.consumers?.[groupId].healthCheck()) ?? KafkaHealthState.Dead;
+    if (kafkaState === KafkaHealthState.Stable) {
+      const paused = this.consumersInstance[groupId].paused();
+      const isPaused = paused.filter(
+        (item) => item.topic === this.topics[groupId][0],
+      );
+      if (isPaused.length !== 0) return KafkaHealthState.Pause;
+    }
+    return KafkaHealthState[kafkaState];
+  }
+
+  /**
+   * Retrieves a consumer instance by group ID.
+   */
+  getConsumerInstanceByGroupId(groupId: string) {
+    return this.consumersInstance[groupId];
+  }
+}