keeyu-azure-service-bus 1.0.33 → 1.0.34

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "keeyu-azure-service-bus",
-  "version": "1.0.33",
+  "version": "1.0.34",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -13,6 +13,7 @@
   "devDependencies": {
     "@typescript-eslint/eslint-plugin": "^6.19.1",
     "@typescript-eslint/parser": "^6.19.1",
+    "@types/winston": "^2.4.4",
     "eslint": "^8.56.0",
     "nodemon": "^3.0.2",
     "ts-node": "^10.9.2",
@@ -22,6 +23,9 @@
     "@azure/identity": "^4.2.0",
     "@azure/service-bus": "^7.9.4",
     "@types/node": "^20.10.6",
-    "joi": "^17.13.1"
+    "axios": "^1.7.2",
+    "joi": "^17.13.1",
+    "winston": "^3.11.0",
+    "winston-daily-rotate-file": "^4.7.1"
   }
 }

package/src/enums/message.ts

@@ -0,0 +1,6 @@
+export enum MessageEnum {
+  on_boarding = "OnBoarding",
+  wider = "Wider",
+  normal = "Normal",
+  chunked = "Chunked"
+}

package/src/index.ts

@@ -1,19 +1,18 @@
 import QueryMetricDataAll from "./models/query-metric-data-all";
 import QueryMetricDataOnboarding from "./models/query-metric-data-onboarding";
 import QueryMetricDataChunked from "./models/query-metric-data-chunked";
-import PersistData from "./models/persist-data.model";
 import ServiceBus from "./services/service-bus/service-bus-client";
 import ServiceBusAdminClient from "./services/service-bus/service-bus-admin-client";
 import { ServiceBusReceivedMessage, ProcessErrorArgs } from "@azure/service-bus";
-
+import PersistenceClient from "./services/persistence/persistence-client";
 
 export {
   ServiceBus,
   ServiceBusAdminClient,
   QueryMetricDataAll,
   QueryMetricDataOnboarding,
-  PersistData,
   ServiceBusReceivedMessage,
   ProcessErrorArgs,
-  QueryMetricDataChunked
+  QueryMetricDataChunked,
+  PersistenceClient
 };

package/src/interfaces/message.ts

@@ -0,0 +1,59 @@
+import JobInfo from "../models/job-info.model";
+import RequestBody from "../models/request-body.model";
+import RequestsInfo from "../models/request-info.model";
+import RequestParams from "../models/request-params.model";
+
+export interface MessageDto {
+  tenantId: string;
+  channelId: string;
+  componentId: string;
+  componentName: string;
+  url: string;
+  connectorUrl: string;
+  headers: HeadersInfo[];
+  connectorId: string;
+  connectorInfo:  ConnectorInfo;
+  scheduleFields: ScheduleFields;
+  metricDetailedConfiguration?: any;
+  componentOptions?: Record<string, string>;
+  params?: any;
+}
+
+export interface ChunkedMessageDto {
+  jobInfo: JobInfo;
+  requestInfo: RequestsInfo;
+  requestParams: RequestParams;
+  requestBody: RequestBody;
+}
+
+export interface HeadersInfo {
+  key: string;
+  value: string | boolean | number;
+}
+
+export interface ConnectorInfo {
+  _id: string;
+  name: string;
+  queueName: string;
+}
+
+export interface ScheduleFields {
+  componentId: string;
+  channelId?: string;
+  tenantId?: string;
+  normalPullSchedule: string;
+  widerPullSchedule: string;
+  normalTimeRange: number;
+  widerTimeRange: number;
+  persistenceCollection: string;
+  isProcessing: boolean;
+  lastRunTime: string | null;
+  requireOnboarding: boolean;
+  scheduleActive: boolean;
+  lastSuccessfulCompletionDate?: string | null;
+}
+
+export interface PostRequestAttributes {
+  params: Record<string, string>
+  data: unknown
+}

package/src/logger/logger.ts

@@ -0,0 +1,37 @@
+import winston from 'winston';
+import DailyRotateFile from 'winston-daily-rotate-file';
+import fs from 'fs';
+
+const logFolder = 'logs';
+
+// Ensure the logs directory exists
+if (!fs.existsSync(logFolder)) {
+  fs.mkdirSync(logFolder);
+}
+
+// Define log formats
+const logFormat = winston.format.printf(({ level, message, timestamp }) => {
+  return `${timestamp} ${level}: ${message}`;
+});
+
+// Define transports
+const consoleTransport = new winston.transports.Console();
+const fileTransport = new DailyRotateFile({
+  dirname: logFolder,
+  filename: 'app-%DATE%.log',
+  datePattern: 'YYYY-MM-DD',
+  zippedArchive: true,
+  maxSize: '20m', // Set the maximum size for a log file
+  maxFiles: '1d', // Retain logs for 1 day
+});
+
+// Create the logger
+const logger = winston.createLogger({
+  format: winston.format.combine(
+    winston.format.timestamp(),
+    logFormat
+  ),
+  transports: [consoleTransport, fileTransport],
+});
+
+export default logger;

package/src/models/job-info.model.ts

@@ -1,4 +1,5 @@
 import Joi from "joi";
+import { MessageDto } from "../interfaces/message";
 
 export default class JobInfo {
   daysToPull: string;
@@ -9,14 +10,19 @@ export default class JobInfo {
   startDate: string;
   endDate: string;
   persistenceCollection: string;
+  lastRunTime: string | null;
+  isProcessing: boolean;
+  queueName: string;
 
-  constructor(data: any) {
+  constructor(data: MessageDto) {
     const schema = Joi.object({
       scheduleFields: Joi.object({
         normalTimeRange: Joi.number().positive().required(),
         widerTimeRange: Joi.number().positive().required(),
         requireOnboarding: Joi.boolean().required(),
         persistenceCollection: Joi.required(),
+        lastRunTime: Joi.optional(),
+        isProcessing: Joi.required()
       }).unknown(true),
       tenantId: Joi.required(),
       channelId: Joi.required(),
@@ -28,19 +34,20 @@ export default class JobInfo {
 
     if (error) throw new Error(error.message);
 
-    const { scheduleFields: { normalTimeRange, widerTimeRange, requireOnboarding, persistenceCollection }, tenantId, channelId, componentId, connectorId } = value;
+    const { scheduleFields: { normalTimeRange, widerTimeRange, requireOnboarding, persistenceCollection, isProcessing, lastRunTime }, tenantId, channelId, componentId, connectorId } = value;
 
+    this.queueName = data?.connectorInfo?.queueName;
     const today = new Date();
-    const startDate = new Date();
-    startDate.setUTCHours(0,0,0,0);
     this.daysToPull = requireOnboarding ? widerTimeRange : normalTimeRange;
     this.tenantId = tenantId;
     this.channelId = channelId;
     this.connectorId = connectorId;
     this.componentId = componentId;
     this.persistenceCollection = persistenceCollection;
-    this.startDate = this.getFormatedDate(new Date(startDate.getTime() - (requireOnboarding ? widerTimeRange : normalTimeRange) * 24 * 60 * 60 * 1000));
+    this.startDate = this.getFormatedDate(new Date(today.getTime() - (requireOnboarding ? widerTimeRange : normalTimeRange) * 24 * 60 * 60 * 1000));
     this.endDate = this.getFormatedDate(today);
+    this.isProcessing = isProcessing;
+    this.lastRunTime = lastRunTime;
   }
 
   private getFormatedDate(date: Date) {

package/src/models/persist-data.model.ts

@@ -1,17 +1,17 @@
-export default class PersistData {
-  connectorId: string;
-  channelId: string;
-  tenantId: string;
-  componentId: string;
-  idKey: string;
-  data: any[];
+// export default class PersistData {
+//   connectorId: string;
+//   channelId: string;
+//   tenantId: string;
+//   componentId: string;
+//   idKey: string;
+//   data: any[];
 
-  constructor(data: any) {
-    this.connectorId = data.connectorId;
-    this.componentId = data.componentId;
-    this.channelId = data.channelId;
-    this.tenantId = data.tenantId;
-    this.idKey = data.idKey;
-    this.data = data.record_list;
-  }
-}
+//   constructor(data: any) {
+//     this.connectorId = data.connectorId;
+//     this.componentId = data.componentId;
+//     this.channelId = data.channelId;
+//     this.tenantId = data.tenantId;
+//     this.idKey = data.idKey;
+//     this.data = data.record_list;
+//   }
+// }

package/src/models/query-metric-data-all.ts

@@ -1,18 +1,22 @@
+import { MessageEnum } from "../enums/message";
+import { MessageDto } from "../interfaces/message";
 import JobInfo from "./job-info.model";
 import RequestBody from "./request-body.model";
 import RequestsInfo from "./request-info.model";
 import RequestParams from "./request-params.model";
 
 export default class QueryMetricDataAll {
+  type: string;
   jobInfo: JobInfo;
   requestInfo: RequestsInfo;
   requestParams: RequestParams;
   requestBody: RequestBody;
 
-  constructor(data: any) {
+  constructor(data: MessageDto) {
+    this.type = MessageEnum.normal;
     this.jobInfo = new JobInfo(data);
-    this.requestInfo = new RequestsInfo(data);
-    this.requestParams = new RequestParams(data);
-    this.requestBody = new RequestBody(data);
+    this.requestInfo = new RequestsInfo(data.headers, data.url);
+    this.requestParams = new RequestParams(data?.params);
+    this.requestBody = new RequestBody(data?.metricDetailedConfiguration, data?.componentOptions);
   }
 }

package/src/models/query-metric-data-chunked.ts

@@ -1,44 +1,22 @@
-import Joi from "joi";
+import { MessageEnum } from "../enums/message";
+import { ChunkedMessageDto } from "../interfaces/message";
+import JobInfo from "./job-info.model";
 import RequestBody from "./request-body.model";
+import RequestsInfo from "./request-info.model";
+import RequestParams from "./request-params.model";
 
 export default class QueryMetricDataChunked {
-  tenantId: string;
-  channelId: string;
-  componentId: string;
-  connectorId: string;
-  persistenceCollection: string;
-  startDate: string;
-  endDate: string;
-  pageNo: number;
+  type: string;
+  jobInfo: JobInfo;
+  requestInfo: RequestsInfo;
+  requestParams: RequestParams;
   requestBody: RequestBody;
 
-  constructor(data: any) {
-    const schema = Joi.object({
-      tenantId: Joi.required(),
-      channelId: Joi.required(),
-      componentId: Joi.required(),
-      connectorId: Joi.required(),
-      persistenceCollection: Joi.required(),
-      startDate: Joi.required(),
-      endDate: Joi.required(),
-      pageNo: Joi.required(),
-    }).unknown(true);
-
-    const { error, value } = schema.validate(data);
-
-    if (error) throw new Error(error.message);
-
-    const { tenantId, channelId, componentId, connectorId, persistenceCollection, startDate, endDate, pageNo } = value;
-
-    this.tenantId = tenantId;
-    this.channelId = channelId;
-    this.connectorId = connectorId;
-    this.componentId = componentId;
-    this.connectorId = connectorId;
-    this.persistenceCollection = persistenceCollection;
-    this.startDate = startDate;
-    this.endDate = endDate;
-    this.pageNo = Number(pageNo);
-    this.requestBody = data.headers;
+  constructor(data: ChunkedMessageDto) {
+    this.type = MessageEnum.chunked;
+    this.jobInfo = data.jobInfo;
+    this.requestInfo = data.requestInfo;
+    this.requestParams = data.requestParams;
+    this.requestBody = data.requestBody;
   }
 }

package/src/models/query-metric-data-onboarding.ts

@@ -1,18 +1,22 @@
+import { MessageEnum } from "../enums/message";
+import { MessageDto } from "../interfaces/message";
 import JobInfo from "./job-info.model";
 import RequestBody from "./request-body.model";
 import RequestsInfo from "./request-info.model";
 import RequestParams from "./request-params.model";
 
 export default class QueryMetricDataOnboarding {
+  type: string;
   jobInfo: JobInfo;
   requestInfo: RequestsInfo;
   requestParams: RequestParams;
   requestBody: RequestBody;
 
-  constructor(data: any) {
+  constructor(data: MessageDto) {
+    this.type = MessageEnum.on_boarding;
     this.jobInfo = new JobInfo(data);
-    this.requestInfo = new RequestsInfo(data);
-    this.requestParams = new RequestParams(data);
-    this.requestBody = new RequestBody(data);
+    this.requestInfo = new RequestsInfo(data.headers, data.url);
+    this.requestParams = new RequestParams(data.params);
+    this.requestBody = new RequestBody(data?.metricDetailedConfiguration, data?.componentOptions);
   }
 }

package/src/models/request-body.model.ts

@@ -1,9 +1,9 @@
-export default class RequestParams {
+export default class RequestBody {
   metricDetailedConfiguration: any;
-  componentOptions: Record<string, string>;
+  componentOptions: Record<string, string> | null | undefined;
 
-  constructor(data: any) {
-    this.metricDetailedConfiguration = data?.metricDetailedConfiguration;
-    this.componentOptions = data?.componentOptions;
+  constructor(metricDetailedConfiguration: any, componentOptions: Record<string, string> | null | undefined) {
+    this.metricDetailedConfiguration = metricDetailedConfiguration;
+    this.componentOptions = componentOptions;
   }
 }

package/src/models/request-info.model.ts

@@ -1,9 +1,11 @@
+import { HeadersInfo } from "../interfaces/message";
+
 export default class RequestsInfo {
-  kcHeaders: Record<string, string|boolean|number>;
+  kcHeaders: HeadersInfo[];
   url: string;
 
-  constructor(data: any) {
-    this.kcHeaders = data.headers;
-    this.url = data.url;
+  constructor(headers: HeadersInfo[], url: string) {
+    this.kcHeaders = headers;
+    this.url = url;
   }
 }

package/src/models/request-params.model.ts

@@ -1,4 +1,7 @@
 export default class RequestParams {
+  params: Record<string, string | number | boolean> | null | undefined;
 
-  constructor(data: any) {};
+  constructor(params: Record<string, string | number | boolean> | null | undefined) {
+    this.params = params;
+  };
 }

package/src/services/persistence/api-request-service.ts

@@ -0,0 +1,63 @@
+import axios, { AxiosError, AxiosInstance, AxiosResponse } from "axios";
+import { PostRequestAttributes } from "../../interfaces/message";
+import QueryMetricDataAll from "../../models/query-metric-data-all";
+import QueryMetricDataChunked from "../../models/query-metric-data-chunked";
+import QueryMetricDataOnboarding from "../../models/query-metric-data-onboarding";
+import logger from "../../logger/logger";
+import ServiceBus from "../service-bus/service-bus-client";
+
+export class ApiRequestService {
+  MaxRetryAttempts = 3;
+
+  protected readonly axiosInstance: AxiosInstance;
+  protected sbClient: ServiceBus;
+  protected message:  QueryMetricDataAll | QueryMetricDataChunked | QueryMetricDataOnboarding;
+
+  constructor(baseUrl: string, message:  QueryMetricDataAll | QueryMetricDataChunked | QueryMetricDataOnboarding, sbClient: ServiceBus) {
+    this.message = message;
+    this.sbClient = sbClient;
+    this.axiosInstance = axios.create({
+      baseURL: baseUrl,
+      headers: {
+        "Content-Type": "application/json",
+        maxContentLength: 100000000,
+        maxBodyLength: 1000000000
+      }
+    });
+
+    this.axiosInstance.interceptors.response.use(this.handleResponse, this.handleError);
+  }
+
+  public async post<T>(url: string, requestAttributes: PostRequestAttributes): Promise<T> {
+    const response: AxiosResponse<T> = await this.axiosInstance.post(url, requestAttributes);
+    return response?.data;
+  }
+
+  private async handleResponse(response: AxiosResponse) {
+    return response;
+  }
+
+  private async handleError(error: AxiosError) {
+    if (error.response) {
+      // Rate limit exceeded, retry after the specified delay
+      const resetTimeInSeconds = parseInt(
+        error.response.headers['x-rate-limit-reset']
+      );
+      const delayInSeconds = resetTimeInSeconds + 1;
+
+      logger.error(`got error, wait time is ${delayInSeconds}, header value is: ${error.response.headers['x-rate-limit-reset']}`)
+
+      if (delayInSeconds > 0 && this.MaxRetryAttempts > 0) {
+        await new Promise((resolve) => setTimeout(resolve, delayInSeconds * 1000));
+        this.MaxRetryAttempts--;
+        return this.axiosInstance.post(error.request.config.url, { params: error.request.config.params, data: error.request.config.body } )
+      }else{
+        const message: QueryMetricDataChunked = new QueryMetricDataChunked(this.message);
+        const result = await this.sbClient.scheduleMessage(this.message.jobInfo.queueName, 2, message)
+        logger.info(`Message is reschedule ${JSON.stringify(message)}`);
+        logger.info(`Response of schdule message method ${result}`);
+      }
+    }
+    throw error;
+  }
+}

package/src/services/persistence/persistence-client.ts

@@ -0,0 +1,37 @@
+import logger from "../../logger/logger";
+import QueryMetricDataAll from "../../models/query-metric-data-all";
+import QueryMetricDataChunked from "../../models/query-metric-data-chunked";
+import QueryMetricDataOnboarding from "../../models/query-metric-data-onboarding";
+import ServiceBus from "../service-bus/service-bus-client";
+import { ApiRequestService } from "./api-request-service";
+
+class PersistenceClient {
+  protected axiosClient: ApiRequestService;
+  protected uniqueKey: string;
+  protected message:  QueryMetricDataAll | QueryMetricDataChunked | QueryMetricDataOnboarding;
+
+  constructor(baseUrl: string, uniqueKey: string, message: QueryMetricDataAll | QueryMetricDataChunked | QueryMetricDataOnboarding, sbClient: ServiceBus) {
+    this.uniqueKey   = uniqueKey;
+    this.message = message;
+    this.axiosClient = new ApiRequestService(baseUrl, message, sbClient);
+  }
+
+  protected async persistData (data: unknown) {
+    const queryParams: Record<string, string> = {
+      startDate: this.message.jobInfo.startDate,
+      endDate: this.message.jobInfo.endDate,
+      tenantId: this.message.jobInfo.tenantId,
+      componentId: this.message.jobInfo.componentId,
+      connectorId: this.message.jobInfo.connectorId,
+      channelId: this.message.jobInfo.channelId,
+      persistenceCollection: this.message.jobInfo.persistenceCollection,
+      idKey: this.uniqueKey,
+    }
+
+    logger.info("API call to persistence service to save data");
+    const response = await this.axiosClient.post("/save-data", { params: queryParams, data })
+    logger.info(`Response of persistence service after API call ${JSON.stringify(response)}`)
+  }
+
+}
+export default PersistenceClient;

package/src/services/service-bus/service-bus-client.ts

@@ -1,8 +1,5 @@
 import { ProcessErrorArgs, ServiceBusClient, ServiceBusReceivedMessage } from "@azure/service-bus";
 import QueryMetricDataAll from "../../models/query-metric-data-all";
-import QueryMetricDataOnboarding from "../../models/query-metric-data-onboarding";
-import PersistData from "../../models/persist-data.model";
-import QueryMetricDataChunked from "../../models/query-metric-data-chunked";
 
 type MessageHandler = (message: ServiceBusReceivedMessage) => void;
 type ErrorHandler = (message: ProcessErrorArgs) => void;
@@ -13,7 +10,7 @@ class ServiceBus {
     this.client = new ServiceBusClient(connectionString);
   }
 
-  async sendMessage(queueName: string, message: QueryMetricDataAll|QueryMetricDataOnboarding|PersistData|QueryMetricDataChunked) {
+  async sendMessage(queueName: string, message: QueryMetricDataAll) {
     const sender = this.client.createSender(queueName);
     try {
       return await sender.sendMessages({ body: message });
@@ -28,10 +25,10 @@ class ServiceBus {
     const receiver = this.client.createReceiver(queueName, { receiveMode: "peekLock" });
     try {
       await receiver.subscribe({
-        processMessage: async (messageReceived) => {
+        processMessage: async (messageReceived: ServiceBusReceivedMessage) => {
           onMessage(messageReceived);
         },
-        processError: async (err) => {
+        processError: async (err: ProcessErrorArgs) => {
           onError(err);
         }
       });
@@ -51,7 +48,7 @@ class ServiceBus {
     }
   }
 
-  async scheduleMessage(queueName: string, scheduleTime: number, message: QueryMetricDataAll|QueryMetricDataOnboarding|PersistData|QueryMetricDataChunked) {
+  async scheduleMessage(queueName: string, scheduleTime: number, message: QueryMetricDataAll) {
     const sender = this.client.createSender(queueName);
     const scheduledEnqueueTimeUtc = new Date(Date.now() + scheduleTime * 60 * 10000);
     return await sender.scheduleMessages({