dt-common-device 3.0.10 → 3.0.11

package/src/queue/entities/HybridHttpQueue.ts

@@ -1,274 +0,0 @@
-import { HttpCallOption } from "../types/http.types";
-import { getConfig } from "../../config/config";
-import axios from "axios";
-import { publishAudit } from "dt-audit-library";
-import { Service } from "typedi";
-import { RateLimitUtils } from "../utils/rateLimit.utils";
-import { JobUtils } from "../utils/jobUtils";
-import { QueueUtils } from "../utils/queueUtils";
-import {
-  IRateLimitConfig,
-  IJobResult,
-  IHttpRequestJob,
-  IQueueResponse,
-} from "../interfaces";
-import { Queue } from "bullmq";
-
-@Service()
-export class HybridHttpQueue {
-  private readonly queues = new Map<string, any>();
-  private readonly workers = new Map<string, any>();
-  private readonly rateLimitConfigs: Map<string, IRateLimitConfig>;
-  private readonly jobResults = new Map<string, IJobResult>();
-
-  constructor() {
-    this.rateLimitConfigs = RateLimitUtils.initializeRateLimitConfigs();
-  }
-
-  private async handleRateLimitAndQueue(
-    url: string,
-    method: string,
-    options: HttpCallOption
-  ): Promise<IQueueResponse> {
-    const { connectionId, provider, microservice } =
-      JobUtils.extractConnectionDetails(options);
-    const key = `rate_limit:${provider}:${connectionId}`;
-    const config = this.rateLimitConfigs.get(provider);
-    const windowMs = config?.windowMs ?? 60000;
-
-    const timestamps = await RateLimitUtils.getRawRequestTimestamps(key);
-    const now = Date.now();
-    const windowStart = now - windowMs;
-    const recentRequests = timestamps.filter((t) => t > windowStart);
-    const nextAvailableTime =
-      recentRequests.length > 0 ? recentRequests[0] + windowMs : now + 1000;
-    const delay = Math.max(nextAvailableTime - now, 1000); // at least 1s delay
-
-    // Create job data
-    const jobData = {
-      microservice,
-      connectionId,
-      provider,
-      url,
-      method,
-      options,
-      timestamp: Date.now(),
-    };
-
-    // Add job to queue with delay (background processing)
-    const queueKey = QueueUtils.getQueueKey(
-      microservice,
-      connectionId,
-      provider
-    );
-    const queue = QueueUtils.getOrCreateQueue(queueKey, this.queues);
-
-    QueueUtils.getOrCreateWorker(
-      queueKey,
-      this.workers,
-      this.processHttpRequest.bind(this),
-      this.jobResults
-    );
-
-    const job = await queue.add("http-request", jobData, {
-      delay,
-      attempts: 1,
-      removeOnComplete: { age: 300, count: 1 },
-      removeOnFail: { age: 300, count: 1 },
-    });
-
-    await publishAudit({
-      eventType: "http.request.rateLimitQueued",
-      properties: {
-        resource: microservice,
-        connectionId,
-        provider,
-        endpoint: url,
-        method,
-        timestamp: Date.now(),
-        queueId: job.id,
-        reason: "rate_limit_exceeded_queued",
-        delay,
-        estimatedProcessingTime: now + delay,
-      },
-    });
-
-    // Return immediate response to controller
-    return {
-      success: true,
-      queued: true,
-      estimatedProcessingTime: now + delay,
-      jobId: job.id,
-    };
-  }
-
-  private async processHttpRequest(job: any): Promise<any> {
-    const { connectionId, provider, url, method, options } = job.data;
-
-    const allowed = await RateLimitUtils.isRateLimitAllowed(
-      connectionId,
-      provider,
-      this.rateLimitConfigs
-    );
-
-    if (!allowed) {
-      // This shouldn't happen since we check before queuing, but handle it gracefully
-      getConfig().LOGGER.warn(
-        `Job ${job.id} still rate limited after delay, skipping`
-      );
-      return;
-    }
-
-    await RateLimitUtils.recordRequest(connectionId, provider);
-
-    try {
-      getConfig().LOGGER.info(
-        `Executing HTTP request: ${method} ${url} for ${provider}`
-      );
-
-      const response = await axios({
-        method: method.toLowerCase(),
-        url: url,
-        headers: options.headers || {},
-        timeout: 30000,
-        ...(options.body && { data: options.body }),
-        ...(options.params && { params: options.params }),
-      });
-
-      getConfig().LOGGER.info(
-        `HTTP request successful: ${method} ${url} for ${provider}`
-      );
-      return response.data;
-    } catch (error: any) {
-      getConfig().LOGGER.error(
-        `HTTP request failed ${job.id}: ${error.message}`
-      );
-
-      await publishAudit({
-        eventType: "http.request.error",
-        properties: {
-          connectionId,
-          provider,
-          endpoint: url,
-          method,
-          timestamp: Date.now(),
-          queueId: job.id,
-          reason: "execution_error",
-          errorMessage: error.message,
-        },
-      });
-
-      throw new Error(`HTTP request failed: ${error.message}`);
-    }
-  }
-
-  async request(options: {
-    method: string;
-    url: string;
-    body?: any;
-    params?: Record<string, any>;
-    headers?: Record<string, string>;
-    queueOptions?: {
-      connectionId: string;
-      connectionProvider: string;
-      microservice: string;
-    };
-  }): Promise<IQueueResponse> {
-    const { method, url, body, params, headers, queueOptions } = options;
-    // Create HttpCallOption object
-    const httpCallOption: HttpCallOption = {
-      headers,
-      body,
-      params,
-      queueOptions,
-    };
-    // Call handleRequest with the constructed parameters
-    return this.handleRequest(url, method, httpCallOption);
-  }
-
-  async handleRequest(
-    url: string,
-    method: string,
-    options: HttpCallOption
-  ): Promise<IQueueResponse> {
-    const { connectionId, provider, microservice } =
-      JobUtils.extractConnectionDetails(options);
-
-    // Check rate limit first
-    const allowed = await RateLimitUtils.isRateLimitAllowed(
-      connectionId,
-      provider,
-      this.rateLimitConfigs
-    );
-
-    if (!allowed) {
-      // Rate limited - queue the request and return immediate response
-      return this.handleRateLimitAndQueue(url, method, options);
-    }
-
-    // Not rate limited - process immediately
-    getConfig().LOGGER.info(
-      `Processing immediately: ${method} ${url} -> ${provider} [${connectionId}]`
-    );
-
-    try {
-      // Record the request first
-      await RateLimitUtils.recordRequest(connectionId, provider);
-
-      // Execute the HTTP request
-      const response = await axios({
-        method: method.toLowerCase(),
-        url: url,
-        headers: options.headers || {},
-        timeout: 30000,
-        ...(options.body && { data: options.body }),
-        ...(options.params && { params: options.params }),
-      });
-
-      getConfig().LOGGER.info(
-        `HTTP request successful: ${method} ${url} for ${provider}`
-      );
-
-      return {
-        success: true,
-        data: response.data,
-        queued: false,
-      };
-    } catch (error: any) {
-      getConfig().LOGGER.error(`HTTP request failed: ${error.message}`);
-
-      await publishAudit({
-        eventType: "http.request.error",
-        properties: {
-          connectionId,
-          provider,
-          endpoint: url,
-          method,
-          timestamp: Date.now(),
-          reason: "execution_error",
-          errorMessage: error.message,
-        },
-      });
-
-      return {
-        success: false,
-        error: `HTTP request failed: ${error.message}`,
-        queued: false,
-      };
-    }
-  }
-
-  async shutdown(): Promise<void> {
-    getConfig().LOGGER.info("Shutting down HTTP queues...");
-
-    await Promise.all([
-      ...Array.from(this.workers.values()).map((worker: any) => worker.close()),
-      ...Array.from(this.queues.values()).map((queue: any) => queue.close()),
-    ]);
-
-    this.workers.clear();
-    this.queues.clear();
-    this.jobResults.clear();
-    getConfig().LOGGER.info("HTTP queues shutdown complete");
-  }
-}

package/src/queue/entities/index.ts

@@ -1 +0,0 @@
-export * from "./HybridHttpQueue";

package/src/queue/index.ts

@@ -1,6 +0,0 @@
-// Main queue exports
-export * from "./entities";
-export * from "./interfaces";
-export * from "./services";
-export * from "./types";
-export * from "./utils";

package/src/queue/interfaces/IHttpRequestJob.ts

@@ -1,10 +0,0 @@
-import { HttpCallOption } from "../types/http.types";
-
-export interface IHttpRequestJob {
-  connectionId: string;
-  provider: string;
-  url: string;
-  method: string;
-  options: HttpCallOption;
-  timestamp: number;
-}

package/src/queue/interfaces/IHybridHttpQueue.ts

@@ -1,25 +0,0 @@
-import { HttpCallOption } from "../types/http.types";
-import { IQueueResponse } from "./IJobResult";
-
-export interface IHybridHttpQueue {
-  request(options: {
-    method: string;
-    url: string;
-    body?: any;
-    params?: Record<string, any>;
-    headers?: Record<string, string>;
-    queueOptions?: {
-      connectionId: string;
-      connectionProvider: string;
-      microservice: string;
-    };
-  }): Promise<IQueueResponse>;
-
-  handleRequest(
-    url: string,
-    method: string,
-    options: HttpCallOption
-  ): Promise<IQueueResponse>;
-
-  shutdown(): Promise<void>;
-}

package/src/queue/interfaces/IJobResult.ts

@@ -1,15 +0,0 @@
-export interface IJobResult {
-  result?: any;
-  error?: string;
-  resolved: boolean;
-  timestamp: number;
-}
-
-export interface IQueueResponse {
-  success: boolean;
-  data?: any;
-  error?: string;
-  queued?: boolean;
-  estimatedProcessingTime?: number;
-  jobId?: string;
-}

package/src/queue/interfaces/IRateLimitConfig.ts

@@ -1,5 +0,0 @@
-export interface IRateLimitConfig {
-  maxRequests: number;
-  windowMs: number;
-  provider: string;
-}

package/src/queue/interfaces/index.ts

@@ -1,4 +0,0 @@
-export * from "./IHybridHttpQueue";
-export * from "./IHttpRequestJob";
-export * from "./IJobResult";
-export * from "./IRateLimitConfig";

package/src/queue/services/QueueService.ts

@@ -1,40 +0,0 @@
-import { Service } from "typedi";
-import { HybridHttpQueue } from "../entities/HybridHttpQueue";
-import { IHybridHttpQueue, IQueueResponse } from "../interfaces";
-import { HttpCallOption } from "../types/http.types";
-
-@Service()
-export class QueueService implements IHybridHttpQueue {
-  private readonly hybridQueue: HybridHttpQueue;
-
-  constructor() {
-    this.hybridQueue = new HybridHttpQueue();
-  }
-
-  async request(options: {
-    method: string;
-    url: string;
-    body?: any;
-    params?: Record<string, any>;
-    headers?: Record<string, string>;
-    queueOptions?: {
-      connectionId: string;
-      connectionProvider: string;
-      microservice: string;
-    };
-  }): Promise<IQueueResponse> {
-    return this.hybridQueue.request(options);
-  }
-
-  async handleRequest(
-    url: string,
-    method: string,
-    options: HttpCallOption
-  ): Promise<IQueueResponse> {
-    return this.hybridQueue.handleRequest(url, method, options);
-  }
-
-  async shutdown(): Promise<void> {
-    return this.hybridQueue.shutdown();
-  }
-}

package/src/queue/services/index.ts

@@ -1 +0,0 @@
-export * from "./QueueService";

package/src/queue/types/http.types.ts

@@ -1,23 +0,0 @@
-export interface HttpCallOption {
-  headers?: Record<string, string>;
-  body?: any;
-  params?: Record<string, any>;
-  queueOptions?: {
-    connectionId: string;
-    connectionProvider: string;
-    microservice: string;
-  };
-}
-
-export interface HttpRequestOptions {
-  method: string;
-  url: string;
-  body?: any;
-  params?: Record<string, any>;
-  headers?: Record<string, string>;
-  queueOptions?: {
-    connectionId: string;
-    connectionProvider: string;
-    microservice: string;
-  };
-}

package/src/queue/types/index.ts

@@ -1,2 +0,0 @@
-export * from "./http.types";
-export * from "./queue.types";

package/src/queue/types/queue.types.ts

@@ -1,21 +0,0 @@
-export interface QueueConfig {
-  concurrency: number;
-  removeOnComplete: { age: number; count: number };
-  removeOnFail: { age: number; count: number };
-  lockDuration: number;
-  stalledInterval: number;
-}
-
-export interface JobOptions {
-  attempts: number;
-  removeOnComplete: { age: number; count: number };
-  removeOnFail: { age: number; count: number };
-}
-
-export interface QueueMetrics {
-  totalJobs: number;
-  completedJobs: number;
-  failedJobs: number;
-  pendingJobs: number;
-  activeJobs: number;
-}

package/src/queue/utils/index.ts

@@ -1,3 +0,0 @@
-export * from "./rateLimit.utils";
-export * from "./jobUtils";
-export * from "./queueUtils";

package/src/queue/utils/jobUtils.ts

@@ -1,79 +0,0 @@
-import { Job } from "bullmq";
-import { IJobResult } from "../interfaces";
-
-export class JobUtils {
-  static async waitForJobCompletion(
-    job: Job,
-    queueKey: string,
-    jobResults: Map<string, IJobResult>
-  ): Promise<any> {
-    return new Promise((resolve, reject) => {
-      let timeoutId: NodeJS.Timeout;
-      let checkCount = 0;
-      const maxChecks = 600;
-
-      const checkJob = async () => {
-        if (++checkCount >= maxChecks) {
-          clearTimeout(timeoutId);
-          return reject(new Error("Job timeout: Request took too long"));
-        }
-
-        try {
-          const state = await job.getState();
-
-          if (state === "completed") {
-            clearTimeout(timeoutId);
-            const memoryResult = jobResults.get(job.id!);
-
-            if (memoryResult?.resolved) {
-              return memoryResult.result !== undefined
-                ? resolve(memoryResult.result)
-                : reject(new Error(memoryResult.error || "Unknown error"));
-            }
-
-            // Fallback to job result
-            resolve(await job.returnvalue);
-          } else if (state === "failed") {
-            clearTimeout(timeoutId);
-            const memoryResult = jobResults.get(job.id!);
-            const errorMsg =
-              memoryResult?.error ||
-              (await job.failedReason) ||
-              "Unknown error";
-            reject(new Error(`Job failed: ${errorMsg}`));
-          } else {
-            timeoutId = setTimeout(checkJob, checkCount < 10 ? 50 : 100);
-          }
-        } catch (error: any) {
-          clearTimeout(timeoutId);
-          reject(new Error(`Error checking job: ${error.message}`));
-        }
-      };
-
-      checkJob();
-
-      // Backup timeout
-      setTimeout(() => {
-        clearTimeout(timeoutId);
-        reject(new Error("Request timeout: Maximum wait time exceeded"));
-      }, 60000);
-    });
-  }
-
-  static extractConnectionDetails(options: any): {
-    connectionId: string;
-    provider: string;
-    microservice: string;
-  } {
-    const {
-      connectionId,
-      connectionProvider: provider,
-      microservice,
-    } = options?.queueOptions ?? {};
-
-    if (!connectionId) throw new Error("Connection ID not found in options");
-    if (!provider) throw new Error("Connection provider not found in options");
-    if (!microservice) throw new Error("Microservice not found in options");
-    return { connectionId, provider, microservice };
-  }
-}

package/src/queue/utils/queueUtils.ts

@@ -1,84 +0,0 @@
-import { getConfig } from "../../config/config";
-import { getRedisClient } from "../../db/redis";
-
-export class QueueUtils {
-  static getQueueKey(
-    microservice: string,
-    connectionId: string,
-    provider: string
-  ): string {
-    return `${microservice}_${provider}_${connectionId}`;
-  }
-
-  static getOrCreateQueue(queueKey: string, queues: Map<string, any>): any {
-    return (
-      queues.get(queueKey) ??
-      queues
-        .set(
-          queueKey,
-          new (require("bullmq").Queue)(queueKey, {
-            connection: getRedisClient(),
-          })
-        )
-        .get(queueKey)!
-    );
-  }
-
-  static getOrCreateWorker(
-    queueKey: string,
-    workers: Map<string, any>,
-    processFunction: (job: any) => Promise<any>,
-    jobResults: Map<string, any>
-  ): void {
-    if (workers.has(queueKey)) return;
-
-    const { Worker } = require("bullmq");
-    const worker = new Worker(queueKey, processFunction, {
-      connection: getRedisClient(),
-      concurrency: 1,
-      removeOnComplete: true,
-      removeOnFail: true,
-      lockDuration: 300000,
-      stalledInterval: 60000,
-    });
-
-    // Simplified event handlers
-    worker.on("completed", (job: any) => {
-      getConfig().LOGGER.info(
-        `HTTP request completed: ${job.id} [${queueKey}]`
-      );
-
-      // job.returnvalue is the actual result, not a Promise
-      const result = job.returnvalue;
-      jobResults.set(job.id!, {
-        result,
-        resolved: true,
-        timestamp: Date.now(),
-      });
-    });
-
-    worker.on("failed", (job: any, err: any) => {
-      getConfig().LOGGER.error(
-        `HTTP request failed: ${job?.id} [${queueKey}], Error: ${err.message}`
-      );
-      jobResults.set(job!.id!, {
-        error: err.message,
-        resolved: true,
-        timestamp: Date.now(),
-      });
-    });
-
-    worker.on("error", (err: any) =>
-      getConfig().LOGGER.error(`Worker error for ${queueKey}: ${err.message}`)
-    );
-    worker.on("stalled", (jobId: string) =>
-      getConfig().LOGGER.warn(`Job ${jobId} stalled in worker ${queueKey}`)
-    );
-    worker.on("active", (job: any) =>
-      getConfig().LOGGER.info(`HTTP request started: ${job.id} [${queueKey}]`)
-    );
-
-    workers.set(queueKey, worker);
-    getConfig().LOGGER.info(`Worker initialized for queue: ${queueKey}`);
-  }
-}