dt-common-device 3.0.0 → 3.0.2

package/src/queue/entities/HybridHttpQueue.ts

@@ -6,7 +6,13 @@ import { Service } from "typedi";
 import { RateLimitUtils } from "../utils/rateLimit.utils";
 import { JobUtils } from "../utils/jobUtils";
 import { QueueUtils } from "../utils/queueUtils";
-import { IRateLimitConfig, IJobResult, IHttpRequestJob } from "../interfaces";
+import {
+  IRateLimitConfig,
+  IJobResult,
+  IHttpRequestJob,
+  IQueueResponse,
+} from "../interfaces";
+import { Queue } from "bullmq";
 
 @Service()
 export class HybridHttpQueue {
@@ -19,56 +25,102 @@ export class HybridHttpQueue {
     this.rateLimitConfigs = RateLimitUtils.initializeRateLimitConfigs();
   }
 
-  private async handleRateLimit(job: any): Promise<void> {
-    const { connectionId, provider, url, method, options } = job.data;
-    const microservice = options.queueOptions?.microservice || "default";
-    const isMaxRetries = job.attemptsMade >= 2;
+  private async handleRateLimitAndQueue(
+    url: string,
+    method: string,
+    options: HttpCallOption
+  ): Promise<IQueueResponse> {
+    const { connectionId, provider, microservice } =
+      JobUtils.extractConnectionDetails(options);
+    const key = `rate_limit:${provider}:${connectionId}`;
+    const config = this.rateLimitConfigs.get(provider);
+    const windowMs = config?.windowMs ?? 60000;
+
+    const timestamps = await RateLimitUtils.getRawRequestTimestamps(key);
+    const now = Date.now();
+    const windowStart = now - windowMs;
+    const recentRequests = timestamps.filter((t) => t > windowStart);
+    const nextAvailableTime =
+      recentRequests.length > 0 ? recentRequests[0] + windowMs : now + 1000;
+    const delay = Math.max(nextAvailableTime - now, 1000); // at least 1s delay
+
+    // Create job data
+    const jobData = {
+      microservice,
+      connectionId,
+      provider,
+      url,
+      method,
+      options,
+      timestamp: Date.now(),
+    };
+
+    // Add job to queue with delay (background processing)
+    const queueKey = QueueUtils.getQueueKey(
+      microservice,
+      connectionId,
+      provider
+    );
+    const queue = QueueUtils.getOrCreateQueue(queueKey, this.queues);
+
+    QueueUtils.getOrCreateWorker(
+      queueKey,
+      this.workers,
+      this.processHttpRequest.bind(this),
+      this.jobResults
+    );
+
+    const job = await queue.add("http-request", jobData, {
+      delay,
+      attempts: 1,
+      removeOnComplete: { age: 300, count: 1 },
+      removeOnFail: { age: 300, count: 1 },
+    });
 
     await publishAudit({
-      eventType: isMaxRetries
-        ? "http.request.failed"
-        : "http.request.rateLimitExceeded",
+      eventType: "http.request.rateLimitQueued",
       properties: {
+        resource: microservice,
         connectionId,
         provider,
         endpoint: url,
         method,
         timestamp: Date.now(),
         queueId: job.id,
-        reason: isMaxRetries ? "max_retries_exceeded" : "rate_limit_exceeded",
-        ...(!isMaxRetries && {
-          retryCount: job.attemptsMade + 1,
-          maxRetries: 3,
-        }),
+        reason: "rate_limit_exceeded_queued",
+        delay,
+        estimatedProcessingTime: now + delay,
       },
     });
 
-    if (isMaxRetries) {
-      await this.queues
-        .get(QueueUtils.getQueueKey(microservice, connectionId, provider))
-        ?.obliterate({ force: true });
-      // Don't throw error for max retries - just return
-      return;
-    }
-
-    throw new Error("Rate limit exceeded");
+    // Return immediate response to controller
+    return {
+      success: true,
+      queued: true,
+      estimatedProcessingTime: now + delay,
+      jobId: job.id,
+    };
   }
 
   private async processHttpRequest(job: any): Promise<any> {
     const { connectionId, provider, url, method, options } = job.data;
 
-    // Check rate limit
-    if (
-      !(await RateLimitUtils.checkRateLimit(
-        connectionId,
-        provider,
-        this.rateLimitConfigs
-      ))
-    ) {
-      await this.handleRateLimit(job);
+    const allowed = await RateLimitUtils.isRateLimitAllowed(
+      connectionId,
+      provider,
+      this.rateLimitConfigs
+    );
+
+    if (!allowed) {
+      // This shouldn't happen since we check before queuing, but handle it gracefully
+      getConfig().LOGGER.warn(
+        `Job ${job.id} still rate limited after delay, skipping`
+      );
       return;
     }
 
+    await RateLimitUtils.recordRequest(connectionId, provider);
+
     try {
       getConfig().LOGGER.info(
         `Executing HTTP request: ${method} ${url} for ${provider}`
@@ -114,18 +166,20 @@ export class HybridHttpQueue {
     method: string;
     url: string;
     body?: any;
+    params?: Record<string, any>;
     headers?: Record<string, string>;
     queueOptions?: {
       connectionId: string;
       connectionProvider: string;
       microservice: string;
     };
-  }): Promise<any> {
-    const { method, url, body, headers, queueOptions } = options;
+  }): Promise<IQueueResponse> {
+    const { method, url, body, params, headers, queueOptions } = options;
     // Create HttpCallOption object
     const httpCallOption: HttpCallOption = {
       headers,
       body,
+      params,
       queueOptions,
     };
     // Call handleRequest with the constructed parameters
@@ -136,48 +190,72 @@ export class HybridHttpQueue {
     url: string,
     method: string,
     options: HttpCallOption
-  ): Promise<any> {
+  ): Promise<IQueueResponse> {
     const { connectionId, provider, microservice } =
       JobUtils.extractConnectionDetails(options);
-    const queueKey = QueueUtils.getQueueKey(
-      microservice,
+
+    // Check rate limit first
+    const allowed = await RateLimitUtils.isRateLimitAllowed(
       connectionId,
-      provider
+      provider,
+      this.rateLimitConfigs
     );
 
+    if (!allowed) {
+      // Rate limited - queue the request and return immediate response
+      return this.handleRateLimitAndQueue(url, method, options);
+    }
+
+    // Not rate limited - process immediately
     getConfig().LOGGER.info(
-      `Queueing: ${method} ${url} -> ${provider} [${connectionId}]`
+      `Processing immediately: ${method} ${url} -> ${provider} [${connectionId}]`
     );
 
-    QueueUtils.getOrCreateWorker(
-      queueKey,
-      this.workers,
-      this.processHttpRequest.bind(this),
-      this.jobResults
-    );
-    const queue = QueueUtils.getOrCreateQueue(queueKey, this.queues);
+    try {
+      // Record the request first
+      await RateLimitUtils.recordRequest(connectionId, provider);
 
-    const job = await queue.add(
-      "http-request",
-      {
-        microservice,
-        connectionId,
-        provider,
-        url,
-        method,
-        options,
-        timestamp: Date.now(),
-      },
-      {
-        attempts: 3,
-        backoff: { type: "exponential", delay: 5000 },
-        removeOnComplete: { age: 300, count: 1 },
-        removeOnFail: { age: 300, count: 1 },
-      }
-    );
+      // Execute the HTTP request
+      const response = await axios({
+        method: method.toLowerCase(),
+        url: url,
+        headers: options.headers || {},
+        timeout: 30000,
+        ...(options.body && { data: options.body }),
+        ...(options.params && { params: options.params }),
+      });
+
+      getConfig().LOGGER.info(
+        `HTTP request successful: ${method} ${url} for ${provider}`
+      );
 
-    getConfig().LOGGER.info(`Job ${job.id} queued, waiting for completion...`);
-    return JobUtils.waitForJobCompletion(job, queueKey, this.jobResults);
+      return {
+        success: true,
+        data: response.data,
+        queued: false,
+      };
+    } catch (error: any) {
+      getConfig().LOGGER.error(`HTTP request failed: ${error.message}`);
+
+      await publishAudit({
+        eventType: "http.request.error",
+        properties: {
+          connectionId,
+          provider,
+          endpoint: url,
+          method,
+          timestamp: Date.now(),
+          reason: "execution_error",
+          errorMessage: error.message,
+        },
+      });
+
+      return {
+        success: false,
+        error: `HTTP request failed: ${error.message}`,
+        queued: false,
+      };
+    }
   }
 
   async shutdown(): Promise<void> {

package/src/queue/interfaces/IHybridHttpQueue.ts

@@ -1,23 +1,25 @@
 import { HttpCallOption } from "../types/http.types";
+import { IQueueResponse } from "./IJobResult";
 
 export interface IHybridHttpQueue {
   request(options: {
     method: string;
     url: string;
     body?: any;
+    params?: Record<string, any>;
     headers?: Record<string, string>;
     queueOptions?: {
       connectionId: string;
       connectionProvider: string;
       microservice: string;
     };
-  }): Promise<any>;
+  }): Promise<IQueueResponse>;
 
   handleRequest(
     url: string,
     method: string,
     options: HttpCallOption
-  ): Promise<any>;
+  ): Promise<IQueueResponse>;
 
   shutdown(): Promise<void>;
 }

package/src/queue/interfaces/IJobResult.ts

@@ -4,3 +4,12 @@ export interface IJobResult {
   resolved: boolean;
   timestamp: number;
 }
+
+export interface IQueueResponse {
+  success: boolean;
+  data?: any;
+  error?: string;
+  queued?: boolean;
+  estimatedProcessingTime?: number;
+  jobId?: string;
+}

package/src/queue/services/QueueService.ts

@@ -1,6 +1,6 @@
 import { Service } from "typedi";
 import { HybridHttpQueue } from "../entities/HybridHttpQueue";
-import { IHybridHttpQueue } from "../interfaces";
+import { IHybridHttpQueue, IQueueResponse } from "../interfaces";
 import { HttpCallOption } from "../types/http.types";
 
 @Service()
@@ -15,13 +15,14 @@ export class QueueService implements IHybridHttpQueue {
     method: string;
     url: string;
     body?: any;
+    params?: Record<string, any>;
     headers?: Record<string, string>;
     queueOptions?: {
       connectionId: string;
       connectionProvider: string;
       microservice: string;
     };
-  }): Promise<any> {
+  }): Promise<IQueueResponse> {
     return this.hybridQueue.request(options);
   }
 
@@ -29,7 +30,7 @@ export class QueueService implements IHybridHttpQueue {
     url: string,
     method: string,
     options: HttpCallOption
-  ): Promise<any> {
+  ): Promise<IQueueResponse> {
     return this.hybridQueue.handleRequest(url, method, options);
   }
 

package/src/queue/types/http.types.ts

@@ -13,6 +13,7 @@ export interface HttpRequestOptions {
   method: string;
   url: string;
   body?: any;
+  params?: Record<string, any>;
   headers?: Record<string, string>;
   queueOptions?: {
     connectionId: string;

package/src/queue/types/queue.types.ts

@@ -8,7 +8,6 @@ export interface QueueConfig {
 
 export interface JobOptions {
   attempts: number;
-  backoff: { type: string; delay: number };
   removeOnComplete: { age: number; count: number };
   removeOnFail: { age: number; count: number };
 }

package/src/queue/utils/queueUtils.ts

@@ -1,4 +1,5 @@
 import { getConfig } from "../../config/config";
+import { getRedisClient } from "../../db/redis";
 
 export class QueueUtils {
   static getQueueKey(
@@ -16,7 +17,7 @@ export class QueueUtils {
         .set(
           queueKey,
           new (require("bullmq").Queue)(queueKey, {
-            connection: require("../../db/redis").getRedisClient(),
+            connection: getRedisClient(),
           })
         )
         .get(queueKey)!
@@ -33,7 +34,7 @@ export class QueueUtils {
 
     const { Worker } = require("bullmq");
     const worker = new Worker(queueKey, processFunction, {
-      connection: require("../../db/redis").getRedisClient(),
+      connection: getRedisClient(),
       concurrency: 1,
       removeOnComplete: { age: 300, count: 1 },
       removeOnFail: { age: 300, count: 1 },

package/src/queue/utils/rateLimit.utils.ts

@@ -48,11 +48,84 @@ export class RateLimitUtils {
 
     // Configure rate limits for different providers
     configs.set("Sensibo", {
-      maxRequests: 40,
+      maxRequests: 5,
       windowMs: 60000,
       provider: "Sensibo",
     });
 
     return configs;
   }
+
+  static async isRateLimitAllowed(
+    connectionId: string,
+    provider: string,
+    rateLimitConfigs: Map<string, IRateLimitConfig>
+  ): Promise<boolean> {
+    const config = rateLimitConfigs.get(provider);
+    if (!config) {
+      getConfig().LOGGER.warn(
+        `No rate limit config found for provider: ${provider}`
+      );
+      return true;
+    }
+
+    const key = `rate_limit:${provider}:${connectionId}`;
+    const now = Date.now();
+    const windowStart = now - config.windowMs;
+
+    try {
+      const data = await this.redisClient.get(key);
+      const requests = data
+        ? JSON.parse(data).filter((t: number) => t > windowStart)
+        : [];
+
+      return requests.length < config.maxRequests;
+    } catch (error) {
+      getConfig().LOGGER.error(`Rate limit check error: ${error}`);
+      return true;
+    }
+  }
+  static async recordRequest(
+    connectionId: string,
+    provider: string
+  ): Promise<void> {
+    const config = this.getRateLimitConfig(provider);
+    if (!config) return;
+
+    const key = `rate_limit:${provider}:${connectionId}`;
+    const now = Date.now();
+    const windowStart = now - config.windowMs;
+
+    try {
+      const data = await this.redisClient.get(key);
+      const requests = data
+        ? JSON.parse(data).filter((t: number) => t > windowStart)
+        : [];
+
+      requests.push(now);
+      await this.redisClient.setex(
+        key,
+        Math.ceil(config.windowMs / 1000),
+        JSON.stringify(requests)
+      );
+    } catch (error) {
+      getConfig().LOGGER.error(`Rate limit record error: ${error}`);
+    }
+  }
+
+  static async getRawRequestTimestamps(key: string): Promise<number[]> {
+    try {
+      const data = await this.redisClient.get(key);
+      return data ? JSON.parse(data) : [];
+    } catch (error) {
+      getConfig().LOGGER.error(
+        `Error fetching raw request timestamps: ${error}`
+      );
+      return [];
+    }
+  }
+
+  static getRateLimitConfig(provider: string): IRateLimitConfig | undefined {
+    return this.initializeRateLimitConfigs().get(provider);
+  }
 }