dt-common-device 2.0.6 → 3.0.0

package/src/index.ts

@@ -1,6 +1,6 @@
 // Main entry point for dt-common-device
 
-// Cloud exports
+// DEVICE EXPORTS
 export {
   CloudDevice,
   CloudDeviceService,
@@ -9,26 +9,32 @@ export {
 export {
   LocalDeviceService,
   LocalHubService,
-  LocalConnectionService,
-  LocalPropertyService,
   LocalScheduleService,
 } from "./device/local/services";
+export * from "./device/local/interfaces";
+export * from "./device/cloud/interfaces";
 
-// Events exports
-export * from "./events";
-export * from "./events/interfaces";
+// CONNECTION EXPORTS
+export * from "./connection";
 
-export * from "./device/cloud/interfaces";
+// PROPERTY EXPORTS
+export * from "./property";
 
-// Local exports
-export * from "./device/local/interfaces";
+// EVENTS EXPORTS
 export * from "./events";
 export * from "./events/interfaces";
-// Types exports
-export * from "./types";
 
-// Redis utils
+// ALERTS EXPORTS
+export * from "./alerts";
+
+// ISSUES EXPORTS
+export * from "./issues";
+
+// REDIS EXPORTS
 export * from "./utils";
 
-//initialize export
+// QUEUE EXPORTS
+export * from "./queue";
+
+// CONFIG EXPORTS
 export { initialize, getConfig, shutdown } from "./config/config";

package/src/{device/local/models → issues}/Issue.model.ts

@@ -9,7 +9,7 @@ import {
   CreateIssueData,
   UpdateIssueData,
   AddCommentData,
-} from "../../../types/issue.types";
+} from "./issue.types";
 
 // Comment sub-schema
 const CommentSchema = new Schema<IssueComment>(

package/src/{device/local/repository → issues}/Issue.repository.ts

@@ -1,5 +1,5 @@
 import { Service } from "typedi";
-import { IssueModel, IIssueDocument } from "../models/Issue.model";
+import { IssueModel, IIssueDocument } from "./Issue.model";
 import {
   CreateIssueData,
   UpdateIssueData,
@@ -7,7 +7,7 @@ import {
   IssuePriority,
   IssuesCategory,
   EntityType,
-} from "../../../types/issue.types";
+} from "./issue.types";
 
 @Service()
 export class IssueRepository {

package/src/{device/local/services → issues}/Issue.service.ts

@@ -1,6 +1,6 @@
 import { Service } from "typedi";
-import { IssueRepository } from "../repository/Issue.repository";
-import { IssueModel, IIssueDocument } from "../models/Issue.model";
+import { IssueRepository } from "./Issue.repository";
+import { IssueModel, IIssueDocument } from "./Issue.model";
 import {
   CreateIssueData,
   UpdateIssueData,
@@ -9,8 +9,8 @@ import {
   IssuePriority,
   IssuesCategory,
   EntityType,
-} from "../../../types/issue.types";
-import { IssueBuilder } from "../entities/IssueBuilder";
+} from "./issue.types";
+import { IssueBuilder } from "./IssueBuilder";
 
 @Service()
 export class IssueService {

package/src/{device/local/entities → issues}/IssueBuilder.example.ts

@@ -1,5 +1,5 @@
 import { IssueBuilder } from "./IssueBuilder";
-import { IssuesCategory, IssuePriority, EntityType } from "../../../types/issue.types";
+import { IssuesCategory, IssuePriority, EntityType } from "./issue.types";
 
 /**
  * Example usage of IssueBuilder

package/src/{device/local/entities → issues}/IssueBuilder.ts

@@ -3,7 +3,7 @@ import {
   IssuesCategory,
   IssuePriority,
   EntityType,
-} from "../../../types/issue.types";
+} from "./issue.types";
 
 /**
  * IssueBuilder - A builder pattern implementation for constructing CreateIssueData objects

package/src/{device/local/services → issues}/IssueService.example.ts

@@ -1,6 +1,6 @@
 import { IssueService } from "./Issue.service";
-import { IssueBuilder } from "../entities/IssueBuilder";
-import { IssuesCategory, IssuePriority, EntityType } from "../../../types/issue.types";
+import { IssueBuilder } from "./IssueBuilder";
+import { IssuesCategory, IssuePriority, EntityType } from "./issue.types";
 
 /**
  * Example usage of the updated IssueService with IssueBuilder integration
@@ -291,17 +291,18 @@ export class IssueServiceExample {
       category: IssuesCategory.OPERATIONS,
       propertyId: "prop123",
       title: "Legacy Issue",
-      description: "This issue was created using the old CreateIssueData format",
+      description:
+        "This issue was created using the old CreateIssueData format",
       entityId: "device456",
       entityType: EntityType.DEVICE,
       priority: IssuePriority.MEDIUM,
       assignedTo: "tech-support",
       createdBy: "legacy-system",
-      dueDate: new Date("2024-01-20")
+      dueDate: new Date("2024-01-20"),
     };
 
     // This still works with the updated createIssue method
     const issue = await this.issueService.createIssue(issueData);
     return issue;
   }
-} 
+}

package/src/issues/index.ts

@@ -0,0 +1,2 @@
+export * from "./Issue.service";
+export * from "./issue.types";

package/src/{device/local/repository → property}/Property.repository.ts

@@ -1,6 +1,6 @@
-import { getPostgresClient } from "../../../db";
+import { getPostgresClient } from "../db";
 import { Service } from "typedi";
-import { IProperty, IPropertySettings } from "../interfaces/IProperty";
+import { IProperty, IPropertySettings } from "./IProperty";
 
 @Service()
 export class PropertyRepository {

package/src/{device/local/services → property}/Property.service.ts

@@ -1,4 +1,4 @@
-import { PropertyRepository } from "../repository/Property.repository";
+import { PropertyRepository } from "./Property.repository";
 
 export class LocalPropertyService {
   private readonly propertyRepository: PropertyRepository;

package/src/property/index.ts

@@ -0,0 +1,2 @@
+export * from "./IProperty";
+export * from "./Property.service";

package/src/queue/entities/HybridHttpQueue.ts

@@ -0,0 +1,196 @@
+import { HttpCallOption } from "../types/http.types";
+import { getConfig } from "../../config/config";
+import axios from "axios";
+import { publishAudit } from "dt-audit-library";
+import { Service } from "typedi";
+import { RateLimitUtils } from "../utils/rateLimit.utils";
+import { JobUtils } from "../utils/jobUtils";
+import { QueueUtils } from "../utils/queueUtils";
+import { IRateLimitConfig, IJobResult, IHttpRequestJob } from "../interfaces";
+
+@Service()
+export class HybridHttpQueue {
+  private readonly queues = new Map<string, any>();
+  private readonly workers = new Map<string, any>();
+  private readonly rateLimitConfigs: Map<string, IRateLimitConfig>;
+  private readonly jobResults = new Map<string, IJobResult>();
+
+  constructor() {
+    this.rateLimitConfigs = RateLimitUtils.initializeRateLimitConfigs();
+  }
+
+  private async handleRateLimit(job: any): Promise<void> {
+    const { connectionId, provider, url, method, options } = job.data;
+    const microservice = options.queueOptions?.microservice || "default";
+    const isMaxRetries = job.attemptsMade >= 2;
+
+    await publishAudit({
+      eventType: isMaxRetries
+        ? "http.request.failed"
+        : "http.request.rateLimitExceeded",
+      properties: {
+        connectionId,
+        provider,
+        endpoint: url,
+        method,
+        timestamp: Date.now(),
+        queueId: job.id,
+        reason: isMaxRetries ? "max_retries_exceeded" : "rate_limit_exceeded",
+        ...(!isMaxRetries && {
+          retryCount: job.attemptsMade + 1,
+          maxRetries: 3,
+        }),
+      },
+    });
+
+    if (isMaxRetries) {
+      await this.queues
+        .get(QueueUtils.getQueueKey(microservice, connectionId, provider))
+        ?.obliterate({ force: true });
+      // Don't throw error for max retries - just return
+      return;
+    }
+
+    throw new Error("Rate limit exceeded");
+  }
+
+  private async processHttpRequest(job: any): Promise<any> {
+    const { connectionId, provider, url, method, options } = job.data;
+
+    // Check rate limit
+    if (
+      !(await RateLimitUtils.checkRateLimit(
+        connectionId,
+        provider,
+        this.rateLimitConfigs
+      ))
+    ) {
+      await this.handleRateLimit(job);
+      return;
+    }
+
+    try {
+      getConfig().LOGGER.info(
+        `Executing HTTP request: ${method} ${url} for ${provider}`
+      );
+
+      const response = await axios({
+        method: method.toLowerCase(),
+        url: url,
+        headers: options.headers || {},
+        timeout: 30000,
+        ...(options.body && { data: options.body }),
+        ...(options.params && { params: options.params }),
+      });
+
+      getConfig().LOGGER.info(
+        `HTTP request successful: ${method} ${url} for ${provider}`
+      );
+      return response.data;
+    } catch (error: any) {
+      getConfig().LOGGER.error(
+        `HTTP request failed ${job.id}: ${error.message}`
+      );
+
+      await publishAudit({
+        eventType: "http.request.error",
+        properties: {
+          connectionId,
+          provider,
+          endpoint: url,
+          method,
+          timestamp: Date.now(),
+          queueId: job.id,
+          reason: "execution_error",
+          errorMessage: error.message,
+        },
+      });
+
+      throw new Error(`HTTP request failed: ${error.message}`);
+    }
+  }
+
+  async request(options: {
+    method: string;
+    url: string;
+    body?: any;
+    headers?: Record<string, string>;
+    queueOptions?: {
+      connectionId: string;
+      connectionProvider: string;
+      microservice: string;
+    };
+  }): Promise<any> {
+    const { method, url, body, headers, queueOptions } = options;
+    // Create HttpCallOption object
+    const httpCallOption: HttpCallOption = {
+      headers,
+      body,
+      queueOptions,
+    };
+    // Call handleRequest with the constructed parameters
+    return this.handleRequest(url, method, httpCallOption);
+  }
+
+  async handleRequest(
+    url: string,
+    method: string,
+    options: HttpCallOption
+  ): Promise<any> {
+    const { connectionId, provider, microservice } =
+      JobUtils.extractConnectionDetails(options);
+    const queueKey = QueueUtils.getQueueKey(
+      microservice,
+      connectionId,
+      provider
+    );
+
+    getConfig().LOGGER.info(
+      `Queueing: ${method} ${url} -> ${provider} [${connectionId}]`
+    );
+
+    QueueUtils.getOrCreateWorker(
+      queueKey,
+      this.workers,
+      this.processHttpRequest.bind(this),
+      this.jobResults
+    );
+    const queue = QueueUtils.getOrCreateQueue(queueKey, this.queues);
+
+    const job = await queue.add(
+      "http-request",
+      {
+        microservice,
+        connectionId,
+        provider,
+        url,
+        method,
+        options,
+        timestamp: Date.now(),
+      },
+      {
+        attempts: 3,
+        backoff: { type: "exponential", delay: 5000 },
+        removeOnComplete: { age: 300, count: 1 },
+        removeOnFail: { age: 300, count: 1 },
+      }
+    );
+
+    getConfig().LOGGER.info(`Job ${job.id} queued, waiting for completion...`);
+    return JobUtils.waitForJobCompletion(job, queueKey, this.jobResults);
+  }
+
+  async shutdown(): Promise<void> {
+    getConfig().LOGGER.info("Shutting down HTTP queues...");
+
+    await Promise.all([
+      ...Array.from(this.workers.values()).map((worker: any) => worker.close()),
+      ...Array.from(this.queues.values()).map((queue: any) => queue.close()),
+    ]);
+
+    this.workers.clear();
+    this.queues.clear();
+    this.jobResults.clear();
+    getConfig().LOGGER.info("HTTP queues shutdown complete");
+  }
+}

package/src/queue/entities/index.ts

@@ -0,0 +1 @@
+export * from "./HybridHttpQueue";

package/src/queue/index.ts

@@ -0,0 +1,6 @@
+// Main queue exports
+export * from "./entities";
+export * from "./interfaces";
+export * from "./services";
+export * from "./types";
+export * from "./utils";

package/src/queue/interfaces/IHttpRequestJob.ts

@@ -0,0 +1,10 @@
+import { HttpCallOption } from "../types/http.types";
+
+export interface IHttpRequestJob {
+  connectionId: string;
+  provider: string;
+  url: string;
+  method: string;
+  options: HttpCallOption;
+  timestamp: number;
+}

package/src/queue/interfaces/IHybridHttpQueue.ts

@@ -0,0 +1,23 @@
+import { HttpCallOption } from "../types/http.types";
+
+export interface IHybridHttpQueue {
+  request(options: {
+    method: string;
+    url: string;
+    body?: any;
+    headers?: Record<string, string>;
+    queueOptions?: {
+      connectionId: string;
+      connectionProvider: string;
+      microservice: string;
+    };
+  }): Promise<any>;
+
+  handleRequest(
+    url: string,
+    method: string,
+    options: HttpCallOption
+  ): Promise<any>;
+
+  shutdown(): Promise<void>;
+}

package/src/queue/interfaces/IJobResult.ts

@@ -0,0 +1,6 @@
+export interface IJobResult {
+  result?: any;
+  error?: string;
+  resolved: boolean;
+  timestamp: number;
+}

package/src/queue/interfaces/IRateLimitConfig.ts

@@ -0,0 +1,5 @@
+export interface IRateLimitConfig {
+  maxRequests: number;
+  windowMs: number;
+  provider: string;
+}

package/src/queue/interfaces/index.ts

@@ -0,0 +1,4 @@
+export * from "./IHybridHttpQueue";
+export * from "./IHttpRequestJob";
+export * from "./IJobResult";
+export * from "./IRateLimitConfig";

package/src/queue/services/QueueService.ts

@@ -0,0 +1,39 @@
+import { Service } from "typedi";
+import { HybridHttpQueue } from "../entities/HybridHttpQueue";
+import { IHybridHttpQueue } from "../interfaces";
+import { HttpCallOption } from "../types/http.types";
+
+@Service()
+export class QueueService implements IHybridHttpQueue {
+  private readonly hybridQueue: HybridHttpQueue;
+
+  constructor() {
+    this.hybridQueue = new HybridHttpQueue();
+  }
+
+  async request(options: {
+    method: string;
+    url: string;
+    body?: any;
+    headers?: Record<string, string>;
+    queueOptions?: {
+      connectionId: string;
+      connectionProvider: string;
+      microservice: string;
+    };
+  }): Promise<any> {
+    return this.hybridQueue.request(options);
+  }
+
+  async handleRequest(
+    url: string,
+    method: string,
+    options: HttpCallOption
+  ): Promise<any> {
+    return this.hybridQueue.handleRequest(url, method, options);
+  }
+
+  async shutdown(): Promise<void> {
+    return this.hybridQueue.shutdown();
+  }
+}

package/src/queue/services/index.ts

@@ -0,0 +1 @@
+export * from "./QueueService";

package/src/queue/types/http.types.ts

@@ -0,0 +1,22 @@
+export interface HttpCallOption {
+  headers?: Record<string, string>;
+  body?: any;
+  params?: Record<string, any>;
+  queueOptions?: {
+    connectionId: string;
+    connectionProvider: string;
+    microservice: string;
+  };
+}
+
+export interface HttpRequestOptions {
+  method: string;
+  url: string;
+  body?: any;
+  headers?: Record<string, string>;
+  queueOptions?: {
+    connectionId: string;
+    connectionProvider: string;
+    microservice: string;
+  };
+}

package/src/queue/types/index.ts

@@ -0,0 +1,2 @@
+export * from "./http.types";
+export * from "./queue.types";

package/src/queue/types/queue.types.ts

@@ -0,0 +1,22 @@
+export interface QueueConfig {
+  concurrency: number;
+  removeOnComplete: { age: number; count: number };
+  removeOnFail: { age: number; count: number };
+  lockDuration: number;
+  stalledInterval: number;
+}
+
+export interface JobOptions {
+  attempts: number;
+  backoff: { type: string; delay: number };
+  removeOnComplete: { age: number; count: number };
+  removeOnFail: { age: number; count: number };
+}
+
+export interface QueueMetrics {
+  totalJobs: number;
+  completedJobs: number;
+  failedJobs: number;
+  pendingJobs: number;
+  activeJobs: number;
+}

package/src/queue/utils/index.ts

@@ -0,0 +1,3 @@
+export * from "./rateLimit.utils";
+export * from "./jobUtils";
+export * from "./queueUtils";

package/src/queue/utils/jobUtils.ts

@@ -0,0 +1,80 @@
+import { Job } from "bullmq";
+import { IJobResult } from "../interfaces";
+import { getConfig } from "../../config/config";
+
+export class JobUtils {
+  static async waitForJobCompletion(
+    job: Job,
+    queueKey: string,
+    jobResults: Map<string, IJobResult>
+  ): Promise<any> {
+    return new Promise((resolve, reject) => {
+      let timeoutId: NodeJS.Timeout;
+      let checkCount = 0;
+      const maxChecks = 600;
+
+      const checkJob = async () => {
+        if (++checkCount >= maxChecks) {
+          clearTimeout(timeoutId);
+          return reject(new Error("Job timeout: Request took too long"));
+        }
+
+        try {
+          const state = await job.getState();
+
+          if (state === "completed") {
+            clearTimeout(timeoutId);
+            const memoryResult = jobResults.get(job.id!);
+
+            if (memoryResult?.resolved) {
+              return memoryResult.result !== undefined
+                ? resolve(memoryResult.result)
+                : reject(new Error(memoryResult.error || "Unknown error"));
+            }
+
+            // Fallback to job result
+            resolve(await job.returnvalue);
+          } else if (state === "failed") {
+            clearTimeout(timeoutId);
+            const memoryResult = jobResults.get(job.id!);
+            const errorMsg =
+              memoryResult?.error ||
+              (await job.failedReason) ||
+              "Unknown error";
+            reject(new Error(`Job failed: ${errorMsg}`));
+          } else {
+            timeoutId = setTimeout(checkJob, checkCount < 10 ? 50 : 100);
+          }
+        } catch (error: any) {
+          clearTimeout(timeoutId);
+          reject(new Error(`Error checking job: ${error.message}`));
+        }
+      };
+
+      checkJob();
+
+      // Backup timeout
+      setTimeout(() => {
+        clearTimeout(timeoutId);
+        reject(new Error("Request timeout: Maximum wait time exceeded"));
+      }, 60000);
+    });
+  }
+
+  static extractConnectionDetails(options: any): {
+    connectionId: string;
+    provider: string;
+    microservice: string;
+  } {
+    const {
+      connectionId,
+      connectionProvider: provider,
+      microservice,
+    } = options?.queueOptions ?? {};
+
+    if (!connectionId) throw new Error("Connection ID not found in options");
+    if (!provider) throw new Error("Connection provider not found in options");
+    if (!microservice) throw new Error("Microservice not found in options");
+    return { connectionId, provider, microservice };
+  }
+}