@oneuptime/common 10.0.84 → 10.0.85

package/Models/DatabaseModels/LlmProvider.ts

@@ -42,7 +42,7 @@ import LlmType from "../../Types/LLM/LlmType";
   pluralName: "LLM Providers",
   icon: IconProp.Bolt,
   tableDescription:
-    "Manage LLM Provider configurations. Connect to OpenAI, Anthropic, Ollama, or other LLM providers to enable AI features.",
+    "Manage LLM Provider configurations. Connect to OpenAI, Azure OpenAI, Anthropic, Groq, Mistral, Ollama, or other LLM providers to enable AI features.",
 })
 @TableAccessControl({
   create: [
@@ -179,7 +179,8 @@ export default class LlmProvider extends BaseModel {
     required: true,
     type: TableColumnType.ShortText,
     title: "LLM Type",
-    description: "The type of LLM provider (OpenAI, Anthropic, Ollama, etc.)",
+    description:
+      "The type of LLM provider (OpenAI, Azure OpenAI, Anthropic, Groq, Mistral, Ollama, etc.)",
   })
   @Column({
     nullable: false,
@@ -214,7 +215,7 @@ export default class LlmProvider extends BaseModel {
     type: TableColumnType.LongText,
     title: "API Key",
     description:
-      "The API key for the LLM provider. Required for OpenAI and Anthropic.",
+      "The API key for the LLM provider. Required for OpenAI, Azure OpenAI, Anthropic, Groq, and Mistral.",
     encrypted: true,
   })
   @Column({
@@ -276,7 +277,7 @@ export default class LlmProvider extends BaseModel {
     type: TableColumnType.ShortURL,
     title: "Base URL",
     description:
-      "The base URL for the LLM API. Required for Ollama, optional for others.",
+      "The base URL for the LLM API. Required for Azure OpenAI and Ollama, optional for others.",
   })
   @Column({
     nullable: true,

package/Models/DatabaseModels/Project.ts

@@ -2180,4 +2180,44 @@ export default class Project extends TenantModel {
     create: PlanType.Free,
   })
   public auditLogsRetentionInDays?: number = undefined;
+
+  @ColumnAccessControl({
+    create: [Permission.User],
+    read: [
+      Permission.ProjectOwner,
+      Permission.ProjectAdmin,
+      Permission.ProjectMember,
+      Permission.Viewer,
+      Permission.ReadProject,
+      Permission.UnAuthorizedSsoUser,
+      Permission.ProjectUser,
+      Permission.ReadAllProjectResources,
+    ],
+    update: [
+      Permission.ProjectOwner,
+      Permission.ProjectAdmin,
+      Permission.EditProject,
+    ],
+  })
+  @TableColumn({
+    required: true,
+    type: TableColumnType.Boolean,
+    isDefaultValueColumn: true,
+    defaultValue: false,
+    title: "Store System Events",
+    description:
+      "When enabled, audit logs will also include events triggered by the system. By default, only events triggered by users are recorded.",
+  })
+  @Column({
+    type: ColumnType.Boolean,
+    nullable: false,
+    unique: false,
+    default: false,
+  })
+  @ColumnBillingAccessControl({
+    read: PlanType.Free,
+    update: PlanType.Enterprise,
+    create: PlanType.Free,
+  })
+  public storeSystemEventsInAuditLogs?: boolean = undefined;
 }

package/Server/API/KubernetesResourceAPI.ts

@@ -56,14 +56,57 @@ export default class KubernetesResourceAPI extends BaseAPI<
         }
       },
     );
+
+    /*
+     * Latest CPU+memory aggregated by Pod namespace. Powers the
+     * Namespaces list view without a ClickHouse round-trip.
+     */
+    this.router.post(
+      `${new this.entityType()
+        .getCrudApiPath()
+        ?.toString()}/latest-pod-metrics-by-namespace/:clusterId`,
+      UserMiddleware.getUserMiddleware,
+      async (req: ExpressRequest, res: ExpressResponse, next: NextFunction) => {
+        try {
+          await this.getLatestPodMetricsByNamespace(req, res);
+        } catch (err) {
+          next(err);
+        }
+      },
+    );
+
+    /*
+     * Latest CPU+memory aggregated by Pod ownerReferences[].name for
+     * a given owner kind. Powers Deployments/StatefulSets/DaemonSets/
+     * Jobs/CronJobs list views.
+     */
+    this.router.post(
+      `${new this.entityType()
+        .getCrudApiPath()
+        ?.toString()}/latest-pod-metrics-by-owner/:clusterId/:ownerKind`,
+      UserMiddleware.getUserMiddleware,
+      async (req: ExpressRequest, res: ExpressResponse, next: NextFunction) => {
+        try {
+          await this.getLatestPodMetricsByOwner(req, res);
+        } catch (err) {
+          next(err);
+        }
+      },
+    );
   }
 
-  private async getInventorySummary(
-    req: ExpressRequest,
-    res: ExpressResponse,
-  ): Promise<void> {
+  /*
+   * Cluster + auth resolution shared by the cluster-scoped sub-routes.
+   * Returns the (projectId, kubernetesClusterId) tuple after enforcing
+   * the standard ACL chain. Throws NotFound when the cluster is
+   * missing or the caller lacks read access (indistinguishable on
+   * purpose).
+   */
+  private async resolveClusterForRequest(req: ExpressRequest): Promise<{
+    projectId: ObjectID;
+    kubernetesClusterId: ObjectID;
+  }> {
     const clusterIdParam: string | undefined = req.params["clusterId"];
-
     if (!clusterIdParam) {
       throw new BadDataException("Cluster ID is required");
     }
@@ -78,12 +121,6 @@ export default class KubernetesResourceAPI extends BaseAPI<
     const props: DatabaseCommonInteractionProps =
       await CommonAPI.getDatabaseCommonInteractionProps(req);
 
-    /*
-     * Authorize: the caller must be able to read the parent cluster.
-     * findOneById applies the full ACL chain; a null return means 404
-     * (either the cluster doesn't exist or the caller cannot see it —
-     * indistinguishable on purpose).
-     */
     const cluster: KubernetesCluster | null =
       await KubernetesClusterService.findOneById({
         id: kubernetesClusterId,
@@ -98,9 +135,104 @@ export default class KubernetesResourceAPI extends BaseAPI<
       throw new NotFoundException("Kubernetes Cluster not found");
     }
 
-    const summary: InventorySummary = await this.service.getInventorySummary({
+    return {
       projectId: cluster.projectId,
       kubernetesClusterId,
+    };
+  }
+
+  /*
+   * Translate a service-layer Map of aggregates into a JSON dict
+   * { name: { cpuPercent, memoryBytes } } suitable for the wire.
+   * memoryBytes is stringified so values past 2 GiB don't overflow
+   * client-side number parsing in the JSON path; the UI parses it
+   * back to a number for rendering.
+   */
+  private mapAggregatesToJson(
+    aggregates: Map<string, { cpuPercent: number; memoryBytes: number }>,
+  ): JSONObject {
+    const out: JSONObject = {};
+    for (const [name, value] of aggregates.entries()) {
+      out[name] = {
+        cpuPercent: value.cpuPercent,
+        memoryBytes: value.memoryBytes.toString(),
+      };
+    }
+    return out;
+  }
+
+  private async getLatestPodMetricsByNamespace(
+    req: ExpressRequest,
+    res: ExpressResponse,
+  ): Promise<void> {
+    const { projectId, kubernetesClusterId } =
+      await this.resolveClusterForRequest(req);
+
+    const staleAfter: Date = new Date(Date.now() - 15 * 60 * 1000);
+    const aggregates: Map<string, { cpuPercent: number; memoryBytes: number }> =
+      await this.service.getLatestMetricsByNamespace({
+        projectId,
+        kubernetesClusterId,
+        staleAfter,
+      });
+
+    return Response.sendJsonObjectResponse(req, res, {
+      aggregates: this.mapAggregatesToJson(aggregates),
+    });
+  }
+
+  private async getLatestPodMetricsByOwner(
+    req: ExpressRequest,
+    res: ExpressResponse,
+  ): Promise<void> {
+    const ownerKind: string | undefined = req.params["ownerKind"];
+    if (!ownerKind) {
+      throw new BadDataException("Owner kind is required");
+    }
+    /*
+     * Only a small allow-list of owner kinds makes sense here; reject
+     * anything else so the endpoint can't be used to probe arbitrary
+     * jsonb_array_elements paths.
+     */
+    const allowed: Set<string> = new Set([
+      "Deployment",
+      "StatefulSet",
+      "DaemonSet",
+      "Job",
+      "CronJob",
+      "ReplicaSet",
+    ]);
+    if (!allowed.has(ownerKind)) {
+      throw new BadDataException(`Unsupported owner kind: ${ownerKind}`);
+    }
+
+    const { projectId, kubernetesClusterId } =
+      await this.resolveClusterForRequest(req);
+
+    const staleAfter: Date = new Date(Date.now() - 15 * 60 * 1000);
+    const aggregates: Map<string, { cpuPercent: number; memoryBytes: number }> =
+      await this.service.getLatestMetricsByOwner({
+        projectId,
+        kubernetesClusterId,
+        ownerKind,
+        staleAfter,
+      });
+
+    return Response.sendJsonObjectResponse(req, res, {
+      aggregates: this.mapAggregatesToJson(aggregates),
+    });
+  }
+
+  private async getInventorySummary(
+    req: ExpressRequest,
+    res: ExpressResponse,
+  ): Promise<void> {
+    const { projectId, kubernetesClusterId } =
+      await this.resolveClusterForRequest(req);
+
+    const summary: InventorySummary = await this.service.getInventorySummary({
+      projectId,
+      kubernetesClusterId,
     });
 
     const responseBody: JSONObject = {

package/Server/Infrastructure/Postgres/SchemaMigrations/1777550162848-MigrationName.ts

@@ -0,0 +1,29 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MigrationName1777550162848 implements MigrationInterface {
+  public name: string = "MigrationName1777550162848";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "storeSystemEventsInAuditLogs" boolean NOT NULL DEFAULT false`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "rotation" SET DEFAULT '{"_type":"Recurring","value":{"intervalType":"Day","intervalCount":{"_type":"PositiveNumber","value":1}}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "restrictionTimes" SET DEFAULT '{"_type":"RestrictionTimes","value":{"restictionType":"None","dayRestrictionTimes":null,"weeklyRestrictionTimes":[]}}'`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "restrictionTimes" SET DEFAULT '{"_type": "RestrictionTimes", "value": {"restictionType": "None", "dayRestrictionTimes": null, "weeklyRestrictionTimes": []}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "rotation" SET DEFAULT '{"_type": "Recurring", "value": {"intervalType": "Day", "intervalCount": {"_type": "PositiveNumber", "value": 1}}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "storeSystemEventsInAuditLogs"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1777571961028-MigrationName.ts

@@ -0,0 +1,99 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MigrationName1777571961028 implements MigrationInterface {
+  public name: string = "MigrationName1777571961028";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE TABLE "KubernetesContainer" ("_id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "deletedAt" TIMESTAMP WITH TIME ZONE, "version" integer NOT NULL, "projectId" uuid NOT NULL, "kubernetesClusterId" uuid NOT NULL, "podNamespaceKey" character varying(100) NOT NULL DEFAULT '', "podName" character varying(100) NOT NULL, "name" character varying(100) NOT NULL, "image" character varying(500), "state" character varying(100), "reason" character varying(100), "isReady" boolean, "restartCount" integer, "memoryLimitBytes" bigint, "latestCpuPercent" numeric, "latestMemoryBytes" bigint, "metricsUpdatedAt" TIMESTAMP WITH TIME ZONE, "lastSeenAt" TIMESTAMP WITH TIME ZONE NOT NULL, "createdByUserId" uuid, "deletedByUserId" uuid, CONSTRAINT "PK_7e19b5140bc3005a6ea2f8f7aee" PRIMARY KEY ("_id"))`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_fcc7f4bc83564a8c7885233f6e" ON "KubernetesContainer" ("projectId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_5303bcae1a72f9830bd7d15e2c" ON "KubernetesContainer" ("kubernetesClusterId") `,
+    );
+    await queryRunner.query(
+      `CREATE UNIQUE INDEX "IDX_1dcb8fed322a9bddfabb60cbc7" ON "KubernetesContainer" ("projectId", "kubernetesClusterId", "podNamespaceKey", "podName", "name") `,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesResource" ADD "controllerDeploymentName" character varying(100)`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesResource" ADD "controllerCronJobName" character varying(100)`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesResource" ADD "latestCpuPercent" numeric`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesResource" ADD "latestMemoryBytes" bigint`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesResource" ADD "metricsUpdatedAt" TIMESTAMP WITH TIME ZONE`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "rotation" SET DEFAULT '{"_type":"Recurring","value":{"intervalType":"Day","intervalCount":{"_type":"PositiveNumber","value":1}}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "restrictionTimes" SET DEFAULT '{"_type":"RestrictionTimes","value":{"restictionType":"None","dayRestrictionTimes":null,"weeklyRestrictionTimes":[]}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesContainer" ADD CONSTRAINT "FK_fcc7f4bc83564a8c7885233f6e3" FOREIGN KEY ("projectId") REFERENCES "Project"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesContainer" ADD CONSTRAINT "FK_5303bcae1a72f9830bd7d15e2cd" FOREIGN KEY ("kubernetesClusterId") REFERENCES "KubernetesCluster"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesContainer" ADD CONSTRAINT "FK_d0f740eb8fc87c2426d78babf6b" FOREIGN KEY ("createdByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesContainer" ADD CONSTRAINT "FK_eadbc98e53bc5788d8313e52c67" FOREIGN KEY ("deletedByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesContainer" DROP CONSTRAINT "FK_eadbc98e53bc5788d8313e52c67"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesContainer" DROP CONSTRAINT "FK_d0f740eb8fc87c2426d78babf6b"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesContainer" DROP CONSTRAINT "FK_5303bcae1a72f9830bd7d15e2cd"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesContainer" DROP CONSTRAINT "FK_fcc7f4bc83564a8c7885233f6e3"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "restrictionTimes" SET DEFAULT '{"_type": "RestrictionTimes", "value": {"restictionType": "None", "dayRestrictionTimes": null, "weeklyRestrictionTimes": []}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "rotation" SET DEFAULT '{"_type": "Recurring", "value": {"intervalType": "Day", "intervalCount": {"_type": "PositiveNumber", "value": 1}}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesResource" DROP COLUMN "metricsUpdatedAt"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesResource" DROP COLUMN "latestMemoryBytes"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesResource" DROP COLUMN "latestCpuPercent"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesResource" DROP COLUMN "controllerCronJobName"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesResource" DROP COLUMN "controllerDeploymentName"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_1dcb8fed322a9bddfabb60cbc7"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_5303bcae1a72f9830bd7d15e2c"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_fcc7f4bc83564a8c7885233f6e"`,
+    );
+    await queryRunner.query(`DROP TABLE "KubernetesContainer"`);
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts

@@ -293,6 +293,8 @@ import { MigrationName1776940714709 } from "./1776940714709-MigrationName";
 import { AddStatusPageLanguageSettings1776971364783 } from "./1776971364783-AddStatusPageLanguageSettings";
 import { AddTelemetryRetentionSettings1777018175127 } from "./1777018175127-AddTelemetryRetentionSettings";
 import { AddMonitorTemplate1777201966799 } from "./1777201966799-AddMonitorTemplate";
+import { MigrationName1777550162848 } from "./1777550162848-MigrationName";
+import { MigrationName1777571961028 } from "./1777571961028-MigrationName";
 export default [
   InitialMigration,
   MigrationName1717678334852,
@@ -589,4 +591,6 @@ export default [
   AddStatusPageLanguageSettings1776971364783,
   AddTelemetryRetentionSettings1777018175127,
   AddMonitorTemplate1777201966799,
+  MigrationName1777550162848,
+  MigrationName1777571961028,
 ];

package/Server/Infrastructure/Queue.ts

@@ -8,6 +8,8 @@ import { BullMQAdapter } from "@bull-board/api/bullMQAdapter";
 import { ExpressRouter } from "../Utils/Express";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import logger from "../Utils/Logger";
+import Telemetry from "../Utils/Telemetry";
+import type { Attributes, ObservableResult } from "@opentelemetry/api";
 import Redis from "./Redis";
 
 export enum QueueName {
@@ -23,6 +25,7 @@ export default class Queue {
   private static queueDict: Dictionary<BullQueue> = {};
   // track queues we have already run initial cleanup on
   private static cleanedQueueNames: Set<string> = new Set<string>();
+  private static queueSizeMetricRegistered: boolean = false;
   // store repeatable jobs to re-add on reconnect
   private static repeatableJobs: Dictionary<
     Dictionary<{
@@ -99,6 +102,9 @@ export default class Queue {
     // save it to the dictionary
     this.queueDict[queueName] = queue;
 
+    // Register the observable gauge once any queue exists in this process.
+    this.registerQueueSizeMetric();
+
     // Add event listener to re-add repeatable jobs on reconnect
     this.setupReconnectListener(queue, queueName).catch((err: unknown) => {
       logger.error("Error setting up reconnect listener for queue");
@@ -243,6 +249,60 @@ export default class Queue {
     return jobAdded;
   }
 
+  private static registerQueueSizeMetric(): void {
+    if (this.queueSizeMetricRegistered) {
+      return;
+    }
+
+    if (!Telemetry.isMetricsEnabled()) {
+      return;
+    }
+
+    try {
+      Telemetry.getObservableGauge({
+        name: "queue.size",
+        description:
+          "Number of BullMQ jobs in each queue, partitioned by job state.",
+        unit: "1",
+        callback: async (
+          result: ObservableResult<Attributes>,
+        ): Promise<void> => {
+          for (const queueName of Object.keys(this.queueDict)) {
+            try {
+              const stats: {
+                waiting: number;
+                active: number;
+                completed: number;
+                failed: number;
+                delayed: number;
+                total: number;
+              } = await this.getQueueStats(queueName as QueueName);
+
+              const baseAttrs: Attributes = {
+                "messaging.system": "bullmq",
+                "messaging.destination.name": queueName,
+              };
+
+              result.observe(stats.waiting, { ...baseAttrs, state: "waiting" });
+              result.observe(stats.active, { ...baseAttrs, state: "active" });
+              result.observe(stats.delayed, { ...baseAttrs, state: "delayed" });
+              result.observe(stats.failed, { ...baseAttrs, state: "failed" });
+            } catch (err) {
+              // Don't let one queue's stat failure break others.
+              logger.debug("Failed to read queue stats");
+              logger.debug(err);
+            }
+          }
+        },
+      });
+
+      this.queueSizeMetricRegistered = true;
+    } catch (err) {
+      logger.error("Failed to register queue.size metric");
+      logger.error(err);
+    }
+  }
+
   @CaptureSpan()
   public static async getQueueSize(queueName: QueueName): Promise<number> {
     const queue: BullQueue = this.getQueue(queueName);

package/Server/Infrastructure/QueueWorker.ts

@@ -7,6 +7,7 @@ import {
 } from "../../Types/FunctionTypes";
 import { Worker } from "bullmq";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import AppMetrics from "../Utils/Telemetry/AppMetrics";
 import Redis from "./Redis";
 
 export default class QueueWorker {
@@ -29,7 +30,44 @@ export default class QueueWorker {
       maxStalledCount?: number;
     },
   ): Worker {
-    const worker: Worker = new Worker(queueName, onJobInQueue, {
+    const instrumentedJobHandler: (job: QueueJob) => Promise<void> = async (
+      job: QueueJob,
+    ): Promise<void> => {
+      const startNs: bigint = process.hrtime.bigint();
+      const baseAttributes: Record<string, string> = {
+        "messaging.system": "bullmq",
+        "messaging.destination.name": queueName,
+        "messaging.operation.name": job.name || "unknown",
+      };
+
+      AppMetrics.getWorkerJobsInFlight().add(1, baseAttributes);
+
+      let outcome: "success" | "failure" | "timeout" = "success";
+
+      try {
+        await onJobInQueue(job);
+      } catch (err) {
+        outcome =
+          err instanceof TimeoutException ||
+          (err as { name?: string })?.name === "TimeoutException"
+            ? "timeout"
+            : "failure";
+        throw err;
+      } finally {
+        const elapsedNs: bigint = process.hrtime.bigint() - startNs;
+        const durationMs: number = Number(elapsedNs) / 1e6;
+        const attributes: Record<string, string> = {
+          ...baseAttributes,
+          outcome,
+        };
+
+        AppMetrics.getWorkerJobCounter().add(1, attributes);
+        AppMetrics.getWorkerJobDuration().record(durationMs, attributes);
+        AppMetrics.getWorkerJobsInFlight().add(-1, baseAttributes);
+      }
+    };
+
+    const worker: Worker = new Worker(queueName, instrumentedJobHandler, {
       connection: Redis.getRedisOptions(),
       concurrency: options.concurrency,
       // Only set these values if provided so we do not override BullMQ defaults

package/Server/Middleware/HttpMetricsMiddleware.ts

@@ -0,0 +1,92 @@
+import {
+  ExpressRequest,
+  ExpressResponse,
+  NextFunction,
+} from "../Utils/Express";
+import AppMetrics from "../Utils/Telemetry/AppMetrics";
+
+/**
+ * Express middleware that records HTTP server metrics (request count,
+ * duration, and in-flight gauge) for every request.
+ *
+ * Attributes are kept low-cardinality on purpose:
+ *   - http.request.method: GET / POST / ...
+ *   - http.route:          Express route template (e.g. /api/users/:id)
+ *                          or "unmatched" when nothing matched the request.
+ *   - http.response.status_code: full status code (bounded set).
+ *   - status_class: 2xx / 3xx / 4xx / 5xx — handy for fast queries.
+ *
+ * High-cardinality identifiers (raw URL, query string, userId, projectId,
+ * requestId) intentionally stay on traces and logs.
+ */
+const HttpMetricsMiddleware: (
+  req: ExpressRequest,
+  res: ExpressResponse,
+  next: NextFunction,
+) => void = (
+  req: ExpressRequest,
+  res: ExpressResponse,
+  next: NextFunction,
+): void => {
+  const startNs: bigint = process.hrtime.bigint();
+  const method: string = (req.method || "UNKNOWN").toUpperCase();
+
+  const inFlight: ReturnType<typeof AppMetrics.getHttpRequestsInFlight> =
+    AppMetrics.getHttpRequestsInFlight();
+
+  inFlight.add(1, { "http.request.method": method });
+
+  let recorded: boolean = false;
+
+  const recordOnce: () => void = (): void => {
+    if (recorded) {
+      return;
+    }
+    recorded = true;
+
+    const elapsedNs: bigint = process.hrtime.bigint() - startNs;
+    const durationMs: number = Number(elapsedNs) / 1e6;
+    const statusCode: number = res.statusCode || 0;
+    const statusClass: string =
+      statusCode >= 100 && statusCode < 600
+        ? `${Math.floor(statusCode / 100)}xx`
+        : "unknown";
+
+    /*
+     * Express populates req.route once the request has matched a route
+     * handler. For 404s (no match), record the request under a stable
+     * "unmatched" label rather than the raw URL to avoid cardinality blowup.
+     */
+    const routeWithMethod: { path?: string } | undefined = (
+      req as ExpressRequest & { route?: { path?: string } }
+    ).route;
+
+    const baseUrl: string = (req as ExpressRequest & { baseUrl?: string })
+      .baseUrl
+      ? (req as ExpressRequest & { baseUrl: string }).baseUrl
+      : "";
+
+    const routeTemplate: string =
+      routeWithMethod && typeof routeWithMethod.path === "string"
+        ? `${baseUrl}${routeWithMethod.path}`
+        : "unmatched";
+
+    const attributes: Record<string, string | number> = {
+      "http.request.method": method,
+      "http.route": routeTemplate,
+      "http.response.status_code": statusCode,
+      status_class: statusClass,
+    };
+
+    AppMetrics.getHttpRequestCounter().add(1, attributes);
+    AppMetrics.getHttpRequestDuration().record(durationMs, attributes);
+    inFlight.add(-1, { "http.request.method": method });
+  };
+
+  res.on("finish", recordOnce);
+  res.on("close", recordOnce);
+
+  next();
+};
+
+export default HttpMetricsMiddleware;