@oneuptime/common 11.4.1 → 11.5.0

package/Tests/Server/Utils/AI/SentinelAlertGating.test.ts

@@ -0,0 +1,360 @@
+import SentinelAlertInvestigationRunner, {
+  AlertGateDecision,
+} from "../../../../Server/Utils/AI/Sentinel/AlertInvestigationRunner";
+import SentinelInvestigationQueue from "../../../../Server/Utils/AI/Sentinel/InvestigationQueue";
+import AlertService from "../../../../Server/Services/AlertService";
+import AlertSeverityService from "../../../../Server/Services/AlertSeverityService";
+import ProjectService from "../../../../Server/Services/ProjectService";
+import AIRunService from "../../../../Server/Services/AIRunService";
+import Alert from "../../../../Models/DatabaseModels/Alert";
+import AlertSeverity from "../../../../Models/DatabaseModels/AlertSeverity";
+import Project from "../../../../Models/DatabaseModels/Project";
+import ObjectID from "../../../../Types/ObjectID";
+import PositiveNumber from "../../../../Types/PositiveNumber";
+import { describe, expect, test, afterEach } from "@jest/globals";
+
+/*
+ * Cost gates for autonomous alert investigations (Phase 1 / G4):
+ *   - severity floor: explicit per-project minimum severity, defaulting to the
+ *     project's top two tiers (lowest two `order` values; lower order = higher
+ *     severity);
+ *   - per-monitor dedupe window: a monitor already investigated within the
+ *     window is not re-investigated;
+ *   - per-project concurrency cap in the shared engine.
+ *
+ * These tests mock the persistence layer (no Postgres) to lock in the gate
+ * decisions and their fail directions: unknown severity PASSES the severity
+ * gate (it only filters known-low-severity noise), while a failed cap check
+ * SKIPS (a cost gate fails cheap).
+ */
+
+function fakeAlert(data: {
+  monitorId?: ObjectID | undefined;
+  severityOrder?: number | undefined;
+}): Alert {
+  return {
+    id: ObjectID.generate(),
+    monitorId: data.monitorId,
+    alertSeverity:
+      data.severityOrder !== undefined
+        ? ({ order: data.severityOrder } as AlertSeverity)
+        : undefined,
+  } as unknown as Alert;
+}
+
+function fakeProject(
+  minimumSeverityId?: ObjectID | undefined,
+  dedupeWindowMinutes?: number | undefined,
+): Project {
+  return {
+    id: ObjectID.generate(),
+    alertInvestigationMinimumSeverityId: minimumSeverityId,
+    alertInvestigationDedupeWindowMinutes: dedupeWindowMinutes,
+  } as unknown as Project;
+}
+
+function fakeSeverities(orders: Array<number>): Array<AlertSeverity> {
+  return orders.map((order: number) => {
+    return { order } as AlertSeverity;
+  });
+}
+
+describe("SentinelAlertInvestigationRunner.shouldInvestigateAlert", () => {
+  const alertId: ObjectID = ObjectID.generate();
+  const projectId: ObjectID = ObjectID.generate();
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  function mockGates(data: {
+    alert: Alert | null;
+    project?: Project;
+    explicitFloorSeverity?: AlertSeverity | null;
+    topTiers?: Array<AlertSeverity>;
+    recentRunCount?: number;
+  }): void {
+    jest.spyOn(AlertService, "findOneById").mockResolvedValue(data.alert);
+    jest
+      .spyOn(ProjectService, "findOneById")
+      .mockResolvedValue(data.project || fakeProject());
+    jest
+      .spyOn(AlertSeverityService, "findOneById")
+      .mockResolvedValue(data.explicitFloorSeverity ?? null);
+    jest
+      .spyOn(AlertSeverityService, "findBy")
+      .mockResolvedValue(data.topTiers || []);
+    jest
+      .spyOn(AIRunService, "countBy")
+      .mockResolvedValue(new PositiveNumber(data.recentRunCount || 0));
+  }
+
+  test("skips when severity is below an explicitly configured floor", async () => {
+    const floorSeverityId: ObjectID = ObjectID.generate();
+    mockGates({
+      alert: fakeAlert({ severityOrder: 3 }),
+      project: fakeProject(floorSeverityId),
+      explicitFloorSeverity: { order: 2 } as AlertSeverity,
+    });
+
+    const decision: AlertGateDecision =
+      await SentinelAlertInvestigationRunner.shouldInvestigateAlert({
+        alertId,
+        projectId,
+      });
+
+    expect(decision.investigate).toBe(false);
+    expect(decision.reason).toContain("below the investigation floor");
+  });
+
+  test("investigates when severity meets the explicit floor and no recent run exists", async () => {
+    const monitorId: ObjectID = ObjectID.generate();
+    const floorSeverityId: ObjectID = ObjectID.generate();
+    mockGates({
+      alert: fakeAlert({ severityOrder: 2, monitorId }),
+      project: fakeProject(floorSeverityId),
+      explicitFloorSeverity: { order: 2 } as AlertSeverity,
+      recentRunCount: 0,
+    });
+
+    const decision: AlertGateDecision =
+      await SentinelAlertInvestigationRunner.shouldInvestigateAlert({
+        alertId,
+        projectId,
+      });
+
+    expect(decision.investigate).toBe(true);
+    expect(decision.monitorId).toBe(monitorId);
+  });
+
+  test("default floor is the top two tiers: third tier is skipped", async () => {
+    mockGates({
+      alert: fakeAlert({ severityOrder: 3 }),
+      topTiers: fakeSeverities([1, 2]),
+    });
+
+    const decision: AlertGateDecision =
+      await SentinelAlertInvestigationRunner.shouldInvestigateAlert({
+        alertId,
+        projectId,
+      });
+
+    expect(decision.investigate).toBe(false);
+  });
+
+  test("default floor: second tier is investigated", async () => {
+    mockGates({
+      alert: fakeAlert({ severityOrder: 2 }),
+      topTiers: fakeSeverities([1, 2]),
+    });
+
+    const decision: AlertGateDecision =
+      await SentinelAlertInvestigationRunner.shouldInvestigateAlert({
+        alertId,
+        projectId,
+      });
+
+    expect(decision.investigate).toBe(true);
+  });
+
+  test("a deleted explicit floor severity falls back to the top-two default", async () => {
+    // Project points at a severity that no longer exists (findOneById → null).
+    mockGates({
+      alert: fakeAlert({ severityOrder: 3 }),
+      project: fakeProject(ObjectID.generate()),
+      explicitFloorSeverity: null,
+      topTiers: fakeSeverities([1, 2]),
+    });
+
+    const decision: AlertGateDecision =
+      await SentinelAlertInvestigationRunner.shouldInvestigateAlert({
+        alertId,
+        projectId,
+      });
+
+    expect(decision.investigate).toBe(false);
+  });
+
+  test("unknown alert severity passes the severity gate", async () => {
+    mockGates({
+      alert: fakeAlert({ severityOrder: undefined }),
+      topTiers: fakeSeverities([1, 2]),
+    });
+
+    const decision: AlertGateDecision =
+      await SentinelAlertInvestigationRunner.shouldInvestigateAlert({
+        alertId,
+        projectId,
+      });
+
+    expect(decision.investigate).toBe(true);
+  });
+
+  test("skips when the monitor was already investigated within the window", async () => {
+    const monitorId: ObjectID = ObjectID.generate();
+    const countBy: jest.SpyInstance = jest.spyOn(AIRunService, "countBy");
+
+    mockGates({
+      alert: fakeAlert({ severityOrder: 1, monitorId }),
+      topTiers: fakeSeverities([1, 2]),
+      recentRunCount: 1,
+    });
+
+    const decision: AlertGateDecision =
+      await SentinelAlertInvestigationRunner.shouldInvestigateAlert({
+        alertId,
+        projectId,
+      });
+
+    expect(decision.investigate).toBe(false);
+    expect(decision.reason).toContain("already investigated");
+    expect(countBy).toHaveBeenCalledWith(
+      expect.objectContaining({
+        query: expect.objectContaining({
+          monitorId,
+        }),
+      }),
+    );
+  });
+
+  test("alerts without a monitor skip the dedupe check entirely", async () => {
+    const countBy: jest.SpyInstance = jest.spyOn(AIRunService, "countBy");
+
+    mockGates({
+      alert: fakeAlert({ severityOrder: 1 }),
+      topTiers: fakeSeverities([1, 2]),
+    });
+
+    const decision: AlertGateDecision =
+      await SentinelAlertInvestigationRunner.shouldInvestigateAlert({
+        alertId,
+        projectId,
+      });
+
+    expect(decision.investigate).toBe(true);
+    expect(countBy).not.toHaveBeenCalled();
+  });
+
+  test("a cooldown of 0 disables the dedupe check entirely", async () => {
+    const monitorId: ObjectID = ObjectID.generate();
+    const countBy: jest.SpyInstance = jest.spyOn(AIRunService, "countBy");
+
+    mockGates({
+      alert: fakeAlert({ severityOrder: 1, monitorId }),
+      project: fakeProject(undefined, 0),
+      topTiers: fakeSeverities([1, 2]),
+      recentRunCount: 1,
+    });
+
+    const decision: AlertGateDecision =
+      await SentinelAlertInvestigationRunner.shouldInvestigateAlert({
+        alertId,
+        projectId,
+      });
+
+    expect(decision.investigate).toBe(true);
+    expect(countBy).not.toHaveBeenCalled();
+  });
+
+  test("a custom cooldown is used for the dedupe window", async () => {
+    const monitorId: ObjectID = ObjectID.generate();
+
+    mockGates({
+      alert: fakeAlert({ severityOrder: 1, monitorId }),
+      project: fakeProject(undefined, 45),
+      topTiers: fakeSeverities([1, 2]),
+      recentRunCount: 1,
+    });
+
+    const decision: AlertGateDecision =
+      await SentinelAlertInvestigationRunner.shouldInvestigateAlert({
+        alertId,
+        projectId,
+      });
+
+    expect(decision.investigate).toBe(false);
+    expect(decision.reason).toContain("within the last 45 minutes");
+  });
+
+  test("skips when the alert cannot be found", async () => {
+    mockGates({ alert: null });
+
+    const decision: AlertGateDecision =
+      await SentinelAlertInvestigationRunner.shouldInvestigateAlert({
+        alertId,
+        projectId,
+      });
+
+    expect(decision.investigate).toBe(false);
+  });
+});
+
+describe("SentinelInvestigationQueue concurrency cap", () => {
+  beforeEach(() => {
+    // No per-project override => the default cap of 3 applies.
+    jest.spyOn(ProjectService, "findOneById").mockResolvedValue(fakeProject());
+  });
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test("leaves the run queued when the per-project cap is reached", async () => {
+    jest
+      .spyOn(AIRunService, "countBy")
+      .mockResolvedValue(new PositiveNumber(3));
+    const claim: jest.SpyInstance = jest.spyOn(
+      AIRunService,
+      "attemptStatusTransition",
+    );
+
+    await SentinelInvestigationQueue.processRun({
+      id: ObjectID.generate(),
+      projectId: ObjectID.generate(),
+      attemptCount: 0,
+      triggeredByAlertId: ObjectID.generate(),
+    });
+
+    expect(claim).not.toHaveBeenCalled();
+  });
+
+  test("leaves the run queued when the cap check itself fails", async () => {
+    jest.spyOn(AIRunService, "countBy").mockRejectedValue(new Error("db down"));
+    const claim: jest.SpyInstance = jest.spyOn(
+      AIRunService,
+      "attemptStatusTransition",
+    );
+
+    await SentinelInvestigationQueue.processRun({
+      id: ObjectID.generate(),
+      projectId: ObjectID.generate(),
+      attemptCount: 0,
+      triggeredByAlertId: ObjectID.generate(),
+    });
+
+    expect(claim).not.toHaveBeenCalled();
+  });
+
+  test("a per-project cap override of 1 blocks the second concurrent run", async () => {
+    jest.spyOn(ProjectService, "findOneById").mockResolvedValue({
+      id: ObjectID.generate(),
+      aiMaxConcurrentInvestigations: 1,
+    } as unknown as Project);
+    jest
+      .spyOn(AIRunService, "countBy")
+      .mockResolvedValue(new PositiveNumber(1));
+    const claim: jest.SpyInstance = jest.spyOn(
+      AIRunService,
+      "attemptStatusTransition",
+    );
+
+    await SentinelInvestigationQueue.processRun({
+      id: ObjectID.generate(),
+      projectId: ObjectID.generate(),
+      attemptCount: 0,
+      triggeredByAlertId: ObjectID.generate(),
+    });
+
+    expect(claim).not.toHaveBeenCalled();
+  });
+});

package/Tests/Server/Utils/AI/SentinelInvestigationQueue.test.ts

@@ -0,0 +1,295 @@
+import SentinelInvestigationQueue, {
+  MAX_INVESTIGATION_ATTEMPTS,
+} from "../../../../Server/Utils/AI/Sentinel/InvestigationQueue";
+import SentinelIncidentInvestigationRunner from "../../../../Server/Utils/AI/Sentinel/IncidentInvestigationRunner";
+import SentinelAlertInvestigationRunner from "../../../../Server/Utils/AI/Sentinel/AlertInvestigationRunner";
+import AIRunService from "../../../../Server/Services/AIRunService";
+import AIService from "../../../../Server/Services/AIService";
+import ProjectService from "../../../../Server/Services/ProjectService";
+import Project from "../../../../Models/DatabaseModels/Project";
+import AIRun from "../../../../Models/DatabaseModels/AIRun";
+import AIRunStatus from "../../../../Types/AI/AIRunStatus";
+import ObjectID from "../../../../Types/ObjectID";
+import PositiveNumber from "../../../../Types/PositiveNumber";
+import { describe, expect, test, afterEach, beforeEach } from "@jest/globals";
+
+/*
+ * The durable investigation queue (Phase 2's first item — replaces detached
+ * fire-and-forget investigations that a pod restart could orphan, D2).
+ *
+ * The invariants these tests lock in:
+ *   (a) enqueue records a Queued AIRun (the durable intent) before any
+ *       expensive work, and kicks inline processing;
+ *   (b) the claim is a status-guarded CAS: a lost claim never executes,
+ *       a won claim increments attemptCount and dispatches to the right
+ *       subject runner;
+ *   (c) the retry policy (G9): transient failures requeue while attempts
+ *       remain; permanent failures and exhausted attempts finalize as
+ *       Error — and the CAS guard means an already-Completed run is never
+ *       clobbered;
+ *   (d) heartbeat-stale runs requeue while attempts remain, else go Stale;
+ *   (e) the poller expires runs that queued past their usefulness window.
+ */
+
+function mockBudgetOk(): void {
+  jest.spyOn(AIService, "getAutonomousDailyBudgetStatus").mockResolvedValue({
+    exhausted: false,
+    limitInTokens: null,
+    usedTokensToday: 0,
+  });
+}
+
+describe("SentinelInvestigationQueue", () => {
+  beforeEach(() => {
+    mockBudgetOk();
+    // No per-project cap override => default of 3.
+    jest
+      .spyOn(ProjectService, "findOneById")
+      .mockResolvedValue({ id: ObjectID.generate() } as unknown as Project);
+    // No investigations currently running (cap check passes).
+    jest
+      .spyOn(AIRunService, "countBy")
+      .mockResolvedValue(new PositiveNumber(0));
+  });
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test("enqueue records a Queued run with the subject and kicks processing", async () => {
+    const projectId: ObjectID = ObjectID.generate();
+    const incidentId: ObjectID = ObjectID.generate();
+    const createdId: ObjectID = ObjectID.generate();
+
+    const create: jest.SpyInstance = jest
+      .spyOn(AIRunService, "create")
+      .mockResolvedValue({ id: createdId } as unknown as AIRun);
+    const processRun: jest.SpyInstance = jest
+      .spyOn(SentinelInvestigationQueue, "processRun")
+      .mockResolvedValue(undefined);
+
+    await SentinelInvestigationQueue.enqueue({
+      projectId,
+      subjectIncidentId: incidentId,
+    });
+
+    expect(create).toHaveBeenCalledWith(
+      expect.objectContaining({
+        data: expect.objectContaining({
+          status: AIRunStatus.Queued,
+          triggeredByIncidentId: incidentId,
+        }),
+      }),
+    );
+    // The inline kick is detached; give the microtask a beat.
+    await new Promise((resolve: (value: unknown) => void) => {
+      setTimeout(resolve, 0);
+    });
+    expect(processRun).toHaveBeenCalledWith(
+      expect.objectContaining({
+        id: createdId,
+        attemptCount: 0,
+        triggeredByIncidentId: incidentId,
+      }),
+    );
+  });
+
+  test("a lost claim never executes the investigation", async () => {
+    jest.spyOn(AIRunService, "attemptStatusTransition").mockResolvedValue(0);
+    const execute: jest.SpyInstance = jest.spyOn(
+      SentinelIncidentInvestigationRunner,
+      "executeInvestigation",
+    );
+
+    await SentinelInvestigationQueue.processRun({
+      id: ObjectID.generate(),
+      projectId: ObjectID.generate(),
+      attemptCount: 0,
+      triggeredByIncidentId: ObjectID.generate(),
+    });
+
+    expect(execute).not.toHaveBeenCalled();
+  });
+
+  test("a won claim increments attemptCount and dispatches to the incident runner", async () => {
+    const runId: ObjectID = ObjectID.generate();
+    const incidentId: ObjectID = ObjectID.generate();
+    const claim: jest.SpyInstance = jest
+      .spyOn(AIRunService, "attemptStatusTransition")
+      .mockResolvedValue(1);
+    const execute: jest.SpyInstance = jest
+      .spyOn(SentinelIncidentInvestigationRunner, "executeInvestigation")
+      .mockResolvedValue(undefined);
+
+    await SentinelInvestigationQueue.processRun({
+      id: runId,
+      projectId: ObjectID.generate(),
+      attemptCount: 0,
+      triggeredByIncidentId: incidentId,
+    });
+
+    expect(claim).toHaveBeenCalledWith(
+      expect.objectContaining({
+        aiRunId: runId,
+        fromStatus: AIRunStatus.Queued,
+        // Guards against stale queue snapshots re-claiming or re-numbering.
+        expectedAttemptCount: 0,
+        set: expect.objectContaining({
+          status: AIRunStatus.Running,
+          attemptCount: 1,
+        }),
+      }),
+    );
+    expect(execute).toHaveBeenCalledWith(
+      expect.objectContaining({
+        aiRunId: runId,
+        incidentId,
+        attemptCount: 1,
+      }),
+    );
+  });
+
+  test("a won claim dispatches alert runs to the alert runner", async () => {
+    const alertId: ObjectID = ObjectID.generate();
+    jest.spyOn(AIRunService, "attemptStatusTransition").mockResolvedValue(1);
+    const execute: jest.SpyInstance = jest
+      .spyOn(SentinelAlertInvestigationRunner, "executeInvestigation")
+      .mockResolvedValue(undefined);
+
+    await SentinelInvestigationQueue.processRun({
+      id: ObjectID.generate(),
+      projectId: ObjectID.generate(),
+      attemptCount: 0,
+      triggeredByAlertId: alertId,
+    });
+
+    expect(execute).toHaveBeenCalledWith(
+      expect.objectContaining({ alertId, attemptCount: 1 }),
+    );
+  });
+
+  test("a transient failure on the first attempt requeues the run", async () => {
+    const runId: ObjectID = ObjectID.generate();
+    const update: jest.SpyInstance = jest
+      .spyOn(AIRunService, "attemptStatusTransition")
+      .mockResolvedValue(1);
+
+    await SentinelInvestigationQueue.failOrRequeue({
+      aiRunId: runId,
+      attemptCount: 1,
+      errorMessage: "LLM provider timed out",
+      isPermanent: false,
+    });
+
+    expect(update).toHaveBeenCalledWith(
+      expect.objectContaining({
+        aiRunId: runId,
+        fromStatus: AIRunStatus.Running,
+        set: expect.objectContaining({
+          status: AIRunStatus.Queued,
+        }),
+      }),
+    );
+  });
+
+  test("a transient failure on the final attempt finalizes as Error", async () => {
+    const update: jest.SpyInstance = jest
+      .spyOn(AIRunService, "attemptStatusTransition")
+      .mockResolvedValue(1);
+
+    await SentinelInvestigationQueue.failOrRequeue({
+      aiRunId: ObjectID.generate(),
+      attemptCount: MAX_INVESTIGATION_ATTEMPTS,
+      errorMessage: "LLM provider timed out",
+      isPermanent: false,
+    });
+
+    expect(update).toHaveBeenCalledWith(
+      expect.objectContaining({
+        set: expect.objectContaining({
+          status: AIRunStatus.Error,
+        }),
+      }),
+    );
+  });
+
+  test("a permanent failure never retries, even with attempts remaining", async () => {
+    const update: jest.SpyInstance = jest
+      .spyOn(AIRunService, "attemptStatusTransition")
+      .mockResolvedValue(1);
+
+    await SentinelInvestigationQueue.failOrRequeue({
+      aiRunId: ObjectID.generate(),
+      attemptCount: 1,
+      errorMessage: "No LLM provider configured for this project.",
+      isPermanent: true,
+    });
+
+    expect(update).toHaveBeenCalledTimes(1);
+    expect(update).toHaveBeenCalledWith(
+      expect.objectContaining({
+        set: expect.objectContaining({
+          status: AIRunStatus.Error,
+        }),
+      }),
+    );
+  });
+
+  test("a heartbeat-stale run requeues while attempts remain", async () => {
+    jest.spyOn(AIRunService, "attemptStatusTransition").mockResolvedValue(1);
+
+    const outcome: "requeued" | "stale" =
+      await SentinelInvestigationQueue.requeueOrMarkStale({
+        id: ObjectID.generate(),
+        attemptCount: 1,
+      });
+
+    expect(outcome).toBe("requeued");
+  });
+
+  test("a heartbeat-stale run out of attempts is marked Stale", async () => {
+    const update: jest.SpyInstance = jest
+      .spyOn(AIRunService, "attemptStatusTransition")
+      .mockResolvedValue(1);
+
+    const outcome: "requeued" | "stale" =
+      await SentinelInvestigationQueue.requeueOrMarkStale({
+        id: ObjectID.generate(),
+        attemptCount: MAX_INVESTIGATION_ATTEMPTS,
+      });
+
+    expect(outcome).toBe("stale");
+    expect(update).toHaveBeenCalledWith(
+      expect.objectContaining({
+        set: expect.objectContaining({
+          status: AIRunStatus.Stale,
+        }),
+      }),
+    );
+  });
+
+  test("the poller expires runs queued past the usefulness window", async () => {
+    const expiredId: ObjectID = ObjectID.generate();
+    const findBy: jest.SpyInstance = jest
+      .spyOn(AIRunService, "findBy")
+      // First call: expired runs. Second call: nothing left to drain.
+      .mockResolvedValueOnce([{ id: expiredId } as unknown as AIRun])
+      .mockResolvedValueOnce([]);
+    const update: jest.SpyInstance = jest
+      .spyOn(AIRunService, "attemptStatusTransition")
+      .mockResolvedValue(1);
+
+    await SentinelInvestigationQueue.processQueuedRuns();
+
+    expect(findBy).toHaveBeenCalledTimes(2);
+    expect(update).toHaveBeenCalledWith(
+      expect.objectContaining({
+        aiRunId: expiredId,
+        fromStatus: AIRunStatus.Queued,
+        set: expect.objectContaining({
+          status: AIRunStatus.Cancelled,
+        }),
+      }),
+    );
+  });
+});