@oneuptime/common 14.0.11 → 14.0.12

package/Tests/Server/Utils/AI/AIAlertGating.test.ts

@@ -15,13 +15,16 @@ import PositiveNumber from "../../../../Types/PositiveNumber";
 import { describe, expect, test, afterEach } from "@jest/globals";
 
 /*
- * Cost gates for autonomous alert investigations (Phase 1 / G4):
- *   - severity floor: explicit per-project minimum severity, defaulting to the
- *     project's top two tiers (lowest two `order` values; lower order = higher
- *     severity);
- *   - per-monitor dedupe window: a monitor already investigated within the
- *     window is not re-investigated;
- *   - an alert-specific concurrency cap in the shared engine.
+ * Cost gates for autonomous alert investigations (Phase 1 / G4). Every one of
+ * them is opt-in — unset means no limit, so by default every alert is
+ * investigated:
+ *   - severity floor: an explicit per-project minimum severity; unset means
+ *     every severity (it used to default to the project's top two tiers);
+ *   - per-monitor cooldown: a monitor already investigated within a window
+ *     the project set is not re-investigated; unset means no cooldown (it
+ *     used to default to 30 minutes);
+ *   - an alert-specific concurrency cap in the shared queue; unset means no
+ *     cap (it used to default to 3).
  *
  * These tests mock the persistence layer (no Postgres) to lock in the gate
  * decisions and their fail directions: unknown severity PASSES the severity
@@ -54,12 +57,6 @@ function fakeProject(
   } as unknown as Project;
 }
 
-function fakeSeverities(orders: Array<number>): Array<AlertSeverity> {
-  return orders.map((order: number) => {
-    return { order } as AlertSeverity;
-  });
-}
-
 describe("AIAlertInvestigationRunner.shouldInvestigateAlert", () => {
   const alertId: ObjectID = ObjectID.generate();
   const projectId: ObjectID = ObjectID.generate();
@@ -72,257 +69,312 @@ describe("AIAlertInvestigationRunner.shouldInvestigateAlert", () => {
     alert: Alert | null;
     project?: Project;
     explicitFloorSeverity?: AlertSeverity | null;
-    topTiers?: Array<AlertSeverity>;
     recentRunCount?: number;
-  }): void {
+  }): {
+    severityLookup: jest.SpyInstance;
+    severityList: jest.SpyInstance;
+    countBy: jest.SpyInstance;
+  } {
     jest.spyOn(AlertService, "findOneById").mockResolvedValue(data.alert);
     jest
       .spyOn(ProjectService, "findOneById")
       .mockResolvedValue(data.project || fakeProject());
-    jest
+    const severityLookup: jest.SpyInstance = jest
       .spyOn(AlertSeverityService, "findOneById")
       .mockResolvedValue(data.explicitFloorSeverity ?? null);
-    jest
+    // The old top-two-tiers default listed the project's severities.
+    const severityList: jest.SpyInstance = jest
       .spyOn(AlertSeverityService, "findBy")
-      .mockResolvedValue(data.topTiers || []);
-    jest
+      .mockResolvedValue([{ order: 1 }, { order: 2 }] as Array<AlertSeverity>);
+    const countBy: jest.SpyInstance = jest
       .spyOn(AIRunService, "countBy")
       .mockResolvedValue(new PositiveNumber(data.recentRunCount || 0));
+
+    return { severityLookup, severityList, countBy };
   }
 
-  test("skips when severity is below an explicitly configured floor", async () => {
-    const floorSeverityId: ObjectID = ObjectID.generate();
-    mockGates({
-      alert: fakeAlert({ severityOrder: 3 }),
-      project: fakeProject(floorSeverityId),
-      explicitFloorSeverity: { order: 2 } as AlertSeverity,
+  function gate(): Promise<AlertGateDecision> {
+    return AIAlertInvestigationRunner.shouldInvestigateAlert({
+      alertId,
+      projectId,
     });
+  }
 
-    const decision: AlertGateDecision =
-      await AIAlertInvestigationRunner.shouldInvestigateAlert({
-        alertId,
-        projectId,
+  describe("severity floor", () => {
+    test("skips when severity is below an explicitly configured floor", async () => {
+      const floorSeverityId: ObjectID = ObjectID.generate();
+      mockGates({
+        alert: fakeAlert({ severityOrder: 3 }),
+        project: fakeProject(floorSeverityId),
+        explicitFloorSeverity: { order: 2, name: "Major" } as AlertSeverity,
       });
 
-    expect(decision.investigate).toBe(false);
-    expect(decision.reason).toContain("below the investigation floor");
-  });
+      const decision: AlertGateDecision = await gate();
 
-  test("investigates when severity meets the explicit floor and no recent run exists", async () => {
-    const monitorId: ObjectID = ObjectID.generate();
-    const floorSeverityId: ObjectID = ObjectID.generate();
-    mockGates({
-      alert: fakeAlert({ severityOrder: 2, monitorId }),
-      project: fakeProject(floorSeverityId),
-      explicitFloorSeverity: { order: 2 } as AlertSeverity,
-      recentRunCount: 0,
+      expect(decision.investigate).toBe(false);
+      expect(decision.reason).toContain("below the investigation floor");
+      expect(decision.notStartedCode).toBe("severity_below_threshold");
+      expect(decision.notStartedDetails?.minimumSeverityName).toBe("Major");
     });
 
-    const decision: AlertGateDecision =
-      await AIAlertInvestigationRunner.shouldInvestigateAlert({
-        alertId,
-        projectId,
+    test("investigates when severity meets the explicit floor and no recent run exists", async () => {
+      const monitorId: ObjectID = ObjectID.generate();
+      const floorSeverityId: ObjectID = ObjectID.generate();
+      mockGates({
+        alert: fakeAlert({ severityOrder: 2, monitorId }),
+        project: fakeProject(floorSeverityId),
+        explicitFloorSeverity: { order: 2 } as AlertSeverity,
+        recentRunCount: 0,
       });
 
-    expect(decision.investigate).toBe(true);
-    expect(decision.monitorId).toBe(monitorId);
-  });
+      const decision: AlertGateDecision = await gate();
 
-  test("default floor is the top two tiers: third tier is skipped", async () => {
-    mockGates({
-      alert: fakeAlert({ severityOrder: 3 }),
-      topTiers: fakeSeverities([1, 2]),
+      expect(decision.investigate).toBe(true);
+      expect(decision.monitorId).toBe(monitorId);
     });
 
-    const decision: AlertGateDecision =
-      await AIAlertInvestigationRunner.shouldInvestigateAlert({
-        alertId,
-        projectId,
+    /*
+     * The headline change. Unset used to mean "the top two severity tiers",
+     * so a project's third-tier alerts were silently never investigated.
+     * Unset now means what it says: no floor.
+     */
+    test.each([[1], [2], [3], [4], [10]])(
+      "with no configured floor, an alert of severity order %p is investigated",
+      async (severityOrder: number) => {
+        mockGates({ alert: fakeAlert({ severityOrder }) });
+
+        expect((await gate()).investigate).toBe(true);
+      },
+    );
+
+    test("with no configured floor, no severity is looked up at all", async () => {
+      const { severityLookup, severityList } = mockGates({
+        alert: fakeAlert({ severityOrder: 5 }),
       });
 
-    expect(decision.investigate).toBe(false);
-  });
+      await gate();
 
-  test("default floor: second tier is investigated", async () => {
-    mockGates({
-      alert: fakeAlert({ severityOrder: 2 }),
-      topTiers: fakeSeverities([1, 2]),
+      expect(severityLookup).not.toHaveBeenCalled();
+      expect(severityList).not.toHaveBeenCalled();
     });
 
-    const decision: AlertGateDecision =
-      await AIAlertInvestigationRunner.shouldInvestigateAlert({
-        alertId,
-        projectId,
+    test("a deleted explicit floor severity falls back to no floor, not to a default tier", async () => {
+      // Project points at a severity that no longer exists (findOneById → null).
+      const { severityList } = mockGates({
+        alert: fakeAlert({ severityOrder: 3 }),
+        project: fakeProject(ObjectID.generate()),
+        explicitFloorSeverity: null,
       });
 
-    expect(decision.investigate).toBe(true);
-  });
-
-  test("a deleted explicit floor severity falls back to the top-two default", async () => {
-    // Project points at a severity that no longer exists (findOneById → null).
-    mockGates({
-      alert: fakeAlert({ severityOrder: 3 }),
-      project: fakeProject(ObjectID.generate()),
-      explicitFloorSeverity: null,
-      topTiers: fakeSeverities([1, 2]),
+      expect((await gate()).investigate).toBe(true);
+      expect(severityList).not.toHaveBeenCalled();
     });
 
-    const decision: AlertGateDecision =
-      await AIAlertInvestigationRunner.shouldInvestigateAlert({
-        alertId,
-        projectId,
+    test("a floor severity with no order falls back to no floor", async () => {
+      mockGates({
+        alert: fakeAlert({ severityOrder: 9 }),
+        project: fakeProject(ObjectID.generate()),
+        explicitFloorSeverity: { name: "Broken" } as AlertSeverity,
       });
 
-    expect(decision.investigate).toBe(false);
-  });
-
-  test("unknown alert severity passes the severity gate", async () => {
-    mockGates({
-      alert: fakeAlert({ severityOrder: undefined }),
-      topTiers: fakeSeverities([1, 2]),
+      expect((await gate()).investigate).toBe(true);
     });
 
-    const decision: AlertGateDecision =
-      await AIAlertInvestigationRunner.shouldInvestigateAlert({
-        alertId,
-        projectId,
+    test("unknown alert severity passes the severity gate", async () => {
+      mockGates({
+        alert: fakeAlert({ severityOrder: undefined }),
+        project: fakeProject(ObjectID.generate()),
+        explicitFloorSeverity: { order: 1 } as AlertSeverity,
       });
 
-    expect(decision.investigate).toBe(true);
+      expect((await gate()).investigate).toBe(true);
+    });
   });
 
-  test("skips when the monitor was already investigated within the window", async () => {
-    const monitorId: ObjectID = ObjectID.generate();
-    const countBy: jest.SpyInstance = jest.spyOn(AIRunService, "countBy");
+  describe("per-monitor cooldown", () => {
+    /*
+     * The other headline change: unset used to mean a 30-minute cooldown, so
+     * a monitor that fired again within half an hour was not looked at.
+     */
+    test("with no configured cooldown, a repeat alert from a just-investigated monitor is investigated again", async () => {
+      const monitorId: ObjectID = ObjectID.generate();
+      const { countBy } = mockGates({
+        alert: fakeAlert({ severityOrder: 1, monitorId }),
+        recentRunCount: 1,
+      });
 
-    mockGates({
-      alert: fakeAlert({ severityOrder: 1, monitorId }),
-      topTiers: fakeSeverities([1, 2]),
-      recentRunCount: 1,
+      const decision: AlertGateDecision = await gate();
+
+      expect(decision.investigate).toBe(true);
+      expect(decision.monitorId).toBe(monitorId);
+      // No cooldown, so there is nothing to look up.
+      expect(countBy).not.toHaveBeenCalled();
     });
 
-    const decision: AlertGateDecision =
-      await AIAlertInvestigationRunner.shouldInvestigateAlert({
-        alertId,
-        projectId,
+    test("skips when the monitor was already investigated within a configured window", async () => {
+      const monitorId: ObjectID = ObjectID.generate();
+      const { countBy } = mockGates({
+        alert: fakeAlert({ severityOrder: 1, monitorId }),
+        project: fakeProject(undefined, 30),
+        recentRunCount: 1,
       });
 
-    expect(decision.investigate).toBe(false);
-    expect(decision.reason).toContain("already investigated");
-    expect(countBy).toHaveBeenCalledWith(
-      expect.objectContaining({
-        query: expect.objectContaining({
-          monitorId,
+      const decision: AlertGateDecision = await gate();
+
+      expect(decision.investigate).toBe(false);
+      expect(decision.reason).toContain("already investigated");
+      expect(decision.reason).toContain("within the last 30 minutes");
+      expect(decision.notStartedCode).toBe("monitor_cooldown");
+      expect(decision.notStartedDetails).toEqual({ cooldownWindowMinutes: 30 });
+      expect(countBy).toHaveBeenCalledWith(
+        expect.objectContaining({
+          query: expect.objectContaining({
+            monitorId,
+          }),
         }),
-      }),
-    );
-  });
-
-  test("the cooldown counts alert investigations only, so incident runs cannot suppress an alert", async () => {
-    const monitorId: ObjectID = ObjectID.generate();
-    const alertSubjectFilter: Record<string, string> = {
-      operator: "alert-not-null",
-    };
-    const incidentSubjectFilter: Record<string, string> = {
-      operator: "incident-null",
-    };
-    jest.spyOn(QueryHelper, "notNull").mockReturnValue(alertSubjectFilter);
-    jest.spyOn(QueryHelper, "isNull").mockReturnValue(incidentSubjectFilter);
-    const countBy: jest.SpyInstance = jest.spyOn(AIRunService, "countBy");
-
-    mockGates({
-      alert: fakeAlert({ severityOrder: 1, monitorId }),
-      topTiers: fakeSeverities([1, 2]),
-      recentRunCount: 0,
+      );
     });
 
-    await AIAlertInvestigationRunner.shouldInvestigateAlert({
-      alertId,
-      projectId,
+    test("investigates inside a configured window when the monitor has no recent run", async () => {
+      mockGates({
+        alert: fakeAlert({ severityOrder: 1, monitorId: ObjectID.generate() }),
+        project: fakeProject(undefined, 30),
+        recentRunCount: 0,
+      });
+
+      expect((await gate()).investigate).toBe(true);
     });
 
-    const query: Record<string, unknown> = (
-      countBy.mock.calls[0]![0] as { query: Record<string, unknown> }
-    ).query;
-    expect(query["triggeredByAlertId"]).toBe(alertSubjectFilter);
-    expect(query["triggeredByIncidentId"]).toBe(incidentSubjectFilter);
-  });
+    test("the cooldown counts alert investigations only, so incident runs cannot suppress an alert", async () => {
+      const monitorId: ObjectID = ObjectID.generate();
+      const alertSubjectFilter: Record<string, string> = {
+        operator: "alert-not-null",
+      };
+      const incidentSubjectFilter: Record<string, string> = {
+        operator: "incident-null",
+      };
+      jest.spyOn(QueryHelper, "notNull").mockReturnValue(alertSubjectFilter);
+      jest.spyOn(QueryHelper, "isNull").mockReturnValue(incidentSubjectFilter);
+
+      const { countBy } = mockGates({
+        alert: fakeAlert({ severityOrder: 1, monitorId }),
+        project: fakeProject(undefined, 30),
+        recentRunCount: 0,
+      });
 
-  test("alerts without a monitor skip the dedupe check entirely", async () => {
-    const countBy: jest.SpyInstance = jest.spyOn(AIRunService, "countBy");
+      await gate();
 
-    mockGates({
-      alert: fakeAlert({ severityOrder: 1 }),
-      topTiers: fakeSeverities([1, 2]),
+      const query: Record<string, unknown> = (
+        countBy.mock.calls[0]![0] as { query: Record<string, unknown> }
+      ).query;
+      expect(query["triggeredByAlertId"]).toBe(alertSubjectFilter);
+      expect(query["triggeredByIncidentId"]).toBe(incidentSubjectFilter);
     });
 
-    const decision: AlertGateDecision =
-      await AIAlertInvestigationRunner.shouldInvestigateAlert({
-        alertId,
-        projectId,
+    test("alerts without a monitor skip the dedupe check entirely, even with a cooldown set", async () => {
+      const { countBy } = mockGates({
+        alert: fakeAlert({ severityOrder: 1 }),
+        project: fakeProject(undefined, 30),
       });
 
-    expect(decision.investigate).toBe(true);
-    expect(countBy).not.toHaveBeenCalled();
-  });
+      const decision: AlertGateDecision = await gate();
 
-  test("a cooldown of 0 disables the dedupe check entirely", async () => {
-    const monitorId: ObjectID = ObjectID.generate();
-    const countBy: jest.SpyInstance = jest.spyOn(AIRunService, "countBy");
+      expect(decision.investigate).toBe(true);
+      expect(countBy).not.toHaveBeenCalled();
+    });
+
+    test("a cooldown of 0 disables the dedupe check entirely", async () => {
+      const { countBy } = mockGates({
+        alert: fakeAlert({ severityOrder: 1, monitorId: ObjectID.generate() }),
+        project: fakeProject(undefined, 0),
+        recentRunCount: 1,
+      });
 
-    mockGates({
-      alert: fakeAlert({ severityOrder: 1, monitorId }),
-      project: fakeProject(undefined, 0),
-      topTiers: fakeSeverities([1, 2]),
-      recentRunCount: 1,
+      expect((await gate()).investigate).toBe(true);
+      expect(countBy).not.toHaveBeenCalled();
     });
 
-    const decision: AlertGateDecision =
-      await AIAlertInvestigationRunner.shouldInvestigateAlert({
-        alertId,
-        projectId,
+    test("a negative cooldown is read as none rather than inverting the query", async () => {
+      const { countBy } = mockGates({
+        alert: fakeAlert({ severityOrder: 1, monitorId: ObjectID.generate() }),
+        project: fakeProject(undefined, -30),
+        recentRunCount: 1,
       });
 
-    expect(decision.investigate).toBe(true);
-    expect(countBy).not.toHaveBeenCalled();
-  });
+      expect((await gate()).investigate).toBe(true);
+      expect(countBy).not.toHaveBeenCalled();
+    });
 
-  test("a custom cooldown is used for the dedupe window", async () => {
-    const monitorId: ObjectID = ObjectID.generate();
+    test("a custom cooldown is used for the dedupe window", async () => {
+      mockGates({
+        alert: fakeAlert({ severityOrder: 1, monitorId: ObjectID.generate() }),
+        project: fakeProject(undefined, 45),
+        recentRunCount: 1,
+      });
+
+      const decision: AlertGateDecision = await gate();
 
-    mockGates({
-      alert: fakeAlert({ severityOrder: 1, monitorId }),
-      project: fakeProject(undefined, 45),
-      topTiers: fakeSeverities([1, 2]),
-      recentRunCount: 1,
+      expect(decision.investigate).toBe(false);
+      expect(decision.reason).toContain("within the last 45 minutes");
     });
 
-    const decision: AlertGateDecision =
-      await AIAlertInvestigationRunner.shouldInvestigateAlert({
-        alertId,
-        projectId,
+    test("an absurd cooldown is clamped to a day", async () => {
+      mockGates({
+        alert: fakeAlert({ severityOrder: 1, monitorId: ObjectID.generate() }),
+        project: fakeProject(undefined, 60 * 24 * 365),
+        recentRunCount: 1,
       });
 
-    expect(decision.investigate).toBe(false);
-    expect(decision.reason).toContain("within the last 45 minutes");
+      const decision: AlertGateDecision = await gate();
+
+      expect(decision.investigate).toBe(false);
+      expect(decision.reason).toContain(`${60 * 24} minutes`);
+    });
   });
 
   test("skips when the alert cannot be found", async () => {
     mockGates({ alert: null });
 
-    const decision: AlertGateDecision =
-      await AIAlertInvestigationRunner.shouldInvestigateAlert({
-        alertId,
-        projectId,
-      });
+    const decision: AlertGateDecision = await gate();
 
     expect(decision.investigate).toBe(false);
+    expect(decision.notStartedCode).toBe("eligibility_check_failed");
+  });
+
+  /*
+   * A project that never touched its alert AI settings: every gate is open,
+   * whatever the alert looks like.
+   */
+  test("a project with no alert AI settings investigates a low-severity repeat alert", async () => {
+    const monitorId: ObjectID = ObjectID.generate();
+    const { severityLookup, severityList, countBy } = mockGates({
+      alert: fakeAlert({ severityOrder: 7, monitorId }),
+      recentRunCount: 3,
+    });
+
+    const decision: AlertGateDecision = await gate();
+
+    expect(decision).toEqual({
+      investigate: true,
+      reason: "passed severity and dedupe gates",
+      monitorId,
+    });
+    expect(severityLookup).not.toHaveBeenCalled();
+    expect(severityList).not.toHaveBeenCalled();
+    expect(countBy).not.toHaveBeenCalled();
   });
 });
 
 describe("AIInvestigationQueue concurrency cap", () => {
+  function projectWithAlertCap(cap: number | undefined): Project {
+    return {
+      id: ObjectID.generate(),
+      alertAiMaxConcurrentInvestigations: cap,
+    } as unknown as Project;
+  }
+
   beforeEach(() => {
-    // No per-project override => the default cap of 3 applies.
+    // No per-project cap: the alert lane has no concurrency limit.
     jest.spyOn(ProjectService, "findOneById").mockResolvedValue(fakeProject());
   });
 
@@ -330,7 +382,34 @@ describe("AIInvestigationQueue concurrency cap", () => {
     jest.restoreAllMocks();
   });
 
-  test("leaves the run queued when the alert cap is reached", async () => {
+  /*
+   * Unset used to mean a cap of 3, so the fourth alert in a burst waited in
+   * the queue — and expired there after 30 minutes if the burst kept coming.
+   */
+  test("with no cap set, an alert run is claimed however many are already running", async () => {
+    const countBy: jest.SpyInstance = jest
+      .spyOn(AIRunService, "countBy")
+      .mockResolvedValue(new PositiveNumber(500));
+    const claim: jest.SpyInstance = jest
+      .spyOn(AIRunService, "attemptStatusTransition")
+      .mockResolvedValue(0);
+
+    await AIInvestigationQueue.processRun({
+      id: ObjectID.generate(),
+      projectId: ObjectID.generate(),
+      attemptCount: 0,
+      triggeredByAlertId: ObjectID.generate(),
+    });
+
+    // Nothing to count against, so nothing is counted.
+    expect(countBy).not.toHaveBeenCalled();
+    expect(claim).toHaveBeenCalledTimes(1);
+  });
+
+  test("leaves the run queued when a configured alert cap is reached", async () => {
+    jest
+      .spyOn(ProjectService, "findOneById")
+      .mockResolvedValue(projectWithAlertCap(3));
     jest
       .spyOn(AIRunService, "countBy")
       .mockResolvedValue(new PositiveNumber(3));
@@ -350,6 +429,9 @@ describe("AIInvestigationQueue concurrency cap", () => {
   });
 
   test("leaves the run queued when the cap check itself fails", async () => {
+    jest
+      .spyOn(ProjectService, "findOneById")
+      .mockResolvedValue(projectWithAlertCap(3));
     jest.spyOn(AIRunService, "countBy").mockRejectedValue(new Error("db down"));
     const claim: jest.SpyInstance = jest.spyOn(
       AIRunService,
@@ -389,4 +471,55 @@ describe("AIInvestigationQueue concurrency cap", () => {
 
     expect(claim).not.toHaveBeenCalled();
   });
+
+  // A cap set above the old ceiling of 25 used to be clamped back to 25.
+  test("a configured cap above 25 is honoured", async () => {
+    jest
+      .spyOn(ProjectService, "findOneById")
+      .mockResolvedValue(projectWithAlertCap(40));
+    jest
+      .spyOn(AIRunService, "countBy")
+      .mockResolvedValue(new PositiveNumber(30));
+    const claim: jest.SpyInstance = jest
+      .spyOn(AIRunService, "attemptStatusTransition")
+      .mockResolvedValue(0);
+
+    await AIInvestigationQueue.processRun({
+      id: ObjectID.generate(),
+      projectId: ObjectID.generate(),
+      attemptCount: 0,
+      triggeredByAlertId: ObjectID.generate(),
+    });
+
+    expect(claim).toHaveBeenCalledTimes(1);
+  });
+
+  test("a configured cap of 0 reads as 1, not as paused", async () => {
+    jest
+      .spyOn(ProjectService, "findOneById")
+      .mockResolvedValue(projectWithAlertCap(0));
+    const countBy: jest.SpyInstance = jest
+      .spyOn(AIRunService, "countBy")
+      .mockResolvedValue(new PositiveNumber(0));
+    const claim: jest.SpyInstance = jest
+      .spyOn(AIRunService, "attemptStatusTransition")
+      .mockResolvedValue(0);
+
+    await AIInvestigationQueue.processRun({
+      id: ObjectID.generate(),
+      projectId: ObjectID.generate(),
+      attemptCount: 0,
+      triggeredByAlertId: ObjectID.generate(),
+    });
+    expect(claim).toHaveBeenCalledTimes(1);
+
+    countBy.mockResolvedValue(new PositiveNumber(1));
+    await AIInvestigationQueue.processRun({
+      id: ObjectID.generate(),
+      projectId: ObjectID.generate(),
+      attemptCount: 0,
+      triggeredByAlertId: ObjectID.generate(),
+    });
+    expect(claim).toHaveBeenCalledTimes(1);
+  });
 });