@oneuptime/common 14.0.11 → 14.0.12

package/Tests/Server/Utils/AI/Insights/InvestigationQueueInsightSubject.test.ts

@@ -26,11 +26,12 @@ import { describe, expect, test, afterEach, beforeEach } from "@jest/globals";
  *   (c) dispatch recognizes insight runs as subject-BEARING and routes
  *       them to InsightTriageRunner.executeTriage with the claimed attempt;
  *   (d) a run with no subject at all is still failed at dispatch;
- *   (e) LANE PRIORITY: triage (the preventive lane) may hold at most
- *       (cap - INSIGHT_TRIAGE_RESERVED_SLOTS) of the project's concurrency
- *       slots, so a scan's triage backlog can never starve the interactive
- *       lane (incident/alert RCA, where a human is waiting) — while the
- *       interactive lane's own gating is left byte-for-byte unchanged.
+ *   (e) LANE PRIORITY: triage is subjectless, has no setting and so no
+ *       cap — it never waits on anything but the budget. Inside a lane a
+ *       project HAS capped, background work (remediation plans and
+ *       executions) may hold at most (cap - INSIGHT_TRIAGE_RESERVED_SLOTS)
+ *       of the slots, so it can never starve the interactive lane
+ *       (incident/alert RCA, where a human is waiting).
  */
 
 function mockBudget(exhausted: boolean): void {
@@ -156,20 +157,23 @@ describe("AIInvestigationQueue — insight subject", () => {
 });
 
 /*
- * The starvation scenario this locks down: a scan files 10 insights, so 10
- * triage runs queue. Without the lane sub-cap they would fill every slot of
- * the default cap of 3 and the incident that fires a minute later would wait
- * behind them. With it, triage tops out at (cap - 1) and the incident's
- * inline kick at enqueue always finds the reserved slot free.
+ * The starvation scenario this locks down. With no caps (the default),
+ * nothing waits on anything: a scan's ten triage runs and the incident that
+ * fires a minute later all start at once. When a project caps a lane, the
+ * background work in it tops out at (cap - 1), so the incident's inline kick
+ * at enqueue always finds the reserved slot free.
  */
 describe("AIInvestigationQueue — lane priority (triage never starves RCA)", () => {
   const projectId: ObjectID = ObjectID.generate();
 
+  // The cap a project set on its incident lane, for the capped-lane tests.
+  const INCIDENT_CAP: number = 3;
+
   /*
-   * The claim gates count Running investigations twice: once globally, and
-   * once filtered to the triage lane (the query carrying
-   * triggeredByAiInsightId). Answer each query independently so the
-   * two caps can be driven apart.
+   * The claim gates count Running investigations in the lane, then the
+   * background kinds within it: the triage-lane query carries
+   * triggeredByAiInsightId, the remediation one filters runType to the plan
+   * kinds. Answer each query independently so the caps can be driven apart.
    */
   function mockRunningCounts(counts: {
     total: number;
@@ -193,9 +197,9 @@ describe("AIInvestigationQueue — lane priority (triage never starves RCA)", ()
           query["triggeredByAiInsightId"] !== undefined;
 
         /*
-         * Both the global and the remediation-lane counts filter runType
-         * with QueryHelper.any(...), so they are told apart by what is IN
-         * the list: only the global one counts Investigations.
+         * Both the lane and the remediation counts filter runType with
+         * QueryHelper.any(...), so they are told apart by what is IN the
+         * list: only the lane one counts Investigations.
          */
         const runTypeFilter: string = JSON.stringify(query["runType"] ?? null);
 
@@ -215,33 +219,59 @@ describe("AIInvestigationQueue — lane priority (triage never starves RCA)", ()
       });
   }
 
-  beforeEach(() => {
-    mockBudget(false);
-    jest.spyOn(AIRunService, "attemptStatusTransition").mockResolvedValue(1);
-    /*
-     * No lane overrides: incident work falls back to the default cap of 3,
-     * and triage is subjectless, so it always runs under the default.
-     */
+  function capIncidentLane(cap: number | undefined): void {
     jest.spyOn(ProjectService, "findOneById").mockResolvedValue({
       id: projectId,
+      incidentAiMaxConcurrentInvestigations: cap,
     } as unknown as Project);
+  }
+
+  beforeEach(() => {
+    mockBudget(false);
+    jest.spyOn(AIRunService, "attemptStatusTransition").mockResolvedValue(1);
+    // No lane overrides: no lane has a cap.
+    capIncidentLane(undefined);
   });
 
   afterEach(() => {
     jest.restoreAllMocks();
   });
 
-  test("the reserved slot is real: with the triage lane full (2 of cap 3), an incident investigation still claims and dispatches", async () => {
-    // Both running investigations are triage runs — the lane's sub-cap.
-    mockRunningCounts({ total: 2, triage: 2 });
-    const claim: jest.SpyInstance = jest.spyOn(
-      AIRunService,
-      "attemptStatusTransition",
+  /*
+   * Triage used to run under a fixed cap of 3 with one slot held back, so a
+   * scan's backlog drained two at a time. It has no setting, so no cap.
+   */
+  test("triage has no cap: a triage run claims however many triage runs are running, without counting them", async () => {
+    const countBy: jest.SpyInstance = mockRunningCounts({
+      total: 40,
+      triage: 40,
+    });
+    const executeTriage: jest.SpyInstance = jest
+      .spyOn(InsightTriageRunner, "executeTriage")
+      .mockResolvedValue(undefined);
+
+    await AIInvestigationQueue.processRun({
+      id: ObjectID.generate(),
+      projectId,
+      attemptCount: 0,
+      triggeredByAiInsightId: ObjectID.generate(),
+    });
+
+    expect(countBy).not.toHaveBeenCalled();
+    expect(executeTriage).toHaveBeenCalledWith(
+      expect.objectContaining({ attemptCount: 1 }),
     );
+  });
+
+  test("with no caps, an incident investigation claims straight away whatever background work is running", async () => {
+    const countBy: jest.SpyInstance = mockRunningCounts({
+      total: 10,
+      triage: 5,
+      plans: 5,
+    });
     const executeInvestigation: jest.SpyInstance = jest
       .spyOn(AIIncidentInvestigationRunner, "executeInvestigation")
       .mockResolvedValue(undefined);
-
     const incidentId: ObjectID = ObjectID.generate();
 
     await AIInvestigationQueue.processRun({
@@ -251,41 +281,49 @@ describe("AIInvestigationQueue — lane priority (triage never starves RCA)", ()
       triggeredByIncidentId: incidentId,
     });
 
-    expect(claim).toHaveBeenCalledWith(
-      expect.objectContaining({ fromStatus: AIRunStatus.Queued }),
-    );
+    expect(countBy).not.toHaveBeenCalled();
     expect(executeInvestigation).toHaveBeenCalledWith(
       expect.objectContaining({ incidentId, attemptCount: 1 }),
     );
   });
 
-  test("a triage run at the lane sub-cap is left Queued — the claim is never even attempted", async () => {
-    // 2 triage running = cap(3) - RESERVED(1): the lane is full.
-    mockRunningCounts({ total: 2, triage: 3 - INSIGHT_TRIAGE_RESERVED_SLOTS });
+  test("the reserved slot is real: in a capped lane with its background share full, an incident investigation still claims and dispatches", async () => {
+    capIncidentLane(INCIDENT_CAP);
+    // Both running runs are remediation work — the background share of 3.
+    mockRunningCounts({ total: 2, triage: 0, plans: 2 });
     const claim: jest.SpyInstance = jest.spyOn(
       AIRunService,
       "attemptStatusTransition",
     );
-    const executeTriage: jest.SpyInstance = jest.spyOn(
-      InsightTriageRunner,
-      "executeTriage",
-    );
+    const executeInvestigation: jest.SpyInstance = jest
+      .spyOn(AIIncidentInvestigationRunner, "executeInvestigation")
+      .mockResolvedValue(undefined);
+
+    const incidentId: ObjectID = ObjectID.generate();
 
     await AIInvestigationQueue.processRun({
       id: ObjectID.generate(),
       projectId,
       attemptCount: 0,
-      triggeredByAiInsightId: ObjectID.generate(),
+      triggeredByIncidentId: incidentId,
     });
 
-    // Left Queued for the poller/TTL — never claimed, never executed.
-    expect(claim).not.toHaveBeenCalled();
-    expect(executeTriage).not.toHaveBeenCalled();
+    expect(claim).toHaveBeenCalledWith(
+      expect.objectContaining({ fromStatus: AIRunStatus.Queued }),
+    );
+    expect(executeInvestigation).toHaveBeenCalledWith(
+      expect.objectContaining({ incidentId, attemptCount: 1 }),
+    );
   });
 
-  test("remediation plans share the background sub-cap: triage + plans together at cap - reserved leave a plan run Queued", async () => {
-    // 1 triage + 1 plan running = cap(3) - RESERVED(1): background lane full.
-    mockRunningCounts({ total: 2, triage: 1, plans: 1 });
+  test("in a capped lane, background work at cap - reserved leaves a remediation plan Queued", async () => {
+    capIncidentLane(INCIDENT_CAP);
+    // 2 plans running = cap(3) - RESERVED(1): the background share is full.
+    mockRunningCounts({
+      total: 2,
+      triage: 0,
+      plans: INCIDENT_CAP - INSIGHT_TRIAGE_RESERVED_SLOTS,
+    });
     const claim: jest.SpyInstance = jest.spyOn(
       AIRunService,
       "attemptStatusTransition",
@@ -296,6 +334,7 @@ describe("AIInvestigationQueue — lane priority (triage never starves RCA)", ()
       projectId,
       attemptCount: 0,
       runType: AIRunType.RemediationPlan,
+      triggeredByIncidentId: ObjectID.generate(),
       triggeredByAutoRemediationSuggestionId: ObjectID.generate(),
     });
 
@@ -303,25 +342,30 @@ describe("AIInvestigationQueue — lane priority (triage never starves RCA)", ()
     expect(claim).not.toHaveBeenCalled();
   });
 
-  test("triage still runs while under the lane sub-cap — the sub-cap throttles, it does not disable", async () => {
-    mockRunningCounts({ total: 1, triage: 1 });
-    const executeTriage: jest.SpyInstance = jest
-      .spyOn(InsightTriageRunner, "executeTriage")
-      .mockResolvedValue(undefined);
+  test("in a capped lane, background work still runs while under its share — the sub-cap throttles, it does not disable", async () => {
+    capIncidentLane(INCIDENT_CAP);
+    mockRunningCounts({ total: 1, triage: 0, plans: 1 });
+    const claim: jest.SpyInstance = jest.spyOn(
+      AIRunService,
+      "attemptStatusTransition",
+    );
 
     await AIInvestigationQueue.processRun({
       id: ObjectID.generate(),
       projectId,
       attemptCount: 0,
-      triggeredByAiInsightId: ObjectID.generate(),
+      runType: AIRunType.RemediationPlan,
+      triggeredByIncidentId: ObjectID.generate(),
+      triggeredByAutoRemediationSuggestionId: ObjectID.generate(),
     });
 
-    expect(executeTriage).toHaveBeenCalledWith(
-      expect.objectContaining({ attemptCount: 1 }),
+    expect(claim).toHaveBeenCalledWith(
+      expect.objectContaining({ fromStatus: AIRunStatus.Queued }),
     );
   });
 
-  test("interactive gating is unchanged when no insight runs exist: one global count query, no lane query, and the run claims", async () => {
+  test("interactive gating in a capped lane: one lane count query, no background query, and the run claims", async () => {
+    capIncidentLane(INCIDENT_CAP);
     const countBy: jest.SpyInstance = mockRunningCounts({
       total: 0,
       triage: 0,
@@ -338,7 +382,7 @@ describe("AIInvestigationQueue — lane priority (triage never starves RCA)", ()
     });
 
     expect(executeInvestigation).toHaveBeenCalled();
-    // The lane sub-cap query is only paid for by the lane it caps.
+    // The background sub-cap queries are only paid for by background work.
     expect(countBy).toHaveBeenCalledTimes(1);
     expect(countBy).toHaveBeenCalledWith(
       expect.objectContaining({

package/Tests/Server/Utils/AI/InvestigationEligibility.test.ts

@@ -988,6 +988,30 @@ describe("investigation diagnostic boundaries", () => {
     expect(cooldown.description).toContain("75-minute");
     expect(cooldown.title).not.toContain("covers");
   });
+  /*
+   * Alerts used to be investigated from the top two severity tiers only
+   * unless a floor was set, and the explanation said so. There is no floor
+   * by default now, so a severity skip only ever comes from a floor a
+   * project set — the explanation must not claim a default.
+   */
+  it.each(subjects)(
+    "explains a severity skip by the configured floor alone, never a default, for %o",
+    (subject: InvestigationSubject) => {
+      for (const source of [
+        InvestigationEligibility.reason(
+          "severity_below_threshold",
+          subject,
+          undefined,
+          { severityName: "Low", minimumSeverityName: "High" },
+        ),
+        InvestigationEligibility.reason("severity_below_threshold", subject),
+      ]) {
+        expect(source.description).not.toMatch(/by default/i);
+        expect(source.description).not.toMatch(/two highest/i);
+        expect(source.description).not.toMatch(/top two/i);
+      }
+    },
+  );
   it("does not let a failed caller diagnostic callback make enqueue throw", async () => {
     jest.spyOn(AIService, "getAutonomousDailyBudgetStatus").mockResolvedValue({
       exhausted: true,

package/Tests/Server/Utils/AI/InvestigationQueueRemediationExecution.test.ts

@@ -28,8 +28,9 @@ import { afterEach, beforeEach, describe, expect, test } from "@jest/globals";
  * - a RemediationExecution run without a suggestion subject can never
  *   execute — it finalizes as Error;
  * - RemediationExecution runs count in the BACKGROUND lane: when the
- *   background sub-cap is full the run stays Queued (no claim, no
- *   dispatch) for the poller / TTL to deal with.
+ *   project set a concurrency cap for the lane and the background sub-cap
+ *   is full, the run stays Queued (no claim, no dispatch) for the poller /
+ *   TTL to deal with. With no cap set (the default) nothing holds it back.
  */
 
 const RUN_ID: ObjectID = new ObjectID("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa");
@@ -61,7 +62,7 @@ function flushDetachedKick(): Promise<unknown> {
 describe("AIInvestigationQueue — RemediationExecution runs", () => {
   beforeEach(() => {
     mockBudgetOk();
-    // No per-project cap override => default of 3.
+    // No per-project cap override => no cap.
     jest
       .spyOn(ProjectService, "findOneById")
       .mockResolvedValue({ id: PROJECT_ID } as unknown as Project);
@@ -203,10 +204,14 @@ describe("AIInvestigationQueue — RemediationExecution runs", () => {
     );
   });
 
-  test("a RemediationExecution run stays Queued when the background lane is full", async () => {
+  test("a RemediationExecution run stays Queued when the background lane of a capped lane is full", async () => {
+    jest.spyOn(ProjectService, "findOneById").mockResolvedValue({
+      id: PROJECT_ID,
+      incidentAiMaxConcurrentInvestigations: 3,
+    } as unknown as Project);
     /*
      * Cap 3 => background sub-cap 2 (one slot reserved for interactive
-     * RCA). Global count 2 passes the cap; triage 1 + plans 1 = 2 fills
+     * RCA). Lane count 2 passes the cap; triage 1 + plans 1 = 2 fills
      * the background lane.
      */
     jest
@@ -226,6 +231,7 @@ describe("AIInvestigationQueue — RemediationExecution runs", () => {
       projectId: PROJECT_ID,
       attemptCount: 0,
       runType: AIRunType.RemediationExecution,
+      triggeredByIncidentId: INCIDENT_ID,
       triggeredByAutoRemediationSuggestionId: SUGGESTION_ID,
     });
 
@@ -233,4 +239,34 @@ describe("AIInvestigationQueue — RemediationExecution runs", () => {
     expect(claim).not.toHaveBeenCalled();
     expect(execute).not.toHaveBeenCalled();
   });
+
+  test("a RemediationExecution run in a lane with no cap is never held back by the background lane", async () => {
+    const count: jest.SpyInstance = jest
+      .spyOn(AIRunService, "countBy")
+      .mockResolvedValue(new PositiveNumber(25));
+    const claim: jest.SpyInstance = jest
+      .spyOn(AIRunService, "attemptStatusTransition")
+      .mockResolvedValue(1);
+    const execute: jest.SpyInstance = jest
+      .spyOn(RemediationExecutionRunner, "executeRemediation")
+      .mockResolvedValue(undefined);
+
+    await AIInvestigationQueue.processRun({
+      id: RUN_ID,
+      projectId: PROJECT_ID,
+      attemptCount: 0,
+      runType: AIRunType.RemediationExecution,
+      triggeredByIncidentId: INCIDENT_ID,
+      triggeredByAutoRemediationSuggestionId: SUGGESTION_ID,
+    });
+
+    expect(count).not.toHaveBeenCalled();
+    expect(claim).toHaveBeenCalledTimes(1);
+    expect(execute).toHaveBeenCalledWith({
+      aiRunId: RUN_ID,
+      projectId: PROJECT_ID,
+      suggestionId: SUGGESTION_ID,
+      attemptCount: 1,
+    });
+  });
 });

package/Tests/Server/Utils/AI/OpenPullRequestCap.test.ts

@@ -1,5 +1,4 @@
 import OpenPullRequestCap, {
-  DEFAULT_MAX_OPEN_FIX_PULL_REQUESTS,
   OpenPullRequestCapDecision,
 } from "../../../../Server/Utils/AI/CodeFix/OpenPullRequestCap";
 import AIAgentTaskPullRequestService from "../../../../Server/Services/AIAgentTaskPullRequestService";
@@ -14,32 +13,53 @@ import { describe, expect, test, afterEach } from "@jest/globals";
  * AI fix pull requests refuses the token — no token, no push, no PR. The
  * open counts come from AIAgentTaskPullRequest rows kept current by the
  * SyncPullRequestStates worker, so merging/closing a PR frees a slot.
+ *
+ * The cap is opt-in: a repository with no cap set has no limit on open AI
+ * fix pull requests (it used to default to 5).
  */
 
 const codeRepositoryId: ObjectID = ObjectID.generate();
 
 describe("OpenPullRequestCap.evaluate (pure decision)", () => {
-  test("unset cap uses the default — under it is allowed", () => {
-    const decision: OpenPullRequestCapDecision = OpenPullRequestCap.evaluate({
-      configuredLimit: null,
-      openCount: DEFAULT_MAX_OPEN_FIX_PULL_REQUESTS - 1,
-    });
+  test.each([[null], [undefined]])(
+    "an unset (%p) cap means no cap",
+    (configuredLimit: null | undefined) => {
+      expect(
+        OpenPullRequestCap.evaluate({ configuredLimit, openCount: 0 }),
+      ).toEqual({
+        allowed: true,
+        limit: null,
+        paused: false,
+        openCount: 0,
+      });
+    },
+  );
+
+  // The old default cap was 5 open AI pull requests per repository.
+  test.each([[4], [5], [6], [200]])(
+    "an unset cap allows another AI fix PR with %p already open",
+    (openCount: number) => {
+      const decision: OpenPullRequestCapDecision = OpenPullRequestCap.evaluate({
+        configuredLimit: null,
+        openCount,
+      });
 
-    expect(decision.allowed).toBe(true);
-    expect(decision.limit).toBe(DEFAULT_MAX_OPEN_FIX_PULL_REQUESTS);
-    expect(decision.paused).toBe(false);
-  });
+      expect(decision.allowed).toBe(true);
+      expect(decision.limit).toBeNull();
+      expect(decision.paused).toBe(false);
+    },
+  );
 
-  test("AT the cap is rejected — the cap is a maximum of open PRs, not a trigger", () => {
+  test("AT a set cap is rejected — the cap is a maximum of open PRs, not a trigger", () => {
     expect(
       OpenPullRequestCap.evaluate({
-        configuredLimit: undefined,
-        openCount: DEFAULT_MAX_OPEN_FIX_PULL_REQUESTS,
+        configuredLimit: 5,
+        openCount: 5,
       }).allowed,
     ).toBe(false);
   });
 
-  test("a custom cap overrides the default in both directions", () => {
+  test("a custom cap is enforced in both directions", () => {
     expect(
       OpenPullRequestCap.evaluate({ configuredLimit: 1, openCount: 0 }).allowed,
     ).toBe(true);
@@ -47,10 +67,8 @@ describe("OpenPullRequestCap.evaluate (pure decision)", () => {
       OpenPullRequestCap.evaluate({ configuredLimit: 1, openCount: 1 }).allowed,
     ).toBe(false);
     expect(
-      OpenPullRequestCap.evaluate({
-        configuredLimit: 20,
-        openCount: DEFAULT_MAX_OPEN_FIX_PULL_REQUESTS,
-      }).allowed,
+      OpenPullRequestCap.evaluate({ configuredLimit: 20, openCount: 19 })
+        .allowed,
     ).toBe(true);
   });
 
@@ -120,6 +138,29 @@ describe("OpenPullRequestCap.checkForRepository (IO wiring)", () => {
     expect(decision.allowed).toBe(false);
   });
 
+  test("a repository with no cap set short-circuits without the count query", async () => {
+    const countBy: jest.SpyInstance = jest
+      .spyOn(AIAgentTaskPullRequestService, "countBy")
+      .mockResolvedValue(new PositiveNumber(50));
+
+    for (const configuredLimit of [null, undefined]) {
+      const decision: OpenPullRequestCapDecision =
+        await OpenPullRequestCap.checkForRepository({
+          codeRepositoryId,
+          configuredLimit,
+        });
+
+      expect(decision).toEqual({
+        allowed: true,
+        limit: null,
+        paused: false,
+        openCount: 0,
+      });
+    }
+
+    expect(countBy).not.toHaveBeenCalled();
+  });
+
   test("a blocked repository (cap 0) short-circuits without the count query", async () => {
     const countBy: jest.SpyInstance = jest.spyOn(
       AIAgentTaskPullRequestService,
@@ -152,7 +193,7 @@ describe("OpenPullRequestCap.describeRejection", () => {
     expect(message).toMatch(/Max Open Fix Pull Requests/);
   });
 
-  test("at-cap rejection names the counts, the setting and the default", () => {
+  test("at-cap rejection names the counts, the setting and that clearing it lifts the cap", () => {
     const message: string = OpenPullRequestCap.describeRejection({
       decision: OpenPullRequestCap.evaluate({
         configuredLimit: 5,
@@ -163,8 +204,7 @@ describe("OpenPullRequestCap.describeRejection", () => {
 
     expect(message).toMatch(/5 open of a maximum 5/);
     expect(message).toMatch(/Max Open Fix Pull Requests/);
-    expect(message).toMatch(
-      new RegExp(String(DEFAULT_MAX_OPEN_FIX_PULL_REQUESTS)),
-    );
+    expect(message).toMatch(/clear it for no cap/);
+    expect(message).not.toMatch(/default/i);
   });
 });

package/Tests/Server/Utils/AI/RemediationCommandToolsLiveCluster.test.ts

@@ -353,29 +353,33 @@ describe("RemediationCommandToolkit re-reads the cluster's AI page before every
     expect(enqueueKubectl).toHaveBeenCalledTimes(1);
   });
 
-  it("refuses when the project's AI switches went off (the status folds them into readiness)", async () => {
+  it("refuses when the project's Enable AI switch went off (the status folds it into readiness)", async () => {
     liveStatus.mockResolvedValue(
       cluster({
+        isInvestigationReady: false,
         isRemediationReady: false,
         gaps: [
           {
-            code: "project_ai_command_execution_disabled",
-            title: "AI command execution is off for this project",
-            description: "",
-            nextStep: "",
-            blocks: "remediation",
+            code: "project_ai_disabled",
+            title: "AI is disabled for this project",
+            description: "OneUptime AI is switched off at the project level.",
+            nextStep: "Enable AI under Project Settings → AI Features.",
+            blocks: "both",
           },
         ],
       }),
     );
-    const toolkit: RemediationCommandToolkit = buildToolkit();
+    // A cluster round, which keeps changes a human's click could allow.
+    const toolkit: RemediationCommandToolkit = buildToolkit({
+      proposesRefusedCommands: true,
+    });
 
     const outcome: ToolCallOutcome = await execute(toolkit, kubectlArgs());
 
     expectNothingRanOrRecorded(toolkit, outcome);
-    expect(outcome.textForLlm).toContain(
-      "AI command execution is off for this project",
-    );
+    expect(outcome.textForLlm).toContain("AI is disabled for this project");
+    // No click can allow it while AI is off — nothing is kept for a proposal.
+    expect(toolkit.getCommandsNeedingApproval()).toHaveLength(0);
   });
 
   it("refuses — and keeps the change for a proposal — once the cluster was moved to Ask for approval mid-run", async () => {