@oneuptime/common 14.0.7 → 14.0.9

package/Tests/Server/Utils/AI/RemediationExecutionRunner.test.ts

@@ -75,9 +75,11 @@ import { afterEach, beforeEach, describe, expect, it } from "@jest/globals";
  * - a retried run whose earlier attempt already executed commands (the
  *   durable commandPlan record proves it) settles what happened and NEVER
  *   re-runs the agent loop;
- * - the project gates (enableAi, enableAutoRemediation, and the strict
- *   opt-in enableAiCommandExecution === true) and the rule gates
- *   (exists, enabled, aiComposesCommands) are re-checked at execution time;
+ * - the project gates (enableAi, enableAutoRemediation, and — for rule
+ *   rounds only — the strict opt-in enableAiCommandExecution === true) and
+ *   the rule gates (exists, enabled, aiComposesCommands) are re-checked at
+ *   execution time; a cluster round (a cluster, no rule) needs only the
+ *   kill switches and its cluster's own readiness;
  * - FullAuto requires the rule to say FullAuto AND a non-empty operator
  *   allowlist AND breaker headroom — every other combination (including a
  *   failing breaker query) fails safe to Suggest, whose tools cannot
@@ -524,6 +526,64 @@ describe("RemediationExecutionRunner.executeRemediation", () => {
       expect(executeRun).not.toHaveBeenCalled();
       expectSettledNoneApplicable("AI command execution is not enabled");
     });
+
+    it("still refuses a RULE round with the opt-in off even when its signal has a remediation-ready cluster — nothing about kubectl exempts a rule", async () => {
+      mockSuggestion();
+      mockProject({ enableAiCommandExecution: false });
+      const subjectStatuses: jest.SpyInstance = jest
+        .spyOn(KubernetesClusterAiAccessService, "getStatusesForSubject")
+        .mockResolvedValue([clusterStatus()]);
+      const clusterStatusRead: jest.SpyInstance = jest.spyOn(
+        KubernetesClusterAiAccessService,
+        "getStatusForCluster",
+      );
+      const ruleRead: jest.SpyInstance = jest.spyOn(
+        AutoRemediationRuleService,
+        "findOneById",
+      );
+      const executeRun: jest.SpyInstance = jest
+        .spyOn(AIInvestigationEngine, "executeRun")
+        .mockResolvedValue(undefined as never);
+
+      await run();
+
+      expect(executeRun).not.toHaveBeenCalled();
+      // Refused on the project switch, before the rule or any cluster is read.
+      expect(ruleRead).not.toHaveBeenCalled();
+      expect(subjectStatuses).not.toHaveBeenCalled();
+      expect(clusterStatusRead).not.toHaveBeenCalled();
+      expectSettledNoneApplicable("AI command execution is not enabled");
+    });
+
+    it("gates a row that names a rule AND a cluster as a rule round — the exemption fails closed", async () => {
+      mockSuggestion({ kubernetesClusterId: CLUSTER_ID });
+      mockProject({ enableAiCommandExecution: undefined });
+      const clusterStatusRead: jest.SpyInstance = jest.spyOn(
+        KubernetesClusterAiAccessService,
+        "getStatusForCluster",
+      );
+      const executeRun: jest.SpyInstance = jest
+        .spyOn(AIInvestigationEngine, "executeRun")
+        .mockResolvedValue(undefined as never);
+
+      await run();
+
+      expect(executeRun).not.toHaveBeenCalled();
+      expect(clusterStatusRead).not.toHaveBeenCalled();
+      expectSettledNoneApplicable("AI command execution is not enabled");
+    });
+
+    it("sends people to Project Settings → AI Features, the page every install shows", async () => {
+      mockSuggestion();
+      mockProject({ enableAiCommandExecution: false });
+      jest
+        .spyOn(AIInvestigationEngine, "executeRun")
+        .mockResolvedValue(undefined as never);
+
+      await run();
+
+      expectSettledNoneApplicable("(Project Settings → AI Features)");
+    });
   });
 
   describe("rule gates re-checked at execution time", () => {
@@ -960,6 +1020,8 @@ function clusterStatus(
       posture: { inCluster: true, allowWrites: true },
     },
     accessMethod: "in_cluster",
+    aiAgent: null,
+    automaticInvestigation: { incidents: false, alerts: false },
     kubectlAllowlist: [],
     isInvestigationEnabled: true,
     isInvestigationReady: true,
@@ -2546,6 +2608,193 @@ describe("RemediationExecutionRunner.executeRemediation — cluster rounds", ()
     });
   });
 
+  /*
+   * The project's "Enable AI command execution" opt-in gates rule rounds.
+   * A cluster round's consent is the cluster's own: its Fixes mode, and the
+   * chart's write RBAC on the in-cluster agent. The kill switches still
+   * stop it, and a cluster reached through an advanced Runner keeps needing
+   * the opt-in — through its status gap, which the round re-reads.
+   */
+  describe("the project's AI command execution opt-in and cluster rounds", () => {
+    it("runs an Ask-for-approval round when the opt-in was never set", async () => {
+      mockProject({ enableAiCommandExecution: undefined });
+      mockSuggestionHonouringSelect(
+        clusterRow({ executionMode: AutoRemediationExecutionMode.Suggest }),
+      );
+      (
+        KubernetesClusterAiAccessService.getStatusForCluster as unknown as jest.SpyInstance
+      ).mockResolvedValue(
+        clusterStatus({
+          remediationMode: KubernetesAiRemediationMode.RequireApproval,
+        }),
+      );
+      const request: { get: () => InvestigationRequest } = captureRequest();
+
+      await run();
+
+      expect(toolNames(request.get())).toContain(
+        "propose_remediation_commands",
+      );
+      // Nothing settled it early: the run went to the engine.
+      expect(suggestionCas).not.toHaveBeenCalled();
+    });
+
+    it("runs a Bypass-approval round FullAuto with the opt-in explicitly off", async () => {
+      mockProject({ enableAiCommandExecution: false });
+      mockSuggestionHonouringSelect(clusterRow());
+      const request: { get: () => InvestigationRequest } = captureRequest();
+
+      await run();
+
+      expect(toolNames(request.get())).toContain("execute_remediation_command");
+      expect(suggestionCas).not.toHaveBeenCalled();
+    });
+
+    it.each([
+      ["Enable AI", { enableAi: false }],
+      ["Enable auto-remediation", { enableAutoRemediation: false }],
+    ])(
+      "is still stopped by %s being off, before its cluster is read",
+      async (_label: string, overrides: Record<string, unknown>) => {
+        mockProject({ enableAiCommandExecution: false, ...overrides });
+        mockSuggestionHonouringSelect(clusterRow());
+        const statusRead: jest.SpyInstance =
+          KubernetesClusterAiAccessService.getStatusForCluster as unknown as jest.SpyInstance;
+        const executeRun: jest.SpyInstance = jest
+          .spyOn(AIInvestigationEngine, "executeRun")
+          .mockResolvedValue(undefined as never);
+
+        await run();
+
+        expect(executeRun).not.toHaveBeenCalled();
+        expect(statusRead).not.toHaveBeenCalled();
+        const set: ReturnType<typeof settleSet> = settleSet();
+        expect(set.status).toBe(AutoRemediationSuggestionStatus.NoneApplicable);
+        expect(set.rationaleMarkdown).toContain(
+          "AI or auto-remediation was disabled for this project",
+        );
+        expect(set.rationaleMarkdown).not.toContain(
+          "AI command execution is not enabled",
+        );
+      },
+    );
+
+    it("a cluster reached through an advanced Runner is still held back by the opt-in — via its status gap", async () => {
+      mockProject({ enableAiCommandExecution: false });
+      mockSuggestionHonouringSelect(clusterRow());
+      (
+        KubernetesClusterAiAccessService.getStatusForCluster as unknown as jest.SpyInstance
+      ).mockResolvedValue(
+        clusterStatus({
+          runner: {
+            id: CLUSTER_RUNNER_ID.toString(),
+            name: "ops-runner",
+            kind: "runner",
+            isOnline: true,
+            canRunAiCommands: true,
+          },
+          accessMethod: "credential",
+          credentialId: "99999999-9999-4999-8999-999999999999",
+          isRemediationReady: false,
+          gaps: [
+            {
+              code: "project_ai_command_execution_disabled",
+              title: "AI command execution is off for this project",
+              description:
+                "This cluster is reached through a Runner with a Kubernetes credential.",
+              nextStep:
+                'Turn on "Enable AI command execution" under Project Settings → AI Features.',
+              blocks: "remediation",
+            },
+          ],
+        }),
+      );
+      const executeRun: jest.SpyInstance = jest
+        .spyOn(AIInvestigationEngine, "executeRun")
+        .mockResolvedValue(undefined as never);
+
+      await run();
+
+      expect(executeRun).not.toHaveBeenCalled();
+      const set: ReturnType<typeof settleSet> = settleSet();
+      expect(set.status).toBe(AutoRemediationSuggestionStatus.NoneApplicable);
+      expect(set.rationaleMarkdown).toContain(
+        "AI command execution is off for this project",
+      );
+      expect(set.rationaleMarkdown).toContain(
+        "Review the cluster's AI agent page (AI → Agent).",
+      );
+      expect(set.rationaleMarkdown).not.toContain("cluster's AI page");
+    });
+
+    it("negative control: the same advanced-Runner cluster without the gap runs — the round follows the status, not the target kind", async () => {
+      mockProject({ enableAiCommandExecution: true });
+      mockSuggestionHonouringSelect(clusterRow());
+      (
+        KubernetesClusterAiAccessService.getStatusForCluster as unknown as jest.SpyInstance
+      ).mockResolvedValue(
+        clusterStatus({
+          runner: {
+            id: CLUSTER_RUNNER_ID.toString(),
+            name: "ops-runner",
+            kind: "runner",
+            isOnline: true,
+            canRunAiCommands: true,
+            posture: { inCluster: false, allowWrites: true },
+          },
+          accessMethod: "credential",
+          credentialId: "99999999-9999-4999-8999-999999999999",
+        }),
+      );
+      const request: { get: () => InvestigationRequest } = captureRequest();
+
+      await run();
+
+      expect(toolNames(request.get())).toContain("execute_remediation_command");
+    });
+
+    it("a round whose cluster was deleted is closed as such with the opt-in off — never blamed on an opt-in it did not need", async () => {
+      mockProject({ enableAiCommandExecution: false });
+      mockSuggestionHonouringSelect(
+        clusterRow({ kubernetesClusterId: undefined }),
+      );
+      jest
+        .spyOn(AIInvestigationEngine, "executeRun")
+        .mockResolvedValue(undefined as never);
+
+      await run();
+
+      const set: ReturnType<typeof settleSet> = settleSet();
+      expect(set.status).toBe(AutoRemediationSuggestionStatus.NoneApplicable);
+      expect(set.rationaleMarkdown).toContain(
+        'The Kubernetes cluster "prod-us" was deleted before OneUptime AI could remediate it',
+      );
+      expect(set.rationaleMarkdown).not.toContain("AI command execution");
+    });
+
+    it("a round whose cluster was deleted is closed as such even with AI off — nothing is left to gate", async () => {
+      mockProject({ enableAi: false });
+      mockSuggestionHonouringSelect(
+        clusterRow({ kubernetesClusterId: undefined }),
+      );
+      const projectRead: jest.SpyInstance = jest.spyOn(
+        ProjectService,
+        "findOneById",
+      );
+      jest
+        .spyOn(AIInvestigationEngine, "executeRun")
+        .mockResolvedValue(undefined as never);
+
+      await run();
+
+      const set: ReturnType<typeof settleSet> = settleSet();
+      expect(set.rationaleMarkdown).toContain(
+        'The Kubernetes cluster "prod-us" was deleted',
+      );
+      expect(projectRead).not.toHaveBeenCalled();
+    });
+  });
+
   describe("a round whose cluster was deleted while it waited", () => {
     it("closes with an accurate message about the cluster — never a rule that never existed", async () => {
       mockSuggestionHonouringSelect(

package/Tests/Server/Utils/AI/RemediationExecutionRunnerClusterMode.test.ts

@@ -78,6 +78,8 @@ function cluster(
       posture: { inCluster: true, allowWrites: true },
     },
     accessMethod: "in_cluster",
+    aiAgent: null,
+    automaticInvestigation: { incidents: false, alerts: false },
     kubectlAllowlist: [],
     isInvestigationEnabled: true,
     isInvestigationReady: true,

package/Tests/Server/Utils/AI/RemediationExecutionRunnerProjectGates.test.ts

@@ -0,0 +1,240 @@
+import RemediationExecutionRunner, {
+  isClusterRemediationRound,
+} from "../../../../Server/Utils/AI/Remediation/RemediationExecutionRunner";
+import ProjectService from "../../../../Server/Services/ProjectService";
+import Project from "../../../../Models/DatabaseModels/Project";
+import ObjectID from "../../../../Types/ObjectID";
+import { afterEach, describe, expect, it } from "@jest/globals";
+
+/*
+ * Contract under test — which project switches a remediation round must
+ * still pass when it runs (RemediationExecutionRunner.checkProjectGates),
+ * and which rounds count as cluster rounds (isClusterRemediationRound):
+ *
+ * - Enable AI and Enable auto-remediation are kill switches for EVERY
+ *   round, cluster or rule (read as "off only when === false");
+ * - the project's "Enable AI command execution" opt-in (=== true) gates
+ *   rule rounds only. A cluster round's consent is the cluster's own Fixes
+ *   mode (and the chart's write RBAC on the in-cluster agent); a cluster
+ *   reached through an advanced Runner is held back by its status gap, not
+ *   here;
+ * - a cluster round is a suggestion that names a cluster and NO rule — a
+ *   row naming a rule is a rule round whatever else it carries, so the
+ *   exemption can never be reached from the rule lane;
+ * - the refusals send people to the always-visible Project Settings → AI
+ *   Features page, never to AI Credits (billing-only).
+ */
+
+const PROJECT_ID: ObjectID = new ObjectID(
+  "22222222-2222-4222-8222-222222222222",
+);
+const CLUSTER_ID: ObjectID = new ObjectID(
+  "44444444-4444-4444-8444-444444444444",
+);
+const RULE_ID: ObjectID = new ObjectID("11111111-1111-4111-8111-111111111111");
+
+const KILL_SWITCH_MESSAGE: string =
+  "AI or auto-remediation was disabled for this project before the run started";
+const OPT_IN_MESSAGE: string =
+  "AI command execution is not enabled for this project";
+
+function mockProject(
+  project: Record<string, unknown> | null,
+): jest.SpyInstance {
+  return jest
+    .spyOn(ProjectService, "findOneById")
+    .mockResolvedValue(project as unknown as Project);
+}
+
+function allOn(
+  overrides: Record<string, unknown> = {},
+): Record<string, unknown> {
+  return {
+    id: PROJECT_ID,
+    enableAi: true,
+    enableAutoRemediation: true,
+    enableAiCommandExecution: true,
+    ...overrides,
+  };
+}
+
+async function gates(isClusterRound: boolean): Promise<string | null> {
+  return RemediationExecutionRunner.checkProjectGates({
+    projectId: PROJECT_ID,
+    isClusterRound,
+  });
+}
+
+afterEach(() => {
+  jest.restoreAllMocks();
+});
+
+describe("isClusterRemediationRound", () => {
+  it("is a cluster round when the suggestion names a cluster and no rule", () => {
+    expect(
+      isClusterRemediationRound({
+        kubernetesClusterId: CLUSTER_ID,
+        autoRemediationRuleId: undefined,
+      }),
+    ).toBe(true);
+  });
+
+  it("is a rule round when the suggestion names a rule and no cluster", () => {
+    expect(
+      isClusterRemediationRound({
+        kubernetesClusterId: undefined,
+        autoRemediationRuleId: RULE_ID,
+      }),
+    ).toBe(false);
+  });
+
+  it("is a rule round when the suggestion names BOTH — a rule id always wins, so the exemption fails closed", () => {
+    expect(
+      isClusterRemediationRound({
+        kubernetesClusterId: CLUSTER_ID,
+        autoRemediationRuleId: RULE_ID,
+      }),
+    ).toBe(false);
+  });
+
+  it("is not a cluster round when the suggestion names neither (a deleted cluster, or a deleted rule)", () => {
+    expect(isClusterRemediationRound({})).toBe(false);
+    expect(
+      isClusterRemediationRound({
+        kubernetesClusterId: undefined,
+        autoRemediationRuleId: undefined,
+      }),
+    ).toBe(false);
+  });
+});
+
+describe("RemediationExecutionRunner.checkProjectGates", () => {
+  it("reads the project as root, by id, with exactly the three switches", async () => {
+    const find: jest.SpyInstance = mockProject(allOn());
+
+    await gates(false);
+
+    expect(find).toHaveBeenCalledTimes(1);
+    expect(find).toHaveBeenCalledWith({
+      id: PROJECT_ID,
+      select: {
+        enableAi: true,
+        enableAutoRemediation: true,
+        enableAiCommandExecution: true,
+      },
+      props: { isRoot: true },
+    });
+  });
+
+  describe.each([
+    ["a rule round", false],
+    ["a cluster round", true],
+  ])("the kill switches stop %s", (_label: string, isClusterRound: boolean) => {
+    it("when the project row is gone", async () => {
+      mockProject(null);
+
+      expect(await gates(isClusterRound)).toContain(KILL_SWITCH_MESSAGE);
+    });
+
+    it("when Enable AI is off", async () => {
+      mockProject(allOn({ enableAi: false }));
+
+      expect(await gates(isClusterRound)).toContain(KILL_SWITCH_MESSAGE);
+    });
+
+    it("when Enable auto-remediation is off", async () => {
+      mockProject(allOn({ enableAutoRemediation: false }));
+
+      expect(await gates(isClusterRound)).toContain(KILL_SWITCH_MESSAGE);
+    });
+
+    it("but not when the kill switches were never set — they are on unless explicitly false", async () => {
+      mockProject({
+        id: PROJECT_ID,
+        enableAi: undefined,
+        enableAutoRemediation: null,
+        enableAiCommandExecution: true,
+      });
+
+      expect(await gates(isClusterRound)).toBeNull();
+    });
+
+    it("and a kill switch wins over the opt-in: the message names the kill switch, not command execution", async () => {
+      mockProject(allOn({ enableAi: false, enableAiCommandExecution: false }));
+
+      const failure: string | null = await gates(isClusterRound);
+
+      expect(failure).toContain(KILL_SWITCH_MESSAGE);
+      expect(failure).not.toContain(OPT_IN_MESSAGE);
+    });
+  });
+
+  describe("a rule round needs the AI command execution opt-in", () => {
+    it.each([
+      ["false", false],
+      ["never set (undefined)", undefined],
+      ["null", null],
+    ])(
+      "is refused when the opt-in is %s — only an explicit true opts in",
+      async (_label: string, value: boolean | null | undefined) => {
+        mockProject(allOn({ enableAiCommandExecution: value }));
+
+        expect(await gates(false)).toContain(OPT_IN_MESSAGE);
+      },
+    );
+
+    it("passes when the opt-in is explicitly true", async () => {
+      mockProject(allOn());
+
+      expect(await gates(false)).toBeNull();
+    });
+  });
+
+  describe("a cluster round does not need the AI command execution opt-in", () => {
+    it.each([
+      ["false", false],
+      ["never set (undefined)", undefined],
+      ["null", null],
+      ["true", true],
+    ])(
+      "passes when the opt-in is %s",
+      async (_label: string, value: boolean | null | undefined) => {
+        mockProject(allOn({ enableAiCommandExecution: value }));
+
+        expect(await gates(true)).toBeNull();
+      },
+    );
+  });
+
+  describe("the refusals point at the page that owns the switches", () => {
+    it.each([
+      ["a kill switch", allOn({ enableAi: false }), true],
+      [
+        "a kill switch (rule round)",
+        allOn({ enableAutoRemediation: false }),
+        false,
+      ],
+      [
+        "the command execution opt-in",
+        allOn({ enableAiCommandExecution: false }),
+        false,
+      ],
+    ])(
+      "%s names Project Settings → AI Features and says nothing ran",
+      async (
+        _label: string,
+        project: Record<string, unknown>,
+        isClusterRound: boolean,
+      ) => {
+        mockProject(project);
+
+        const failure: string = (await gates(isClusterRound)) || "";
+
+        expect(failure).toContain("Project Settings → AI Features");
+        expect(failure).not.toContain("AI Credits");
+        expect(failure).not.toMatch(/Project Settings → AI\)/);
+        expect(failure).toContain("nothing was run or proposed");
+      },
+    );
+  });
+});

package/Tests/Server/Utils/AI/RemediationKubectlCopyParity.test.ts

@@ -355,6 +355,8 @@ describe("the tools the model reads use the shared summaries", () => {
         canRunAiCommands: true,
       },
       accessMethod: "in_cluster",
+      aiAgent: null,
+      automaticInvestigation: { incidents: false, alerts: false },
       kubectlAllowlist: ["kubectl set image deployment/web * -n web"],
       isInvestigationEnabled: true,
       isInvestigationReady: true,

package/Tests/Server/Utils/AutoRemediation/CommandPlanExecutor.test.ts

@@ -1421,6 +1421,8 @@ function clusterStatus(
       posture: { inCluster: true, allowWrites: true },
     },
     accessMethod: "in_cluster",
+    aiAgent: null,
+    automaticInvestigation: { incidents: false, alerts: false },
     kubectlAllowlist: [],
     isInvestigationEnabled: true,
     isInvestigationReady: true,

package/Tests/Server/Utils/AutoRemediation/CommandPlanExecutorUnknownRun.test.ts

@@ -133,6 +133,8 @@ function clusterStatus(): KubernetesClusterAiAccessStatus {
       posture: { inCluster: true, allowWrites: true },
     },
     accessMethod: "in_cluster",
+    aiAgent: null,
+    automaticInvestigation: { incidents: false, alerts: false },
     kubectlAllowlist: [],
     isInvestigationEnabled: true,
     isInvestigationReady: true,

package/Tests/Server/Utils/AutoRemediation/CommandPlanVerification.test.ts

@@ -987,6 +987,8 @@ describe("RemediationVerifier.resumeInterruptedRollbacks — the follow-up after
           canRunAiCommands: true,
         },
         accessMethod: "in_cluster",
+        aiAgent: null,
+        automaticInvestigation: { incidents: false, alerts: false },
         kubectlAllowlist: [],
         isInvestigationEnabled: true,
         isInvestigationReady: true,