@oneuptime/common 14.0.8 → 14.0.10

package/Tests/Server/Services/RunnerJobKubectlWriteScope.test.ts

@@ -2,6 +2,7 @@ import RunnerJobService, {
   Service as RunnerJobServiceClass,
 } from "../../../Server/Services/RunnerJobService";
 import AIRunService from "../../../Server/Services/AIRunService";
+import KubernetesAiAgentService from "../../../Server/Services/KubernetesAiAgentService";
 import KubernetesClusterService from "../../../Server/Services/KubernetesClusterService";
 import RunbookCredentialService from "../../../Server/Services/RunbookCredentialService";
 import RunnerService from "../../../Server/Services/RunnerService";
@@ -94,6 +95,10 @@ describe("RunnerJobService.enqueueAiKubectlCommand refuses writes the Runner's p
     jest
       .spyOn(RunnerJobService, "countBy")
       .mockResolvedValue(new PositiveNumber(0));
+    // No Kubernetes AI agent: the bound Runner is the access target.
+    jest
+      .spyOn(KubernetesAiAgentService, "findForCluster")
+      .mockResolvedValue(null);
     jest.spyOn(AIRunService, "findOneBy").mockResolvedValue({
       runType: AIRunType.RemediationExecution,
     } as unknown as AIRun);
@@ -471,7 +476,7 @@ describe("RunnerJobService.getRunnerWriteScopeRefusal words the Runner's rule fo
     expect(message).toContain('"web": aiAccess.remediation.namespaces');
     expect(message).toContain("ONEUPTIME_KUBECTL_WRITE_NAMESPACES");
     expect(message).toContain(
-      'To let OneUptime AI change "staging", add it to aiAccess.remediation.namespaces',
+      'To let OneUptime AI change "staging", upgrade the Kubernetes agent chart to the Kubernetes AI agent (it replaces the in-cluster Runner) and add it to aiAgent.remediation.namespaces',
     );
     expect(message).not.toContain("-n <namespace>");
   });
@@ -710,7 +715,7 @@ describe("RunnerJobService names where each kind of Runner sets its write scope"
         "its Kubernetes agent was installed with aiAccess.remediation.nodeOperations=false (ONEUPTIME_KUBECTL_ALLOW_NODE_OPERATIONS)",
       );
       expect(message).toContain(
-        "To let OneUptime AI change nodes, upgrade the agent with --set aiAccess.remediation.nodeOperations=true.",
+        "To let OneUptime AI change nodes, upgrade the Kubernetes agent chart to the Kubernetes AI agent (it replaces the in-cluster Runner) with --set aiAgent.remediation.nodeOperations=true.",
       );
       expect(message).not.toContain("on that Runner's host");
     });
@@ -731,7 +736,7 @@ describe("RunnerJobService names where each kind of Runner sets its write scope"
         '("prod": aiAccess.remediation.namespaces on the Kubernetes agent chart, ONEUPTIME_KUBECTL_WRITE_NAMESPACES)',
       );
       expect(outside).toContain(
-        'To let OneUptime AI change "payments", add it to aiAccess.remediation.namespaces and upgrade the agent.',
+        'To let OneUptime AI change "payments", upgrade the Kubernetes agent chart to the Kubernetes AI agent (it replaces the in-cluster Runner) and add it to aiAgent.remediation.namespaces.',
       );
       expect(clusterScoped).toContain(
         "aiAccess.remediation.namespaces on the Kubernetes agent chart",
@@ -741,6 +746,164 @@ describe("RunnerJobService names where each kind of Runner sets its write scope"
         expect(message).not.toContain("on that Runner's host");
       }
     });
+
+    it("says the Kubernetes AI agent replaces it, and never says 'this Runner'", () => {
+      for (const command of [
+        "kubectl cordon n1",
+        "kubectl rollout restart deployment/pay -n payments",
+      ]) {
+        const message: string = refusalFor(command, AGENT_RUNNER, false);
+
+        expect(message).toContain("Kubernetes AI agent");
+        expect(message).toContain("the cluster's Runner");
+        expect(message).not.toContain("this Runner");
+      }
+    });
+
+    it("absent executorKind reads as the in-cluster Runner, the same as naming it", () => {
+      for (const command of [
+        "kubectl cordon n1",
+        "kubectl rollout restart deployment/pay -n payments",
+        "kubectl rollout restart deployment/web",
+      ]) {
+        const policy: ReturnType<typeof KubectlPolicy.evaluateCommand> =
+          KubectlPolicy.evaluateCommand(command);
+
+        expect(
+          RunnerJobServiceClass.getRunnerWriteScopeRefusal({
+            policy,
+            posture: AGENT_RUNNER,
+            usesCredential: false,
+          }),
+        ).toBe(
+          RunnerJobServiceClass.getRunnerWriteScopeRefusal({
+            policy,
+            posture: AGENT_RUNNER,
+            usesCredential: false,
+            executorKind: "legacy_runner",
+          }),
+        );
+      }
+    });
+  });
+
+  /*
+   * The Kubernetes AI agent: its chart values are aiAgent.remediation.*,
+   * and a refusal names the agent, never a Runner.
+   */
+  describe("the Kubernetes AI agent", () => {
+    function agentRefusal(command: string): string {
+      const message: string | null =
+        RunnerJobServiceClass.getRunnerWriteScopeRefusal({
+          policy: KubectlPolicy.evaluateCommand(command),
+          posture: AGENT_RUNNER,
+          usesCredential: false,
+          executorKind: "ai_agent",
+        });
+
+      expect({ command, refused: message !== null }).toEqual({
+        command,
+        refused: true,
+      });
+
+      return message!;
+    }
+
+    it("node operations: names aiAgent.remediation.nodeOperations and the chart upgrade", () => {
+      const message: string = agentRefusal("kubectl cordon n1");
+
+      expect(message).toContain(
+        "the cluster's Kubernetes AI agent does not allow node operations",
+      );
+      expect(message).toContain(
+        "it was installed with aiAgent.remediation.nodeOperations=false (ONEUPTIME_KUBECTL_ALLOW_NODE_OPERATIONS)",
+      );
+      expect(message).toContain(
+        "To let OneUptime AI change nodes, upgrade the Kubernetes agent chart with --set aiAgent.remediation.nodeOperations=true.",
+      );
+      expect(message).toContain("so the AI agent would refuse it");
+    });
+
+    it("outside its scope: names aiAgent.remediation.namespaces", () => {
+      const message: string = agentRefusal(
+        "kubectl rollout restart deployment/pay -n payments",
+      );
+
+      expect(message).toContain(
+        '("prod": aiAgent.remediation.namespaces on the Kubernetes agent chart, ONEUPTIME_KUBECTL_WRITE_NAMESPACES)',
+      );
+      expect(message).toContain(
+        'To let OneUptime AI change "payments", add it to aiAgent.remediation.namespaces and upgrade the Kubernetes agent chart.',
+      );
+    });
+
+    it("its own namespace (a write with no -n lands there) names the agent", () => {
+      const message: string = agentRefusal(
+        "kubectl rollout restart deployment/web",
+      );
+
+      expect(message).toContain('namespace "oneuptime-agent"');
+      expect(message).toContain(
+        "the namespace the cluster's Kubernetes AI agent itself runs in",
+      );
+    });
+
+    it("never names aiAccess, a Runner's host or 'the cluster's Runner'", () => {
+      for (const command of [
+        "kubectl cordon n1",
+        "kubectl rollout restart deployment/pay -n payments",
+        "kubectl rollout restart deployment/web",
+        "kubectl label namespace staging team=a -n prod",
+        "kubectl annotate ingressclass nginx note=x -n prod",
+      ]) {
+        const message: string = agentRefusal(command);
+
+        for (const word of [
+          "aiAccess",
+          "that Runner's host",
+          "the cluster's Runner",
+          "in-cluster Runner",
+        ]) {
+          expect({ command, word, said: message.includes(word) }).toEqual({
+            command,
+            word,
+            said: false,
+          });
+        }
+      }
+    });
+
+    it("refuses exactly what the in-cluster Runner refuses: only the words differ", () => {
+      for (const command of [
+        "kubectl cordon n1",
+        "kubectl rollout restart deployment/pay -n payments",
+        "kubectl rollout restart deployment/pay -n prod",
+        "kubectl rollout restart deployment/web",
+        "kubectl get pods -n payments",
+      ]) {
+        const policy: ReturnType<typeof KubectlPolicy.evaluateCommand> =
+          KubectlPolicy.evaluateCommand(command);
+
+        expect({
+          command,
+          refused:
+            RunnerJobServiceClass.getRunnerWriteScopeRefusal({
+              policy,
+              posture: AGENT_RUNNER,
+              usesCredential: false,
+              executorKind: "ai_agent",
+            }) !== null,
+        }).toEqual({
+          command,
+          refused:
+            RunnerJobServiceClass.getRunnerWriteScopeRefusal({
+              policy,
+              posture: AGENT_RUNNER,
+              usesCredential: false,
+            }) !== null,
+        });
+      }
+    });
   });
 });
 
@@ -770,6 +933,10 @@ describe("RunnerJobService.enqueueAiKubectlCommand refuses a credential Runner's
     jest
       .spyOn(RunnerJobService, "countBy")
       .mockResolvedValue(new PositiveNumber(0));
+    // No Kubernetes AI agent: the bound Runner is the access target.
+    jest
+      .spyOn(KubernetesAiAgentService, "findForCluster")
+      .mockResolvedValue(null);
     jest.spyOn(AIRunService, "findOneBy").mockResolvedValue({
       runType: AIRunType.RemediationExecution,
     } as unknown as AIRun);

package/Tests/Server/Services/RunnerJobResourceCommandLane.test.ts

@@ -0,0 +1,211 @@
+import RunnerJobService, {
+  MAX_AI_COMMAND_JOBS_PER_PROJECT_PER_HOUR,
+} from "../../../Server/Services/RunnerJobService";
+import RunnerService from "../../../Server/Services/RunnerService";
+import RunnerJob from "../../../Models/DatabaseModels/RunnerJob";
+import RunbookStepType, {
+  AI_AGENT_EXECUTED_STEP_TYPES,
+  RUNNER_EXECUTED_STEP_TYPES,
+} from "../../../Types/Runbook/RunbookStepType";
+import { AI_COMMAND_STEP_TYPES } from "../../../Types/AutoRemediation/AiRemediationCommandPlan";
+import RunnerJobOrigin from "../../../Types/Runbook/RunnerJobOrigin";
+import BadDataException from "../../../Types/Exception/BadDataException";
+import ObjectID from "../../../Types/ObjectID";
+import PositiveNumber from "../../../Types/PositiveNumber";
+import logger from "../../../Server/Utils/Logger";
+import {
+  fakeRunnerJobCountBy,
+  fakeRunnerJobRows,
+} from "../TestingUtils/Services/FakeRunnerJobCount";
+import { afterEach, beforeEach, describe, expect, it } from "@jest/globals";
+
+/*
+ * Contract under test — a ResourceCommand job never reaches a Runner.
+ *
+ * ResourceCommand is the step type of a command for an infrastructure
+ * resource's AI agent (a ResourceAiAgent row). It is an AI command step type
+ * (AI_COMMAND_STEP_TYPES), so the generic AI-command lane must refuse it
+ * explicitly: that lane writes targetAgentId — a Runner — and a Runner
+ * must never be handed a resource command. The runbook lane (enqueue)
+ * refuses it because it is not a runner-executed type. Both refusals happen
+ * before any row is written.
+ */
+
+const PROJECT_ID: ObjectID = new ObjectID(
+  "22222222-2222-4222-8222-222222222222",
+);
+const RUNNER_ID: ObjectID = new ObjectID(
+  "44444444-4444-4444-8444-444444444444",
+);
+
+describe("ResourceCommand never enters a Runner lane", () => {
+  let create: jest.SpyInstance;
+  let runnerLookup: jest.SpyInstance;
+  let countBy: jest.SpyInstance;
+
+  beforeEach(() => {
+    // @CaptureSpan logs the expected throws at error level.
+    jest.spyOn(logger, "error").mockImplementation((): void => {
+      return undefined;
+    });
+    create = jest
+      .spyOn(RunnerJobService, "create")
+      .mockImplementation(async (data: unknown): Promise<RunnerJob> => {
+        return (data as { data: RunnerJob }).data;
+      });
+    runnerLookup = jest.spyOn(RunnerService, "findOneBy").mockResolvedValue({
+      id: RUNNER_ID,
+      _id: RUNNER_ID.toString(),
+      projectId: PROJECT_ID,
+      name: "office-runner",
+    } as never);
+    countBy = jest
+      .spyOn(RunnerJobService, "countBy")
+      .mockResolvedValue(new PositiveNumber(0));
+  });
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  it("is an AI command step type but not a Runner-executed one", () => {
+    expect(AI_COMMAND_STEP_TYPES).toContain(RunbookStepType.ResourceCommand);
+    expect(AI_AGENT_EXECUTED_STEP_TYPES).toContain(
+      RunbookStepType.ResourceCommand,
+    );
+    expect(RUNNER_EXECUTED_STEP_TYPES).not.toContain(
+      RunbookStepType.ResourceCommand,
+    );
+  });
+
+  it("enqueue (the runbook lane) refuses it before writing a row", async () => {
+    await expect(
+      RunnerJobService.enqueue({
+        projectId: PROJECT_ID,
+        runbookExecutionId: new ObjectID(
+          "33333333-3333-4333-8333-333333333333",
+        ),
+        stepId: "step-1",
+        stepType: RunbookStepType.ResourceCommand,
+        targetAgentId: RUNNER_ID,
+        script: "",
+        payload: { program: "docker", args: ["ps"] },
+        timeoutInMs: 30000,
+      }),
+    ).rejects.toThrow(
+      new BadDataException(
+        'Runner does not execute step type "ResourceCommand".',
+      ),
+    );
+
+    expect(create).not.toHaveBeenCalled();
+  });
+
+  it("enqueueAiCommand (the Runner AI-command lane) refuses it before looking anything up", async () => {
+    let caught: unknown = null;
+
+    try {
+      await RunnerJobService.enqueueAiCommand({
+        projectId: PROJECT_ID,
+        aiRunId: new ObjectID("88888888-8888-4888-8888-888888888888"),
+        autoRemediationSuggestionId: new ObjectID(
+          "77777777-7777-4777-8777-777777777777",
+        ),
+        stepId: "ai-approved-1",
+        stepType: RunbookStepType.ResourceCommand,
+        targetAgentId: RUNNER_ID,
+        command: "docker restart web",
+        timeoutInMs: 30000,
+      });
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(BadDataException);
+    expect((caught as Error).message).toContain("never on a Runner");
+    expect((caught as Error).message).toContain("enqueueAiResourceCommand");
+    expect(runnerLookup).not.toHaveBeenCalled();
+    expect(countBy).not.toHaveBeenCalled();
+    expect(create).not.toHaveBeenCalled();
+  });
+
+  it("negative control: a Bash command still goes through the same lane", async () => {
+    await RunnerJobService.enqueueAiCommand({
+      projectId: PROJECT_ID,
+      aiRunId: new ObjectID("88888888-8888-4888-8888-888888888888"),
+      autoRemediationSuggestionId: new ObjectID(
+        "77777777-7777-4777-8777-777777777777",
+      ),
+      stepId: "ai-approved-1",
+      stepType: RunbookStepType.Bash,
+      targetAgentId: RUNNER_ID,
+      command: "systemctl restart nginx",
+      timeoutInMs: 30000,
+    });
+
+    expect(create).toHaveBeenCalledTimes(1);
+  });
+
+  /*
+   * Resource commands are braked on their own ResourceCommand rows, so the
+   * Runner lane's hourly brake must not count them: a full hour of fixes on
+   * Docker hosts or database servers leaves every Bash and SSH fix its own
+   * budget. Counted against an in-memory RunnerJob table.
+   */
+  describe("the Runner lane's hourly brake never counts resource-command rows", () => {
+    function bashCommand(): Parameters<
+      typeof RunnerJobService.enqueueAiCommand
+    >[0] {
+      return {
+        projectId: PROJECT_ID,
+        aiRunId: new ObjectID("88888888-8888-4888-8888-888888888888"),
+        autoRemediationSuggestionId: new ObjectID(
+          "77777777-7777-4777-8777-777777777777",
+        ),
+        stepId: "ai-approved-1",
+        stepType: RunbookStepType.Bash,
+        targetAgentId: RUNNER_ID,
+        command: "systemctl restart nginx",
+        timeoutInMs: 30000,
+      };
+    }
+
+    it("runs a Bash fix after a full hour of resource fixes", async () => {
+      countBy.mockImplementation(
+        fakeRunnerJobCountBy(
+          fakeRunnerJobRows(MAX_AI_COMMAND_JOBS_PER_PROJECT_PER_HOUR, {
+            projectId: PROJECT_ID,
+            origin: RunnerJobOrigin.AiRemediation,
+            stepType: RunbookStepType.ResourceCommand,
+          }),
+        ) as never,
+      );
+
+      await RunnerJobService.enqueueAiCommand(bashCommand());
+
+      expect(create).toHaveBeenCalledTimes(1);
+    });
+
+    it("negative control: a full hour of kubectl and SSH fixes still stops it", async () => {
+      countBy.mockImplementation(
+        fakeRunnerJobCountBy([
+          ...fakeRunnerJobRows(MAX_AI_COMMAND_JOBS_PER_PROJECT_PER_HOUR - 1, {
+            projectId: PROJECT_ID,
+            origin: RunnerJobOrigin.AiRemediation,
+            stepType: RunbookStepType.Kubectl,
+          }),
+          ...fakeRunnerJobRows(1, {
+            projectId: PROJECT_ID,
+            origin: RunnerJobOrigin.AiRemediation,
+            stepType: RunbookStepType.SSH,
+          }),
+        ]) as never,
+      );
+
+      await expect(
+        RunnerJobService.enqueueAiCommand(bashCommand()),
+      ).rejects.toThrow(/AI remediation commands in the last hour/);
+      expect(create).not.toHaveBeenCalled();
+    });
+  });
+});