@oneuptime/common 14.0.8 → 14.0.10

package/Tests/Server/Utils/AI/KubectlInvestigationToolkit.test.ts

@@ -91,6 +91,8 @@ function readyCluster(
       posture: { inCluster: true, allowWrites: false },
     },
     accessMethod: "in_cluster",
+    aiAgent: null,
+    automaticInvestigation: { incidents: false, alerts: false },
     kubectlAllowlist: [],
     isInvestigationEnabled: true,
     isInvestigationReady: true,
@@ -1248,6 +1250,74 @@ describe("KubectlInvestigationToolkit run_kubectl when kubectl never ran", () =>
     expect(toolkit.getCommandsRun()).toBe(1);
   });
 
+  /*
+   * The same unclaimed command on a cluster reached through its Kubernetes
+   * AI agent: the text names the agent, never "its Runner".
+   */
+  it("names the Kubernetes AI agent, not a Runner, when the agent left the command unclaimed", async () => {
+    const toolkit: KubectlInvestigationToolkit =
+      new KubectlInvestigationToolkit({
+        projectId: PROJECT_ID,
+        aiRunId: RUN_ID,
+        clusters: [
+          readyCluster({
+            runner: {
+              id: RUNNER_ID.toString(),
+              name: "Kubernetes AI agent",
+              kind: "ai_agent",
+              isOnline: true,
+              canRunAiCommands: true,
+              posture: { inCluster: true, allowWrites: false },
+            },
+          }),
+        ],
+      });
+
+    const before: ToolCallOutcome = await getTool(
+      toolkit,
+      LIST_CLUSTER_ACCESS_TOOL_NAME,
+    ).execute({});
+    expect(before.textForLlm).toContain(
+      "read-only kubectl via the Kubernetes AI agent",
+    );
+
+    const outcome: ToolCallOutcome = await runOn(toolkit);
+
+    expect(outcome.errorMessage).toBe(
+      'kubectl was not run on cluster "prod-us": its Kubernetes AI agent did not pick up the command in time.',
+    );
+    expect(outcome.textForLlm).toContain(
+      "its Kubernetes AI agent did not pick up",
+    );
+    expect(outcome.textForLlm).toContain(
+      "the cluster's Kubernetes AI agent did not respond",
+    );
+    expect(outcome.textForLlm).not.toContain("Runner");
+
+    // The breaker speaks of the agent too.
+    const second: ToolCallOutcome = await runOn(toolkit);
+    expect(second.textForLlm).toMatch(/^The Kubernetes AI agent, which serves/);
+    expect(second.errorMessage).toBe(
+      'kubectl was not run on cluster "prod-us": its Kubernetes AI agent did not pick up an earlier command, so the cluster was unreachable for the rest of this investigation.',
+    );
+
+    const list: ToolCallOutcome = await getTool(
+      toolkit,
+      LIST_CLUSTER_ACCESS_TOOL_NAME,
+    ).execute({});
+    expect(list.textForLlm).toContain(
+      "its Kubernetes AI agent did not pick up an earlier command",
+    );
+
+    // The command went to the agent's row id with no credential.
+    const data: Record<string, unknown> = enqueue.mock.calls[0]![0] as Record<
+      string,
+      unknown
+    >;
+    expect(String(data["targetAgentId"])).toBe(RUNNER_ID.toString());
+    expect(data["credentialId"]).toBeUndefined();
+  });
+
   it("trips a per-cluster breaker: the next command fails at once, enqueues nothing and spends nothing", async () => {
     const toolkit: KubectlInvestigationToolkit = singleClusterToolkit();
 

package/Tests/Server/Utils/AI/KubectlJobRunnerFinishedJob.test.ts

@@ -230,6 +230,64 @@ describe("KubectlJobRunner.readFinishedJob", () => {
   });
 });
 
+/*
+ * A job enqueued for the cluster's Kubernetes AI agent (the row carries
+ * targetKubernetesAiAgentId) is worded for the agent: "Runner" would send
+ * the operator looking for a Runner that does not exist.
+ */
+describe("KubectlJobRunner.readFinishedJob for a Kubernetes AI agent job", () => {
+  function readAgentJob(terminalJob: RunnerJob): Promise<KubectlJobOutcome> {
+    return KubectlJobRunner.readFinishedJob({
+      job: {
+        ...enqueuedJob({ displayCommand: "kubectl get pods -n web" }),
+        targetKubernetesAiAgentId: ObjectID.generate(),
+      } as unknown as RunnerJob,
+      terminalJob,
+      command: "kubectl get pods -n web",
+      claimTimeoutInMs: KUBECTL_CLAIM_TIMEOUT_MS,
+      executionTimeoutInMs: EXECUTION_TIMEOUT_MS,
+    });
+  }
+
+  it("never picked up: names the agent", async () => {
+    claimRead.mockResolvedValue(terminal({}));
+
+    const outcome: KubectlJobOutcome = await readAgentJob(
+      terminal({ status: RunnerJobStatus.TimedOut }),
+    );
+
+    expect(outcome.errorMessage).toMatch(
+      /^The cluster's Kubernetes AI agent did not pick up this kubectl command within \d+s/,
+    );
+    expect(outcome.runState).toBe(KubectlRunState.NotRun);
+  });
+
+  it("taken and gone silent: names the agent", async () => {
+    claimRead.mockResolvedValue(terminal({ claimedAt: new Date() }));
+
+    const outcome: KubectlJobOutcome = await readAgentJob(
+      terminal({ status: RunnerJobStatus.TimedOut }),
+    );
+
+    expect(outcome.errorMessage).toMatch(
+      /^The Kubernetes AI agent took this kubectl command but did not report a result in time/,
+    );
+    expect(outcome.errorMessage).not.toContain("Runner");
+  });
+
+  it("claim unreadable: names the agent", async () => {
+    claimRead.mockRejectedValue(new Error("db down"));
+
+    const outcome: KubectlJobOutcome = await readAgentJob(
+      terminal({ status: RunnerJobStatus.TimedOut }),
+    );
+
+    expect(outcome.errorMessage).toContain(
+      "whether the Kubernetes AI agent picked it up could not be read",
+    );
+  });
+});
+
 describe("KubectlJobRunner.recordOutcomeOnCluster", () => {
   let recordOutcome: jest.SpyInstance;
 

package/Tests/Server/Utils/AI/RemediationCommandTools.test.ts

@@ -25,10 +25,15 @@ import {
   MAX_PLAN_COMMANDS,
 } from "../../../../Types/AutoRemediation/AiRemediationCommandPlan";
 import RunbookStepType from "../../../../Types/Runbook/RunbookStepType";
+import RunnerJobOrigin from "../../../../Types/Runbook/RunnerJobOrigin";
 import RunnerJobStatus from "../../../../Types/Runbook/RunnerJobStatus";
 import { JSONObject } from "../../../../Types/JSON";
 import ObjectID from "../../../../Types/ObjectID";
 import PositiveNumber from "../../../../Types/PositiveNumber";
+import {
+  fakeRunnerJobCountBy,
+  fakeRunnerJobRows,
+} from "../../TestingUtils/Services/FakeRunnerJobCount";
 import { afterEach, beforeEach, describe, expect, it } from "@jest/globals";
 
 /*
@@ -504,7 +509,36 @@ describe("RemediationCommandToolkit execute_remediation_command", () => {
     ).execute(bashArgs({ stepType: "Kubernetes" }));
 
     expect(outcome.success).toBe(false);
-    expect(outcome.textForLlm).toContain("stepType must be one of");
+    // Exactly the step types a rule round offers — never ResourceCommand.
+    expect(outcome.textForLlm).toBe(
+      "stepType must be one of: Bash, SSH, Kubectl.",
+    );
+  });
+
+  it("never runs a ResourceCommand step on a Runner", async () => {
+    /*
+     * ResourceCommand is an AI command step type, but it belongs to a
+     * resource's own AI agent: it must never fall through to the Bash/SSH
+     * path, which would enqueue it for a Runner.
+     */
+    const enqueue: jest.SpyInstance = jest.spyOn(
+      RunnerJobService,
+      "enqueueAiCommand",
+    );
+
+    const outcome: ToolCallOutcome = await getTool(
+      buildToolkit(),
+      "execute_remediation_command",
+    ).execute(
+      bashArgs({ stepType: "ResourceCommand", command: "docker restart web" }),
+    );
+
+    expect(outcome.success).toBe(false);
+    // This round never offered it: refused with the step types it offers.
+    expect(outcome.textForLlm).toBe(
+      "stepType must be one of: Bash, SSH, Kubectl.",
+    );
+    expect(enqueue).not.toHaveBeenCalled();
   });
 
   it("rejects a missing command", async () => {
@@ -653,6 +687,65 @@ describe("RemediationCommandToolkit execute_remediation_command", () => {
     expect(outcome.textForLlm).toContain("hourly AI-command limit");
     expect(enqueue).not.toHaveBeenCalled();
   });
+
+  /*
+   * Resource commands have their own hourly brake, so a full hour of fixes
+   * on the project's Docker hosts or database servers must not refuse a
+   * Runner command here. Counted against an in-memory RunnerJob table.
+   */
+  it("does not count resource-command jobs toward the hourly AI-command cap", async () => {
+    jest.spyOn(RunnerJobService, "countBy").mockImplementation(
+      fakeRunnerJobCountBy(
+        fakeRunnerJobRows(MAX_AI_COMMAND_JOBS_PER_PROJECT_PER_HOUR, {
+          projectId: PROJECT_ID,
+          origin: RunnerJobOrigin.AiRemediation,
+          stepType: RunbookStepType.ResourceCommand,
+        }),
+      ) as never,
+    );
+    const enqueue: jest.SpyInstance = jest.spyOn(
+      RunnerJobService,
+      "enqueueAiCommand",
+    );
+
+    const outcome: ToolCallOutcome = await getTool(
+      buildToolkit(),
+      "execute_remediation_command",
+    ).execute(bashArgs());
+
+    expect(outcome.success).toBe(true);
+    expect(enqueue).toHaveBeenCalledTimes(1);
+  });
+
+  it("negative control: a full hour of Runner and kubectl jobs still hits the cap", async () => {
+    jest.spyOn(RunnerJobService, "countBy").mockImplementation(
+      fakeRunnerJobCountBy([
+        ...fakeRunnerJobRows(MAX_AI_COMMAND_JOBS_PER_PROJECT_PER_HOUR - 1, {
+          projectId: PROJECT_ID,
+          origin: RunnerJobOrigin.AiRemediation,
+          stepType: RunbookStepType.Bash,
+        }),
+        ...fakeRunnerJobRows(1, {
+          projectId: PROJECT_ID,
+          origin: RunnerJobOrigin.AiRemediation,
+          stepType: RunbookStepType.Kubectl,
+        }),
+      ]) as never,
+    );
+    const enqueue: jest.SpyInstance = jest.spyOn(
+      RunnerJobService,
+      "enqueueAiCommand",
+    );
+
+    const outcome: ToolCallOutcome = await getTool(
+      buildToolkit(),
+      "execute_remediation_command",
+    ).execute(bashArgs());
+
+    expect(outcome.success).toBe(false);
+    expect(outcome.textForLlm).toContain("hourly AI-command limit");
+    expect(enqueue).not.toHaveBeenCalled();
+  });
 });
 
 describe("RemediationCommandToolkit propose_remediation_commands", () => {

package/Tests/Server/Utils/AI/RemediationCommandToolsKubectl.test.ts

@@ -107,6 +107,8 @@ function cluster(
       posture: { inCluster: true, allowWrites: true },
     },
     accessMethod: "in_cluster",
+    aiAgent: null,
+    automaticInvestigation: { incidents: false, alerts: false },
     kubectlAllowlist: [],
     isInvestigationEnabled: true,
     isInvestigationReady: true,

package/Tests/Server/Utils/AI/RemediationCommandToolsKubectlOutcome.test.ts

@@ -79,6 +79,8 @@ function cluster(): KubernetesClusterAiAccessStatus {
       posture: { inCluster: true, allowWrites: true },
     },
     accessMethod: "in_cluster",
+    aiAgent: null,
+    automaticInvestigation: { incidents: false, alerts: false },
     kubectlAllowlist: [],
     isInvestigationEnabled: true,
     isInvestigationReady: true,

package/Tests/Server/Utils/AI/RemediationCommandToolsLiveCluster.test.ts

@@ -103,6 +103,8 @@ function cluster(
       posture: { inCluster: true, allowWrites: true },
     },
     accessMethod: "in_cluster",
+    aiAgent: null,
+    automaticInvestigation: { incidents: false, alerts: false },
     kubectlAllowlist: [],
     isInvestigationEnabled: true,
     isInvestigationReady: true,