@oneuptime/common 14.0.9 → 14.0.10

package/Tests/Server/Services/RunnerJobResourceCommandLane.test.ts

@@ -0,0 +1,211 @@
+import RunnerJobService, {
+  MAX_AI_COMMAND_JOBS_PER_PROJECT_PER_HOUR,
+} from "../../../Server/Services/RunnerJobService";
+import RunnerService from "../../../Server/Services/RunnerService";
+import RunnerJob from "../../../Models/DatabaseModels/RunnerJob";
+import RunbookStepType, {
+  AI_AGENT_EXECUTED_STEP_TYPES,
+  RUNNER_EXECUTED_STEP_TYPES,
+} from "../../../Types/Runbook/RunbookStepType";
+import { AI_COMMAND_STEP_TYPES } from "../../../Types/AutoRemediation/AiRemediationCommandPlan";
+import RunnerJobOrigin from "../../../Types/Runbook/RunnerJobOrigin";
+import BadDataException from "../../../Types/Exception/BadDataException";
+import ObjectID from "../../../Types/ObjectID";
+import PositiveNumber from "../../../Types/PositiveNumber";
+import logger from "../../../Server/Utils/Logger";
+import {
+  fakeRunnerJobCountBy,
+  fakeRunnerJobRows,
+} from "../TestingUtils/Services/FakeRunnerJobCount";
+import { afterEach, beforeEach, describe, expect, it } from "@jest/globals";
+
+/*
+ * Contract under test — a ResourceCommand job never reaches a Runner.
+ *
+ * ResourceCommand is the step type of a command for an infrastructure
+ * resource's AI agent (a ResourceAiAgent row). It is an AI command step type
+ * (AI_COMMAND_STEP_TYPES), so the generic AI-command lane must refuse it
+ * explicitly: that lane writes targetAgentId — a Runner — and a Runner
+ * must never be handed a resource command. The runbook lane (enqueue)
+ * refuses it because it is not a runner-executed type. Both refusals happen
+ * before any row is written.
+ */
+
+const PROJECT_ID: ObjectID = new ObjectID(
+  "22222222-2222-4222-8222-222222222222",
+);
+const RUNNER_ID: ObjectID = new ObjectID(
+  "44444444-4444-4444-8444-444444444444",
+);
+
+describe("ResourceCommand never enters a Runner lane", () => {
+  let create: jest.SpyInstance;
+  let runnerLookup: jest.SpyInstance;
+  let countBy: jest.SpyInstance;
+
+  beforeEach(() => {
+    // @CaptureSpan logs the expected throws at error level.
+    jest.spyOn(logger, "error").mockImplementation((): void => {
+      return undefined;
+    });
+    create = jest
+      .spyOn(RunnerJobService, "create")
+      .mockImplementation(async (data: unknown): Promise<RunnerJob> => {
+        return (data as { data: RunnerJob }).data;
+      });
+    runnerLookup = jest.spyOn(RunnerService, "findOneBy").mockResolvedValue({
+      id: RUNNER_ID,
+      _id: RUNNER_ID.toString(),
+      projectId: PROJECT_ID,
+      name: "office-runner",
+    } as never);
+    countBy = jest
+      .spyOn(RunnerJobService, "countBy")
+      .mockResolvedValue(new PositiveNumber(0));
+  });
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  it("is an AI command step type but not a Runner-executed one", () => {
+    expect(AI_COMMAND_STEP_TYPES).toContain(RunbookStepType.ResourceCommand);
+    expect(AI_AGENT_EXECUTED_STEP_TYPES).toContain(
+      RunbookStepType.ResourceCommand,
+    );
+    expect(RUNNER_EXECUTED_STEP_TYPES).not.toContain(
+      RunbookStepType.ResourceCommand,
+    );
+  });
+
+  it("enqueue (the runbook lane) refuses it before writing a row", async () => {
+    await expect(
+      RunnerJobService.enqueue({
+        projectId: PROJECT_ID,
+        runbookExecutionId: new ObjectID(
+          "33333333-3333-4333-8333-333333333333",
+        ),
+        stepId: "step-1",
+        stepType: RunbookStepType.ResourceCommand,
+        targetAgentId: RUNNER_ID,
+        script: "",
+        payload: { program: "docker", args: ["ps"] },
+        timeoutInMs: 30000,
+      }),
+    ).rejects.toThrow(
+      new BadDataException(
+        'Runner does not execute step type "ResourceCommand".',
+      ),
+    );
+
+    expect(create).not.toHaveBeenCalled();
+  });
+
+  it("enqueueAiCommand (the Runner AI-command lane) refuses it before looking anything up", async () => {
+    let caught: unknown = null;
+
+    try {
+      await RunnerJobService.enqueueAiCommand({
+        projectId: PROJECT_ID,
+        aiRunId: new ObjectID("88888888-8888-4888-8888-888888888888"),
+        autoRemediationSuggestionId: new ObjectID(
+          "77777777-7777-4777-8777-777777777777",
+        ),
+        stepId: "ai-approved-1",
+        stepType: RunbookStepType.ResourceCommand,
+        targetAgentId: RUNNER_ID,
+        command: "docker restart web",
+        timeoutInMs: 30000,
+      });
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(BadDataException);
+    expect((caught as Error).message).toContain("never on a Runner");
+    expect((caught as Error).message).toContain("enqueueAiResourceCommand");
+    expect(runnerLookup).not.toHaveBeenCalled();
+    expect(countBy).not.toHaveBeenCalled();
+    expect(create).not.toHaveBeenCalled();
+  });
+
+  it("negative control: a Bash command still goes through the same lane", async () => {
+    await RunnerJobService.enqueueAiCommand({
+      projectId: PROJECT_ID,
+      aiRunId: new ObjectID("88888888-8888-4888-8888-888888888888"),
+      autoRemediationSuggestionId: new ObjectID(
+        "77777777-7777-4777-8777-777777777777",
+      ),
+      stepId: "ai-approved-1",
+      stepType: RunbookStepType.Bash,
+      targetAgentId: RUNNER_ID,
+      command: "systemctl restart nginx",
+      timeoutInMs: 30000,
+    });
+
+    expect(create).toHaveBeenCalledTimes(1);
+  });
+
+  /*
+   * Resource commands are braked on their own ResourceCommand rows, so the
+   * Runner lane's hourly brake must not count them: a full hour of fixes on
+   * Docker hosts or database servers leaves every Bash and SSH fix its own
+   * budget. Counted against an in-memory RunnerJob table.
+   */
+  describe("the Runner lane's hourly brake never counts resource-command rows", () => {
+    function bashCommand(): Parameters<
+      typeof RunnerJobService.enqueueAiCommand
+    >[0] {
+      return {
+        projectId: PROJECT_ID,
+        aiRunId: new ObjectID("88888888-8888-4888-8888-888888888888"),
+        autoRemediationSuggestionId: new ObjectID(
+          "77777777-7777-4777-8777-777777777777",
+        ),
+        stepId: "ai-approved-1",
+        stepType: RunbookStepType.Bash,
+        targetAgentId: RUNNER_ID,
+        command: "systemctl restart nginx",
+        timeoutInMs: 30000,
+      };
+    }
+
+    it("runs a Bash fix after a full hour of resource fixes", async () => {
+      countBy.mockImplementation(
+        fakeRunnerJobCountBy(
+          fakeRunnerJobRows(MAX_AI_COMMAND_JOBS_PER_PROJECT_PER_HOUR, {
+            projectId: PROJECT_ID,
+            origin: RunnerJobOrigin.AiRemediation,
+            stepType: RunbookStepType.ResourceCommand,
+          }),
+        ) as never,
+      );
+
+      await RunnerJobService.enqueueAiCommand(bashCommand());
+
+      expect(create).toHaveBeenCalledTimes(1);
+    });
+
+    it("negative control: a full hour of kubectl and SSH fixes still stops it", async () => {
+      countBy.mockImplementation(
+        fakeRunnerJobCountBy([
+          ...fakeRunnerJobRows(MAX_AI_COMMAND_JOBS_PER_PROJECT_PER_HOUR - 1, {
+            projectId: PROJECT_ID,
+            origin: RunnerJobOrigin.AiRemediation,
+            stepType: RunbookStepType.Kubectl,
+          }),
+          ...fakeRunnerJobRows(1, {
+            projectId: PROJECT_ID,
+            origin: RunnerJobOrigin.AiRemediation,
+            stepType: RunbookStepType.SSH,
+          }),
+        ]) as never,
+      );
+
+      await expect(
+        RunnerJobService.enqueueAiCommand(bashCommand()),
+      ).rejects.toThrow(/AI remediation commands in the last hour/);
+      expect(create).not.toHaveBeenCalled();
+    });
+  });
+});

package/Tests/Server/Services/RunnerJobResourceCommandTimeoutAndRedaction.test.ts

@@ -0,0 +1,352 @@
+import RunnerJobService, {
+  RunnerJobTimeoutKind,
+  Service as RunnerJobServiceClass,
+  describeRunnerJobTimeout,
+} from "../../../Server/Services/RunnerJobService";
+import PostgresAppInstance from "../../../Server/Infrastructure/PostgresDatabase";
+import RunnerJob from "../../../Models/DatabaseModels/RunnerJob";
+import ObjectID from "../../../Types/ObjectID";
+import AiResourceType from "../../../Types/ResourceAiAgent/AiResourceType";
+import RunbookStepType from "../../../Types/Runbook/RunbookStepType";
+import RunnerJobOrigin from "../../../Types/Runbook/RunnerJobOrigin";
+import RunnerJobStatus from "../../../Types/Runbook/RunnerJobStatus";
+import { afterEach, beforeEach, describe, expect, it } from "@jest/globals";
+
+/*
+ * Contract under test — the two RunnerJobService readings of a resource
+ * command job's row besides its enqueue:
+ *
+ * - a timed-out ResourceCommand job is worded for the resource's AI agent
+ *   ("the Docker AI agent did not pick up the command in time ... Nothing
+ *   was run on the Docker host"), on the row, while every Kubectl and
+ *   runbook reason stays exactly what it was;
+ * - submitResult, given the job's resource type and program, stores the
+ *   output through the resource command redactor as well, and without them
+ *   stores exactly what it always has.
+ */
+
+const JOB_ID: ObjectID = new ObjectID("11111111-1111-4111-8111-111111111111");
+const AGENT_ID: ObjectID = new ObjectID("22222222-2222-4222-8222-222222222222");
+const CLAIM_TIMEOUT_MS: number = 60_000;
+const EXECUTION_TIMEOUT_MS: number = 30_000;
+
+function reason(data: {
+  kind: RunnerJobTimeoutKind;
+  wasClaimed?: boolean | undefined;
+  origin?: RunnerJobOrigin | undefined;
+  stepType?: RunbookStepType | undefined;
+  resourceType?: AiResourceType | null | undefined;
+  isForKubernetesAiAgent?: boolean | undefined;
+}): string {
+  return describeRunnerJobTimeout({
+    kind: data.kind,
+    wasClaimed: data.wasClaimed,
+    origin: data.origin ?? RunnerJobOrigin.AiInvestigation,
+    stepType: data.stepType ?? RunbookStepType.ResourceCommand,
+    claimTimeoutInMs: CLAIM_TIMEOUT_MS,
+    executionTimeoutInMs: EXECUTION_TIMEOUT_MS,
+    isForKubernetesAiAgent: data.isForKubernetesAiAgent,
+    resourceType: data.resourceType,
+  });
+}
+
+describe("describeRunnerJobTimeout for a resource command", () => {
+  it("says the agent did not pick up the command in time, and that nothing ran", () => {
+    expect(
+      reason({ kind: "unclaimed", resourceType: AiResourceType.DockerHost }),
+    ).toBe(
+      "The Docker AI agent did not pick up the command in time (within 60s) — it may be offline, restarting or busy with other work. Nothing was run on the Docker host.",
+    );
+  });
+
+  it("names each type's agent and resource", () => {
+    expect(
+      reason({ kind: "unclaimed", resourceType: AiResourceType.Host }),
+    ).toContain("The Host AI agent did not pick up the command in time");
+    expect(
+      reason({ kind: "unclaimed", resourceType: AiResourceType.Host }),
+    ).toContain("Nothing was run on the host.");
+    expect(
+      reason({
+        kind: "unclaimed",
+        resourceType: AiResourceType.DatabaseServer,
+      }),
+    ).toContain("Nothing was run on the database server.");
+    expect(
+      reason({
+        kind: "unclaimed",
+        resourceType: AiResourceType.VMwareVCenter,
+      }),
+    ).toContain("The VMware AI agent did not pick up the command in time");
+  });
+
+  it("reads a job still Pending at the overall deadline as never picked up", () => {
+    expect(
+      reason({
+        kind: "overall",
+        wasClaimed: false,
+        resourceType: AiResourceType.CephCluster,
+      }),
+    ).toContain("The Ceph AI agent did not pick up the command in time");
+  });
+
+  it("says what a command the agent took did is unknown", () => {
+    expect(
+      reason({
+        kind: "lease_expired",
+        wasClaimed: true,
+        resourceType: AiResourceType.ProxmoxCluster,
+      }),
+    ).toBe(
+      "The Proxmox AI agent stopped responding while this command was running — it may have restarted or lost its connection. What the command did is unknown.",
+    );
+    expect(
+      reason({
+        kind: "overall",
+        wasClaimed: true,
+        resourceType: AiResourceType.ProxmoxCluster,
+      }),
+    ).toBe(
+      "The Proxmox AI agent did not report a result for this command in time — the command may have outlived its 30s timeout. What the command did is unknown.",
+    );
+  });
+
+  it("tells a remediation to check the resource before running a change again", () => {
+    for (const kind of [
+      "lease_expired",
+      "overall",
+    ] as Array<RunnerJobTimeoutKind>) {
+      expect(
+        reason({
+          kind,
+          wasClaimed: true,
+          origin: RunnerJobOrigin.AiRemediation,
+          resourceType: AiResourceType.DockerSwarmCluster,
+        }),
+      ).toContain(
+        "Check the Docker Swarm cluster before running a change again.",
+      );
+    }
+
+    // Nothing ran: nothing to check.
+    expect(
+      reason({
+        kind: "unclaimed",
+        origin: RunnerJobOrigin.AiRemediation,
+        resourceType: AiResourceType.DockerSwarmCluster,
+      }),
+    ).not.toContain("Check the");
+  });
+
+  it("falls back to generic words for a row without a readable type", () => {
+    expect(reason({ kind: "unclaimed", resourceType: null })).toBe(
+      "The resource's AI agent did not pick up the command in time (within 60s) — it may be offline, restarting or busy with other work. Nothing was run on the resource.",
+    );
+  });
+
+  it("leaves the kubectl wording exactly as it was", () => {
+    expect(
+      reason({
+        kind: "unclaimed",
+        stepType: RunbookStepType.Kubectl,
+        isForKubernetesAiAgent: true,
+        resourceType: AiResourceType.DockerHost,
+      }),
+    ).toBe(
+      "The cluster's Kubernetes AI agent did not pick up this kubectl command within 60s — it may be offline, restarting or busy with other work. Nothing was run on the cluster.",
+    );
+    expect(
+      reason({ kind: "unclaimed", stepType: RunbookStepType.Kubectl }),
+    ).toBe(
+      "The cluster's Runner did not pick up this kubectl command within 60s — it may be offline, restarting or busy with other work. Nothing was run on the cluster.",
+    );
+  });
+
+  it("leaves a runbook step's wording as it was", () => {
+    expect(
+      reason({
+        kind: "unclaimed",
+        origin: RunnerJobOrigin.Runbook,
+        stepType: RunbookStepType.Bash,
+      }),
+    ).toBe(
+      "No runbook agent picked up this step before the wait window expired. The agent may be offline — check that it is running and reachable, then try again.",
+    );
+  });
+});
+
+describe("RunnerJobService.pollUntilTerminal on a resource command", () => {
+  let row: Record<string, unknown>;
+  let findOneById: jest.SpyInstance;
+  let timeoutSpy: jest.SpyInstance;
+
+  beforeEach(() => {
+    row = {};
+    findOneById = jest
+      .spyOn(RunnerJobService, "findOneById")
+      .mockImplementation(async (): Promise<RunnerJob> => {
+        return row as unknown as RunnerJob;
+      });
+    timeoutSpy = jest
+      .spyOn(RunnerJobService, "timeoutJob")
+      .mockImplementation(
+        async (data: { reason: string }): Promise<RunnerJob> => {
+          return {
+            status: RunnerJobStatus.TimedOut,
+            errorMessage: data.reason,
+          } as unknown as RunnerJob;
+        },
+      );
+  });
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  it("reads the resource agent and its type off the row and words the timeout", async () => {
+    row = {
+      status: RunnerJobStatus.Pending,
+      origin: RunnerJobOrigin.AiInvestigation,
+      stepType: RunbookStepType.ResourceCommand,
+      targetResourceAiAgentId: AGENT_ID,
+      resourceType: AiResourceType.PodmanHost,
+      claimDeadlineAt: new Date(Date.now() - 1000),
+      leaseExpiresAt: new Date(Date.now() + 60_000),
+    };
+
+    const job: RunnerJob = await RunnerJobService.pollUntilTerminal({
+      jobId: JOB_ID,
+      claimTimeoutInMs: CLAIM_TIMEOUT_MS,
+      executionTimeoutInMs: EXECUTION_TIMEOUT_MS,
+    });
+
+    const select: Record<string, unknown> = (
+      findOneById.mock.calls[0]![0] as { select: Record<string, unknown> }
+    ).select;
+
+    expect(select["targetResourceAiAgentId"]).toBe(true);
+    expect(select["resourceType"]).toBe(true);
+    // The Kubernetes target is still read for kubectl rows.
+    expect(select["targetKubernetesAiAgentId"]).toBe(true);
+    expect(timeoutSpy).toHaveBeenCalledTimes(1);
+    expect(job.errorMessage).toContain(
+      "The Podman AI agent did not pick up the command in time",
+    );
+  });
+
+  it("returns a terminal resource job as it is", async () => {
+    row = {
+      status: RunnerJobStatus.Succeeded,
+      stepType: RunbookStepType.ResourceCommand,
+      output: "[stdout]\nok",
+    };
+
+    const job: RunnerJob = await RunnerJobService.pollUntilTerminal({
+      jobId: JOB_ID,
+      claimTimeoutInMs: CLAIM_TIMEOUT_MS,
+      executionTimeoutInMs: EXECUTION_TIMEOUT_MS,
+    });
+
+    expect(job.status).toBe(RunnerJobStatus.Succeeded);
+    expect(timeoutSpy).not.toHaveBeenCalled();
+  });
+});
+
+describe("RunnerJobService.submitResult with the job's resource command", () => {
+  let queries: Array<{ sql: string; params: Array<unknown> }>;
+
+  beforeEach(() => {
+    queries = [];
+    jest.spyOn(PostgresAppInstance, "getDataSource").mockReturnValue({
+      query: jest
+        .fn()
+        .mockImplementation(
+          async (sql: string, params: Array<unknown>): Promise<unknown> => {
+            queries.push({ sql, params });
+            return [[{ _id: JOB_ID.toString() }], 1];
+          },
+        ),
+    } as never);
+  });
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  const INSPECT_OUTPUT: string =
+    '[stdout]\n[{"Id":"abc","Config":{"Env":["FOO_SETTING=plainvalue42","PATH=/usr/bin"]}}]';
+
+  // $10 is the AI-origin output, $11 the AI-origin error (see the UPDATE).
+  function aiOriginOutput(): string {
+    return queries[queries.length - 1]!.params[9] as string;
+  }
+
+  function aiOriginError(): string {
+    return queries[queries.length - 1]!.params[10] as string;
+  }
+
+  function verbatimOutput(): string {
+    return queries[queries.length - 1]!.params[1] as string;
+  }
+
+  it("masks every docker inspect environment value for a Docker host's job", async () => {
+    await RunnerJobService.submitResult({
+      jobId: JOB_ID,
+      agentId: AGENT_ID,
+      success: true,
+      exitCode: 0,
+      output: INSPECT_OUTPUT,
+      resourceCommand: {
+        resourceType: AiResourceType.DockerHost,
+        program: "docker",
+      },
+    });
+
+    expect(aiOriginOutput()).not.toContain("plainvalue42");
+    expect(aiOriginOutput()).toContain("FOO_SETTING=[redacted]");
+    // The runbook branch of the CASE keeps the raw text, as always.
+    expect(verbatimOutput()).toBe(INSPECT_OUTPUT);
+  });
+
+  it("redacts the error message the same way", async () => {
+    await RunnerJobService.submitResult({
+      jobId: JOB_ID,
+      agentId: AGENT_ID,
+      success: false,
+      errorMessage:
+        '{"query":"SELECT * FROM users WHERE email = \'bob@example.com\' AND pin = 1234"}',
+      resourceCommand: {
+        resourceType: AiResourceType.DatabaseServer,
+        program: "db",
+      },
+    });
+
+    expect(aiOriginError()).not.toContain("bob@example.com");
+    expect(aiOriginError()).not.toContain("1234");
+  });
+
+  it("stores exactly what it always has without a resource command", async () => {
+    await RunnerJobService.submitResult({
+      jobId: JOB_ID,
+      agentId: AGENT_ID,
+      success: true,
+      exitCode: 0,
+      output: INSPECT_OUTPUT,
+    });
+
+    expect(aiOriginOutput()).toBe(
+      RunnerJobServiceClass.redactAiJobText(INSPECT_OUTPUT),
+    );
+    expect(aiOriginOutput()).toContain("plainvalue42");
+  });
+
+  it("redactResourceCommandText is the identity without a resource command", () => {
+    expect(RunnerJobServiceClass.redactResourceCommandText("x=1")).toBe("x=1");
+    expect(
+      RunnerJobServiceClass.redactResourceCommandText(INSPECT_OUTPUT, {
+        resourceType: AiResourceType.PodmanHost,
+        program: "docker",
+      }),
+    ).toContain("FOO_SETTING=[redacted]");
+  });
+});