@oneuptime/common 14.0.9 → 14.0.11

package/Server/Utils/AI/Remediation/RemediationPlanRunner.ts

@@ -682,6 +682,12 @@ export default class RemediationPlanRunner {
           suggestionType: true,
           commandPlan: true,
           verificationWindowMinutes: true,
+          /*
+           * So a resource round's notes name its resource: this sweep's
+           * own (describeStrandedSource) and settleInterruptedExecution's.
+           */
+          resourceType: true,
+          resourceId: true,
         },
         limit: 100,
         skip: 0,
@@ -745,7 +751,7 @@ export default class RemediationPlanRunner {
 
         await this.postFeedItem({
           suggestion,
-          markdown: `⚡ **Auto Remediation Rule "${suggestion.ruleNameSnapshot || "Auto Remediation Rule"}": AI planning did not complete** (${reason}) — no runbook was proposed.`,
+          markdown: `⚡ **${this.describeStrandedSource(suggestion)}: AI planning did not complete** (${reason}) — no runbook was proposed.`,
           pingWorkspace: false,
         });
       } catch (error) {
@@ -756,6 +762,24 @@ export default class RemediationPlanRunner {
     }
   }
 
+  /*
+   * How the sweeper's feed note names a stranded round. A resource round
+   * (a Docker host's, a database server's, ...) names its resource —
+   * 'AI remediation for Docker host "web-1"' — as the execution runner's
+   * notes do, never "Auto Remediation Rule ...": only a resource round
+   * carries resourceType and resourceId (a rule round never sets them).
+   * Every other round keeps its label unchanged.
+   */
+  private static describeStrandedSource(
+    suggestion: AutoRemediationSuggestion,
+  ): string {
+    if (suggestion.resourceType && suggestion.resourceId) {
+      return suggestion.ruleNameSnapshot || "AI remediation";
+    }
+
+    return `Auto Remediation Rule "${suggestion.ruleNameSnapshot || "Auto Remediation Rule"}"`;
+  }
+
   // Why a Planning suggestion counts as stranded — or null if it does not.
   private static async getStrandedReason(
     suggestion: AutoRemediationSuggestion,

package/Server/Utils/AI/ResourceAccess/InfrastructureInvestigationToolkit.ts

@@ -0,0 +1,622 @@
+import ObjectID from "../../../../Types/ObjectID";
+import { JSONObject } from "../../../../Types/JSON";
+import RunnerJobOrigin from "../../../../Types/Runbook/RunnerJobOrigin";
+import AiResourceType, {
+  AI_RESOURCE_TYPE_INFO,
+  ALL_AI_RESOURCE_TYPES,
+  AiResourceTypeInfo,
+  isAiResourceType,
+} from "../../../../Types/ResourceAiAgent/AiResourceType";
+import {
+  DEFAULT_RESOURCE_COMMAND_TIMEOUT_MS,
+  MAX_RESOURCE_COMMANDS_PER_INVESTIGATION,
+  MAX_RESOURCE_COMMAND_TIMEOUT_MS,
+  ResourceAiAccessStatus,
+  ResourceCommandTier,
+} from "../../../../Types/ResourceAiAgent/ResourceAiAccess";
+import ResourceCommandPolicy from "../../../../Utils/AiRemediation/Resource/ResourceCommandPolicy";
+import { ResourceCommandPolicyResult } from "../../../../Utils/AiRemediation/Resource/ResourceCommandPolicyCore";
+import { redactResourceCommandOutput } from "../../../../Utils/AiRemediation/Resource/ResourceOutputRedactor";
+import KubectlWaitBudget, {
+  KubectlWaitBudgetResult,
+  MIN_KUBECTL_TIMEOUT_MS,
+} from "../../../../Utils/AiRemediation/KubectlWaitBudget";
+import { describeResourceNoun } from "../../../Services/ResourceAiAccessService";
+import { ToolCallOutcome } from "../Toolbox/Index";
+import { ToolArgs } from "../Toolbox/ToolTypes";
+import { ObservabilityAssistantExtraTool } from "../Chat/ObservabilityAssistant";
+import ResourceCommandJobRunner, {
+  RESOURCE_COMMAND_CLAIM_TIMEOUT_MS,
+  ResourceCommandJobOutcome,
+  ResourceCommandRunState,
+} from "./ResourceCommandJobRunner";
+import {
+  INFRASTRUCTURE_RESULT_UNKNOWN_EVENT_PREFIX,
+  LIST_INFRASTRUCTURE_ACCESS_TOOL_NAME,
+  RUN_INFRASTRUCTURE_COMMAND_TOOL_NAME,
+} from "./ResourceAccessToolNames";
+import ResourceAccessContext, {
+  describeResource,
+} from "./ResourceAccessContext";
+import ToolOutputPager, {
+  hasAgentTruncationMarker,
+  PagedToolOutput,
+  READ_TOOL_OUTPUT_TOOL_NAME,
+} from "../Chat/ToolOutputPager";
+
+/*
+ * The run-scoped, READ-ONLY infrastructure command toolkit for an
+ * investigation — the sibling of KubectlInvestigationToolkit for every
+ * resource with a resource AI agent (Docker and Podman hosts, Docker Swarm,
+ * Proxmox, VMware and Ceph clusters, database servers, hosts).
+ *
+ * One instance backs one AIRun. It is handed to the agent loop as
+ * extraTools and only ever offers resources the access service called
+ * investigation-ready. Three independent guards keep it read-only: the
+ * resource command policy here refuses anything but the Read tier, the
+ * enqueue chokepoint (RunnerJobService.enqueueAiResourceCommand) refuses a
+ * non-Read investigation-origin job, and the agent refuses the same before
+ * it runs anything. "Read-only — nothing in your systems was changed" stays
+ * literally true.
+ *
+ * What else the toolkit owns, exactly as the kubectl one does:
+ *  - the run's time limit, when its project configured one (by default a
+ *    run has none): every command's claim window and timeout are planned
+ *    against the run's deadline (KubectlWaitBudget, which is generic in
+ *    all but name) and a command the time left can no longer hold is
+ *    refused before anything is enqueued;
+ *  - what the model sees: output reaches it only through the shared
+ *    ResourceCommandJobRunner redaction, and is never cut — the whole
+ *    redacted output is kept and a long one is paged (ToolOutputPager);
+ *  - what counts as evidence: a command that never ran is a failed call
+ *    that mints no citation, one whose result never came back is "result
+ *    unknown" (never "did not run"), and one that ran and exited non-zero
+ *    is still evidence, cited with rowCount 0;
+ *  - a per-agent breaker: once one command goes unclaimed, that resource's
+ *    agent is unreachable for the rest of the run.
+ */
+
+export interface InfrastructureInvestigationToolkitOptions {
+  projectId: ObjectID;
+  aiRunId: ObjectID;
+  resources: Array<ResourceAiAccessStatus>;
+  /*
+   * Commands one run may send. Defaults to the runaway guard: an
+   * investigation runs as many commands as it needs.
+   */
+  maxCommands?: number | undefined;
+  /*
+   * Where long outputs are kept so the model can page through them
+   * (read_tool_output). Shared with the run's other toolkits so one tool
+   * reads them all; the toolkit makes its own when none is given.
+   */
+  outputPager?: ToolOutputPager | undefined;
+  /*
+   * Which readiness admits a resource. Investigations require the
+   * investigation switch; a remediation run may read a resource it is
+   * allowed to fix even when its operator left investigation off.
+   */
+  readinessCheck?: "investigation" | "remediation" | undefined;
+  /*
+   * When the run's configured time limit ends (epoch milliseconds). Every
+   * command's wait is planned to end before it; absent (the default — no
+   * time limit), commands are only bounded by their own timeouts.
+   */
+  runDeadlineAtMs?: number | undefined;
+}
+
+// A reason that already ends a sentence.
+const SENTENCE_END_PATTERN: RegExp = /[.!?]$/;
+
+// The agent an unclaimed command tripped the breaker for.
+interface UnreachableAgent {
+  // The claim window the unclaimed command was given.
+  claimWindowMs: number;
+}
+
+export default class InfrastructureInvestigationToolkit {
+  private options: InfrastructureInvestigationToolkitOptions;
+  private commandsRun: number = 0;
+  // Agents (by id) that left a command unclaimed in this run.
+  private unreachableAgents: Map<string, UnreachableAgent> = new Map<
+    string,
+    UnreachableAgent
+  >();
+
+  private outputPager: ToolOutputPager;
+
+  public getOutputPager(): ToolOutputPager {
+    return this.outputPager;
+  }
+
+  public constructor(options: InfrastructureInvestigationToolkitOptions) {
+    this.outputPager = options.outputPager ?? new ToolOutputPager();
+    this.options = options;
+  }
+
+  public getReadyResources(): Array<ResourceAiAccessStatus> {
+    return this.options.resources.filter(
+      (resource: ResourceAiAccessStatus): boolean => {
+        const isReady: boolean =
+          this.options.readinessCheck === "remediation"
+            ? resource.isRemediationReady
+            : resource.isInvestigationReady;
+
+        return (
+          isReady &&
+          isAiResourceType(resource.resourceType) &&
+          resource.agent !== null &&
+          Boolean(resource.agent.agentId)
+        );
+      },
+    );
+  }
+
+  public getCommandsRun(): number {
+    return this.commandsRun;
+  }
+
+  // Whether this resource's agent tripped the breaker.
+  public isResourceUnreachable(resourceId: string): boolean {
+    const resource: ResourceAiAccessStatus | undefined =
+      this.options.resources.find(
+        (candidate: ResourceAiAccessStatus): boolean => {
+          return candidate.resourceId === resourceId;
+        },
+      );
+
+    return Boolean(
+      resource?.agent && this.unreachableAgents.has(resource.agent.agentId),
+    );
+  }
+
+  /*
+   * No ready resource, no tools: the model is told in its context why each
+   * resource is unreachable, and a tool that always fails would only burn
+   * its budget.
+   */
+  public buildTools(): Array<ObservabilityAssistantExtraTool> {
+    if (this.getReadyResources().length === 0) {
+      return [];
+    }
+
+    return [this.buildListAccessTool(), this.buildRunCommandTool()];
+  }
+
+  // One line per ready resource: what it is, its id, and its programs.
+  private describeReadyResources(): string {
+    return this.getReadyResources()
+      .map((resource: ResourceAiAccessStatus): string => {
+        const info: AiResourceTypeInfo =
+          AI_RESOURCE_TYPE_INFO[resource.resourceType];
+
+        return `- ${describeResource(resource)} — resourceId: ${resource.resourceId} — programs: ${info.programs.join(
+          ", ",
+        )}`;
+      })
+      .join("\n");
+  }
+
+  /*
+   * The read-command cheat-sheet of every tool policy the ready resources
+   * use, once per policy (a Docker host and a Podman host share one).
+   */
+  private describeReadCommandGuides(): string {
+    const presentTypes: Array<AiResourceType> = ALL_AI_RESOURCE_TYPES.filter(
+      (type: AiResourceType): boolean => {
+        return this.getReadyResources().some(
+          (resource: ResourceAiAccessStatus): boolean => {
+            return resource.resourceType === type;
+          },
+        );
+      },
+    );
+
+    const byPolicy: Map<string, Array<AiResourceType>> = new Map<
+      string,
+      Array<AiResourceType>
+    >();
+
+    for (const type of presentTypes) {
+      const policyName: string = ResourceCommandPolicy.getToolPolicy(type).name;
+      byPolicy.set(policyName, [...(byPolicy.get(policyName) || []), type]);
+    }
+
+    const sections: Array<string> = [];
+
+    for (const types of byPolicy.values()) {
+      const heading: string = types
+        .map((type: AiResourceType): string => {
+          return AI_RESOURCE_TYPE_INFO[type].displayName;
+        })
+        .join(" / ");
+
+      sections.push(
+        `${heading} — read-only commands:\n${ResourceCommandPolicy.getReadCommandGuide(
+          types[0]!,
+        )}`,
+      );
+    }
+
+    return sections.join("\n\n");
+  }
+
+  private buildListAccessTool(): ObservabilityAssistantExtraTool {
+    return {
+      definition: {
+        name: LIST_INFRASTRUCTURE_ACCESS_TOOL_NAME,
+        description: `List the infrastructure resources (Docker and Podman hosts, Docker Swarm, Proxmox, VMware and Ceph clusters, database servers, hosts) linked to this signal that OneUptime AI may inspect with read-only commands through their AI agents, with their resourceId and the programs each one runs. Call this once before ${RUN_INFRASTRUCTURE_COMMAND_TOOL_NAME} if you are unsure which resource to target.`,
+        inputSchema: { type: "object", properties: {} },
+      },
+      execute: async (): Promise<ToolCallOutcome> => {
+        const ready: Array<ResourceAiAccessStatus> = this.getReadyResources();
+
+        const text: string = ready
+          .map((resource: ResourceAiAccessStatus): string => {
+            const info: AiResourceTypeInfo =
+              AI_RESOURCE_TYPE_INFO[resource.resourceType];
+
+            return `- resourceId: ${resource.resourceId} — ${describeResource(
+              resource,
+            )} — ${
+              this.isResourceUnreachable(resource.resourceId)
+                ? `UNREACHABLE for the rest of this investigation: its ${info.agentDisplayName} did not pick up an earlier command. Do not call ${RUN_INFRASTRUCTURE_COMMAND_TOOL_NAME} on it again.`
+                : `read-only commands via its ${info.agentDisplayName} (programs: ${info.programs.join(
+                    ", ",
+                  )})`
+            }`;
+          })
+          .join("\n");
+
+        return {
+          success: true,
+          textForLlm: text,
+          result: {
+            dataForLlm: text,
+            rowCount: ready.length,
+            citationLabel: "Infrastructure OneUptime AI can inspect",
+            redactionCount: 0,
+            isTruncated: false,
+          },
+        };
+      },
+    };
+  }
+
+  private buildRunCommandTool(): ObservabilityAssistantExtraTool {
+    const maxCommands: number =
+      this.options.maxCommands ?? MAX_RESOURCE_COMMANDS_PER_INVESTIGATION;
+
+    return {
+      definition: {
+        name: RUN_INFRASTRUCTURE_COMMAND_TOOL_NAME,
+        description: `Run ONE read-only command on a linked infrastructure resource through its AI agent and get its output. The resources you may inspect:\n${this.describeReadyResources()}\n\nWhat each kind of resource accepts:\n\n${this.describeReadCommandGuides()}\n\nOne command per call, written as the program followed by its arguments — never a shell line: pipes, redirects, ;, &&, $( ) and sudo are refused. Anything that changes a resource is refused here — this is an investigation — and so is anything that would read credentials; credential-looking values (passwords, tokens, keys, connection strings) are redacted from every output before you see it, so do not spend commands on them. Output is never cut off: a long output shows its first page and tells you how to read the rest with ${READ_TOOL_OUTPUT_TOOL_NAME}. Run as many commands as the investigation needs.${
+          maxCommands < MAX_RESOURCE_COMMANDS_PER_INVESTIGATION
+            ? ` At most ${maxCommands} commands in this run.`
+            : ""
+        }`,
+        inputSchema: {
+          type: "object",
+          properties: {
+            resourceId: {
+              type: "string",
+              description: `The resourceId of the linked resource to inspect (from the context or ${LIST_INFRASTRUCTURE_ACCESS_TOOL_NAME}).`,
+            },
+            command: {
+              type: "string",
+              description:
+                'The command, starting with one of the resource\'s programs, e.g. "docker ps -a", "docker logs --tail 100 web", "systemctl status nginx --no-pager", "pvesh get /cluster/resources", "ceph health detail" or "db sessions --limit 20". One line, no shell operators.',
+            },
+            rationale: {
+              type: "string",
+              description:
+                "One sentence on what you expect this command to tell you — shown to humans on the timeline.",
+            },
+            timeoutInMs: {
+              type: "number",
+              description: `Timeout in milliseconds (default ${DEFAULT_RESOURCE_COMMAND_TIMEOUT_MS}, max ${MAX_RESOURCE_COMMAND_TIMEOUT_MS}).${
+                this.options.runDeadlineAtMs !== undefined
+                  ? " Shortened automatically when the investigation's configured time limit is close."
+                  : ""
+              }`,
+            },
+          },
+          required: ["resourceId", "command", "rationale"],
+        },
+      },
+      execute: async (args: JSONObject): Promise<ToolCallOutcome> => {
+        return this.runCommand(args, maxCommands);
+      },
+    };
+  }
+
+  private async runCommand(
+    args: JSONObject,
+    maxCommands: number,
+  ): Promise<ToolCallOutcome> {
+    if (this.commandsRun >= maxCommands) {
+      return this.failure(
+        `This run has already sent ${maxCommands} infrastructure commands, the most one run may send. Finish your analysis with what you have.`,
+      );
+    }
+
+    const resourceIdRaw: string | undefined = ToolArgs.getString(
+      args,
+      "resourceId",
+    );
+    const resource: ResourceAiAccessStatus | undefined =
+      this.getReadyResources().find(
+        (candidate: ResourceAiAccessStatus): boolean => {
+          return (
+            Boolean(resourceIdRaw) &&
+            candidate.resourceId.toLowerCase() === resourceIdRaw!.toLowerCase()
+          );
+        },
+      );
+
+    if (!resource || !resource.agent) {
+      return this.failure(
+        `resourceId is not one of the resources OneUptime AI may inspect for this signal. Use ${LIST_INFRASTRUCTURE_ACCESS_TOOL_NAME}.`,
+      );
+    }
+
+    const info: AiResourceTypeInfo =
+      AI_RESOURCE_TYPE_INFO[resource.resourceType];
+    const label: string = describeResource(resource);
+
+    const command: string = ToolArgs.getString(args, "command") || "";
+
+    if (!command) {
+      return this.failure("command is required.");
+    }
+
+    const policy: ResourceCommandPolicyResult =
+      ResourceCommandPolicy.evaluateCommand({
+        resourceType: resource.resourceType,
+        command,
+      });
+
+    if (policy.tier === ResourceCommandTier.Denied) {
+      return this.failure(
+        `Refused by the ${info.displayName} command policy: ${policy.reason}. Nothing was run. What ${label} accepts:\n${ResourceCommandPolicy.getReadCommandGuide(
+          resource.resourceType,
+        )}`,
+        `Refused by the ${info.displayName} command policy on ${label}. Nothing was run.`,
+      );
+    }
+
+    if (policy.tier !== ResourceCommandTier.Read) {
+      return this.failure(
+        `"${policy.displayCommand}" would change ${label} (${policy.tier}) and this is a read-only investigation, so it was NOT run. If a change is the fix, put it in your Suggested next steps for a human. Read-only commands for it:\n${ResourceCommandPolicy.getReadCommandGuide(
+          resource.resourceType,
+        )}`,
+        `"${policy.displayCommand}" would change ${label} and this is a read-only investigation, so it was not run.`,
+      );
+    }
+
+    /*
+     * The breaker: checked before the budget is planned, a command is spent
+     * or anything is enqueued, so a dead agent costs one claim window per
+     * run rather than one per command.
+     */
+    const tripped: UnreachableAgent | undefined = this.unreachableAgents.get(
+      resource.agent.agentId,
+    );
+
+    if (tripped !== undefined) {
+      return this.failure(
+        `The ${info.agentDisplayName} of ${label} did not pick up an earlier command within ${InfrastructureInvestigationToolkit.describeSeconds(
+          tripped.claimWindowMs,
+        )}, so the ${describeResourceNoun(resource.resourceType)} is treated as unreachable for the rest of this investigation. Nothing was run. Do not call ${RUN_INFRASTRUCTURE_COMMAND_TOOL_NAME} on it again: continue with OneUptime telemetry and say in **${ResourceAccessContext.REPORT_SECTION_HEADING}** that its ${info.agentDisplayName} did not respond.`,
+        `No command was run on ${label}: its ${info.agentDisplayName} did not pick up an earlier command, so it was unreachable for the rest of this investigation.`,
+      );
+    }
+
+    const requestedTimeoutInMs: number = ToolArgs.getNumber(
+      args,
+      "timeoutInMs",
+      {
+        defaultValue: DEFAULT_RESOURCE_COMMAND_TIMEOUT_MS,
+        min: MIN_KUBECTL_TIMEOUT_MS,
+        max: MAX_RESOURCE_COMMAND_TIMEOUT_MS,
+      },
+    );
+
+    /*
+     * Planned before the command counts against the budget or anything is
+     * enqueued: a command the run's wall clock can no longer hold is
+     * refused outright rather than started and abandoned.
+     */
+    const budget: KubectlWaitBudgetResult = KubectlWaitBudget.plan({
+      requestedTimeoutInMs,
+      maxClaimTimeoutInMs: RESOURCE_COMMAND_CLAIM_TIMEOUT_MS,
+      deadlineAtMs: this.options.runDeadlineAtMs,
+    });
+
+    if (!budget.ok) {
+      return this.failure(
+        `Not enough time is left before this investigation's configured time limit to run another infrastructure command (about ${InfrastructureInvestigationToolkit.describeSeconds(
+          budget.refusal.remainingBudgetMs,
+        )} remain; a command needs at least ${InfrastructureInvestigationToolkit.describeSeconds(
+          budget.refusal.minimumBudgetMs,
+        )} including the wait for the agent). Nothing was run. Finish your analysis with what you have.`,
+      );
+    }
+
+    /*
+     * Counted once a job is enqueued, whether or not the agent then runs
+     * it: the per-investigation cap bounds the agent work this run can ask
+     * for, not the commands that happened to succeed.
+     */
+    this.commandsRun++;
+
+    let outcome: ResourceCommandJobOutcome;
+
+    try {
+      outcome = await ResourceCommandJobRunner.run({
+        projectId: this.options.projectId,
+        aiRunId: this.options.aiRunId,
+        origin: RunnerJobOrigin.AiInvestigation,
+        resourceType: resource.resourceType,
+        resourceId: new ObjectID(resource.resourceId),
+        targetResourceAiAgentId: new ObjectID(resource.agent.agentId),
+        /*
+         * The command exactly as evaluated above: the chokepoint and the
+         * agent evaluate the same text, so all three verdicts agree.
+         */
+        command,
+        stepId: `ai-investigation-resource-${this.commandsRun}`,
+        timeoutInMs: budget.plan.timeoutInMs,
+        claimTimeoutInMs: budget.plan.claimTimeoutInMs,
+        // Everything the agent returned; long output is paged below.
+        maxOutputChars: Number.MAX_SAFE_INTEGER,
+      });
+    } catch (error) {
+      const message: string = redactResourceCommandOutput({
+        resourceType: resource.resourceType,
+        program: policy.program,
+        text: error instanceof Error ? error.message : String(error),
+      }).trim();
+
+      return this.failure(
+        `The command could not be run on ${label}: ${message} Continue with OneUptime telemetry and mention this in your report.`,
+        `No command was run on ${label}: it was refused before it reached the ${info.agentDisplayName}.`,
+      );
+    }
+
+    if (outcome.claimTimedOut === true) {
+      this.unreachableAgents.set(resource.agent.agentId, {
+        claimWindowMs: budget.plan.claimTimeoutInMs,
+      });
+
+      return this.failure(
+        `The command was NOT run on ${label}: its ${info.agentDisplayName} did not pick up "${outcome.displayCommand}" within ${InfrastructureInvestigationToolkit.describeSeconds(
+          budget.plan.claimTimeoutInMs,
+        )} (it may be offline, restarting or busy). The ${describeResourceNoun(
+          resource.resourceType,
+        )} is treated as unreachable for the rest of this investigation — do not call ${RUN_INFRASTRUCTURE_COMMAND_TOOL_NAME} on it again. Continue with OneUptime telemetry and say in **${ResourceAccessContext.REPORT_SECTION_HEADING}** that its ${info.agentDisplayName} did not respond.`,
+        `No command was run on ${label}: the ${info.agentDisplayName} did not pick up the command in time.`,
+      );
+    }
+
+    /*
+     * The agent took the command and no result came back: it may have run
+     * or not. There is no output to cite, so it is a failed call — but it
+     * is never recorded as "did not run", and the panel counts it apart.
+     */
+    if (outcome.runState === ResourceCommandRunState.Unknown) {
+      const reason: string = InfrastructureInvestigationToolkit.toSentence(
+        redactResourceCommandOutput({
+          resourceType: resource.resourceType,
+          program: policy.program,
+          text: outcome.errorMessage || "No result came back for this command.",
+        }),
+      );
+
+      return this.failure(
+        `No result came back from ${label} for "${outcome.displayCommand}": ${reason} Nothing from this command is evidence. Continue with OneUptime telemetry and mention in **${ResourceAccessContext.REPORT_SECTION_HEADING}** that its ${info.agentDisplayName} stopped responding.`,
+        `${INFRASTRUCTURE_RESULT_UNKNOWN_EVENT_PREFIX} the ${info.agentDisplayName} of ${label} took the command, but no result came back, so whether it ran is unknown.`,
+      );
+    }
+
+    /*
+     * Nothing reached the program, so there is nothing to cite: a refusal
+     * by the server or the agent, or a program that could not start. The
+     * model reads why; the persisted event carries only the category,
+     * because the agent's reason can name its configuration and every
+     * reader of the incident sees the event.
+     */
+    if (outcome.executed === false) {
+      const reason: string = InfrastructureInvestigationToolkit.toSentence(
+        redactResourceCommandOutput({
+          resourceType: resource.resourceType,
+          program: policy.program,
+          text:
+            outcome.errorMessage ||
+            "The job ended without running the command.",
+        }),
+      );
+
+      return this.failure(
+        `No result came back from ${label} for "${outcome.displayCommand}": ${reason} Nothing from this command is evidence. Continue with OneUptime telemetry and mention in **${ResourceAccessContext.REPORT_SECTION_HEADING}** that the command could not run.`,
+        `No result came back from ${label}: the command was refused, or the ${info.agentDisplayName} did not run it.`,
+      );
+    }
+
+    const citationLabel: string =
+      InfrastructureInvestigationToolkit.getCitationLabel({
+        displayCommand: outcome.displayCommand,
+        resource,
+      });
+
+    /*
+     * The whole output came back (see maxOutputChars above). A long one is
+     * shown a page at a time: the first page now, the rest through
+     * read_tool_output — nothing is cut away.
+     */
+    const paged: PagedToolOutput = this.outputPager.paginate({
+      label: citationLabel,
+      text: outcome.output,
+    });
+
+    const text: string = `${ResourceCommandJobRunner.describeForLlm({
+      outcome: { ...outcome, output: paged.firstPage },
+      resourceType: resource.resourceType,
+    })}${paged.continuationNote ? `\n${paged.continuationNote}` : ""}`;
+
+    return {
+      success: true,
+      textForLlm: text,
+      result: {
+        dataForLlm: text,
+        rowCount: outcome.succeeded ? 1 : 0,
+        citationLabel,
+        redactionCount: outcome.redactionCount ?? 0,
+        // Paged output is complete; only an output the agent cut is not.
+        isTruncated:
+          outcome.isTruncated === true ||
+          hasAgentTruncationMarker(outcome.output),
+      },
+    };
+  }
+
+  /*
+   * How a command that ran is cited: `docker ps -a` on Docker host "web-1".
+   * The report's evidence list and the panel show it as is.
+   */
+  public static getCitationLabel(data: {
+    displayCommand: string;
+    resource: ResourceAiAccessStatus;
+  }): string {
+    const displayName: string = isAiResourceType(data.resource.resourceType)
+      ? AI_RESOURCE_TYPE_INFO[data.resource.resourceType].displayName
+      : "resource";
+
+    return `\`${data.displayCommand}\` on ${displayName} "${data.resource.resourceName}"`;
+  }
+
+  // A redacted reason that ends a sentence.
+  private static toSentence(text: string): string {
+    const trimmed: string = text.trim();
+
+    return SENTENCE_END_PATTERN.test(trimmed) ? trimmed : `${trimmed}.`;
+  }
+
+  private static describeSeconds(milliseconds: number): string {
+    return `${Math.max(0, Math.round(milliseconds / 1000))}s`;
+  }
+
+  /*
+   * `errorMessage` is what the run's persisted event (and so every reader
+   * of the incident's investigation panel) carries; it defaults to the
+   * model's text, which is right for failures that name nothing beyond the
+   * command and the resource.
+   */
+  private failure(text: string, errorMessage?: string): ToolCallOutcome {
+    return {
+      success: false,
+      textForLlm: text,
+      errorMessage: errorMessage ?? text,
+    };
+  }
+}