@oneuptime/common 14.0.9 → 14.0.11

package/Server/Utils/AI/ClusterAccess/KubectlInvestigationToolkit.ts

@@ -22,7 +22,6 @@ import { ToolArgs } from "../Toolbox/ToolTypes";
 import { ObservabilityAssistantExtraTool } from "../Chat/ObservabilityAssistant";
 import KubectlJobRunner, {
   KUBECTL_CLAIM_TIMEOUT_MS,
-  KUBECTL_OUTPUT_TRUNCATED_SUFFIX,
   KubectlJobOutcome,
   KubectlRunState,
 } from "./KubectlJobRunner";
@@ -36,6 +35,11 @@ import {
   describeClusterAccessTargetOfCluster,
   describeClusterAccessTargetRole,
 } from "./ClusterAccessContext";
+import ToolOutputPager, {
+  hasAgentTruncationMarker,
+  PagedToolOutput,
+  READ_TOOL_OUTPUT_TOOL_NAME,
+} from "../Chat/ToolOutputPager";
 
 /*
  * The run-scoped, READ-ONLY kubectl toolkit for an investigation.
@@ -49,15 +53,17 @@ import {
  * changed" stays literally true.
  *
  * Two more things the toolkit owns:
- *  - the run's wall clock. The agent loop checks its budget only between
- *    tool calls and cannot interrupt one, so the toolkit plans every
- *    command's claim window and execution timeout against the run's
- *    deadline (KubectlWaitBudget) and refuses a command the budget can no
+ *  - the run's time limit, when its project configured one (by default a
+ *    run has none). The agent loop checks it only between tool calls and
+ *    cannot interrupt one, so the toolkit plans every command's claim
+ *    window and execution timeout against the run's deadline
+ *    (KubectlWaitBudget) and refuses a command the time left can no
  *    longer hold, before anything is enqueued;
  *  - what the model sees. Output reaches it only through the shared
  *    KubectlJobRunner redaction, and Secret reads are refused by the
  *    policy outright, so the tool says both up front instead of letting
- *    the model burn calls finding out;
+ *    the model burn calls finding out. Output is never cut: the whole
+ *    redacted output is kept and a long one is paged (ToolOutputPager);
  *  - what counts as evidence. A command that never reached kubectl (no
  *    Runner claimed it, or the server or Runner refused it) is a failed
  *    tool call: it mints no citation and is not counted as run anywhere.
@@ -90,24 +96,21 @@ export {
   RUN_KUBECTL_TOOL_NAME,
 } from "../../../../Types/Kubernetes/KubernetesClusterAiAccessToolNames";
 
-/*
- * The wall clock an investigation run gets, for the runners to hand to
- * BOTH the engine (its maxWallClockMs) and this toolkit (as an absolute
- * deadline), so the loop's budget and the kubectl budget can never
- * disagree. It IS the engine's default, re-exported under the name the
- * runners use rather than restated. A re-export (a live binding) rather
- * than a copied const: the engine's module graph reaches this file
- * through the services and the investigation runners, so a value copied
- * at load time could read an engine module that has not finished
- * evaluating.
- */
-export { MAX_WALL_CLOCK_MS as INVESTIGATION_MAX_WALL_CLOCK_MS } from "../SRE/AIInvestigationEngine";
-
 export interface KubectlInvestigationToolkitOptions {
   projectId: ObjectID;
   aiRunId: ObjectID;
   clusters: Array<KubernetesClusterAiAccessStatus>;
+  /*
+   * Commands one run may send. Defaults to the runaway guard: an
+   * investigation runs as many kubectl commands as it needs.
+   */
   maxCommands?: number | undefined;
+  /*
+   * Where long outputs are kept so the model can page through them
+   * (read_tool_output). Shared with the run's other toolkits so one tool
+   * reads them all; the toolkit makes its own when none is given.
+   */
+  outputPager?: ToolOutputPager | undefined;
   /*
    * Which readiness admits a cluster. Investigations require the
    * investigation switch; a remediation run may read a cluster it is
@@ -115,9 +118,9 @@ export interface KubectlInvestigationToolkitOptions {
    */
   readinessCheck?: "investigation" | "remediation" | undefined;
   /*
-   * When the run's wall-clock budget ends (epoch milliseconds). Every
-   * command's wait is planned to end before it; absent, commands are only
-   * bounded by their own timeouts.
+   * When the run's configured time limit ends (epoch milliseconds). Every
+   * command's wait is planned to end before it; absent (the default — no
+   * time limit), commands are only bounded by their own timeouts.
    */
   runDeadlineAtMs?: number | undefined;
 }
@@ -145,8 +148,15 @@ export default class KubectlInvestigationToolkit {
     UnreachableRunner
   >();
 
+  private outputPager: ToolOutputPager;
+
   public constructor(options: KubectlInvestigationToolkitOptions) {
     this.options = options;
+    this.outputPager = options.outputPager ?? new ToolOutputPager();
+  }
+
+  public getOutputPager(): ToolOutputPager {
+    return this.outputPager;
   }
 
   // Whether the Runner serving this cluster tripped the breaker.
@@ -266,7 +276,11 @@ export default class KubectlInvestigationToolkit {
     return {
       definition: {
         name: RUN_KUBECTL_TOOL_NAME,
-        description: `Run ONE read-only kubectl command on a linked cluster and get its output. Allowed: get, describe, logs, events, top, rollout status/history, api-resources, explain, auth can-i, cluster-info. Anything that changes the cluster (delete, scale, patch, apply, exec, ...) is refused here — this is an investigation. Reading Secrets is refused too, and credential-looking values (Secret data, passwords, tokens, keys) are redacted from every output before you see it, so do not spend commands on them. Always pass -n <namespace> for namespaced objects and keep output small (use --tail, -o wide, field selectors). At most ${maxCommands} commands per investigation.`,
+        description: `Run ONE read-only kubectl command on a linked cluster and get its output. Allowed: get, describe, logs, events, top, rollout status/history, api-resources, explain, auth can-i, cluster-info. Anything that changes the cluster (delete, scale, patch, apply, exec, ...) is refused here — this is an investigation. Reading Secrets is refused too, and credential-looking values (Secret data, passwords, tokens, keys) are redacted from every output before you see it, so do not spend commands on them. Always pass -n <namespace> for namespaced objects. Output is never cut off: a long output shows its first page and tells you how to read the rest with ${READ_TOOL_OUTPUT_TOOL_NAME}. Run as many commands as the investigation needs.${
+          maxCommands < MAX_KUBECTL_COMMANDS_PER_INVESTIGATION
+            ? ` At most ${maxCommands} commands in this run.`
+            : ""
+        }`,
         inputSchema: {
           type: "object",
           properties: {
@@ -287,7 +301,11 @@ export default class KubectlInvestigationToolkit {
             },
             timeoutInMs: {
               type: "number",
-              description: `Timeout in milliseconds (default ${DEFAULT_KUBECTL_TIMEOUT_MS}, max ${MAX_KUBECTL_TIMEOUT_MS}). Shortened automatically when the investigation's time budget is nearly spent.`,
+              description: `Timeout in milliseconds (default ${DEFAULT_KUBECTL_TIMEOUT_MS}, max ${MAX_KUBECTL_TIMEOUT_MS}).${
+                this.options.runDeadlineAtMs !== undefined
+                  ? " Shortened automatically when the investigation's configured time limit is close."
+                  : ""
+              }`,
             },
           },
           required: ["clusterId", "command", "rationale"],
@@ -305,7 +323,7 @@ export default class KubectlInvestigationToolkit {
   ): Promise<ToolCallOutcome> {
     if (this.commandsRun >= maxCommands) {
       return this.failure(
-        `The per-investigation kubectl budget (${maxCommands} commands) is spent. Finish your analysis with what you have.`,
+        `This run has already sent ${maxCommands} kubectl commands, the most one run may send. Finish your analysis with what you have.`,
       );
     }
 
@@ -396,7 +414,7 @@ export default class KubectlInvestigationToolkit {
 
     if (!budget.ok) {
       return this.failure(
-        `Not enough time is left in this investigation's budget to run another kubectl command (about ${KubectlInvestigationToolkit.describeSeconds(
+        `Not enough time is left before this investigation's configured time limit to run another kubectl command (about ${KubectlInvestigationToolkit.describeSeconds(
           budget.refusal.remainingBudgetMs,
         )} remain; a command needs at least ${KubectlInvestigationToolkit.describeSeconds(
           budget.refusal.minimumBudgetMs,
@@ -425,6 +443,8 @@ export default class KubectlInvestigationToolkit {
         stepId: `ai-investigation-kubectl-${this.commandsRun}`,
         timeoutInMs: budget.plan.timeoutInMs,
         claimTimeoutInMs: budget.plan.claimTimeoutInMs,
+        // Everything the agent returned; long output is paged below.
+        maxOutputChars: Number.MAX_SAFE_INTEGER,
       });
     } catch (error) {
       const message: string =
@@ -517,7 +537,26 @@ export default class KubectlInvestigationToolkit {
       );
     }
 
-    const text: string = KubectlJobRunner.describeForLlm(outcome);
+    const citationLabel: string = `${outcome.displayCommand} on cluster "${cluster.clusterName}"`;
+
+    /*
+     * The whole output came back (see maxOutputChars above). A long one is
+     * shown a page at a time: the first page now, the rest through
+     * read_tool_output — nothing is cut away.
+     */
+    const paged: PagedToolOutput = this.outputPager.paginate({
+      label: citationLabel,
+      text: outcome.output,
+      continuationHint: KubectlInvestigationToolkit.describeStderrPosition(
+        outcome.output,
+        this.outputPager.getPageChars(),
+      ),
+    });
+
+    const text: string = `${KubectlJobRunner.describeForLlm({
+      ...outcome,
+      output: paged.firstPage,
+    })}${paged.continuationNote ? `\n${paged.continuationNote}` : ""}`;
 
     return {
       success: true,
@@ -525,15 +564,34 @@ export default class KubectlInvestigationToolkit {
       result: {
         dataForLlm: text,
         rowCount: outcome.succeeded ? 1 : 0,
-        citationLabel: `${outcome.displayCommand} on cluster "${cluster.clusterName}"`,
+        citationLabel,
         redactionCount: outcome.redactionCount ?? 0,
+        // Paged output is complete; only an output the agent cut is not.
         isTruncated:
-          outcome.isTruncated ??
-          outcome.output.endsWith(KUBECTL_OUTPUT_TRUNCATED_SUFFIX.trim()),
+          outcome.isTruncated === true ||
+          hasAgentTruncationMarker(outcome.output),
       },
     };
   }
 
+  /*
+   * kubectl says why it failed at the END of its output (the Runner puts
+   * its "[stderr]" section last). When that section falls past the first
+   * page, say where it is so the model can read it directly.
+   */
+  public static describeStderrPosition(
+    output: string,
+    pageChars: number,
+  ): string | undefined {
+    const stderrIndex: number = output.lastIndexOf("\n[stderr]\n");
+
+    if (stderrIndex < 0 || stderrIndex < pageChars) {
+      return undefined;
+    }
+
+    return `kubectl's stderr starts at offset ${stderrIndex + 1}.`;
+  }
+
   private static describeSeconds(milliseconds: number): string {
     return `${Math.max(0, Math.round(milliseconds / 1000))}s`;
   }

package/Server/Utils/AI/ClusterAccess/KubectlJobRunner.ts

@@ -274,6 +274,12 @@ export default class KubectlJobRunner {
      * every other kubectl job's.
      */
     isAccessTest?: boolean | undefined;
+    /*
+     * How much of the redacted output to keep. Defaults to the shared cap;
+     * a caller that pages long output itself (the investigation and
+     * conversation toolkits) asks for all of it.
+     */
+    maxOutputChars?: number | undefined;
   }): Promise<KubectlJobOutcome> {
     const claimTimeoutInMs: number =
       data.claimTimeoutInMs ?? KUBECTL_CLAIM_TIMEOUT_MS;
@@ -306,6 +312,7 @@ export default class KubectlJobRunner {
       command: data.command,
       claimTimeoutInMs,
       executionTimeoutInMs: data.timeoutInMs,
+      maxOutputChars: data.maxOutputChars,
     });
 
     await KubectlJobRunner.recordOutcomeOnCluster({
@@ -334,6 +341,8 @@ export default class KubectlJobRunner {
     command: string;
     claimTimeoutInMs: number;
     executionTimeoutInMs: number;
+    // How much of the redacted output to keep (default: the shared cap).
+    maxOutputChars?: number | undefined;
   }): Promise<KubectlJobOutcome> {
     const { job, terminalJob } = data;
 
@@ -375,6 +384,7 @@ export default class KubectlJobRunner {
 
     const redacted: RedactedKubectlOutput = this.redactAndCap(
       terminalJob.output || "",
+      data.maxOutputChars,
     );
 
     return {
@@ -734,8 +744,10 @@ export default class KubectlJobRunner {
    */
   public static redactAndCap(
     output: string,
-    maxChars: number = MAX_KUBECTL_OUTPUT_CHARS_FOR_LLM,
+    maxCharsOverride?: number | undefined,
   ): RedactedKubectlOutput {
+    const maxChars: number =
+      maxCharsOverride ?? MAX_KUBECTL_OUTPUT_CHARS_FOR_LLM;
     const structured: { text: string; redactionCount: number } =
       KubectlOutputRedactor.redact(output || "");
     const generic: { text: string; count: number } =

package/Server/Utils/AI/CodeFix/CodeFixReadiness.ts

@@ -1,5 +1,4 @@
 import AIAgentService from "../../../Services/AIAgentService";
-import AIService, { AutonomousBudgetStatus } from "../../../Services/AIService";
 import LlmProviderService from "../../../Services/LlmProviderService";
 import ProjectService from "../../../Services/ProjectService";
 import SubjectCodeFixRun from "../SRE/SubjectCodeFixRun";
@@ -93,37 +92,6 @@ export default class CodeFixReadiness {
       }
     }
 
-    /*
-     * The project must also be ALLOWED to spend, not just able to pay.
-     * executeWithLogging enforces two gates back to back: the balance check
-     * above, and the daily autonomous token budget — and AI_CODE_FIX_FEATURE
-     * is one of AUTONOMOUS_AI_FEATURES, so every fix completion goes through
-     * it. Unlike the balance, this one has no billing condition: it fires on
-     * self-hosted too, where a project-owned provider makes the balance gate
-     * moot and this becomes the ONLY thing that can kill a run.
-     *
-     * These readiness surfaces create subjectless exception/general tasks, so
-     * they use the "Other AI Workload" token lane. Incident/alert fix paths
-     * enforce their own lane when they are triggered and on every completion.
-     * A limit of 0 is a documented kill-switch for this lane, so without this
-     * a paused project would read "ready" forever while every subjectless run
-     * died at its first completion call.
-     */
-    const budget: AutonomousBudgetStatus =
-      await AIService.getAutonomousDailyBudgetStatus(params.projectId);
-
-    if (budget.exhausted) {
-      return {
-        id: "llmProvider",
-        ok: false,
-        title: "LLM provider",
-        detail:
-          budget.limitInTokens !== null && budget.limitInTokens <= 0
-            ? "Other autonomous AI work is paused for this project: the daily background AI token limit is set to 0. Raise or unset it under Project Settings > AI > AI Guardrails to let subjectless fix tasks run."
-            : `The daily background AI token budget is exhausted (${budget.usedTokensToday.toLocaleString()} of ${budget.limitInTokens?.toLocaleString()} tokens used today). Subjectless fix tasks resume tomorrow (UTC) — raise or unset the limit under Project Settings > AI > AI Guardrails.`,
-      };
-    }
-
     let detail: string = "";
 
     if (!isGlobal) {

package/Server/Utils/AI/CodeFix/FixRunBudget.ts

@@ -13,8 +13,8 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
  *
  * Every CodeFix AIRun counts against the daily cap for its lane: incident,
  * alert, or subjectless. Incident and alert runs use their independent
- * settings; recipes with neither subject retain `Project.aiDailyFixTaskLimit`
- * as a fallback. Null/unset means the default below and 0 pauses that lane.
+ * settings; recipes with neither subject always use the default below.
+ * Null/unset means the default and 0 pauses that lane.
  *
  * Enforced centrally at BOTH creation paths:
  *   - TelemetryExceptionService.createCodeFixRunForException (the
@@ -93,23 +93,23 @@ export default class FixRunBudget {
   ): Promise<FixRunBudgetDecision> {
     const lane: FixRunBudgetLane = this.getLane(subject);
 
-    const project: Project | null = await ProjectService.findOneById({
-      id: projectId,
-      select:
-        lane === "incident"
-          ? { incidentAiDailyFixTaskLimit: true }
-          : lane === "alert"
-            ? { alertAiDailyFixTaskLimit: true }
-            : { aiDailyFixTaskLimit: true },
-      props: { isRoot: true },
-    });
+    let configuredLimit: number | null = null;
 
-    const configuredLimit: number | null =
-      lane === "incident"
-        ? project?.incidentAiDailyFixTaskLimit ?? null
-        : lane === "alert"
-          ? project?.alertAiDailyFixTaskLimit ?? null
-          : project?.aiDailyFixTaskLimit ?? null;
+    if (lane !== "other") {
+      const project: Project | null = await ProjectService.findOneById({
+        id: projectId,
+        select:
+          lane === "incident"
+            ? { incidentAiDailyFixTaskLimit: true }
+            : { alertAiDailyFixTaskLimit: true },
+        props: { isRoot: true },
+      });
+
+      configuredLimit =
+        lane === "incident"
+          ? project?.incidentAiDailyFixTaskLimit ?? null
+          : project?.alertAiDailyFixTaskLimit ?? null;
+    }
 
     // Paused short-circuits the count query (mirrors the token budget).
     const pausedCheck: FixRunBudgetDecision = this.evaluate({
@@ -177,19 +177,21 @@ export default class FixRunBudget {
     subject?: FixRunBudgetSubject | undefined,
   ): string {
     const lane: FixRunBudgetLane = this.getLane(subject);
+
+    // Subjectless fix tasks have a fixed daily cap with no setting.
+    if (lane === "other") {
+      return `The project's daily fix task limit for AI work outside incidents and alerts has been reached (${decision.runsToday} of ${decision.limit} fix tasks created today, UTC). New fix tasks can be created tomorrow.`;
+    }
+
     const settingTitle: string =
       lane === "incident"
         ? "Daily Incident AI Fix Task Limit"
-        : lane === "alert"
-          ? "Daily Alert AI Fix Task Limit"
-          : "Daily Other AI Fix Task Limit";
+        : "Daily Alert AI Fix Task Limit";
     const settingsLocation: string =
       lane === "incident"
         ? "Incidents > AI > Investigation"
-        : lane === "alert"
-          ? "Alerts > AI > Investigation"
-          : "Project Settings > AI > AI Guardrails";
-    const laneLabel: string = lane === "other" ? "other AI" : `${lane} AI`;
+        : "Alerts > AI > Investigation";
+    const laneLabel: string = `${lane} AI`;
 
     if (decision.paused) {
       return `${laneLabel} fix tasks are paused for this project — the "${settingTitle}" is set to 0. Raise or unset it under ${settingsLocation} to resume.`;