@oneuptime/common 14.0.8 → 14.0.9

package/Server/Utils/AI/ClusterAccess/ClusterAccessContext.ts

@@ -2,6 +2,7 @@ import {
   KubernetesAiAccessGap,
   KubernetesAiRemediationMode,
   KubernetesClusterAiAccessStatus,
+  getKubernetesAiAccessTargetKind,
 } from "../../../../Types/Kubernetes/KubernetesClusterAiAccess";
 import { KUBECTL_ALWAYS_ASKS_SUMMARY } from "../../../../Types/AutoRemediation/AiRemediationCommandPlan";
 
@@ -20,6 +21,46 @@ function capitalizeFirst(text: string): string {
   return text.charAt(0).toUpperCase() + text.slice(1);
 }
 
+/*
+ * Who runs kubectl for a cluster, by name, for copy the model and humans
+ * read: "the Kubernetes AI agent", or `Runner "<name>"`. Told apart by
+ * runner.kind only — accessMethod is "in_cluster" for both the agent and
+ * the chart's previous in-cluster Runner.
+ */
+export function describeClusterAccessTarget(
+  status: Pick<KubernetesClusterAiAccessStatus, "runner">,
+): string {
+  if (getKubernetesAiAccessTargetKind(status.runner) === "ai_agent") {
+    return "the Kubernetes AI agent";
+  }
+
+  return `Runner "${status.runner?.name || "Runner"}"`;
+}
+
+/*
+ * The same with "its" ("its Kubernetes AI agent", "its Runner"), for copy
+ * that has already named the cluster.
+ */
+export function describeClusterAccessTargetOfCluster(
+  status: Pick<KubernetesClusterAiAccessStatus, "runner">,
+): string {
+  return getKubernetesAiAccessTargetKind(status.runner) === "ai_agent"
+    ? "its Kubernetes AI agent"
+    : "its Runner";
+}
+
+/*
+ * The same as a role, for copy about the cluster in general: "the
+ * cluster's Kubernetes AI agent" or "the cluster's Runner".
+ */
+export function describeClusterAccessTargetRole(
+  status: Pick<KubernetesClusterAiAccessStatus, "runner">,
+): string {
+  return getKubernetesAiAccessTargetKind(status.runner) === "ai_agent"
+    ? "the cluster's Kubernetes AI agent"
+    : "the cluster's Runner";
+}
+
 /*
  * The prompt-side view of cluster access.
  *
@@ -55,7 +96,7 @@ export default class ClusterAccessContext {
           })
           .join(
             ", ",
-          )}. Use run_kubectl for READ-ONLY inspection — kubectl get/describe/events/logs/top/rollout status — the way an on-call engineer would open a terminal: describe the failing pod, read its recent events, check node capacity and pending-pod reasons, tail the crashing container's logs. Prefer direct cluster inspection over guessing from metrics when the two disagree. Every kubectl command that ran is cited like any other tool result; a command that could not run comes back as an error, is not evidence, and must never be described as inspected. The Runner that runs kubectl lives in the cluster and can itself be down: if run_kubectl says a cluster's Runner did not pick up a command, that cluster is unreachable for the rest of this investigation — do not call run_kubectl on it again, continue with OneUptime telemetry, and say so in your report. Reading Secrets is refused and credential-looking values (Secret data, passwords, tokens, keys) are redacted from every output before you see it — never ask for them and never treat a redaction marker as a finding.`,
+          )}. Use run_kubectl for READ-ONLY inspection — kubectl get/describe/events/logs/top/rollout status — the way an on-call engineer would open a terminal: describe the failing pod, read its recent events, check node capacity and pending-pod reasons, tail the crashing container's logs. Prefer direct cluster inspection over guessing from metrics when the two disagree. Every kubectl command that ran is cited like any other tool result; a command that could not run comes back as an error, is not evidence, and must never be described as inspected. The agent (or Runner) that runs kubectl lives in the cluster and can itself be down: if run_kubectl says a cluster's agent or Runner did not pick up a command, that cluster is unreachable for the rest of this investigation — do not call run_kubectl on it again, continue with OneUptime telemetry, and say so in your report. Reading Secrets is refused and credential-looking values (Secret data, passwords, tokens, keys) are redacted from every output before you see it — never ask for them and never treat a redaction marker as a finding.`,
       );
     }
 
@@ -78,7 +119,7 @@ export default class ClusterAccessContext {
     }
 
     lines.push(
-      `Add a section **${ClusterAccessContext.REPORT_SECTION_HEADING}** before Suggested next steps: one or two sentences on what you inspected directly on the cluster (or that you could not — access not set up, or the cluster's Runner not responding — and that the human should check the cluster's AI page).`,
+      `Add a section **${ClusterAccessContext.REPORT_SECTION_HEADING}** before Suggested next steps: one or two sentences on what you inspected directly on the cluster (or that you could not — access not set up, or the cluster's AI agent or Runner not responding — and that the human should check the cluster's AI agent page (AI → Agent)).`,
     );
 
     return lines.join("\n");
@@ -97,13 +138,9 @@ export default class ClusterAccessContext {
     for (const status of statuses) {
       if (status.isInvestigationReady) {
         lines.push(
-          `- Cluster "${status.clusterName}" (clusterId: ${status.clusterId}): kubectl READ access available via run_kubectl${
-            status.accessMethod === "in_cluster"
-              ? " (in-cluster Runner)"
-              : status.credentialName
-                ? ` (Runner "${status.runner?.name}" with credential "${status.credentialName}")`
-                : ""
-          }. Remediation: ${ClusterAccessContext.describeRemediationMode(status)}.`,
+          `- Cluster "${status.clusterName}" (clusterId: ${status.clusterId}): kubectl READ access available via run_kubectl${ClusterAccessContext.describeAccessPath(
+            status,
+          )}. Remediation: ${ClusterAccessContext.describeRemediationMode(status)}.`,
         );
         continue;
       }
@@ -126,6 +163,27 @@ export default class ClusterAccessContext {
     return lines.join("\n");
   }
 
+  /*
+   * How kubectl reaches a ready cluster, as a parenthetical: the Kubernetes
+   * AI agent, the chart's previous in-cluster Runner, or a Runner with a
+   * credential.
+   */
+  private static describeAccessPath(
+    status: KubernetesClusterAiAccessStatus,
+  ): string {
+    if (getKubernetesAiAccessTargetKind(status.runner) === "ai_agent") {
+      return " (via the in-cluster Kubernetes AI agent)";
+    }
+
+    if (status.accessMethod === "in_cluster") {
+      return " (in-cluster Runner)";
+    }
+
+    return status.credentialName
+      ? ` (Runner "${status.runner?.name}" with credential "${status.credentialName}")`
+      : "";
+  }
+
   /*
    * The sentence the investigation panel shows humans when AI investigated
    * with OneUptime data only. Deterministic so it reads the same on every

package/Server/Utils/AI/ClusterAccess/KubectlInvestigationToolkit.ts

@@ -7,6 +7,7 @@ import {
   KubernetesClusterAiAccessStatus,
   MAX_KUBECTL_COMMANDS_PER_INVESTIGATION,
   MAX_KUBECTL_TIMEOUT_MS,
+  getKubernetesAiAccessTargetKind,
 } from "../../../../Types/Kubernetes/KubernetesClusterAiAccess";
 import KubectlPolicy, {
   KubectlPolicyResult,
@@ -30,6 +31,11 @@ import {
   LIST_CLUSTER_ACCESS_TOOL_NAME,
   RUN_KUBECTL_TOOL_NAME,
 } from "../../../../Types/Kubernetes/KubernetesClusterAiAccessToolNames";
+import {
+  describeClusterAccessTarget,
+  describeClusterAccessTargetOfCluster,
+  describeClusterAccessTargetRole,
+} from "./ClusterAccessContext";
 
 /*
  * The run-scoped, READ-ONLY kubectl toolkit for an investigation.
@@ -227,8 +233,13 @@ export default class KubectlInvestigationToolkit {
                 : ""
             } — ${
               this.isClusterUnreachable(cluster.clusterId)
-                ? `UNREACHABLE for the rest of this investigation: its Runner "${cluster.runner?.name}" did not pick up an earlier command. Do not call run_kubectl on it again.`
-                : `read-only kubectl via Runner "${cluster.runner?.name}"`
+                ? `UNREACHABLE for the rest of this investigation: its ${describeClusterAccessTarget(
+                    cluster,
+                  ).replace(
+                    /^the /,
+                    "",
+                  )} did not pick up an earlier command. Do not call run_kubectl on it again.`
+                : `read-only kubectl via ${describeClusterAccessTarget(cluster)}`
             }`;
           })
           .join("\n");
@@ -345,15 +356,20 @@ export default class KubectlInvestigationToolkit {
     );
 
     if (tripped !== undefined) {
+      const target: string = describeClusterAccessTarget(cluster);
+      const role: string = describeClusterAccessTargetRole(cluster);
+
       return this.failure(
-        `Runner "${cluster.runner.name}", which serves cluster "${cluster.clusterName}", did not pick up an earlier command${
+        `${target.charAt(0).toUpperCase()}${target.slice(1)}, which serves cluster "${cluster.clusterName}", did not pick up an earlier command${
           tripped.clusterName !== cluster.clusterName
             ? ` (for cluster "${tripped.clusterName}")`
             : ""
         } within ${KubectlInvestigationToolkit.describeSeconds(
           tripped.claimWindowMs,
-        )}, so the cluster is treated as unreachable for the rest of this investigation. Nothing was run. Do not call run_kubectl on this cluster again: continue with OneUptime telemetry and say in **Cluster access** that the cluster's Runner did not respond.`,
-        `kubectl was not run on cluster "${cluster.clusterName}": its Runner did not pick up an earlier command, so the cluster was unreachable for the rest of this investigation.`,
+        )}, so the cluster is treated as unreachable for the rest of this investigation. Nothing was run. Do not call run_kubectl on this cluster again: continue with OneUptime telemetry and say in **Cluster access** that ${role} did not respond.`,
+        `kubectl was not run on cluster "${cluster.clusterName}": ${describeClusterAccessTargetOfCluster(
+          cluster,
+        )} did not pick up an earlier command, so the cluster was unreachable for the rest of this investigation.`,
       );
     }
 
@@ -427,8 +443,12 @@ export default class KubectlInvestigationToolkit {
       const alsoUnreachable: Array<KubernetesClusterAiAccessStatus> =
         this.getOtherClustersOnRunner(cluster);
 
+      const role: string = describeClusterAccessTargetRole(cluster);
+
       return this.failure(
-        `kubectl was NOT run on cluster "${cluster.clusterName}": its Runner did not pick up "${outcome.displayCommand}" within ${KubectlInvestigationToolkit.describeSeconds(
+        `kubectl was NOT run on cluster "${cluster.clusterName}": ${describeClusterAccessTargetOfCluster(
+          cluster,
+        )} did not pick up "${outcome.displayCommand}" within ${KubectlInvestigationToolkit.describeSeconds(
           budget.plan.claimTimeoutInMs,
         )} (it may be offline, restarting or busy). The cluster is treated as unreachable for the rest of this investigation — do not call run_kubectl on it again.${
           alsoUnreachable.length > 0
@@ -440,8 +460,10 @@ export default class KubectlInvestigationToolkit {
                   ", ",
                 )}, so ${alsoUnreachable.length === 1 ? "that cluster is" : "those clusters are"} unreachable too.`
             : ""
-        } Continue with OneUptime telemetry and say in **Cluster access** that the cluster's Runner did not respond.`,
-        `kubectl was not run on cluster "${cluster.clusterName}": its Runner did not pick up the command in time.`,
+        } Continue with OneUptime telemetry and say in **Cluster access** that ${role} did not respond.`,
+        `kubectl was not run on cluster "${cluster.clusterName}": ${describeClusterAccessTargetOfCluster(
+          cluster,
+        )} did not pick up the command in time.`,
       );
     }
 
@@ -460,9 +482,16 @@ export default class KubectlInvestigationToolkit {
         ? redactedReason
         : `${redactedReason}.`;
 
+      const isAgent: boolean =
+        getKubernetesAiAccessTargetKind(cluster.runner) === "ai_agent";
+
       return this.failure(
-        `No kubectl result came back from cluster "${cluster.clusterName}" for "${outcome.displayCommand}": ${reason} Nothing from this command is evidence. Continue with OneUptime telemetry and mention in **Cluster access** that the cluster's Runner stopped responding.`,
-        `${KUBECTL_RESULT_UNKNOWN_EVENT_PREFIX} the Runner of cluster "${cluster.clusterName}" took the command, but no result came back, so whether it ran is unknown.`,
+        `No kubectl result came back from cluster "${cluster.clusterName}" for "${outcome.displayCommand}": ${reason} Nothing from this command is evidence. Continue with OneUptime telemetry and mention in **Cluster access** that ${describeClusterAccessTargetRole(
+          cluster,
+        )} stopped responding.`,
+        `${KUBECTL_RESULT_UNKNOWN_EVENT_PREFIX} the ${
+          isAgent ? "Kubernetes AI agent" : "Runner"
+        } of cluster "${cluster.clusterName}" took the command, but no result came back, so whether it ran is unknown.`,
       );
     }
 

package/Server/Utils/AI/ClusterAccess/KubectlJobRunner.ts

@@ -392,6 +392,9 @@ export default class KubectlJobRunner {
                   wasClaimed: claimState.wasClaimed,
                   claimTimeoutInMs: data.claimTimeoutInMs,
                   executionTimeoutInMs: data.executionTimeoutInMs,
+                  isForKubernetesAiAgent: Boolean(
+                    job.targetKubernetesAiAgentId,
+                  ),
                 })
               : terminalJob.errorMessage ||
                   `Command ended with status ${terminalJob.status}.`,
@@ -612,18 +615,26 @@ export default class KubectlJobRunner {
     wasClaimed: boolean | undefined;
     claimTimeoutInMs: number;
     executionTimeoutInMs: number;
+    // The job was for the cluster's Kubernetes AI agent, not a Runner.
+    isForKubernetesAiAgent?: boolean | undefined;
   }): string {
+    const executor: string = data.isForKubernetesAiAgent
+      ? "Kubernetes AI agent"
+      : "Runner";
+
     if (data.wasClaimed === false) {
-      return `The cluster's Runner did not pick up this kubectl command within ${KubectlJobRunner.describeSeconds(
+      return `The cluster's ${executor} did not pick up this kubectl command within ${KubectlJobRunner.describeSeconds(
         data.claimTimeoutInMs,
       )} — it may be offline, restarting or busy with other work. Nothing was run on the cluster.`;
     }
 
     if (data.wasClaimed === undefined) {
-      return "No result came back for this kubectl command in time, and whether a Runner picked it up could not be read. What the command did is unknown.";
+      return `No result came back for this kubectl command in time, and whether ${
+        data.isForKubernetesAiAgent ? "the Kubernetes AI agent" : "a Runner"
+      } picked it up could not be read. What the command did is unknown.`;
     }
 
-    return `The Runner took this kubectl command but did not report a result in time — it stopped responding, or kubectl outlived its ${KubectlJobRunner.describeSeconds(
+    return `The ${executor} took this kubectl command but did not report a result in time — it stopped responding, or kubectl outlived its ${KubectlJobRunner.describeSeconds(
       data.executionTimeoutInMs,
     )} timeout. Whether it ran, and what it did, is unknown.`;
   }

package/Server/Utils/AI/Remediation/RemediationCommandTools.ts

@@ -38,6 +38,7 @@ import {
   KubernetesRunnerPosture,
   isKubernetesAgentRunnerName,
   isKubernetesAgentRunnerPosture,
+  getKubernetesAiAccessTargetKind,
   isUnattendedRemediationMode,
   parseKubernetesRunnerPosture,
 } from "../../../../Types/Kubernetes/KubernetesClusterAiAccess";
@@ -82,7 +83,10 @@ import KubectlJobRunner, {
   RedactedKubectlOutput,
 } from "../ClusterAccess/KubectlJobRunner";
 import KubectlOutputRedactor from "../../../../Utils/AiRemediation/KubectlOutputRedactor";
-import { UNATTENDED_ROUND_BECOMES_PROPOSAL_SUMMARY } from "../ClusterAccess/ClusterAccessContext";
+import {
+  UNATTENDED_ROUND_BECOMES_PROPOSAL_SUMMARY,
+  describeClusterAccessTarget,
+} from "../ClusterAccess/ClusterAccessContext";
 import logger from "../../Logger";
 
 /*
@@ -404,7 +408,7 @@ export default class RemediationCommandToolkit {
       definition: {
         name: "list_command_targets",
         description:
-          "List where this remediation may run commands: Runners (Bash runs on the Runner's host; SSH runs on an assigned credential's host) and Kubernetes clusters (Kubectl runs through the cluster's Runner). Call this before composing any command.",
+          "List where this remediation may run commands: Runners (Bash runs on the Runner's host; SSH runs on an assigned credential's host) and Kubernetes clusters (Kubectl runs through the cluster's Kubernetes AI agent or Runner). Call this before composing any command.",
         inputSchema: {
           type: "object",
           properties: {},
@@ -497,7 +501,7 @@ export default class RemediationCommandToolkit {
         kubernetesClusterId: cluster.clusterId,
         name: cluster.clusterName,
         stepTypes: "Kubectl",
-        via: `Runner "${cluster.runner?.name}"${
+        via: `${describeClusterAccessTarget(cluster)}${
           cluster.accessMethod === "in_cluster" ? " (in-cluster)" : ""
         }`,
         /*
@@ -785,7 +789,10 @@ export default class RemediationCommandToolkit {
     return refusal
       ? RemediationCommandToolkit.describeRunnerScopeRefusal({
           refusal,
-          runnerLabel: `the Runner of cluster "${data.cluster.clusterName}"`,
+          runnerLabel:
+            getKubernetesAiAccessTargetKind(data.cluster.runner) === "ai_agent"
+              ? `the Kubernetes AI agent of cluster "${data.cluster.clusterName}"`
+              : `the Runner of cluster "${data.cluster.clusterName}"`,
           settings: RemediationCommandToolkit.getRunnerScopeSettings(
             data.cluster,
           ),
@@ -794,14 +801,29 @@ export default class RemediationCommandToolkit {
   }
 
   /*
-   * Where the bound Runner's write scope is set, for a refusal to name: the
-   * Kubernetes agent chart's values for the agent's in-cluster Runner (by
-   * its server-owned name, or its posture), the Runner's own environment
-   * for any other — a credential Runner, which no chart configures.
+   * Where the target's write scope is set, for a refusal to name — three
+   * ways, by who runs kubectl for the cluster:
+   *
+   * - the Kubernetes AI agent (runner.kind "ai_agent"): the chart's
+   *   aiAgent.remediation.* values;
+   * - the chart's previous in-cluster Runner (by its server-owned name, or
+   *   its posture): its values are aiAccess.remediation.*, and changing
+   *   them now means upgrading the chart to the AI agent, which replaces it;
+   * - any other Runner — a credential Runner, which no chart configures:
+   *   its own environment.
    */
-  private static getRunnerScopeSettings(
-    cluster: KubernetesClusterAiAccessStatus,
+  public static getRunnerScopeSettings(
+    cluster: Pick<KubernetesClusterAiAccessStatus, "runner">,
   ): { namespaces: string; nodeOperations: string } {
+    if (getKubernetesAiAccessTargetKind(cluster.runner) === "ai_agent") {
+      return {
+        namespaces:
+          "aiAgent.remediation.namespaces on the Kubernetes agent chart",
+        nodeOperations:
+          "aiAgent.remediation.nodeOperations=false on the Kubernetes agent chart",
+      };
+    }
+
     const isAgentRunner: boolean =
       isKubernetesAgentRunnerName(cluster.runner?.name) ||
       isKubernetesAgentRunnerPosture(cluster.runner?.posture);
@@ -809,9 +831,9 @@ export default class RemediationCommandToolkit {
     if (isAgentRunner) {
       return {
         namespaces:
-          "aiAccess.remediation.namespaces on the Kubernetes agent chart",
+          "aiAccess.remediation.namespaces on the Kubernetes agent chart; after upgrading to the Kubernetes AI agent, aiAgent.remediation.namespaces",
         nodeOperations:
-          "aiAccess.remediation.nodeOperations=false on the Kubernetes agent chart",
+          "aiAccess.remediation.nodeOperations=false on the Kubernetes agent chart; after upgrading to the Kubernetes AI agent, aiAgent.remediation.nodeOperations",
       };
     }
 
@@ -1783,7 +1805,7 @@ export default class RemediationCommandToolkit {
       return {
         text: `Cluster "${status.clusterName}" no longer allows AI remediation${
           gap ? ` (${gap.title})` : ""
-        } — its AI page changed during this run. ${stopText}`,
+        } — its AI agent page changed during this run. ${stopText}`,
       };
     }
 
@@ -1794,7 +1816,7 @@ export default class RemediationCommandToolkit {
     ) {
       this.revokeCluster(clusterId);
       return {
-        text: `Cluster "${status.clusterName}" was re-bound to a different Runner or credential during this run. ${stopText}`,
+        text: `Cluster "${status.clusterName}" was re-bound to a different Runner or credential during this run (or its Kubernetes AI agent took over from its Runner). ${stopText}`,
       };
     }
 
@@ -2001,7 +2023,7 @@ export default class RemediationCommandToolkit {
     return {
       definition: {
         name: "propose_remediation_commands",
-        description: `Propose an ordered plan of at most ${MAX_PLAN_COMMANDS} remediation commands for one-click human approval. Nothing executes until a human approves the whole plan. Call this at most once with your final plan (a later call replaces the earlier one). Provide a rollbackCommand for every state-changing command that has an undo (for Kubectl, e.g. kubectl rollout undo deployment/<name> -n <namespace>). Kubectl commands run through the cluster's Runner, and a write outside its writeScope (list_command_targets) is refused; ${KUBECTL_NEVER_RUNS_SUMMARY}.`,
+        description: `Propose an ordered plan of at most ${MAX_PLAN_COMMANDS} remediation commands for one-click human approval. Nothing executes until a human approves the whole plan. Call this at most once with your final plan (a later call replaces the earlier one). Provide a rollbackCommand for every state-changing command that has an undo (for Kubectl, e.g. kubectl rollout undo deployment/<name> -n <namespace>). Kubectl commands run through the cluster's Kubernetes AI agent or Runner, and a write outside its writeScope (list_command_targets) is refused; ${KUBECTL_NEVER_RUNS_SUMMARY}.`,
         inputSchema: {
           type: "object",
           properties: {

package/Server/Utils/AI/Remediation/RemediationExecutionRunner.ts

@@ -107,6 +107,26 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
  * suggestion settles what happened instead of re-executing anything.
  */
 
+/*
+ * A cluster round: remediation a cluster's AI agent page asked for (its
+ * Fixes mode), not an auto-remediation rule — it names a cluster and no
+ * rule. The project's "Enable AI command execution" opt-in gates rule
+ * rounds only; a cluster round's consent lives on the cluster (see
+ * RemediationExecutionRunner.checkProjectGates). Keyed on the suggestion
+ * row, never on the plan's step types: a rule can compose an all-kubectl
+ * plan, and it was composed under the opt-in, so it stays under it. Shared
+ * with the approve route so execution and approval can never disagree about
+ * which rounds need the opt-in.
+ */
+export function isClusterRemediationRound(suggestion: {
+  kubernetesClusterId?: ObjectID | undefined;
+  autoRemediationRuleId?: ObjectID | undefined;
+}): boolean {
+  return (
+    Boolean(suggestion.kubernetesClusterId) && !suggestion.autoRemediationRuleId
+  );
+}
+
 const MAX_SIGNAL_TITLE_CHARS: number = 500;
 const MAX_SIGNAL_DESCRIPTION_CHARS: number = 4000;
 const MAX_POSTED_ANALYSIS_CHARS: number = 6000;
@@ -507,25 +527,16 @@ export default class RemediationExecutionRunner {
         return;
       }
 
-      // Gates that must still hold at execution time, not just at rule match.
-      const gateFailure: string | null =
-        await this.checkProjectGates(projectId);
-      if (gateFailure) {
-        await this.settleNoneApplicable({
-          suggestion,
-          rationaleMarkdown: gateFailure,
-        });
-        await this.completeRunQuietly(aiRunId);
-        return;
-      }
-
       /*
        * A cluster-level round whose cluster was deleted while it waited in
        * the queue: the delete nulled kubernetesClusterId (ON DELETE SET
        * NULL), and a cluster round never had a rule — so without this it
        * would fall into the rule lane below and be closed as "the rule
        * behind this suggestion was deleted", naming a rule that never
-       * existed. Its server-written name is what is left to tell.
+       * existed. Its server-written name is what is left to tell. Checked
+       * before the project gates: nothing is left to run, and with no
+       * cluster on the row it no longer reads as a cluster round, so the
+       * gates would otherwise blame an opt-in it never needed.
        */
       const deletedClusterRound: { clusterName: string } | null =
         !suggestion.kubernetesClusterId && !suggestion.autoRemediationRuleId
@@ -541,6 +552,20 @@ export default class RemediationExecutionRunner {
         return;
       }
 
+      // Gates that must still hold at execution time, not just at rule match.
+      const gateFailure: string | null = await this.checkProjectGates({
+        projectId,
+        isClusterRound: isClusterRemediationRound(suggestion),
+      });
+      if (gateFailure) {
+        await this.settleNoneApplicable({
+          suggestion,
+          rationaleMarkdown: gateFailure,
+        });
+        await this.completeRunQuietly(aiRunId);
+        return;
+      }
+
       if (suggestion.kubernetesClusterId) {
         /*
          * Cluster-level remediation: the cluster's AI page plays the rule's
@@ -565,7 +590,7 @@ export default class RemediationExecutionRunner {
             suggestion,
             rationaleMarkdown: `OneUptime AI can no longer remediate cluster "${
               clusterTarget?.clusterName || "(deleted)"
-            }"${firstGap ? `: ${firstGap}` : ""}. Nothing was run or proposed. Review the cluster's AI page.`,
+            }"${firstGap ? `: ${firstGap}` : ""}. Nothing was run or proposed. Review the cluster's AI agent page (AI → Agent).`,
           });
           await this.completeRunQuietly(aiRunId);
           return;
@@ -1119,7 +1144,7 @@ export default class RemediationExecutionRunner {
     if (live.withdrawnReason) {
       await this.settleNoneApplicable({
         suggestion,
-        rationaleMarkdown: `${live.withdrawnReason} Review the cluster's AI page.\n\n${data.rationaleMarkdown}`,
+        rationaleMarkdown: `${live.withdrawnReason} Review the cluster's AI agent page (AI → Agent).\n\n${data.rationaleMarkdown}`,
       });
       return;
     }
@@ -1386,12 +1411,26 @@ export default class RemediationExecutionRunner {
       });
   }
 
-  // Null when all gates pass; otherwise the rationale for settling quietly.
-  private static async checkProjectGates(
-    projectId: ObjectID,
-  ): Promise<string | null> {
+  /*
+   * The project switches a round must still pass when it runs. Null when
+   * they do; otherwise the rationale for settling quietly.
+   *
+   * Enable AI and Enable auto-remediation are kill switches for every
+   * round. The "Enable AI command execution" opt-in gates rule rounds
+   * (Bash/SSH on Runners, and kubectl a rule composes) but not a cluster
+   * round (isClusterRemediationRound): its consent is the cluster's own
+   * Fixes mode plus, for the in-cluster Kubernetes AI agent, the chart's
+   * write RBAC. A cluster reached through an advanced Runner with a
+   * Kubernetes credential has no chart RBAC behind it, so its status keeps
+   * the project_ai_command_execution_disabled gap while the opt-in is off —
+   * and the cluster round stops at isRemediationReady instead.
+   */
+  public static async checkProjectGates(data: {
+    projectId: ObjectID;
+    isClusterRound: boolean;
+  }): Promise<string | null> {
     const project: Project | null = await ProjectService.findOneById({
-      id: projectId,
+      id: data.projectId,
       select: {
         enableAi: true,
         enableAutoRemediation: true,
@@ -1405,15 +1444,19 @@ export default class RemediationExecutionRunner {
       project.enableAi === false ||
       project.enableAutoRemediation === false
     ) {
-      return "AI or auto-remediation was disabled for this project before the run started — nothing was run or proposed.";
+      return "AI or auto-remediation was disabled for this project before the run started (Project Settings → AI Features) — nothing was run or proposed.";
+    }
+
+    if (data.isClusterRound) {
+      return null;
     }
 
     /*
-     * Opt-in semantics (=== true): AI command execution never runs in a
-     * project that has not explicitly turned it on.
+     * Opt-in semantics (=== true): a rule's AI command execution never runs
+     * in a project that has not explicitly turned it on.
      */
     if (project.enableAiCommandExecution !== true) {
-      return "AI command execution is not enabled for this project (Project Settings → AI) — nothing was run or proposed.";
+      return "AI command execution is not enabled for this project (Project Settings → AI Features) — nothing was run or proposed.";
     }
 
     return null;

package/Server/Utils/AI/SRE/AIInvestigationEngine.ts

@@ -18,6 +18,7 @@ import Project from "../../../../Models/DatabaseModels/Project";
 import LlmProvider from "../../../../Models/DatabaseModels/LlmProvider";
 import AIRunService from "../../../Services/AIRunService";
 import AIRunEventService from "../../../Services/AIRunEventService";
+import AIService from "../../../Services/AIService";
 import ProjectService from "../../../Services/ProjectService";
 import LlmProviderService from "../../../Services/LlmProviderService";
 import AIInvestigationQueue from "./InvestigationQueue";
@@ -205,10 +206,10 @@ export interface InvestigationRequest {
 
 export default class AIInvestigationEngine {
   /*
-   * Shared gate: AI enabled, the subject's auto-investigation opt-in on, and an
-   * LLM provider configured. Incidents and alerts each have their own opt-in so
-   * they can be enabled independently. Runs before any (subject-specific)
-   * context assembly.
+   * Shared gate: AI enabled, the subject's auto-investigation opt-in on, an
+   * LLM provider configured, and AI credits to pay for it. Incidents and
+   * alerts each have their own opt-in so they can be enabled independently.
+   * Runs before any (subject-specific) context assembly.
    */
   @CaptureSpan()
   public static async isEnabledForProject(
@@ -249,16 +250,58 @@ export default class AIInvestigationEngine {
       return "automatic_investigation_disabled";
     }
 
+    return this.getProviderOrBalanceReason(projectId);
+  }
+
+  /*
+   * The half of the gate that is not a switch: a model to call, and AI
+   * credits to pay for it. Shared with the other autonomous AI work that
+   * has its own switch (the postmortem draft on resolve), so the two can
+   * never disagree about whether AI can run at all.
+   *
+   * The balance check is the same predicate as the Kubernetes cluster's
+   * ai_balance_insufficient gap (AIService.getAiBalanceBlocker): on
+   * OneUptime's own billed provider, with no credits left and auto-recharge
+   * off, every model call would be refused with "Insufficient AI balance" —
+   * so a run started now would only fail, be retried, and fail again, and
+   * nobody would see why. It fails OPEN: a balance that cannot be read
+   * never blocks, because the model call itself still enforces the balance.
+   */
+  public static async getProviderOrBalanceReason(
+    projectId: ObjectID,
+  ): Promise<"provider_missing" | "insufficient_ai_balance" | null> {
     const llmProvider: LlmProvider | null =
       await LlmProviderService.getLLMProviderForProject(projectId);
 
     if (!llmProvider) {
       logger.debug(
-        `AI: skipping investigation for project ${projectId.toString()} — no LLM provider configured.`,
+        `AI: skipping autonomous AI work for project ${projectId.toString()} — no LLM provider configured.`,
       );
       return "provider_missing";
     }
 
+    let balanceBlocker: string | null = null;
+
+    try {
+      balanceBlocker = await AIService.getAiBalanceBlocker({
+        projectId,
+        // Already resolved above — spares the predicate a second lookup.
+        llmProvider,
+      });
+    } catch (error) {
+      logger.error(
+        `AI: could not check the AI balance for project ${projectId.toString()}; not blocking on it: ${error}`,
+      );
+      balanceBlocker = null;
+    }
+
+    if (balanceBlocker) {
+      logger.debug(
+        `AI: skipping autonomous AI work for project ${projectId.toString()} — ${balanceBlocker}`,
+      );
+      return "insufficient_ai_balance";
+    }
+
     return null;
   }