@oneuptime/common 14.0.8 → 14.0.10

package/Server/API/AutoRemediationAPI.ts

@@ -24,6 +24,7 @@ import RunbookStepType from "../../Types/Runbook/RunbookStepType";
 import {
   KubernetesAiAccessGap,
   KubernetesClusterAiAccessStatus,
+  getKubernetesAiAccessTargetKind,
   isKubernetesAgentRunnerName,
   isKubernetesAgentRunnerPosture,
   parseKubernetesRunnerPosture,
@@ -53,6 +54,25 @@ import RunnerService from "../Services/RunnerService";
 import KubernetesClusterAiAccessService from "../Services/KubernetesClusterAiAccessService";
 import CommandPlanExecutor from "../Utils/AutoRemediation/CommandPlanExecutor";
 import RemediationCommandToolkit from "../Utils/AI/Remediation/RemediationCommandTools";
+import {
+  isClusterRemediationRound,
+  isResourceRemediationRound,
+} from "../Utils/AI/Remediation/RemediationExecutionRunner";
+import ResourceAiAccessService, {
+  describeResourceNoun,
+  getResourceAiAgentPage,
+} from "../Services/ResourceAiAccessService";
+import AiResourceType, {
+  AI_RESOURCE_TYPE_INFO,
+  isAiResourceType,
+} from "../../Types/ResourceAiAgent/AiResourceType";
+import {
+  ResourceAiAccessGap,
+  ResourceAiAccessStatus,
+  ResourceCommandTier,
+} from "../../Types/ResourceAiAgent/ResourceAiAccess";
+import ResourceCommandPolicy from "../../Utils/AiRemediation/Resource/ResourceCommandPolicy";
+import { ResourceCommandPolicyResult } from "../../Utils/AiRemediation/Resource/ResourceCommandPolicyCore";
 import logger from "../Utils/Logger";
 
 const router: ExpressRouter = Express.getRouter();
@@ -112,24 +132,158 @@ async function findAccessibleSuggestion(
   return suggestion.id;
 }
 
+/*
+ * One approval reads each cluster's AI access status at most once: the
+ * Runner loop reads it to recognise the Kubernetes AI agent, and the kubectl
+ * re-check reuses the same answer.
+ */
+type ClusterStatusCache = Map<string, KubernetesClusterAiAccessStatus | null>;
+
+async function getClusterStatusOnce(data: {
+  clusterId: string;
+  projectId: ObjectID;
+  statusByClusterId: ClusterStatusCache;
+}): Promise<KubernetesClusterAiAccessStatus | null> {
+  if (data.statusByClusterId.has(data.clusterId)) {
+    return data.statusByClusterId.get(data.clusterId) || null;
+  }
+
+  const status: KubernetesClusterAiAccessStatus | null =
+    await KubernetesClusterAiAccessService.getStatusForCluster({
+      clusterId: new ObjectID(data.clusterId),
+      projectId: data.projectId,
+    });
+
+  data.statusByClusterId.set(data.clusterId, status);
+
+  return status;
+}
+
+/*
+ * The cluster status whose access target is the Kubernetes AI agent this
+ * plan names as `runnerId`, or null when `runnerId` is not the resolved AI
+ * agent of any cluster its kubectl commands target. Only a Kubectl command
+ * can reach the agent, and only the cluster's current status (not the plan)
+ * says what the target is — so a Runner id never passes for the agent, and
+ * an agent that is no longer the cluster's target falls back to the Runner
+ * re-read, which refuses it.
+ */
+async function findAiAgentTargetStatus(data: {
+  plan: AiRemediationCommandPlan;
+  runnerId: string;
+  projectId: ObjectID;
+  statusByClusterId: ClusterStatusCache;
+}): Promise<KubernetesClusterAiAccessStatus | null> {
+  for (const command of data.plan.commands) {
+    if (
+      command.stepType !== RunbookStepType.Kubectl ||
+      command.runnerId !== data.runnerId ||
+      !command.kubernetesClusterId
+    ) {
+      continue;
+    }
+
+    const status: KubernetesClusterAiAccessStatus | null =
+      await getClusterStatusOnce({
+        clusterId: command.kubernetesClusterId,
+        projectId: data.projectId,
+        statusByClusterId: data.statusByClusterId,
+      });
+
+    if (
+      status?.runner &&
+      getKubernetesAiAccessTargetKind(status.runner) === "ai_agent" &&
+      status.runner.id === data.runnerId
+    ) {
+      return status;
+    }
+  }
+
+  return null;
+}
+
+/*
+ * The AI agent's side of the Runner consent re-check. The agent is not a
+ * Runner row, so there is no canRunAiCommands flag to re-read — the chart
+ * that installed it is its consent. What can still have changed is whether
+ * it is connected, and it only ever runs Kubectl steps, so a Bash or SSH
+ * step aimed at it fails the click for the same reason as on the legacy
+ * in-cluster Runner.
+ */
+function assertAiAgentCanRunPlan(data: {
+  plan: AiRemediationCommandPlan;
+  runnerId: string;
+  status: KubernetesClusterAiAccessStatus;
+}): void {
+  const hostStep: AiRemediationCommand | undefined = data.plan.commands.find(
+    (command: AiRemediationCommand) => {
+      return (
+        command.runnerId === data.runnerId &&
+        command.stepType !== RunbookStepType.Kubectl
+      );
+    },
+  );
+
+  if (hostStep) {
+    throw new BadDataException(
+      `Command ${hostStep.sequence} is a ${hostStep.stepType} step on the Kubernetes AI agent of cluster "${data.status.clusterName}", which runs only Kubectl steps. The plan cannot be run — dismiss it and let a new suggestion be composed.`,
+    );
+  }
+
+  if (!data.status.runner?.isOnline) {
+    throw new BadDataException(
+      `The Kubernetes AI agent of cluster "${data.status.clusterName}" is offline, so this plan cannot be run. Check the agent on the cluster's AI agent page (AI → Agent) and approve again, or dismiss the suggestion.`,
+    );
+  }
+}
+
+/*
+ * What to do about a write-scope refusal, three ways by who runs kubectl for
+ * the cluster — the same split as RemediationCommandToolkit
+ * .getRunnerScopeSettings, which words the refusal itself:
+ *
+ * - the Kubernetes AI agent: its chart values; after a helm upgrade the
+ *   same agent serves the cluster, so the plan can be approved again;
+ * - the chart's previous in-cluster Runner: the chart no longer ships it,
+ *   so the way to widen its scope is to upgrade to the AI agent — which
+ *   replaces the Runner, so this plan (composed for the Runner) cannot be
+ *   approved again and a new one is needed;
+ * - any other Runner (a credential Runner): no chart configures it, its
+ *   scope is set where it runs.
+ */
+export function getScopeRefusalNextStep(
+  runner: KubernetesClusterAiAccessStatus["runner"],
+): string {
+  if (getKubernetesAiAccessTargetKind(runner) === "ai_agent") {
+    return "Dismiss the suggestion and let a new plan be composed, or change the AI agent's write access (aiAgent.remediation.*) on the Kubernetes agent chart and approve again.";
+  }
+
+  if (
+    isKubernetesAgentRunnerName(runner?.name) ||
+    isKubernetesAgentRunnerPosture(runner?.posture)
+  ) {
+    return "Dismiss the suggestion and let a new plan be composed. To allow changes like this, upgrade the Kubernetes agent chart to the AI agent and set aiAgent.remediation.*.";
+  }
+
+  return "Dismiss the suggestion and let a new plan be composed, or change the Runner's write scope on the Runner's host and approve again.";
+}
+
 /*
  * Approval-time re-check of a Kubectl command's cluster. The plan named the
  * cluster, the Runner and the credential it was composed for; any of them
  * can have changed since (the operator turned remediation off on the
- * cluster's AI page, re-pointed it at another Runner, swapped or removed
- * the credential, the agent went offline). CommandPlanExecutor hard-stops
- * on the same conditions right before each command runs — this makes the
- * Approve click fail up front, with the first blocking gap named, instead
- * of claiming the plan and having it stop one command in. Each cluster is
- * checked once however many commands target it.
+ * cluster's AI agent page, re-pointed it at another Runner, swapped or
+ * removed the credential, the agent went offline). CommandPlanExecutor
+ * hard-stops on the same conditions right before each command runs — this
+ * makes the Approve click fail up front, with the first blocking gap named,
+ * instead of claiming the plan and having it stop one command in. Each
+ * cluster is checked once however many commands target it.
  */
 async function assertKubectlCommandsStillRunnable(data: {
   plan: AiRemediationCommandPlan;
   projectId: ObjectID;
+  statusByClusterId: ClusterStatusCache;
 }): Promise<void> {
-  const statusByClusterId: Map<string, KubernetesClusterAiAccessStatus | null> =
-    new Map<string, KubernetesClusterAiAccessStatus | null>();
-
   for (const command of data.plan.commands) {
     if (command.stepType !== RunbookStepType.Kubectl) {
       continue;
@@ -146,16 +300,12 @@ async function assertKubectlCommandsStillRunnable(data: {
       );
     }
 
-    let status: KubernetesClusterAiAccessStatus | null | undefined =
-      statusByClusterId.get(command.kubernetesClusterId);
-
-    if (status === undefined) {
-      status = await KubernetesClusterAiAccessService.getStatusForCluster({
-        clusterId: new ObjectID(command.kubernetesClusterId),
+    const status: KubernetesClusterAiAccessStatus | null =
+      await getClusterStatusOnce({
+        clusterId: command.kubernetesClusterId,
         projectId: data.projectId,
+        statusByClusterId: data.statusByClusterId,
       });
-      statusByClusterId.set(command.kubernetesClusterId, status);
-    }
 
     if (!status) {
       throw new BadDataException(
@@ -173,7 +323,7 @@ async function assertKubectlCommandsStillRunnable(data: {
       throw new BadDataException(
         `Cluster "${status.clusterName}" (command ${command.sequence}) no longer allows AI remediation${
           gap ? `: ${gap.title} — ${gap.nextStep}` : ""
-        }. Fix that on the cluster's AI page and approve again, or dismiss the suggestion.`,
+        }. Fix that on the cluster's AI agent page (AI → Agent) and approve again, or dismiss the suggestion.`,
       );
     }
 
@@ -218,7 +368,7 @@ async function assertKubectlCommandsStillRunnable(data: {
 
       if (scopeRefusal) {
         throw new BadDataException(
-          `Command ${command.sequence}${part.label} cannot run on cluster "${status.clusterName}": ${scopeRefusal} Nothing ran. Dismiss the suggestion and let a new plan be composed, or change the Runner's scope on the Kubernetes agent chart and approve again.`,
+          `Command ${command.sequence}${part.label} cannot run on cluster "${status.clusterName}": ${scopeRefusal} Nothing ran. ${getScopeRefusalNextStep(status.runner)}`,
         );
       }
     }
@@ -266,6 +416,329 @@ function assertNotHostStepOnKubernetesAgent(data: {
   }
 }
 
+/*
+ * ------------------------------------------------------------------
+ * Resource commands (a Docker or Podman host, a Docker Swarm, Proxmox,
+ * VMware or Ceph cluster, a database server or a host, reached through its
+ * resource AI agent)
+ * ------------------------------------------------------------------
+ */
+
+// One approval reads each resource's AI access status at most once.
+type ResourceStatusCache = Map<string, ResourceAiAccessStatus | null>;
+
+function getResourceStatusKey(
+  resourceType: AiResourceType,
+  resourceId: string,
+): string {
+  return `${resourceType}:${resourceId.toLowerCase()}`;
+}
+
+async function getResourceStatusOnce(data: {
+  resourceType: AiResourceType;
+  resourceId: string;
+  projectId: ObjectID;
+  statusByResourceKey: ResourceStatusCache;
+}): Promise<ResourceAiAccessStatus | null> {
+  const key: string = getResourceStatusKey(data.resourceType, data.resourceId);
+
+  if (data.statusByResourceKey.has(key)) {
+    return data.statusByResourceKey.get(key) || null;
+  }
+
+  const status: ResourceAiAccessStatus | null =
+    await ResourceAiAccessService.getStatusForResource({
+      projectId: data.projectId,
+      resourceType: data.resourceType,
+      resourceId: new ObjectID(data.resourceId),
+    });
+
+  data.statusByResourceKey.set(key, status);
+
+  return status;
+}
+
+// 'Docker host "web-1"' — a resource command's resource, from its snapshot.
+function describeCommandResource(command: AiRemediationCommand): string {
+  return `${
+    isAiResourceType(command.resourceType)
+      ? describeResourceNoun(command.resourceType)
+      : "resource"
+  } "${command.resourceNameSnapshot || command.resourceId || "(unknown)"}"`;
+}
+
+/*
+ * The resource AI agent's side of the Runner consent re-check — the
+ * resource sibling of assertAiAgentCanRunPlan. A ResourceCommand step names
+ * its resource's AI agent (a ResourceAiAgent row id) where a Runner id
+ * would be. The agent is not a Runner row, so there is no canRunAiCommands
+ * flag to re-read: its consent is ONEUPTIME_AI_ALLOW_WRITES on the agent
+ * and the resource's own Fixes mode. What can have changed is whether it is
+ * still the resource's agent and connected, and it only ever runs
+ * ResourceCommand steps for its one resource — so any other step aimed at
+ * it, or a step for another resource, fails the click.
+ */
+async function assertResourceAgentCanRunPlan(data: {
+  plan: AiRemediationCommandPlan;
+  runnerId: string;
+  projectId: ObjectID;
+  statusByResourceKey: ResourceStatusCache;
+}): Promise<void> {
+  const commands: Array<AiRemediationCommand> = data.plan.commands.filter(
+    (command: AiRemediationCommand): boolean => {
+      return command.runnerId === data.runnerId;
+    },
+  );
+
+  const first: AiRemediationCommand | undefined = commands.find(
+    (command: AiRemediationCommand): boolean => {
+      return command.stepType === RunbookStepType.ResourceCommand;
+    },
+  );
+
+  if (!first) {
+    return;
+  }
+
+  const agentName: string = isAiResourceType(first.resourceType)
+    ? AI_RESOURCE_TYPE_INFO[first.resourceType].agentDisplayName
+    : "resource's AI agent";
+
+  const otherStep: AiRemediationCommand | undefined = commands.find(
+    (command: AiRemediationCommand): boolean => {
+      return command.stepType !== RunbookStepType.ResourceCommand;
+    },
+  );
+
+  if (otherStep) {
+    throw new BadDataException(
+      `Command ${otherStep.sequence} is a ${otherStep.stepType} step on the ${agentName} of ${describeCommandResource(
+        first,
+      )}, which runs only ResourceCommand steps. The plan cannot be run — dismiss it and let a new suggestion be composed.`,
+    );
+  }
+
+  const otherResource: AiRemediationCommand | undefined = commands.find(
+    (command: AiRemediationCommand): boolean => {
+      return (
+        command.resourceType !== first.resourceType ||
+        (command.resourceId || "").toLowerCase() !==
+          (first.resourceId || "").toLowerCase()
+      );
+    },
+  );
+
+  if (otherResource) {
+    throw new BadDataException(
+      `Command ${otherResource.sequence} names a different resource than the ${agentName} it is sent to serves. The plan cannot be run — dismiss it and let a new suggestion be composed.`,
+    );
+  }
+
+  if (
+    !isAiResourceType(first.resourceType) ||
+    !first.resourceId ||
+    !ObjectID.isValidUUID(first.resourceId)
+  ) {
+    throw new BadDataException(
+      `Command ${first.sequence} is a resource command with no valid resource, so this plan cannot be run. Dismiss it and let a new suggestion be composed.`,
+    );
+  }
+
+  const status: ResourceAiAccessStatus | null = await getResourceStatusOnce({
+    resourceType: first.resourceType,
+    resourceId: first.resourceId,
+    projectId: data.projectId,
+    statusByResourceKey: data.statusByResourceKey,
+  });
+
+  if (!status) {
+    throw new BadDataException(
+      `${describeCommandResource(first)} (command ${first.sequence}) no longer exists in this project, so this plan cannot be run. Dismiss it and let a new suggestion be composed.`,
+    );
+  }
+
+  const label: string = `${describeResourceNoun(status.resourceType)} "${status.resourceName}"`;
+
+  if (!status.agent || status.agent.agentId !== data.runnerId) {
+    throw new BadDataException(
+      `${label.charAt(0).toUpperCase()}${label.slice(1)} (command ${first.sequence}) is no longer reached through the ${agentName} this plan was composed for (the agent was reset or replaced). Dismiss the suggestion and let a new plan be composed for the current agent.`,
+    );
+  }
+
+  if (!status.agent.isOnline) {
+    throw new BadDataException(
+      `The ${agentName} of ${label} is offline, so this plan cannot be run. Check the agent on ${getResourceAiAgentPage(
+        status.resourceType,
+      )} and approve again, or dismiss the suggestion.`,
+    );
+  }
+}
+
+/*
+ * Approval-time re-check of every ResourceCommand step — the resource
+ * sibling of assertKubectlCommandsStillRunnable. The resource must still
+ * exist and be remediation-ready (fixes on, agent online and allowed to
+ * write, the project switches), be reached through the agent the plan was
+ * composed for, and every command and its rollback must still pass the
+ * resource command policy and the agent's reported write scope. The
+ * executor hard-stops on the same conditions right before each command
+ * runs — this makes the click fail up front, with the reason and the next
+ * step, instead of claiming the plan and stopping one command in.
+ */
+async function assertResourceCommandsStillRunnable(data: {
+  plan: AiRemediationCommandPlan;
+  projectId: ObjectID;
+  statusByResourceKey: ResourceStatusCache;
+}): Promise<void> {
+  for (const command of data.plan.commands) {
+    if (command.stepType !== RunbookStepType.ResourceCommand) {
+      continue;
+    }
+
+    if (
+      !isAiResourceType(command.resourceType) ||
+      !command.resourceId ||
+      !ObjectID.isValidUUID(command.resourceId)
+    ) {
+      throw new BadDataException(
+        `Command ${command.sequence} is a resource command with no valid resource, so this plan cannot be run. Dismiss it and let a new suggestion be composed.`,
+      );
+    }
+
+    const status: ResourceAiAccessStatus | null = await getResourceStatusOnce({
+      resourceType: command.resourceType,
+      resourceId: command.resourceId,
+      projectId: data.projectId,
+      statusByResourceKey: data.statusByResourceKey,
+    });
+
+    if (!status) {
+      throw new BadDataException(
+        `${describeCommandResource(command)} (command ${command.sequence}) no longer exists in this project, so this plan cannot be run. Dismiss it and let a new suggestion be composed.`,
+      );
+    }
+
+    const noun: string = describeResourceNoun(status.resourceType);
+    const label: string = `${noun} "${status.resourceName}"`;
+    const capitalizedLabel: string = `${label.charAt(0).toUpperCase()}${label.slice(1)}`;
+    const agentName: string =
+      AI_RESOURCE_TYPE_INFO[status.resourceType].agentDisplayName;
+
+    if (!status.isRemediationReady) {
+      const gap: ResourceAiAccessGap | undefined = status.gaps.find(
+        (candidate: ResourceAiAccessGap): boolean => {
+          return candidate.blocksRemediation;
+        },
+      );
+
+      throw new BadDataException(
+        `${capitalizedLabel} (command ${command.sequence}) no longer allows AI remediation${
+          gap ? `: ${gap.title} — ${gap.nextStep}` : ""
+        }. Fix that on ${getResourceAiAgentPage(
+          status.resourceType,
+        )} and approve again, or dismiss the suggestion.`,
+      );
+    }
+
+    if (!status.agent || status.agent.agentId !== command.runnerId) {
+      throw new BadDataException(
+        `${capitalizedLabel} (command ${command.sequence}) is no longer reached through the ${agentName} this plan was composed for (the agent was reset or replaced). Dismiss the suggestion and let a new plan be composed for the current agent.`,
+      );
+    }
+
+    /*
+     * The command and its rollback, re-checked live: the policy (a command
+     * the policy denies now never runs, whoever approves it) and the
+     * agent's write scope as it reports it now — a rollback it would refuse
+     * would otherwise only surface when verification fails and the change
+     * has to be undone.
+     */
+    const parts: Array<{ label: string; text: string | undefined }> = [
+      { label: "", text: command.command },
+      { label: "'s rollback", text: command.rollbackCommand },
+    ];
+
+    for (const part of parts) {
+      if (!part.text) {
+        continue;
+      }
+
+      const policy: ResourceCommandPolicyResult =
+        ResourceCommandPolicy.evaluateCommand({
+          resourceType: status.resourceType,
+          command: part.text,
+        });
+
+      if (policy.tier === ResourceCommandTier.Denied) {
+        throw new BadDataException(
+          `Command ${command.sequence}${part.label} is denied by the ${
+            AI_RESOURCE_TYPE_INFO[status.resourceType].displayName
+          } command policy (${policy.reason}), so it can never run. Nothing ran. Dismiss the suggestion and let a new plan be composed.`,
+        );
+      }
+
+      const scopeRefusal: string | null =
+        RemediationCommandToolkit.getResourceWriteScopeRefusal({
+          resource: status,
+          command: part.text,
+        });
+
+      if (scopeRefusal) {
+        throw new BadDataException(
+          `Command ${command.sequence}${part.label} cannot run on ${label}: ${scopeRefusal} Nothing ran. ${RemediationCommandToolkit.getResourceScopeRefusalNextStep(
+            status.resourceType,
+          )}`,
+        );
+      }
+    }
+  }
+}
+
+/*
+ * Approving a resource command runs it on the resource as root, so the
+ * approver must be someone who may edit that resource — its update ACL,
+ * label-scoped blocks included — not merely someone who may start runbooks
+ * in the project. Each resource is checked once, however many commands
+ * target it. Called after assertResourceCommandsStillRunnable, which has
+ * already refused a command with no valid resource.
+ */
+async function assertApproverMayChangeResources(data: {
+  plan: AiRemediationCommandPlan;
+  props: DatabaseCommonInteractionProps;
+  projectId: ObjectID;
+}): Promise<void> {
+  const checked: Set<string> = new Set<string>();
+
+  for (const command of data.plan.commands) {
+    if (
+      command.stepType !== RunbookStepType.ResourceCommand ||
+      !isAiResourceType(command.resourceType) ||
+      !command.resourceId ||
+      !ObjectID.isValidUUID(command.resourceId)
+    ) {
+      continue;
+    }
+
+    const key: string = getResourceStatusKey(
+      command.resourceType,
+      command.resourceId,
+    );
+
+    if (checked.has(key)) {
+      continue;
+    }
+
+    checked.add(key);
+
+    await ResourceAiAccessService.assertCallerMayChangeResource({
+      props: data.props,
+      projectId: data.projectId,
+      resourceType: command.resourceType,
+      resourceId: new ObjectID(command.resourceId),
+    });
+  }
+}
+
 async function loadSuggestionAsRoot(
   suggestionId: ObjectID,
 ): Promise<AutoRemediationSuggestion> {
@@ -284,6 +757,16 @@ async function loadSuggestionAsRoot(
         verificationWindowMinutes: true,
         suggestionType: true,
         commandPlan: true,
+        /*
+         * Which lane composed a command plan: a cluster round (a cluster,
+         * no rule) does not need the project's AI command execution opt-in,
+         * a rule round does (isClusterRemediationRound).
+         */
+        kubernetesClusterId: true,
+        autoRemediationRuleId: true,
+        // A resource round (a resource, no rule) needs no opt-in either.
+        resourceType: true,
+        resourceId: true,
       },
       props: { isRoot: true },
     });
@@ -371,10 +854,15 @@ router.post(
         assertCanExecuteRunbooks(props, suggestion.projectId);
 
         /*
-         * Re-check the project opt-in at approval time. The plan may have
-         * been composed hours ago; an operator who has since turned AI
-         * command execution off expects that switch to stop pending plans
-         * too, not just new ones.
+         * Re-check the project switches at approval time. The plan may have
+         * been composed hours ago; an operator who has since turned AI,
+         * auto-remediation or AI command execution off expects that switch
+         * to stop pending plans too, not just new ones. The command
+         * execution opt-in covers rule rounds only — the same line
+         * RemediationExecutionRunner.checkProjectGates draws — and is keyed
+         * on the round, never on the plan's step types: a rule's
+         * all-kubectl plan still needs it. A cluster round's consent is the
+         * cluster's own, re-checked per kubectl command below.
          */
         const project: Project | null = await ProjectService.findOneById({
           id: suggestion.projectId,
@@ -390,11 +878,20 @@ router.post(
         if (
           !project ||
           project.enableAi === false ||
-          project.enableAutoRemediation === false ||
+          project.enableAutoRemediation === false
+        ) {
+          throw new BadDataException(
+            "AI or auto-remediation is disabled for this project, so this plan cannot be run. Re-enable it in Project Settings → AI Features, or dismiss the suggestion.",
+          );
+        }
+
+        if (
+          !isClusterRemediationRound(suggestion) &&
+          !isResourceRemediationRound(suggestion) &&
           project.enableAiCommandExecution !== true
         ) {
           throw new BadDataException(
-            "AI command execution is disabled for this project, so this plan cannot be run. Re-enable it in Project Settings → AI, or dismiss the suggestion.",
+            "AI command execution is disabled for this project, so this plan cannot be run. Re-enable it in Project Settings → AI Features, or dismiss the suggestion.",
           );
         }
 
@@ -421,7 +918,64 @@ router.post(
           ),
         );
 
+        const statusByClusterId: ClusterStatusCache = new Map<
+          string,
+          KubernetesClusterAiAccessStatus | null
+        >();
+
+        const statusByResourceKey: ResourceStatusCache = new Map<
+          string,
+          ResourceAiAccessStatus | null
+        >();
+
         for (const runnerId of runnerIds) {
+          /*
+           * A kubectl command composed for a cluster's Kubernetes AI agent
+           * names the agent's id where a Runner id would be. The agent is
+           * not a Runner row — re-reading the Runner table would refuse
+           * every agent plan — so it is recognised from the cluster's
+           * status instead, and must be connected.
+           */
+          const aiAgentStatus: KubernetesClusterAiAccessStatus | null =
+            await findAiAgentTargetStatus({
+              plan,
+              runnerId,
+              projectId: suggestion.projectId,
+              statusByClusterId,
+            });
+
+          if (aiAgentStatus) {
+            assertAiAgentCanRunPlan({
+              plan,
+              runnerId,
+              status: aiAgentStatus,
+            });
+            continue;
+          }
+
+          /*
+           * A ResourceCommand step names its resource's AI agent the same
+           * way; that agent is not a Runner row either, so it is recognised
+           * from the resource's status and must be its current, connected
+           * agent.
+           */
+          if (
+            plan.commands.some((command: AiRemediationCommand): boolean => {
+              return (
+                command.stepType === RunbookStepType.ResourceCommand &&
+                command.runnerId === runnerId
+              );
+            })
+          ) {
+            await assertResourceAgentCanRunPlan({
+              plan,
+              runnerId,
+              projectId: suggestion.projectId,
+              statusByResourceKey,
+            });
+            continue;
+          }
+
           const runner: Runner | null = await RunnerService.findOneBy({
             query: {
               _id: runnerId,
@@ -454,6 +1008,25 @@ router.post(
         await assertKubectlCommandsStillRunnable({
           plan,
           projectId: suggestion.projectId,
+          statusByClusterId,
+        });
+
+        // And for every resource command: its resource, as it is now.
+        await assertResourceCommandsStillRunnable({
+          plan,
+          projectId: suggestion.projectId,
+          statusByResourceKey,
+        });
+
+        /*
+         * And the approver may change every resource the plan changes: the
+         * plan runs as root on the resource, so this is where the
+         * resource's own edit ACL (with its label scope) applies.
+         */
+        await assertApproverMayChangeResources({
+          plan,
+          props,
+          projectId: suggestion.projectId,
         });
 
         const claimedPlan: number =