@oneuptime/common 14.0.8 → 14.0.10

package/build/dist/Server/Services/RunnerJobService.js

@@ -20,13 +20,19 @@ import KubectlPolicy from "../../Utils/AiRemediation/KubectlPolicy";
 import KubectlOutputRedactor from "../../Utils/AiRemediation/KubectlOutputRedactor";
 import KubectlWriteScope from "../../Utils/AiRemediation/KubectlWriteScope";
 import ToolResultSerializer from "../Utils/AI/Toolbox/Serializer";
-import { KUBECTL_ALLOW_NODE_OPERATIONS_ENV, KUBECTL_ALLOW_WRITES_ENV, KUBECTL_WRITE_NAMESPACES_ENV, KubectlCommandTier, KubernetesAiRemediationMode, isInClusterPostureForCluster, parseKubernetesRunnerPosture, } from "../../Types/Kubernetes/KubernetesClusterAiAccess";
+import { KUBECTL_ALLOW_NODE_OPERATIONS_ENV, KUBECTL_ALLOW_WRITES_ENV, KUBECTL_WRITE_NAMESPACES_ENV, KubectlCommandTier, KubernetesAiRemediationMode, isInClusterPostureForCluster, parseKubernetesAgentPosture, parseKubernetesRunnerPosture, } from "../../Types/Kubernetes/KubernetesClusterAiAccess";
 import RunbookCredentialType from "../../Types/Runbook/RunbookCredentialType";
 import AIRunType from "../../Types/AI/AIRunType";
 import AIRunService from "./AIRunService";
+import KubernetesClusterAiAccessService, { getKubernetesAiAccessTargetId, } from "./KubernetesClusterAiAccessService";
 import KubernetesClusterService from "./KubernetesClusterService";
 import RunbookCredentialService from "./RunbookCredentialService";
 import RunnerService, { Service as RunnerServiceClass } from "./RunnerService";
+import ResourceAiAccessService, { describeResourceNoun, getResourceAiAgentInstallNextStep, getResourceAiAgentPage, } from "./ResourceAiAccessService";
+import { AI_RESOURCE_TYPE_INFO, isAiResourceType, } from "../../Types/ResourceAiAgent/AiResourceType";
+import { ResourceAiRemediationMode, ResourceCommandTier, parseResourceAiAgentPosture, } from "../../Types/ResourceAiAgent/ResourceAiAccess";
+import ResourceCommandPolicy from "../../Utils/AiRemediation/Resource/ResourceCommandPolicy";
+import { redactResourceCommandOutput } from "../../Utils/AiRemediation/Resource/ResourceOutputRedactor";
 import QueryHelper from "../Types/Database/QueryHelper";
 /*
  * The cluster remediation modes in which a kubectl job for remediation may
@@ -56,6 +62,31 @@ export const MAX_AI_COMMAND_JOBS_PER_PROJECT_PER_HOUR = 30;
  * ceiling, because every call spends Runner time and LLM budget.
  */
 export const MAX_AI_INVESTIGATION_COMMAND_JOBS_PER_PROJECT_PER_HOUR = 240;
+/*
+ * The same two brakes for commands on infrastructure resources (Docker,
+ * Podman, Swarm, Proxmox, VMware, Ceph, database servers, hosts), counted
+ * on their own ResourceCommand rows so a busy hour on one kind of
+ * infrastructure never starves the cluster lane, or the other way round.
+ */
+export const MAX_AI_INVESTIGATION_RESOURCE_COMMAND_JOBS_PER_PROJECT_PER_HOUR = 240;
+export const MAX_AI_RESOURCE_COMMAND_JOBS_PER_PROJECT_PER_HOUR = 30;
+/*
+ * A refusal of the resource command chokepoint. A BadDataException, so
+ * every caller that already handles the kubectl chokepoint's refusals
+ * handles this one the same way; `refusalCode` says which rule refused it.
+ */
+export class ResourceCommandEnqueueRefusedException extends BadDataException {
+    constructor(refusalCode, message) {
+        super(message);
+        this.refusalCode = refusalCode;
+    }
+}
+// The resource modes in which a remediation command may be enqueued at all.
+const ENABLED_RESOURCE_REMEDIATION_MODES = [
+    ResourceAiRemediationMode.RequireApproval,
+    ResourceAiRemediationMode.Automatic,
+    ResourceAiRemediationMode.BypassApproval,
+];
 import SortOrder from "../../Types/BaseDatabase/SortOrder";
 import OneUptimeDate from "../../Types/Date";
 import PostgresAppInstance from "../Infrastructure/PostgresDatabase";
@@ -133,6 +164,10 @@ function keepSurrogatePairOnly(match) {
         : RUNNER_JOB_UNSTORABLE_CHARACTER_REPLACEMENT;
 }
 export function describeRunnerJobTimeout(data) {
+    if (data.stepType === RunbookStepType.ResourceCommand &&
+        AI_COMMAND_JOB_ORIGINS.includes(data.origin || RunnerJobOrigin.Runbook)) {
+        return describeResourceCommandTimeout(data);
+    }
     const isAiKubectl = data.stepType === RunbookStepType.Kubectl &&
         AI_COMMAND_JOB_ORIGINS.includes(data.origin || RunnerJobOrigin.Runbook);
     if (!isAiKubectl) {
@@ -155,18 +190,49 @@ export function describeRunnerJobTimeout(data) {
     const kind = data.kind === "overall" && data.wasClaimed === false
         ? "unclaimed"
         : data.kind;
+    const executor = data.isForKubernetesAiAgent
+        ? "The cluster's Kubernetes AI agent"
+        : "The cluster's Runner";
     switch (kind) {
         case "unclaimed":
-            return `The cluster's Runner did not pick up this kubectl command within ${describeSeconds(data.claimTimeoutInMs)} — it may be offline, restarting or busy with other work. Nothing was run on the cluster.`;
+            return `${executor} did not pick up this kubectl command within ${describeSeconds(data.claimTimeoutInMs)} — it may be offline, restarting or busy with other work. Nothing was run on the cluster.`;
         case "lease_expired":
-            return `The cluster's Runner stopped responding while this kubectl command was running — it may have restarted or lost its connection. What the command did is unknown.${checkFirst}`;
+            return `${executor} stopped responding while this kubectl command was running — it may have restarted or lost its connection. What the command did is unknown.${checkFirst}`;
         default:
-            return `The cluster's Runner did not report a result for this kubectl command in time — kubectl may have outlived its ${describeSeconds(data.executionTimeoutInMs)} timeout. What the command did is unknown.${checkFirst}`;
+            return `${executor} did not report a result for this kubectl command in time — kubectl may have outlived its ${describeSeconds(data.executionTimeoutInMs)} timeout. What the command did is unknown.${checkFirst}`;
     }
 }
 function describeSeconds(milliseconds) {
     return `${Math.max(1, Math.round(milliseconds / 1000))}s`;
 }
+/*
+ * The same three reasons for a command on an infrastructure resource, run
+ * by that resource's AI agent: "nobody picked it up" (nothing ran) versus
+ * "it was picked up and then went silent" (what it did is unknown), naming
+ * the agent ("the Docker AI agent") and the resource ("the Docker host").
+ */
+function describeResourceCommandTimeout(data) {
+    const agentName = isAiResourceType(data.resourceType)
+        ? AI_RESOURCE_TYPE_INFO[data.resourceType].agentDisplayName
+        : "resource's AI agent";
+    const noun = isAiResourceType(data.resourceType)
+        ? describeResourceNoun(data.resourceType)
+        : "resource";
+    const checkFirst = data.origin === RunnerJobOrigin.AiRemediation
+        ? ` Check the ${noun} before running a change again.`
+        : "";
+    const kind = data.kind === "overall" && data.wasClaimed === false
+        ? "unclaimed"
+        : data.kind;
+    switch (kind) {
+        case "unclaimed":
+            return `The ${agentName} did not pick up the command in time (within ${describeSeconds(data.claimTimeoutInMs)}) — it may be offline, restarting or busy with other work. Nothing was run on the ${noun}.`;
+        case "lease_expired":
+            return `The ${agentName} stopped responding while this command was running — it may have restarted or lost its connection. What the command did is unknown.${checkFirst}`;
+        default:
+            return `The ${agentName} did not report a result for this command in time — the command may have outlived its ${describeSeconds(data.executionTimeoutInMs)} timeout. What the command did is unknown.${checkFirst}`;
+    }
+}
 export class Service extends DatabaseService {
     constructor() {
         super(Model);
@@ -214,39 +280,63 @@ export class Service extends DatabaseService {
             usesCredential: data.usesCredential,
         });
         return refusal
-            ? Service.describeRunnerWriteScopeRefusal(refusal, Service.getRunnerWriteScopeSettings(data.usesCredential))
+            ? Service.describeRunnerWriteScopeRefusal(refusal, Service.getRunnerWriteScopeSettings(data.executorKind ||
+                (data.usesCredential ? "credential_runner" : "legacy_runner")))
             : null;
     }
     /*
-     * Where the target Runner's write scope is set, in the words a refusal
-     * uses. The in-cluster Runner is the Kubernetes agent's: its chart sets
-     * the scope (and the environment variables behind it), and an upgrade
-     * changes it. A Runner that reaches the cluster through a Kubernetes
-     * credential is an ordinary Runner no chart configures: it reports what
-     * it was started with — ONEUPTIME_KUBECTL_WRITE_NAMESPACES and
-     * ONEUPTIME_KUBECTL_ALLOW_NODE_OPERATIONS on its own host — and a
-     * restart picks up a change.
+     * Where the target's write scope is set, in the words a refusal uses —
+     * three ways, by who runs the command:
+     *
+     * - the Kubernetes AI agent: the chart's aiAgent.remediation.* values;
+     * - the previous in-cluster Runner an older chart installed: its values
+     *   are aiAccess.remediation.*, and the way to change them now is to
+     *   upgrade the chart to the AI agent, which replaces the Runner;
+     * - a Runner that reaches the cluster through a Kubernetes credential: an
+     *   ordinary Runner no chart configures, which reports what it was
+     *   started with — ONEUPTIME_KUBECTL_WRITE_NAMESPACES and
+     *   ONEUPTIME_KUBECTL_ALLOW_NODE_OPERATIONS on its own host — and picks
+     *   up a change on restart.
      */
-    static getRunnerWriteScopeSettings(usesCredential) {
-        if (usesCredential) {
+    static getRunnerWriteScopeSettings(executorKind) {
+        if (executorKind === "credential_runner") {
             return {
+                executor: "the cluster's Runner",
+                inClusterExecutor: "the cluster's in-cluster Runner",
+                executorShort: "the Runner",
                 nodeOperationsOff: `it was started with ${KUBECTL_ALLOW_NODE_OPERATIONS_ENV} (or ${KUBECTL_ALLOW_WRITES_ENV}) set to something other than true on its host`,
                 allowNodeOperations: `set ${KUBECTL_ALLOW_NODE_OPERATIONS_ENV}=true on that Runner's host and restart it`,
                 namespaces: `${KUBECTL_WRITE_NAMESPACES_ENV} on that Runner's host`,
                 addNamespace: `add it to ${KUBECTL_WRITE_NAMESPACES_ENV} on that Runner's host and restart it`,
             };
         }
+        if (executorKind === "ai_agent") {
+            return {
+                executor: "the cluster's Kubernetes AI agent",
+                inClusterExecutor: "the cluster's Kubernetes AI agent",
+                executorShort: "the AI agent",
+                nodeOperationsOff: `it was installed with aiAgent.remediation.nodeOperations=false (${KUBECTL_ALLOW_NODE_OPERATIONS_ENV})`,
+                allowNodeOperations: "upgrade the Kubernetes agent chart with --set aiAgent.remediation.nodeOperations=true",
+                namespaces: `aiAgent.remediation.namespaces on the Kubernetes agent chart, ${KUBECTL_WRITE_NAMESPACES_ENV}`,
+                addNamespace: "add it to aiAgent.remediation.namespaces and upgrade the Kubernetes agent chart",
+            };
+        }
         return {
+            executor: "the cluster's Runner",
+            inClusterExecutor: "the cluster's in-cluster Runner",
+            executorShort: "the Runner",
             nodeOperationsOff: `its Kubernetes agent was installed with aiAccess.remediation.nodeOperations=false (${KUBECTL_ALLOW_NODE_OPERATIONS_ENV})`,
-            allowNodeOperations: "upgrade the agent with --set aiAccess.remediation.nodeOperations=true",
+            allowNodeOperations: "upgrade the Kubernetes agent chart to the Kubernetes AI agent (it replaces the in-cluster Runner) with --set aiAgent.remediation.nodeOperations=true",
             namespaces: `aiAccess.remediation.namespaces on the Kubernetes agent chart, ${KUBECTL_WRITE_NAMESPACES_ENV}`,
-            addNamespace: "add it to aiAccess.remediation.namespaces and upgrade the agent",
+            addNamespace: "upgrade the Kubernetes agent chart to the Kubernetes AI agent (it replaces the in-cluster Runner) and add it to aiAgent.remediation.namespaces",
         };
     }
     // A write-scope refusal, worded for whoever enqueued the command.
     static describeRunnerWriteScopeRefusal(refusal, settings) {
         const command = `"${refusal.displayCommand}"`;
-        const wouldRefuse = "so the Runner would refuse it. It was not enqueued.";
+        const wouldRefuse = `so ${settings.executorShort} would refuse it. It was not enqueued.`;
+        const executor = settings.executor;
+        const Executor = `${executor.charAt(0).toUpperCase()}${executor.slice(1)}`;
         const scopeList = `${refusal.writeNamespaces
             .map((allowed) => {
             return `"${allowed}"`;
@@ -258,34 +348,34 @@ export class Service extends DatabaseService {
             case "node_operations":
                 return `${refusal.uncertainty === null
                     ? `${command} changes a node`
-                    : `The cluster's Runner cannot tell for certain whether ${command} changes a node (${refusal.uncertainty})`}, and the cluster's Runner does not allow node operations (cordon, uncordon, drain, taint, or labelling, annotating or patching a Node): ${settings.nodeOperationsOff}, ${wouldRefuse}${fix} To let OneUptime AI change nodes, ${settings.allowNodeOperations}.`;
+                    : `${Executor} cannot tell for certain whether ${command} changes a node (${refusal.uncertainty})`}, and ${executor} does not allow node operations (cordon, uncordon, drain, taint, or labelling, annotating or patching a Node): ${settings.nodeOperationsOff}, ${wouldRefuse}${fix} To let OneUptime AI change nodes, ${settings.allowNodeOperations}.`;
             case "objects_uncertain":
             case "namespace_uncertain":
-                return `The cluster's Runner cannot tell for certain which ${refusal.code === "namespace_uncertain" ? "namespace" : "objects"} ${command} changes (${refusal.uncertainty}), ${wouldRefuse}${fix}`;
+                return `${Executor} cannot tell for certain which ${refusal.code === "namespace_uncertain" ? "namespace" : "objects"} ${command} changes (${refusal.uncertainty}), ${wouldRefuse}${fix}`;
             case "verb_mismatch":
-                return `The cluster's Runner reads the verb of ${command} as "${refusal.readVerb}", but the kubectl policy read "${refusal.policyVerb}", so it cannot tell for certain what the command changes, ${wouldRefuse}`;
+                return `${Executor} reads the verb of ${command} as "${refusal.readVerb}", but the kubectl policy read "${refusal.policyVerb}", so it cannot tell for certain what the command changes, ${wouldRefuse}`;
             case "unnamed_namespace_objects":
-                return `${command} would change Namespace objects without naming them (a selector, --all or no name at all), so the cluster's Runner cannot tell whether one of them is ${refusal.podNamespace
+                return `${command} would change Namespace objects without naming them (a selector, --all or no name at all), so ${executor} cannot tell whether one of them is ${refusal.podNamespace
                     ? `"${refusal.podNamespace}", the namespace it runs in`
                     : "outside the namespaces it lets OneUptime AI change"}, ${wouldRefuse}${fix}`;
             case "cluster_scoped":
-                return `${command} changes ${refusal.clusterScopedKinds.join(", ")} objects, which are cluster-scoped: they live outside every namespace, so outside the namespaces the cluster's Runner lets OneUptime AI change (${scopeList}) whatever -n says, ${wouldRefuse}`;
+                return `${command} changes ${refusal.clusterScopedKinds.join(", ")} objects, which are cluster-scoped: they live outside every namespace, so outside the namespaces ${executor} lets OneUptime AI change (${scopeList}) whatever -n says, ${wouldRefuse}`;
             case "all_namespaces":
-                return `${command} would change every namespace, including ones the cluster's Runner may not change, ${wouldRefuse}${fix}`;
+                return `${command} would change every namespace, including ones ${executor} may not change, ${wouldRefuse}${fix}`;
             case "no_namespace":
-                return `${command} names no namespace, and the cluster's Runner reported no namespace of its own for kubectl to use there, ${wouldRefuse}${fix}`;
+                return `${command} names no namespace, and ${executor} reported no namespace of its own for kubectl to use there, ${wouldRefuse}${fix}`;
             case "own_namespace":
                 return refusal.namespaceSource === "namespace_object"
-                    ? `${command} would change the Namespace object "${namespace}", the namespace the cluster's in-cluster Runner itself runs in. OneUptime AI never changes it — a change there could reconfigure the Kubernetes agent or the Runner — ${wouldRefuse}`
+                    ? `${command} would change the Namespace object "${namespace}", the namespace ${settings.inClusterExecutor} itself runs in. OneUptime AI never changes it — a change there could reconfigure the Kubernetes agent or ${settings.executorShort} — ${wouldRefuse}`
                     : `${command} would change namespace "${namespace}"${refusal.namespaceSource === "default_namespace"
                         ? " (the command names none, so kubectl would use that one)"
-                        : ""}, the namespace the cluster's in-cluster Runner itself runs in. OneUptime AI never changes it — a change there could scale away the Kubernetes agent or the Runner — ${wouldRefuse} Name the namespace of the workload to fix with -n <namespace>.`;
+                        : ""}, the namespace ${settings.inClusterExecutor} itself runs in. OneUptime AI never changes it — a change there could scale away the Kubernetes agent or ${settings.executorShort} — ${wouldRefuse} Name the namespace of the workload to fix with -n <namespace>.`;
             case "outside_scope":
                 return `${command} would change ${refusal.namespaceSource === "namespace_object"
                     ? `the Namespace object "${namespace}" (a Namespace object is judged by its name; -n does not apply to it)`
                     : `namespace "${namespace}"${refusal.namespaceSource === "default_namespace"
                         ? " (the command names none, so kubectl would use that one)"
-                        : ""}`}, which is outside the namespaces the cluster's Runner lets OneUptime AI change (${scopeList}), ${wouldRefuse} To let OneUptime AI change "${namespace}", ${settings.addNamespace}.`;
+                        : ""}`}, which is outside the namespaces ${executor} lets OneUptime AI change (${scopeList}), ${wouldRefuse} To let OneUptime AI change "${namespace}", ${settings.addNamespace}.`;
             default: {
                 const unhandled = refusal.code;
                 return unhandled;
@@ -357,6 +447,14 @@ export class Service extends DatabaseService {
         if (data.stepType === RunbookStepType.Kubectl) {
             throw new BadDataException("Kubectl commands are enqueued through enqueueAiKubectlCommand.");
         }
+        /*
+         * A resource command runs on that resource's own AI agent, never on a
+         * Runner: this lane would target a Runner (targetAgentId), which must
+         * never be handed one.
+         */
+        if (data.stepType === RunbookStepType.ResourceCommand) {
+            throw new BadDataException("Resource commands run on the resource's AI agent, never on a Runner, and are enqueued through enqueueAiResourceCommand.");
+        }
         const command = (data.command || "").trim();
         if (!command) {
             throw new BadDataException("An AI command job needs a command.");
@@ -399,12 +497,15 @@ export class Service extends DatabaseService {
          * approved-plan and rollback paths — not just the FullAuto inline tool
          * that has its own pre-check for a friendlier message to the model.
          * Check-then-act, so a burst of concurrent runs can overshoot slightly;
-         * the cap is a storm brake, not a quota.
+         * the cap is a storm brake, not a quota. Resource commands share the
+         * table and the origin but have their own brake, so they never count
+         * here (see MAX_AI_RESOURCE_COMMAND_JOBS_PER_PROJECT_PER_HOUR).
          */
         const jobsInLastHour = (await this.countBy({
             query: {
                 projectId: data.projectId,
                 origin: RunnerJobOrigin.AiRemediation,
+                stepType: QueryHelper.notEquals(RunbookStepType.ResourceCommand),
                 createdAt: QueryHelper.greaterThan(OneUptimeDate.getSomeHoursAgo(1)),
             },
             props: { isRoot: true },
@@ -460,20 +561,27 @@ export class Service extends DatabaseService {
      * the cluster), and the payload carries the cluster's identifier so the
      * claim path and the Runner can refuse a job that reaches the wrong pod.
      * A credential is never sent to a kubernetes-agent Runner (its identity
-     * is minted with the project's telemetry ingestion key).
+     * is minted with the project's telemetry ingestion key), nor to the
+     * cluster's Kubernetes AI agent.
+     *
+     * The target may be the cluster's Kubernetes AI agent (a KubernetesAiAgent
+     * row, never a Runner): targetAgentId then names the agent row, and the
+     * job is written with targetKubernetesAiAgentId instead, so only that
+     * agent can claim it and no Runner ever sees it.
      *
      * It also re-reads the cluster's CURRENT AI access configuration, because
      * every caller acts on a snapshot (an investigation reads the cluster's
      * status once when it starts, a remediation run when it composes): the
-     * target must still be the cluster's bound Runner, a credential must still
-     * be its bound Kubernetes credential, an investigation's reads need "Let
-     * AI investigate with kubectl" still on, and a remediation run's commands
-     * (its reads included) need the cluster's remediation still enabled. An
-     * operator who revokes or re-points access therefore stops an in-flight
-     * run at its next command.
+     * target must still be the cluster's access target as resolved now
+     * (KubernetesClusterAiAccessService.loadAccessTarget), a credential must
+     * still be its bound Kubernetes credential, an investigation's reads need
+     * "Investigate with kubectl" still on, and a remediation run's commands
+     * (its reads included) need the cluster's fixes still enabled. An
+     * operator who revokes or re-points access — or an agent that takes over
+     * from a Runner — therefore stops an in-flight run at its next command.
      */
     async enqueueAiKubectlCommand(data) {
-        var _a, _b, _c;
+        var _a, _b, _c, _d, _e, _f;
         if (!data.targetAgentId) {
             throw new BadDataException("targetAgentId is required to dispatch a kubectl command to a Runner.");
         }
@@ -515,14 +623,16 @@ export class Service extends DatabaseService {
         /*
          * The access test is bounded by its own route limits instead, and an
          * investigation's brake counts only rows with an AI run, so tests never
-         * spend the budget real investigations share.
+         * spend the budget real investigations share. Resource commands have
+         * their own brake and never count here, so a busy hour on the project's
+         * other infrastructure never starves the cluster lane.
          */
         if (!isAccessTest) {
             const hourlyCap = data.origin === RunnerJobOrigin.AiInvestigation
                 ? MAX_AI_INVESTIGATION_COMMAND_JOBS_PER_PROJECT_PER_HOUR
                 : MAX_AI_COMMAND_JOBS_PER_PROJECT_PER_HOUR;
             const jobsInLastHour = (await this.countBy({
-                query: Object.assign({ projectId: data.projectId, origin: data.origin, createdAt: QueryHelper.greaterThan(OneUptimeDate.getSomeHoursAgo(1)) }, (data.origin === RunnerJobOrigin.AiInvestigation
+                query: Object.assign({ projectId: data.projectId, origin: data.origin, stepType: QueryHelper.notEquals(RunbookStepType.ResourceCommand), createdAt: QueryHelper.greaterThan(OneUptimeDate.getSomeHoursAgo(1)) }, (data.origin === RunnerJobOrigin.AiInvestigation
                     ? { aiRunId: QueryHelper.notNull() }
                     : {})),
                 props: { isRoot: true },
@@ -546,6 +656,7 @@ export class Service extends DatabaseService {
             },
             select: {
                 _id: true,
+                projectId: true,
                 name: true,
                 clusterIdentifier: true,
                 aiAccessRunnerId: true,
@@ -560,9 +671,16 @@ export class Service extends DatabaseService {
         }
         const clusterIdentifier = (cluster.clusterIdentifier || "").trim();
         const clusterLabel = cluster.name || clusterIdentifier || data.kubernetesClusterId.toString();
+        /*
+         * The cluster's access target as it resolves NOW. A target that is not
+         * it — a Runner the cluster was re-pointed away from, or a Runner after
+         * the cluster's Kubernetes AI agent came online — is refused below.
+         */
+        const loaded = await KubernetesClusterAiAccessService.loadAccessTarget({ cluster });
         const bindingRefusal = Service.getClusterBindingRefusal({
             cluster,
             clusterLabel,
+            resolvedTargetId: getKubernetesAiAccessTargetId(loaded.target),
             targetAgentId: data.targetAgentId,
             credentialId: data.credentialId,
             requiredSwitch: await this.getRequiredClusterSwitch({
@@ -575,72 +693,103 @@ export class Service extends DatabaseService {
         if (bindingRefusal) {
             throw new BadDataException(bindingRefusal);
         }
-        const targetRunner = await RunnerService.findOneBy({
-            query: {
-                _id: data.targetAgentId.toString(),
-                projectId: data.projectId,
-            },
-            select: { _id: true, name: true, hostInfo: true },
-            props: { isRoot: true },
-        });
-        if (!targetRunner) {
-            throw new BadDataException("The target Runner was not found or it does not belong to this project.");
-        }
-        if (data.credentialId) {
+        let posture = undefined;
+        let executorKind;
+        let targetKubernetesAiAgentId = undefined;
+        if (loaded.target.type === "ai_agent") {
+            const agent = loaded.target.agent;
+            /*
+             * The agent row was read by (project, cluster), so it is this
+             * cluster's: checked again here so the rule reads in one place.
+             */
+            if (((_a = agent.projectId) === null || _a === void 0 ? void 0 : _a.toString()) !== data.projectId.toString() ||
+                ((_b = agent.kubernetesClusterId) === null || _b === void 0 ? void 0 : _b.toString()) !==
+                    data.kubernetesClusterId.toString()) {
+                throw new BadDataException(`The Kubernetes AI agent this kubectl command names is not cluster "${clusterLabel}"'s, so it was not enqueued.`);
+            }
             /*
-             * A kubernetes-agent Runner is never handed credential material: its
-             * identity is issued with the project's telemetry ingestion key, so
-             * anyone holding that key could come to hold the credential too. The
-             * claim path refuses it as well (by the same name-or-posture rule);
-             * refusing here keeps the job from ever existing.
+             * The agent runs kubectl with its own ServiceAccount only and is
+             * never handed credential material: its identity is issued with the
+             * project's telemetry ingestion key.
              */
-            if (RunnerServiceClass.isKubernetesAgentRunnerRow(targetRunner)) {
-                throw new BadDataException(`Runner "${targetRunner.name}" is a cluster's in-cluster Runner and is never given a credential, so a kubectl command for cluster "${clusterLabel}" that needs one cannot run on it. Create a Runner under Project Settings → Runners, assign the Kubernetes credential to it and select both on the cluster's AI page.`);
+            if (data.credentialId) {
+                throw new BadDataException(`Cluster "${clusterLabel}" is reached through its Kubernetes AI agent, which runs kubectl with its own ServiceAccount and is never given a credential, so a kubectl command that needs one was not enqueued.`);
             }
-            const credential = ObjectID.isValidUUID(data.credentialId)
-                ? await RunbookCredentialService.findOneBy({
-                    query: {
-                        _id: data.credentialId,
-                        projectId: data.projectId,
-                    },
-                    select: { _id: true, name: true, credentialType: true },
-                    props: { isRoot: true },
-                })
-                : null;
-            if (!credential ||
-                credential.credentialType !== RunbookCredentialType.Kubernetes) {
-                throw new BadDataException(`The credential this kubectl command names is not a Kubernetes credential of this project, so it was not enqueued for cluster "${clusterLabel}".`);
+            posture = parseKubernetesAgentPosture(agent.posture);
+            executorKind = "ai_agent";
+            targetKubernetesAiAgentId = agent.id;
+            if (!isInClusterPostureForCluster(posture, clusterIdentifier)) {
+                throw new BadDataException(`The Kubernetes AI agent has not reported that it runs in cluster "${clusterIdentifier || cluster.name || ((_c = cluster.id) === null || _c === void 0 ? void 0 : _c.toString())}", so this kubectl command was not enqueued. Wait for its next report (within a minute), or reset the agent on the cluster's AI agent page (AI → Agent).`);
             }
         }
-        const posture = parseKubernetesRunnerPosture(targetRunner.hostInfo);
-        if (!data.credentialId) {
-            if (!isInClusterPostureForCluster(posture, clusterIdentifier)) {
-                throw new BadDataException(`A kubectl command without a Kubernetes credential can only run on the in-cluster Runner of cluster "${clusterIdentifier || cluster.name || ((_a = cluster.id) === null || _a === void 0 ? void 0 : _a.toString())}", and Runner "${targetRunner.name}" ${(posture === null || posture === void 0 ? void 0 : posture.inCluster)
-                    ? `is the in-cluster Runner of ${((_b = posture.clusterIdentifier) === null || _b === void 0 ? void 0 : _b.trim())
-                        ? `cluster "${posture.clusterIdentifier.trim()}"`
-                        : "an unnamed cluster"}`
-                    : "runs outside the cluster"}. Select a Kubernetes credential for this Runner on the cluster's AI page, or install the in-cluster Runner on this cluster.`);
+        else {
+            /*
+             * A Runner target: the Runner row the resolution read, which is the
+             * cluster's bound Runner and so belongs to this project.
+             */
+            const targetRunner = loaded.target.type === "none" ? null : loaded.target.runner;
+            if (!targetRunner) {
+                throw new BadDataException("The target Runner was not found or it does not belong to this project.");
+            }
+            executorKind = data.credentialId ? "credential_runner" : "legacy_runner";
+            if (data.credentialId) {
+                /*
+                 * A kubernetes-agent Runner is never handed credential material: its
+                 * identity is issued with the project's telemetry ingestion key, so
+                 * anyone holding that key could come to hold the credential too. The
+                 * claim path refuses it as well (by the same name-or-posture rule);
+                 * refusing here keeps the job from ever existing.
+                 */
+                if (RunnerServiceClass.isKubernetesAgentRunnerRow(targetRunner)) {
+                    throw new BadDataException(`Runner "${targetRunner.name}" is a cluster's in-cluster Runner and is never given a credential, so a kubectl command for cluster "${clusterLabel}" that needs one cannot run on it. Create a Runner under Project Settings → Runners, assign the Kubernetes credential to it and select both on the cluster's AI agent page (AI → Agent).`);
+                }
+                const credential = ObjectID.isValidUUID(data.credentialId)
+                    ? await RunbookCredentialService.findOneBy({
+                        query: {
+                            _id: data.credentialId,
+                            projectId: data.projectId,
+                        },
+                        select: { _id: true, name: true, credentialType: true },
+                        props: { isRoot: true },
+                    })
+                    : null;
+                if (!credential ||
+                    credential.credentialType !== RunbookCredentialType.Kubernetes) {
+                    throw new BadDataException(`The credential this kubectl command names is not a Kubernetes credential of this project, so it was not enqueued for cluster "${clusterLabel}".`);
+                }
+            }
+            posture = parseKubernetesRunnerPosture(targetRunner.hostInfo);
+            if (!data.credentialId) {
+                if (!isInClusterPostureForCluster(posture, clusterIdentifier)) {
+                    throw new BadDataException(`A kubectl command without a Kubernetes credential can only run on the in-cluster Runner of cluster "${clusterIdentifier || cluster.name || ((_d = cluster.id) === null || _d === void 0 ? void 0 : _d.toString())}", and Runner "${targetRunner.name}" ${(posture === null || posture === void 0 ? void 0 : posture.inCluster)
+                        ? `is the in-cluster Runner of ${((_e = posture.clusterIdentifier) === null || _e === void 0 ? void 0 : _e.trim())
+                            ? `cluster "${posture.clusterIdentifier.trim()}"`
+                            : "an unnamed cluster"}`
+                        : "runs outside the cluster"}. Select a Kubernetes credential for this Runner on the cluster's AI agent page (AI → Agent), or clear the Runner there to use the cluster's Kubernetes AI agent.`);
+                }
             }
         }
         /*
-         * A remediation write the Runner's own posture says it will refuse — a
+         * A remediation write the target's own posture says it will refuse — a
          * namespace outside its write scope, its own namespace, or a node
          * operation its chart did not grant — is refused here, before it can be
          * approved, enqueued or counted by the cluster's circuit breaker, with
-         * the chart value that would allow it. Reads are never refused for
-         * scope.
+         * the setting that would allow it. Reads are never refused for scope.
+         * A job with no credential lands in the pod's own namespace when it
+         * names none — for the agent exactly as for the in-cluster Runner.
          */
         if (data.origin === RunnerJobOrigin.AiRemediation) {
             const scopeRefusal = Service.getRunnerWriteScopeRefusal({
                 policy,
                 posture,
                 usesCredential: Boolean(data.credentialId),
+                executorKind,
             });
             if (scopeRefusal) {
                 throw new BadDataException(scopeRefusal);
             }
         }
-        const claimDeadlineAt = OneUptimeDate.addRemoveSeconds(OneUptimeDate.getCurrentDate(), Math.ceil(((_c = data.claimTimeoutInMs) !== null && _c !== void 0 ? _c : DEFAULT_CLAIM_TIMEOUT_MS) / 1000));
+        const claimDeadlineAt = OneUptimeDate.addRemoveSeconds(OneUptimeDate.getCurrentDate(), Math.ceil(((_f = data.claimTimeoutInMs) !== null && _f !== void 0 ? _f : DEFAULT_CLAIM_TIMEOUT_MS) / 1000));
         const row = new Model();
         row.projectId = data.projectId;
         row.origin = data.origin;
@@ -653,7 +802,16 @@ export class Service extends DatabaseService {
         row.kubernetesClusterId = data.kubernetesClusterId;
         row.stepId = data.stepId;
         row.stepType = RunbookStepType.Kubectl;
-        row.targetAgentId = data.targetAgentId;
+        /*
+         * Exactly one target: the agent's row, or the Runner. targetAgentId has
+         * a foreign key to Runner, so it stays empty for the agent.
+         */
+        if (targetKubernetesAiAgentId) {
+            row.targetKubernetesAiAgentId = targetKubernetesAiAgentId;
+        }
+        else {
+            row.targetAgentId = data.targetAgentId;
+        }
         row.script = "";
         row.payload = Object.assign(Object.assign({ args: policy.args, displayCommand: policy.displayCommand, tier: policy.tier, kubernetesClusterId: data.kubernetesClusterId.toString() }, (clusterIdentifier ? { clusterIdentifier } : {})), (data.credentialId ? { credentialId: data.credentialId } : {}));
         row.timeoutInMs = data.timeoutInMs;
@@ -694,32 +852,270 @@ export class Service extends DatabaseService {
     /*
      * Why a kubectl job may NOT be enqueued for this cluster as it is
      * configured right now, or null when it may. Pure, so the rule reads in
-     * one place: the job must go through the cluster's CURRENT binding, and
-     * the capability it is for must still be switched on.
+     * one place: the job must go to the cluster's CURRENT access target
+     * (resolvedTargetId, from resolveKubernetesAiAccessTarget), and the
+     * capability it is for must still be switched on.
      *
      * A credential-less job does not require the credential binding to be
-     * empty: the in-cluster Runner of the cluster wins over a bound
-     * credential (the readiness status prefers it), so such a job is
-     * legitimate with a credential still selected.
+     * empty: the in-cluster Runner of the cluster (or its Kubernetes AI
+     * agent) wins over a bound credential (the readiness status prefers it),
+     * so such a job is legitimate with a credential still selected.
      */
     static getClusterBindingRefusal(data) {
         const { cluster, clusterLabel } = data;
-        if (!cluster.aiAccessRunnerId ||
-            cluster.aiAccessRunnerId.toString() !== data.targetAgentId.toString()) {
-            return `Cluster "${clusterLabel}" is no longer reached through this Runner (its AI access binding changed or was cleared on the cluster's AI page), so this kubectl command was not enqueued.`;
+        if (!data.resolvedTargetId ||
+            data.resolvedTargetId !== data.targetAgentId.toString()) {
+            return `Cluster "${clusterLabel}" is no longer reached through this Runner or agent (its AI access changed on the cluster's AI agent page (AI → Agent), or its Kubernetes AI agent took over), so this kubectl command was not enqueued.`;
         }
         if (data.credentialId &&
             (!cluster.aiAccessCredentialId ||
                 cluster.aiAccessCredentialId.toString() !== data.credentialId)) {
-            return `Cluster "${clusterLabel}" is no longer reached with this Kubernetes credential (its AI access binding changed or was cleared on the cluster's AI page), so this kubectl command was not enqueued.`;
+            return `Cluster "${clusterLabel}" is no longer reached with this Kubernetes credential (its AI access changed on the cluster's AI agent page (AI → Agent)), so this kubectl command was not enqueued.`;
         }
         if (data.requiredSwitch === "investigation" &&
             cluster.isAiInvestigationEnabled !== true) {
-            return `"Let AI investigate with kubectl" is turned off for cluster "${clusterLabel}", so this investigation command was not enqueued.`;
+            return `"Investigate with kubectl" is turned off for cluster "${clusterLabel}", so this investigation command was not enqueued.`;
         }
         if (data.requiredSwitch === "remediation" &&
             !ENABLED_REMEDIATION_MODES.includes(cluster.aiRemediationMode)) {
-            return `AI remediation is turned off for cluster "${clusterLabel}", so this kubectl command was not enqueued.`;
+            return `AI fixes are turned off for cluster "${clusterLabel}", so this kubectl command was not enqueued.`;
+        }
+        return null;
+    }
+    /*
+     * Enqueue ONE command OneUptime AI composed for an infrastructure
+     * resource — a Docker or Podman host, a Docker Swarm, Proxmox, VMware or
+     * Ceph cluster, a database server or a host — for that resource's AI
+     * agent (a ResourceAiAgent row, never a Runner).
+     *
+     * The server-side chokepoint between an LLM's output and a resource, in
+     * the order enqueueAiKubectlCommand checks a cluster command:
+     *
+     *  1. the call itself: a resource type and id, an AI origin, the
+     *     suggestion a remediation job belongs to, the run an investigation
+     *     job belongs to (the access test alone has none, and is a read);
+     *  2. the resource command policy, re-run whatever the tool layer
+     *     checked: Denied never enqueues, and an investigation-origin job must
+     *     be Read tier — an investigation can never write;
+     *  3. the project's hourly brake for resource commands (investigations
+     *     and remediation counted apart, on ResourceCommand rows only);
+     *  4. the resource must be this project's;
+     *  5. the target is re-resolved NOW: the resource's online agent
+     *     (ResourceAiAgentService.findOnlineAgentForResource), and the agent
+     *     the caller named, if any, must still be it;
+     *  6. the switch the job needs must still be on: an investigation's reads
+     *     need the resource's investigation switch (a remediation run's reads
+     *     its fixes), and a remediation job needs its fixes (mode not
+     *     Disabled) — so an operator who turns AI off stops an in-flight run
+     *     at its next command;
+     *  7. no credential: the agent uses only its own environment's;
+     *  8. the agent's reported posture must be for this resource type and
+     *     name the identity it registered with (the payload carries it, and
+     *     the agent refuses a job for any other), and a write must fit it:
+     *     ONEUPTIME_AI_ALLOW_WRITES on, ONEUPTIME_AI_WRITE_TARGETS and the
+     *     agent's protected targets respected (ResourceCommandPolicy.
+     *     getWriteScopeRefusal, the rule the agent itself applies).
+     *
+     * The command travels as an argv (never a shell line) in a
+     * ResourceCommandJobPayload, and the row names the agent in
+     * targetResourceAiAgentId, so only that agent can claim it. Every
+     * refusal is a ResourceCommandEnqueueRefusedException saying which rule.
+     */
+    async enqueueAiResourceCommand(data) {
+        var _a;
+        // 1. The call itself.
+        if (!isAiResourceType(data.resourceType)) {
+            throw new ResourceCommandEnqueueRefusedException("invalid_request", `A resource command needs a resource type; "${String(data.resourceType)}" is not one.`);
+        }
+        const info = AI_RESOURCE_TYPE_INFO[data.resourceType];
+        const noun = describeResourceNoun(data.resourceType);
+        if (!data.resourceId) {
+            throw new ResourceCommandEnqueueRefusedException("invalid_request", `resourceId is required for a command on a ${noun}.`);
+        }
+        if (data.origin !== RunnerJobOrigin.AiInvestigation &&
+            data.origin !== RunnerJobOrigin.AiRemediation) {
+            throw new ResourceCommandEnqueueRefusedException("invalid_request", `A resource command job cannot have origin "${String(data.origin)}".`);
+        }
+        if (data.origin === RunnerJobOrigin.AiRemediation &&
+            !data.autoRemediationSuggestionId) {
+            throw new ResourceCommandEnqueueRefusedException("invalid_request", "A resource remediation job needs its auto-remediation suggestion.");
+        }
+        const isAccessTest = data.isAccessTest === true;
+        if (isAccessTest &&
+            (data.origin !== RunnerJobOrigin.AiInvestigation || data.aiRunId)) {
+            throw new ResourceCommandEnqueueRefusedException("invalid_request", "An access test is a read-only check with no AI run behind it.");
+        }
+        if (data.origin === RunnerJobOrigin.AiInvestigation &&
+            !isAccessTest &&
+            !data.aiRunId) {
+            throw new ResourceCommandEnqueueRefusedException("invalid_request", "An investigation resource command job needs the AI run it belongs to.");
+        }
+        // 2. The policy, whatever the tool layer decided.
+        const policy = ResourceCommandPolicy.evaluateCommand({
+            resourceType: data.resourceType,
+            command: data.command,
+        });
+        if (policy.tier === ResourceCommandTier.Denied) {
+            throw new ResourceCommandEnqueueRefusedException("policy_denied", `Denied by the ${info.displayName} command policy: ${policy.reason}.`);
+        }
+        if (data.origin === RunnerJobOrigin.AiInvestigation &&
+            policy.tier !== ResourceCommandTier.Read) {
+            throw new ResourceCommandEnqueueRefusedException("not_read_only", `An investigation may only run read-only commands; "${policy.displayCommand}" would change the ${noun} (${policy.tier}).`);
+        }
+        // 3. The hourly brake, on resource commands only.
+        if (!isAccessTest) {
+            const hourlyCap = data.origin === RunnerJobOrigin.AiInvestigation
+                ? MAX_AI_INVESTIGATION_RESOURCE_COMMAND_JOBS_PER_PROJECT_PER_HOUR
+                : MAX_AI_RESOURCE_COMMAND_JOBS_PER_PROJECT_PER_HOUR;
+            const jobsInLastHour = (await this.countBy({
+                query: Object.assign({ projectId: data.projectId, origin: data.origin, stepType: RunbookStepType.ResourceCommand, createdAt: QueryHelper.greaterThan(OneUptimeDate.getSomeHoursAgo(1)) }, (data.origin === RunnerJobOrigin.AiInvestigation
+                    ? { aiRunId: QueryHelper.notNull() }
+                    : {})),
+                props: { isRoot: true },
+            })).toNumber();
+            if (jobsInLastHour >= hourlyCap) {
+                throw new ResourceCommandEnqueueRefusedException("hourly_limit", data.origin === RunnerJobOrigin.AiInvestigation
+                    ? `This project has run ${jobsInLastHour} AI investigation commands on its infrastructure in the last hour, which is its limit (${hourlyCap}). No further infrastructure commands can run this hour.`
+                    : `This project has run ${jobsInLastHour} AI remediation commands on its infrastructure in the last hour, which is its limit (${hourlyCap}). No further AI commands can run on its infrastructure this hour.`);
+            }
+        }
+        // 4 + 5. The resource, and the agent it is reached through NOW.
+        const loaded = await ResourceAiAccessService.loadAccessTarget({
+            projectId: data.projectId,
+            resourceType: data.resourceType,
+            resourceId: data.resourceId,
+        });
+        if (!loaded.resource) {
+            throw new ResourceCommandEnqueueRefusedException("resource_not_found", `${info.displayName} not found or it does not belong to this project.`);
+        }
+        const bindingRefusal = Service.getResourceBindingRefusal({
+            resource: loaded.resource,
+            agent: loaded.agent,
+            targetResourceAiAgentId: data.targetResourceAiAgentId,
+            credentialId: data.credentialId,
+            // 6. The switch this job needs.
+            requiredSwitch: await this.getRequiredClusterSwitch({
+                projectId: data.projectId,
+                origin: data.origin,
+                aiRunId: data.aiRunId,
+                isAccessTest,
+            }),
+        });
+        if (bindingRefusal) {
+            throw new ResourceCommandEnqueueRefusedException(bindingRefusal.code, bindingRefusal.message);
+        }
+        const agent = loaded.agent;
+        const resourceLabel = loaded.resource.name;
+        // 8. The agent's own posture: this resource, and room for the write.
+        const posture = parseResourceAiAgentPosture(agent.posture);
+        const resourceIdentifier = (agent.resourceIdentifier ||
+            (posture === null || posture === void 0 ? void 0 : posture.resourceIdentifier) ||
+            "").trim();
+        if (!posture ||
+            posture.resourceType !== data.resourceType ||
+            !resourceIdentifier) {
+            throw new ResourceCommandEnqueueRefusedException("agent_posture_mismatch", `The ${info.agentDisplayName} has not reported that it serves ${noun} "${resourceLabel}", so this command was not enqueued. Wait for its next report (within a minute), or reset the agent on ${getResourceAiAgentPage(data.resourceType)}.`);
+        }
+        if (policy.tier !== ResourceCommandTier.Read) {
+            const scopeRefusal = ResourceCommandPolicy.getWriteScopeRefusal({
+                result: policy,
+                allowWrites: posture.allowWrites === true,
+                writeTargets: posture.writeTargets,
+                protectedTargets: posture.protectedTargets,
+                resourceType: data.resourceType,
+            });
+            if (scopeRefusal) {
+                throw new ResourceCommandEnqueueRefusedException("write_scope", `${scopeRefusal} It was not enqueued.`);
+            }
+        }
+        const claimDeadlineAt = OneUptimeDate.addRemoveSeconds(OneUptimeDate.getCurrentDate(), Math.ceil(((_a = data.claimTimeoutInMs) !== null && _a !== void 0 ? _a : DEFAULT_CLAIM_TIMEOUT_MS) / 1000));
+        const payload = {
+            resourceType: data.resourceType,
+            resourceId: loaded.resource.id.toString(),
+            resourceIdentifier,
+            program: policy.program,
+            args: policy.args,
+            displayCommand: policy.displayCommand,
+            tier: policy.tier,
+        };
+        const row = new Model();
+        row.projectId = data.projectId;
+        row.origin = data.origin;
+        if (data.aiRunId) {
+            row.aiRunId = data.aiRunId;
+        }
+        if (data.autoRemediationSuggestionId) {
+            row.autoRemediationSuggestionId = data.autoRemediationSuggestionId;
+        }
+        row.stepId = data.stepId;
+        row.stepType = RunbookStepType.ResourceCommand;
+        /*
+         * The one target: the resource's agent row. targetAgentId (a Runner)
+         * and targetKubernetesAiAgentId stay empty, so no Runner and no
+         * Kubernetes AI agent can ever claim it.
+         */
+        row.targetResourceAiAgentId = agent.id;
+        row.resourceType = data.resourceType;
+        row.resourceId = loaded.resource.id;
+        row.script = "";
+        row.payload = payload;
+        row.timeoutInMs = data.timeoutInMs;
+        row.status = RunnerJobStatus.Pending;
+        row.claimDeadlineAt = claimDeadlineAt;
+        return this.create({ data: row, props: { isRoot: true } });
+    }
+    /*
+     * Why a resource command may NOT be enqueued for this resource as it is
+     * configured right now, or null when it may. Pure, so the rule reads in
+     * one place: the resource must have an online agent (the one the caller
+     * named, when it named one) that is this resource's, the switch the job
+     * needs must still be on, and no credential may travel with it.
+     */
+    static getResourceBindingRefusal(data) {
+        var _a, _b;
+        const { resource, agent } = data;
+        const info = AI_RESOURCE_TYPE_INFO[resource.resourceType];
+        const noun = describeResourceNoun(resource.resourceType);
+        const label = resource.name;
+        if (!agent || !agent.id) {
+            return {
+                code: "agent_not_online",
+                message: `${noun.charAt(0).toUpperCase()}${noun.slice(1)} "${label}" has no online ${info.agentDisplayName}, so this command was not enqueued. ${getResourceAiAgentInstallNextStep({
+                    resourceType: resource.resourceType,
+                    resourceId: resource.id.toString(),
+                    identifier: resource.identifier,
+                })} Already installed? Check that its container is running and can reach your OneUptime URL.`,
+            };
+        }
+        if (((_a = agent.projectId) === null || _a === void 0 ? void 0 : _a.toString()) !== resource.projectId.toString() ||
+            agent.resourceType !== resource.resourceType ||
+            ((_b = agent.resourceId) === null || _b === void 0 ? void 0 : _b.toString()) !== resource.id.toString() ||
+            (data.targetResourceAiAgentId &&
+                data.targetResourceAiAgentId.toString() !== agent.id.toString())) {
+            return {
+                code: "agent_changed",
+                message: `${noun.charAt(0).toUpperCase()}${noun.slice(1)} "${label}" is no longer reached through this ${info.agentDisplayName} (the agent was reset or replaced), so this command was not enqueued.`,
+            };
+        }
+        if (data.requiredSwitch === "investigation" &&
+            resource.isAiInvestigationEnabled !== true) {
+            return {
+                code: "switch_off",
+                message: `AI investigation is turned off for ${noun} "${label}", so this investigation command was not enqueued.`,
+            };
+        }
+        if (data.requiredSwitch === "remediation" &&
+            !ENABLED_RESOURCE_REMEDIATION_MODES.includes(resource.aiRemediationMode)) {
+            return {
+                code: "switch_off",
+                message: `AI fixes are turned off for ${noun} "${label}", so this command was not enqueued.`,
+            };
+        }
+        if (data.credentialId) {
+            return {
+                code: "credential_refused",
+                message: `${noun.charAt(0).toUpperCase()}${noun.slice(1)} "${label}" is reached through its ${info.agentDisplayName}, which uses only the credentials in its own environment and is never given one by OneUptime, so a command that names a credential was not enqueued.`,
+            };
         }
         return null;
     }
@@ -898,6 +1294,21 @@ export class Service extends DatabaseService {
         return ToolResultSerializer.redact(KubectlOutputRedactor.redact(text).text)
             .text;
     }
+    /*
+     * A resource command's text through the resource command redactor (the
+     * program's own rules, then the generic ones), or the text unchanged
+     * when it is not one. Pure; redactAiJobText still runs after it.
+     */
+    static redactResourceCommandText(text, resourceCommand) {
+        if (!resourceCommand) {
+            return text;
+        }
+        return redactResourceCommandOutput({
+            resourceType: resourceCommand.resourceType,
+            program: resourceCommand.program,
+            text,
+        });
+    }
     /*
      * A Runner's text as a Postgres text column can hold it. A NUL byte is
      * ordinary in what a Runner reports — `kubectl logs` of a log that was
@@ -940,10 +1351,12 @@ export class Service extends DatabaseService {
          * write stay one atomic step (and one round trip): the CASE picks the
          * redacted text for an AI-origin row and the verbatim text otherwise.
          */
-        const redactedOutput = rawOutput === null ? null : Service.redactAiJobText(rawOutput);
+        const redactedOutput = rawOutput === null
+            ? null
+            : Service.redactAiJobText(Service.redactResourceCommandText(rawOutput, data.resourceCommand));
         const redactedErrorMessage = rawErrorMessage === null
             ? null
-            : Service.redactAiJobText(rawErrorMessage);
+            : Service.redactAiJobText(Service.redactResourceCommandText(rawErrorMessage, data.resourceCommand));
         /*
          * Only the agent that holds the lease is allowed to write the terminal
          * result. If the lease has already moved on (Worker timed out and
@@ -1015,6 +1428,10 @@ export class Service extends DatabaseService {
                     // The timeout reason is worded for what the job is.
                     origin: true,
                     stepType: true,
+                    targetKubernetesAiAgentId: true,
+                    // A resource command names its resource's AI agent.
+                    targetResourceAiAgentId: true,
+                    resourceType: true,
                 },
                 props: { isRoot: true },
             });
@@ -1052,6 +1469,8 @@ export class Service extends DatabaseService {
                         stepType: job.stepType,
                         claimTimeoutInMs: data.claimTimeoutInMs,
                         executionTimeoutInMs: data.executionTimeoutInMs,
+                        isForKubernetesAiAgent: Boolean(job.targetKubernetesAiAgentId),
+                        resourceType: job.resourceType,
                     }),
                 });
             }
@@ -1155,6 +1574,12 @@ __decorate([
     __metadata("design:paramtypes", [Object]),
     __metadata("design:returntype", Promise)
 ], Service.prototype, "enqueueAiKubectlCommand", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "enqueueAiResourceCommand", null);
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),