@oneuptime/common 14.0.1 → 14.0.3

package/build/dist/Server/Services/RunnerJobService.js

@@ -12,17 +12,50 @@ import ObjectID from "../../Types/ObjectID";
 import BadDataException from "../../Types/Exception/BadDataException";
 import Model from "../../Models/DatabaseModels/RunnerJob";
 import RunnerJobStatus from "../../Types/Runbook/RunnerJobStatus";
-import RunnerJobOrigin from "../../Types/Runbook/RunnerJobOrigin";
+import RunnerJobOrigin, { AI_COMMAND_JOB_ORIGINS, } from "../../Types/Runbook/RunnerJobOrigin";
 import RunbookStepType, { isPayloadCarryingStepType, isRunnerExecutedStepType, } from "../../Types/Runbook/RunbookStepType";
 import { AI_COMMAND_STEP_TYPES } from "../../Types/AutoRemediation/AiRemediationCommandPlan";
 import CommandPolicy from "../../Utils/AiRemediation/CommandPolicy";
+import KubectlPolicy from "../../Utils/AiRemediation/KubectlPolicy";
+import KubectlOutputRedactor from "../../Utils/AiRemediation/KubectlOutputRedactor";
+import KubectlWriteScope from "../../Utils/AiRemediation/KubectlWriteScope";
+import ToolResultSerializer from "../Utils/AI/Toolbox/Serializer";
+import { KUBECTL_ALLOW_NODE_OPERATIONS_ENV, KUBECTL_ALLOW_WRITES_ENV, KUBECTL_WRITE_NAMESPACES_ENV, KubectlCommandTier, KubernetesAiRemediationMode, isInClusterPostureForCluster, parseKubernetesRunnerPosture, } from "../../Types/Kubernetes/KubernetesClusterAiAccess";
+import RunbookCredentialType from "../../Types/Runbook/RunbookCredentialType";
+import AIRunType from "../../Types/AI/AIRunType";
+import AIRunService from "./AIRunService";
+import KubernetesClusterService from "./KubernetesClusterService";
+import RunbookCredentialService from "./RunbookCredentialService";
+import RunnerService, { Service as RunnerServiceClass } from "./RunnerService";
 import QueryHelper from "../Types/Database/QueryHelper";
+/*
+ * The cluster remediation modes in which a kubectl job for remediation may
+ * be enqueued at all. Anything else — Disabled, or a value the dashboard
+ * does not know — reads as "remediation is off".
+ */
+const ENABLED_REMEDIATION_MODES = [
+    KubernetesAiRemediationMode.RequireApproval,
+    KubernetesAiRemediationMode.Automatic,
+    KubernetesAiRemediationMode.BypassApproval,
+];
+// AI runs whose cluster reads are governed by the remediation switch.
+const REMEDIATION_RUN_TYPES = [
+    AIRunType.RemediationExecution,
+    AIRunType.RemediationPlan,
+];
 /*
  * Project-wide hourly ceiling on AI-composed command jobs, counted on the
  * RunnerJob rows themselves so every path (FullAuto inline execution,
  * approved plans, rollbacks) shares one brake.
  */
 export const MAX_AI_COMMAND_JOBS_PER_PROJECT_PER_HOUR = 30;
+/*
+ * Separate, looser brake for READ-ONLY kubectl during investigations: a
+ * busy hour of alerts on a large cluster legitimately runs many describe /
+ * logs / events calls, and none of them can change anything. Still a
+ * ceiling, because every call spends Runner time and LLM budget.
+ */
+export const MAX_AI_INVESTIGATION_COMMAND_JOBS_PER_PROJECT_PER_HOUR = 240;
 import SortOrder from "../../Types/BaseDatabase/SortOrder";
 import OneUptimeDate from "../../Types/Date";
 import PostgresAppInstance from "../Infrastructure/PostgresDatabase";
@@ -72,10 +105,193 @@ function unwrapRows(result) {
     }
     return [];
 }
+/*
+ * What stands in for a character a Postgres text column cannot hold: the
+ * Unicode replacement character, so a reader of the stored output can see
+ * that something was there instead of the bytes silently closing up.
+ */
+export const RUNNER_JOB_UNSTORABLE_CHARACTER_REPLACEMENT = "\uFFFD";
+/*
+ * U+0000. Postgres rejects it anywhere in a text value (22021 "invalid byte
+ * sequence for encoding UTF8: 0x00") — at bind time, so even a parameter
+ * the statement's CASE would not pick fails the whole UPDATE.
+ */
+// eslint-disable-next-line no-control-regex
+const NUL_CHARACTER_PATTERN = /\u0000/g;
+/*
+ * A UTF-16 surrogate pair, or a surrogate half on its own. A lone half has
+ * no UTF-8 encoding at all (a Runner that cut a string between the two
+ * halves sends one); the driver would quietly turn it into U+FFFD on the
+ * wire, so it is replaced here explicitly, before redaction, and the text
+ * the redaction passes read is exactly the text that is stored. A whole
+ * pair is matched first and kept.
+ */
+const SURROGATE_PATTERN = /[\uD800-\uDBFF][\uDC00-\uDFFF]|[\uD800-\uDFFF]/g;
+function keepSurrogatePairOnly(match) {
+    return match.length === 2
+        ? match
+        : RUNNER_JOB_UNSTORABLE_CHARACTER_REPLACEMENT;
+}
+export function describeRunnerJobTimeout(data) {
+    const isAiKubectl = data.stepType === RunbookStepType.Kubectl &&
+        AI_COMMAND_JOB_ORIGINS.includes(data.origin || RunnerJobOrigin.Runbook);
+    if (!isAiKubectl) {
+        switch (data.kind) {
+            case "unclaimed":
+                return "No runbook agent picked up this step before the wait window expired. The agent may be offline — check that it is running and reachable, then try again.";
+            case "lease_expired":
+                return "The runbook agent stopped responding while this step was running. The agent may have crashed or lost its network connection — check that it is still online, then try running the runbook again.";
+            default:
+                return "This step ran longer than the allowed execution window. Increase the timeout on the step or make the script complete faster.";
+        }
+    }
+    /*
+     * A change that may have been applied must not read as "try again": for
+     * a remediation command, say to look first.
+     */
+    const checkFirst = data.origin === RunnerJobOrigin.AiRemediation
+        ? " Check the cluster before running a change again."
+        : "";
+    const kind = data.kind === "overall" && data.wasClaimed === false
+        ? "unclaimed"
+        : data.kind;
+    switch (kind) {
+        case "unclaimed":
+            return `The cluster's Runner did not pick up this kubectl command within ${describeSeconds(data.claimTimeoutInMs)} — it may be offline, restarting or busy with other work. Nothing was run on the cluster.`;
+        case "lease_expired":
+            return `The cluster's Runner stopped responding while this kubectl command was running — it may have restarted or lost its connection. What the command did is unknown.${checkFirst}`;
+        default:
+            return `The cluster's Runner did not report a result for this kubectl command in time — kubectl may have outlived its ${describeSeconds(data.executionTimeoutInMs)} timeout. What the command did is unknown.${checkFirst}`;
+    }
+}
+function describeSeconds(milliseconds) {
+    return `${Math.max(1, Math.round(milliseconds / 1000))}s`;
+}
 export class Service extends DatabaseService {
     constructor() {
         super(Model);
     }
+    /*
+     * Why the target Runner, by its own reported posture, will refuse this
+     * kubectl WRITE — or null when nothing it reported rules the write out.
+     * The rule is the Runner's own (KubectlWriteScope.getRefusal, the one the
+     * Runner asks before it spawns kubectl), asked here with what the Runner
+     * reported instead of what it was started with. Refusing here means
+     * nothing is approved, enqueued or counted by the cluster's circuit
+     * breaker only to be refused on the Runner (or answered Forbidden by the
+     * API server). Each refusal names the setting to change where that
+     * Runner is configured (getRunnerWriteScopeSettings): the Kubernetes
+     * agent chart's values for the in-cluster Runner, the Runner's own
+     * environment on its host for a Runner that reaches the cluster through
+     * a credential.
+     *
+     * - writeNamespaces (aiAccess.remediation.namespaces, or a credential
+     *   Runner's ONEUPTIME_KUBECTL_WRITE_NAMESPACES) and podNamespace are the
+     *   Runner's scope; a Runner that did not report one has none.
+     * - allowNodeOperations === false (aiAccess.remediation.nodeOperations,
+     *   or a credential Runner's ONEUPTIME_KUBECTL_ALLOW_NODE_OPERATIONS): no
+     *   node operation. A Runner that never said (absent) is not
+     *   second-guessed.
+     * - usesCredential: a write that names no namespace lands in "default"
+     *   through a credential's kubeconfig, and in the pod's own namespace
+     *   in-cluster.
+     *
+     * Reads, and commands the policy denies outright, are never refused for
+     * scope.
+     */
+    static getRunnerWriteScopeRefusal(data) {
+        const { policy, posture } = data;
+        if (!posture ||
+            policy.tier === KubectlCommandTier.Read ||
+            policy.tier === KubectlCommandTier.Denied) {
+            return null;
+        }
+        const refusal = KubectlWriteScope.getRefusal({
+            command: policy,
+            writeNamespaces: posture.writeNamespaces || [],
+            podNamespace: posture.podNamespace || null,
+            allowNodeOperations: posture.allowNodeOperations !== false,
+            usesCredential: data.usesCredential,
+        });
+        return refusal
+            ? Service.describeRunnerWriteScopeRefusal(refusal, Service.getRunnerWriteScopeSettings(data.usesCredential))
+            : null;
+    }
+    /*
+     * Where the target Runner's write scope is set, in the words a refusal
+     * uses. The in-cluster Runner is the Kubernetes agent's: its chart sets
+     * the scope (and the environment variables behind it), and an upgrade
+     * changes it. A Runner that reaches the cluster through a Kubernetes
+     * credential is an ordinary Runner no chart configures: it reports what
+     * it was started with — ONEUPTIME_KUBECTL_WRITE_NAMESPACES and
+     * ONEUPTIME_KUBECTL_ALLOW_NODE_OPERATIONS on its own host — and a
+     * restart picks up a change.
+     */
+    static getRunnerWriteScopeSettings(usesCredential) {
+        if (usesCredential) {
+            return {
+                nodeOperationsOff: `it was started with ${KUBECTL_ALLOW_NODE_OPERATIONS_ENV} (or ${KUBECTL_ALLOW_WRITES_ENV}) set to something other than true on its host`,
+                allowNodeOperations: `set ${KUBECTL_ALLOW_NODE_OPERATIONS_ENV}=true on that Runner's host and restart it`,
+                namespaces: `${KUBECTL_WRITE_NAMESPACES_ENV} on that Runner's host`,
+                addNamespace: `add it to ${KUBECTL_WRITE_NAMESPACES_ENV} on that Runner's host and restart it`,
+            };
+        }
+        return {
+            nodeOperationsOff: `its Kubernetes agent was installed with aiAccess.remediation.nodeOperations=false (${KUBECTL_ALLOW_NODE_OPERATIONS_ENV})`,
+            allowNodeOperations: "upgrade the agent with --set aiAccess.remediation.nodeOperations=true",
+            namespaces: `aiAccess.remediation.namespaces on the Kubernetes agent chart, ${KUBECTL_WRITE_NAMESPACES_ENV}`,
+            addNamespace: "add it to aiAccess.remediation.namespaces and upgrade the agent",
+        };
+    }
+    // A write-scope refusal, worded for whoever enqueued the command.
+    static describeRunnerWriteScopeRefusal(refusal, settings) {
+        const command = `"${refusal.displayCommand}"`;
+        const wouldRefuse = "so the Runner would refuse it. It was not enqueued.";
+        const scopeList = `${refusal.writeNamespaces
+            .map((allowed) => {
+            return `"${allowed}"`;
+        })
+            .join(", ")}: ${settings.namespaces}`;
+        const namespace = refusal.namespace || "";
+        const fix = refusal.fix ? ` ${refusal.fix}` : "";
+        switch (refusal.code) {
+            case "node_operations":
+                return `${refusal.uncertainty === null
+                    ? `${command} changes a node`
+                    : `The cluster's Runner cannot tell for certain whether ${command} changes a node (${refusal.uncertainty})`}, and the cluster's Runner does not allow node operations (cordon, uncordon, drain, taint, or labelling, annotating or patching a Node): ${settings.nodeOperationsOff}, ${wouldRefuse}${fix} To let OneUptime AI change nodes, ${settings.allowNodeOperations}.`;
+            case "objects_uncertain":
+            case "namespace_uncertain":
+                return `The cluster's Runner cannot tell for certain which ${refusal.code === "namespace_uncertain" ? "namespace" : "objects"} ${command} changes (${refusal.uncertainty}), ${wouldRefuse}${fix}`;
+            case "verb_mismatch":
+                return `The cluster's Runner reads the verb of ${command} as "${refusal.readVerb}", but the kubectl policy read "${refusal.policyVerb}", so it cannot tell for certain what the command changes, ${wouldRefuse}`;
+            case "unnamed_namespace_objects":
+                return `${command} would change Namespace objects without naming them (a selector, --all or no name at all), so the cluster's Runner cannot tell whether one of them is ${refusal.podNamespace
+                    ? `"${refusal.podNamespace}", the namespace it runs in`
+                    : "outside the namespaces it lets OneUptime AI change"}, ${wouldRefuse}${fix}`;
+            case "cluster_scoped":
+                return `${command} changes ${refusal.clusterScopedKinds.join(", ")} objects, which are cluster-scoped: they live outside every namespace, so outside the namespaces the cluster's Runner lets OneUptime AI change (${scopeList}) whatever -n says, ${wouldRefuse}`;
+            case "all_namespaces":
+                return `${command} would change every namespace, including ones the cluster's Runner may not change, ${wouldRefuse}${fix}`;
+            case "no_namespace":
+                return `${command} names no namespace, and the cluster's Runner reported no namespace of its own for kubectl to use there, ${wouldRefuse}${fix}`;
+            case "own_namespace":
+                return refusal.namespaceSource === "namespace_object"
+                    ? `${command} would change the Namespace object "${namespace}", the namespace the cluster's in-cluster Runner itself runs in. OneUptime AI never changes it — a change there could reconfigure the Kubernetes agent or the Runner — ${wouldRefuse}`
+                    : `${command} would change namespace "${namespace}"${refusal.namespaceSource === "default_namespace"
+                        ? " (the command names none, so kubectl would use that one)"
+                        : ""}, the namespace the cluster's in-cluster Runner itself runs in. OneUptime AI never changes it — a change there could scale away the Kubernetes agent or the Runner — ${wouldRefuse} Name the namespace of the workload to fix with -n <namespace>.`;
+            case "outside_scope":
+                return `${command} would change ${refusal.namespaceSource === "namespace_object"
+                    ? `the Namespace object "${namespace}" (a Namespace object is judged by its name; -n does not apply to it)`
+                    : `namespace "${namespace}"${refusal.namespaceSource === "default_namespace"
+                        ? " (the command names none, so kubectl would use that one)"
+                        : ""}`}, which is outside the namespaces the cluster's Runner lets OneUptime AI change (${scopeList}), ${wouldRefuse} To let OneUptime AI change "${namespace}", ${settings.addNamespace}.`;
+            default: {
+                const unhandled = refusal.code;
+                return unhandled;
+            }
+        }
+    }
     async enqueue(data) {
         var _a;
         if (!data.targetAgentId) {
@@ -134,6 +350,13 @@ export class Service extends DatabaseService {
         if (!AI_COMMAND_STEP_TYPES.includes(data.stepType)) {
             throw new BadDataException(`AI remediation does not execute step type "${data.stepType}".`);
         }
+        /*
+         * kubectl has its own policy and its own payload shape (an argv, a
+         * cluster, an optional credential) — enqueueAiKubectlCommand owns it.
+         */
+        if (data.stepType === RunbookStepType.Kubectl) {
+            throw new BadDataException("Kubectl commands are enqueued through enqueueAiKubectlCommand.");
+        }
         const command = (data.command || "").trim();
         if (!command) {
             throw new BadDataException("An AI command job needs a command.");
@@ -145,6 +368,31 @@ export class Service extends DatabaseService {
         if (data.stepType === RunbookStepType.SSH && !data.credentialId) {
             throw new BadDataException("An SSH command job needs a credentialId.");
         }
+        /*
+         * The target must be one of this project's Runners, and never a
+         * kubernetes-agent Runner. That Runner exists to run policy-tiered
+         * kubectl with its own ServiceAccount: the claim path serves it kubectl
+         * only, so a Bash or SSH job aimed at it would sit unclaimed until its
+         * deadline after a human approved it — and its identity is minted with
+         * the project's telemetry ingestion key, so it must never be handed
+         * AI-composed shell work or an SSH credential. Checked here, at the
+         * enqueue chokepoint, so an already-stored plan is refused too.
+         */
+        const targetRunner = await RunnerService.findOneBy({
+            query: {
+                _id: data.targetAgentId.toString(),
+                projectId: data.projectId,
+            },
+            select: { _id: true, name: true, hostInfo: true },
+            props: { isRoot: true },
+        });
+        if (!targetRunner) {
+            throw new BadDataException("The target Runner was not found or it does not belong to this project.");
+        }
+        // The one "is an agent row" rule: the name marker or an agent posture.
+        if (RunnerServiceClass.isKubernetesAgentRunnerRow(targetRunner)) {
+            throw new BadDataException(`Runner "${targetRunner.name}" is a cluster's in-cluster Runner: it runs kubectl through its cluster only, never ${data.stepType} commands. Target the Kubernetes cluster instead, or pick a Runner you created under Project Settings → Runners.`);
+        }
         /*
          * Project-wide hourly storm brake for the whole AI-command lane. It
          * lives HERE, at the single enqueue chokepoint, so it also bounds the
@@ -193,6 +441,288 @@ export class Service extends DatabaseService {
         row.claimDeadlineAt = claimDeadlineAt;
         return this.create({ data: row, props: { isRoot: true } });
     }
+    /*
+     * Enqueue ONE kubectl command OneUptime AI composed against a cluster.
+     *
+     * This is the server-side chokepoint between an LLM's output and a
+     * cluster's API server, so it re-runs the kubectl policy regardless of
+     * what the tool layer checked: Denied never enqueues, and an
+     * investigation-origin job must be Read tier — an investigation can never
+     * write, no matter which prompt produced the call. The Runner re-checks
+     * the same policy on the argv it receives.
+     *
+     * The command travels as an argv in the payload (never a shell line) plus
+     * the cluster and, for Runners outside the cluster, the credential id the
+     * ingress resolves at claim time. In-cluster Runners carry no credential:
+     * kubectl uses the pod's own ServiceAccount — which reaches whatever
+     * cluster that pod lives in. So a credential-less job is only ever
+     * enqueued for the in-cluster Runner OF THIS CLUSTER (its posture names
+     * the cluster), and the payload carries the cluster's identifier so the
+     * claim path and the Runner can refuse a job that reaches the wrong pod.
+     * A credential is never sent to a kubernetes-agent Runner (its identity
+     * is minted with the project's telemetry ingestion key).
+     *
+     * It also re-reads the cluster's CURRENT AI access configuration, because
+     * every caller acts on a snapshot (an investigation reads the cluster's
+     * status once when it starts, a remediation run when it composes): the
+     * target must still be the cluster's bound Runner, a credential must still
+     * be its bound Kubernetes credential, an investigation's reads need "Let
+     * AI investigate with kubectl" still on, and a remediation run's commands
+     * (its reads included) need the cluster's remediation still enabled. An
+     * operator who revokes or re-points access therefore stops an in-flight
+     * run at its next command.
+     */
+    async enqueueAiKubectlCommand(data) {
+        var _a, _b, _c;
+        if (!data.targetAgentId) {
+            throw new BadDataException("targetAgentId is required to dispatch a kubectl command to a Runner.");
+        }
+        if (!data.kubernetesClusterId) {
+            throw new BadDataException("kubernetesClusterId is required for a kubectl command.");
+        }
+        if (data.origin !== RunnerJobOrigin.AiInvestigation &&
+            data.origin !== RunnerJobOrigin.AiRemediation) {
+            throw new BadDataException(`A kubectl command job cannot have origin "${String(data.origin)}".`);
+        }
+        if (data.origin === RunnerJobOrigin.AiRemediation &&
+            !data.autoRemediationSuggestionId) {
+            throw new BadDataException("A kubectl remediation job needs its auto-remediation suggestion.");
+        }
+        const isAccessTest = data.isAccessTest === true;
+        /*
+         * The access test is the only kubectl job without an AI run, and it is
+         * a read: that is what lets the investigation brake below count real
+         * investigations only (rows with an AI run) without the test slipping
+         * work past it under another name.
+         */
+        if (isAccessTest &&
+            (data.origin !== RunnerJobOrigin.AiInvestigation || data.aiRunId)) {
+            throw new BadDataException("An access test is a read-only check with no AI run behind it.");
+        }
+        if (data.origin === RunnerJobOrigin.AiInvestigation &&
+            !isAccessTest &&
+            !data.aiRunId) {
+            throw new BadDataException("An investigation kubectl job needs the AI run it belongs to.");
+        }
+        const policy = KubectlPolicy.evaluateCommand(data.command);
+        if (policy.tier === KubectlCommandTier.Denied) {
+            throw new BadDataException(`Denied by the kubectl command policy: ${policy.reason}.`);
+        }
+        if (data.origin === RunnerJobOrigin.AiInvestigation &&
+            policy.tier !== KubectlCommandTier.Read) {
+            throw new BadDataException(`An investigation may only run read-only kubectl commands; "${policy.displayCommand}" would change the cluster (${policy.tier}).`);
+        }
+        /*
+         * The access test is bounded by its own route limits instead, and an
+         * investigation's brake counts only rows with an AI run, so tests never
+         * spend the budget real investigations share.
+         */
+        if (!isAccessTest) {
+            const hourlyCap = data.origin === RunnerJobOrigin.AiInvestigation
+                ? MAX_AI_INVESTIGATION_COMMAND_JOBS_PER_PROJECT_PER_HOUR
+                : MAX_AI_COMMAND_JOBS_PER_PROJECT_PER_HOUR;
+            const jobsInLastHour = (await this.countBy({
+                query: Object.assign({ projectId: data.projectId, origin: data.origin, createdAt: QueryHelper.greaterThan(OneUptimeDate.getSomeHoursAgo(1)) }, (data.origin === RunnerJobOrigin.AiInvestigation
+                    ? { aiRunId: QueryHelper.notNull() }
+                    : {})),
+                props: { isRoot: true },
+            })).toNumber();
+            if (jobsInLastHour >= hourlyCap) {
+                throw new BadDataException(data.origin === RunnerJobOrigin.AiInvestigation
+                    ? `This project has run ${jobsInLastHour} AI investigation commands in the last hour, which is its limit (${hourlyCap}). No further cluster commands can run this hour.`
+                    : `This project has run ${jobsInLastHour} AI remediation commands in the last hour, which is its limit (${hourlyCap}). No further AI commands can run this hour.`);
+            }
+        }
+        /*
+         * The cluster and the target Runner must both be THIS project's. The
+         * cluster's identifier travels in the payload; without a credential the
+         * Runner must be that cluster's own in-cluster agent, because its
+         * ServiceAccount reaches only the cluster its pod lives in.
+         */
+        const cluster = await KubernetesClusterService.findOneBy({
+            query: {
+                _id: data.kubernetesClusterId.toString(),
+                projectId: data.projectId,
+            },
+            select: {
+                _id: true,
+                name: true,
+                clusterIdentifier: true,
+                aiAccessRunnerId: true,
+                aiAccessCredentialId: true,
+                isAiInvestigationEnabled: true,
+                aiRemediationMode: true,
+            },
+            props: { isRoot: true },
+        });
+        if (!cluster) {
+            throw new BadDataException("Kubernetes cluster not found or it does not belong to this project.");
+        }
+        const clusterIdentifier = (cluster.clusterIdentifier || "").trim();
+        const clusterLabel = cluster.name || clusterIdentifier || data.kubernetesClusterId.toString();
+        const bindingRefusal = Service.getClusterBindingRefusal({
+            cluster,
+            clusterLabel,
+            targetAgentId: data.targetAgentId,
+            credentialId: data.credentialId,
+            requiredSwitch: await this.getRequiredClusterSwitch({
+                projectId: data.projectId,
+                origin: data.origin,
+                aiRunId: data.aiRunId,
+                isAccessTest,
+            }),
+        });
+        if (bindingRefusal) {
+            throw new BadDataException(bindingRefusal);
+        }
+        const targetRunner = await RunnerService.findOneBy({
+            query: {
+                _id: data.targetAgentId.toString(),
+                projectId: data.projectId,
+            },
+            select: { _id: true, name: true, hostInfo: true },
+            props: { isRoot: true },
+        });
+        if (!targetRunner) {
+            throw new BadDataException("The target Runner was not found or it does not belong to this project.");
+        }
+        if (data.credentialId) {
+            /*
+             * A kubernetes-agent Runner is never handed credential material: its
+             * identity is issued with the project's telemetry ingestion key, so
+             * anyone holding that key could come to hold the credential too. The
+             * claim path refuses it as well (by the same name-or-posture rule);
+             * refusing here keeps the job from ever existing.
+             */
+            if (RunnerServiceClass.isKubernetesAgentRunnerRow(targetRunner)) {
+                throw new BadDataException(`Runner "${targetRunner.name}" is a cluster's in-cluster Runner and is never given a credential, so a kubectl command for cluster "${clusterLabel}" that needs one cannot run on it. Create a Runner under Project Settings → Runners, assign the Kubernetes credential to it and select both on the cluster's AI page.`);
+            }
+            const credential = ObjectID.isValidUUID(data.credentialId)
+                ? await RunbookCredentialService.findOneBy({
+                    query: {
+                        _id: data.credentialId,
+                        projectId: data.projectId,
+                    },
+                    select: { _id: true, name: true, credentialType: true },
+                    props: { isRoot: true },
+                })
+                : null;
+            if (!credential ||
+                credential.credentialType !== RunbookCredentialType.Kubernetes) {
+                throw new BadDataException(`The credential this kubectl command names is not a Kubernetes credential of this project, so it was not enqueued for cluster "${clusterLabel}".`);
+            }
+        }
+        const posture = parseKubernetesRunnerPosture(targetRunner.hostInfo);
+        if (!data.credentialId) {
+            if (!isInClusterPostureForCluster(posture, clusterIdentifier)) {
+                throw new BadDataException(`A kubectl command without a Kubernetes credential can only run on the in-cluster Runner of cluster "${clusterIdentifier || cluster.name || ((_a = cluster.id) === null || _a === void 0 ? void 0 : _a.toString())}", and Runner "${targetRunner.name}" ${(posture === null || posture === void 0 ? void 0 : posture.inCluster)
+                    ? `is the in-cluster Runner of ${((_b = posture.clusterIdentifier) === null || _b === void 0 ? void 0 : _b.trim())
+                        ? `cluster "${posture.clusterIdentifier.trim()}"`
+                        : "an unnamed cluster"}`
+                    : "runs outside the cluster"}. Select a Kubernetes credential for this Runner on the cluster's AI page, or install the in-cluster Runner on this cluster.`);
+            }
+        }
+        /*
+         * A remediation write the Runner's own posture says it will refuse — a
+         * namespace outside its write scope, its own namespace, or a node
+         * operation its chart did not grant — is refused here, before it can be
+         * approved, enqueued or counted by the cluster's circuit breaker, with
+         * the chart value that would allow it. Reads are never refused for
+         * scope.
+         */
+        if (data.origin === RunnerJobOrigin.AiRemediation) {
+            const scopeRefusal = Service.getRunnerWriteScopeRefusal({
+                policy,
+                posture,
+                usesCredential: Boolean(data.credentialId),
+            });
+            if (scopeRefusal) {
+                throw new BadDataException(scopeRefusal);
+            }
+        }
+        const claimDeadlineAt = OneUptimeDate.addRemoveSeconds(OneUptimeDate.getCurrentDate(), Math.ceil(((_c = data.claimTimeoutInMs) !== null && _c !== void 0 ? _c : DEFAULT_CLAIM_TIMEOUT_MS) / 1000));
+        const row = new Model();
+        row.projectId = data.projectId;
+        row.origin = data.origin;
+        if (data.aiRunId) {
+            row.aiRunId = data.aiRunId;
+        }
+        if (data.autoRemediationSuggestionId) {
+            row.autoRemediationSuggestionId = data.autoRemediationSuggestionId;
+        }
+        row.kubernetesClusterId = data.kubernetesClusterId;
+        row.stepId = data.stepId;
+        row.stepType = RunbookStepType.Kubectl;
+        row.targetAgentId = data.targetAgentId;
+        row.script = "";
+        row.payload = Object.assign(Object.assign({ args: policy.args, displayCommand: policy.displayCommand, tier: policy.tier, kubernetesClusterId: data.kubernetesClusterId.toString() }, (clusterIdentifier ? { clusterIdentifier } : {})), (data.credentialId ? { credentialId: data.credentialId } : {}));
+        row.timeoutInMs = data.timeoutInMs;
+        row.status = RunnerJobStatus.Pending;
+        row.claimDeadlineAt = claimDeadlineAt;
+        return this.create({ data: row, props: { isRoot: true } });
+    }
+    /*
+     * Which of the cluster's AI switches must still be on for this kubectl
+     * job, or null for none (the access test, which is how an operator checks
+     * access before turning AI on).
+     *
+     * A remediation job needs remediation. A read-only (investigation-origin)
+     * job follows the run it belongs to: a remediation run reads the cluster
+     * it may fix through the same read-only lane — even when the operator
+     * left investigation off — so its reads need remediation, and every
+     * other run's reads need the investigation switch.
+     */
+    async getRequiredClusterSwitch(data) {
+        if (data.origin === RunnerJobOrigin.AiRemediation) {
+            return "remediation";
+        }
+        if (data.isAccessTest || !data.aiRunId) {
+            return null;
+        }
+        const run = await AIRunService.findOneBy({
+            query: {
+                _id: data.aiRunId.toString(),
+                projectId: data.projectId,
+            },
+            select: { _id: true, runType: true },
+            props: { isRoot: true },
+        });
+        return (run === null || run === void 0 ? void 0 : run.runType) && REMEDIATION_RUN_TYPES.includes(run.runType)
+            ? "remediation"
+            : "investigation";
+    }
+    /*
+     * Why a kubectl job may NOT be enqueued for this cluster as it is
+     * configured right now, or null when it may. Pure, so the rule reads in
+     * one place: the job must go through the cluster's CURRENT binding, and
+     * the capability it is for must still be switched on.
+     *
+     * A credential-less job does not require the credential binding to be
+     * empty: the in-cluster Runner of the cluster wins over a bound
+     * credential (the readiness status prefers it), so such a job is
+     * legitimate with a credential still selected.
+     */
+    static getClusterBindingRefusal(data) {
+        const { cluster, clusterLabel } = data;
+        if (!cluster.aiAccessRunnerId ||
+            cluster.aiAccessRunnerId.toString() !== data.targetAgentId.toString()) {
+            return `Cluster "${clusterLabel}" is no longer reached through this Runner (its AI access binding changed or was cleared on the cluster's AI page), so this kubectl command was not enqueued.`;
+        }
+        if (data.credentialId &&
+            (!cluster.aiAccessCredentialId ||
+                cluster.aiAccessCredentialId.toString() !== data.credentialId)) {
+            return `Cluster "${clusterLabel}" is no longer reached with this Kubernetes credential (its AI access binding changed or was cleared on the cluster's AI page), so this kubectl command was not enqueued.`;
+        }
+        if (data.requiredSwitch === "investigation" &&
+            cluster.isAiInvestigationEnabled !== true) {
+            return `"Let AI investigate with kubectl" is turned off for cluster "${clusterLabel}", so this investigation command was not enqueued.`;
+        }
+        if (data.requiredSwitch === "remediation" &&
+            !ENABLED_REMEDIATION_MODES.includes(cluster.aiRemediationMode)) {
+            return `AI remediation is turned off for cluster "${clusterLabel}", so this kubectl command was not enqueued.`;
+        }
+        return null;
+    }
     /*
      * The job this (execution, step) pair already produced, if any.
      *
@@ -234,7 +764,10 @@ export class Service extends DatabaseService {
      * don't fight over the same row.
      */
     async claimNextJob(data) {
-        var _a;
+        var _a, _b;
+        if (data.allowedStepTypes && data.allowedStepTypes.length === 0) {
+            return null;
+        }
         const dataSource = PostgresAppInstance.getDataSource();
         if (!dataSource) {
             throw new BadDataException("Database is not connected");
@@ -243,6 +776,7 @@ export class Service extends DatabaseService {
         const allowedOrigins = data.allowedOrigins && data.allowedOrigins.length > 0
             ? data.allowedOrigins
             : [RunnerJobOrigin.Runbook];
+        const allowedStepTypes = (_b = data.allowedStepTypes) !== null && _b !== void 0 ? _b : null;
         const sql = `
       WITH claimed AS (
         SELECT "_id" FROM "RunnerJob"
@@ -250,6 +784,7 @@ export class Service extends DatabaseService {
           AND "status" = $2
           AND "targetAgentId" = $3::uuid
           AND "origin" = ANY($6::text[])
+          AND ($7::text[] IS NULL OR "stepType" = ANY($7::text[]))
           AND "claimDeadlineAt" > NOW()
           AND "deletedAt" IS NULL
         ORDER BY "createdAt" ASC
@@ -278,6 +813,7 @@ export class Service extends DatabaseService {
             RunnerJobStatus.Claimed,
             leaseMs.toString(),
             allowedOrigins,
+            allowedStepTypes,
         ]));
         if (!rows || rows.length === 0) {
             return null;
@@ -347,8 +883,39 @@ export class Service extends DatabaseService {
         ]));
         return rows.length > 0;
     }
+    /*
+     * What an AI-origin job's output looks like on the row. The dashboard
+     * renders RunnerJob.output verbatim on the cluster's AI page and the
+     * remediation card, so a `kubectl get secret -o yaml` an investigation
+     * ran must never land there with its data values intact: the same two
+     * passes KubectlJobRunner applies before the model sees the output are
+     * applied here before the row does. Both are pure text transforms (the
+     * redactor is dependency-free by design; importing KubectlJobRunner into
+     * this service would be an import cycle). A runbook step's output is the
+     * operator's own script and is stored as it was.
+     */
+    static redactAiJobText(text) {
+        return ToolResultSerializer.redact(KubectlOutputRedactor.redact(text).text)
+            .text;
+    }
+    /*
+     * A Runner's text as a Postgres text column can hold it. A NUL byte is
+     * ordinary in what a Runner reports — `kubectl logs` of a log that was
+     * rotated with copytruncate, a ConfigMap value `describe` prints
+     * verbatim, a script that writes binary — and Postgres refuses the whole
+     * UPDATE over one, so the result could never be stored: every submission
+     * failed the same way, the Runner retried it as if the server were down,
+     * and the job ended "result unknown" although the command had run. Each
+     * such character becomes U+FFFD instead, for every origin, and before
+     * anything else reads the text.
+     */
+    static toStorableText(text) {
+        return text
+            .replace(NUL_CHARACTER_PATTERN, RUNNER_JOB_UNSTORABLE_CHARACTER_REPLACEMENT)
+            .replace(SURROGATE_PATTERN, keepSurrogatePairOnly);
+    }
     async submitResult(data) {
-        var _a, _b, _c;
+        var _a;
         const dataSource = PostgresAppInstance.getDataSource();
         if (!dataSource) {
             throw new BadDataException("Database is not connected");
@@ -356,6 +923,27 @@ export class Service extends DatabaseService {
         const status = data.success
             ? RunnerJobStatus.Succeeded
             : RunnerJobStatus.Failed;
+        /*
+         * Made storable first: every text parameter below is bound whichever
+         * branch of the CASE the row takes, so one unstorable character in any
+         * of them would fail the write for every origin.
+         */
+        const rawOutput = typeof data.output === "string"
+            ? Service.toStorableText(data.output)
+            : null;
+        const rawErrorMessage = typeof data.errorMessage === "string"
+            ? Service.toStorableText(data.errorMessage)
+            : null;
+        /*
+         * The origin decides which text is stored, and it is read in the same
+         * statement rather than looked up first so the lease check and the
+         * write stay one atomic step (and one round trip): the CASE picks the
+         * redacted text for an AI-origin row and the verbatim text otherwise.
+         */
+        const redactedOutput = rawOutput === null ? null : Service.redactAiJobText(rawOutput);
+        const redactedErrorMessage = rawErrorMessage === null
+            ? null
+            : Service.redactAiJobText(rawErrorMessage);
         /*
          * Only the agent that holds the lease is allowed to write the terminal
          * result. If the lease has already moved on (Worker timed out and
@@ -364,9 +952,15 @@ export class Service extends DatabaseService {
         const sql = `
       UPDATE "RunnerJob"
       SET "status" = $1,
-          "output" = $2,
+          "output" = CASE
+            WHEN "origin" = ANY($9::text[]) THEN $10::text
+            ELSE $2::text
+          END,
           "exitCode" = $3,
-          "errorMessage" = $4,
+          "errorMessage" = CASE
+            WHEN "origin" = ANY($9::text[]) THEN $11::text
+            ELSE $4::text
+          END,
           "completedAt" = NOW(),
           "updatedAt" = NOW(),
           "version" = "version" + 1
@@ -377,13 +971,16 @@ export class Service extends DatabaseService {
     `;
         const rows = unwrapRows(await dataSource.query(sql, [
             status,
-            (_a = data.output) !== null && _a !== void 0 ? _a : null,
-            (_b = data.exitCode) !== null && _b !== void 0 ? _b : null,
-            (_c = data.errorMessage) !== null && _c !== void 0 ? _c : null,
+            rawOutput,
+            (_a = data.exitCode) !== null && _a !== void 0 ? _a : null,
+            rawErrorMessage,
             data.jobId.toString(),
             data.agentId.toString(),
             RunnerJobStatus.Claimed,
             RunnerJobStatus.Running,
+            AI_COMMAND_JOB_ORIGINS,
+            redactedOutput,
+            redactedErrorMessage,
         ]));
         return rows.length > 0;
     }
@@ -415,6 +1012,9 @@ export class Service extends DatabaseService {
                     startedAt: true,
                     completedAt: true,
                     assignedAgentId: true,
+                    // The timeout reason is worded for what the job is.
+                    origin: true,
+                    stepType: true,
                 },
                 props: { isRoot: true },
             });
@@ -425,29 +1025,34 @@ export class Service extends DatabaseService {
                 return job;
             }
             const now = OneUptimeDate.getCurrentDate();
-            // Pending with claim deadline elapsed -> no agent picked it up.
+            let timeoutKind = null;
             if (job.status === RunnerJobStatus.Pending &&
                 job.claimDeadlineAt &&
                 now > job.claimDeadlineAt) {
-                return this.timeoutJob({
-                    jobId: data.jobId,
-                    reason: "No runbook agent picked up this step before the wait window expired. The agent may be offline — check that it is running and reachable, then try again.",
-                });
+                // Pending with claim deadline elapsed -> no agent picked it up.
+                timeoutKind = "unclaimed";
             }
-            // Claimed/Running with lease elapsed -> agent went silent.
-            if ((job.status === RunnerJobStatus.Claimed ||
+            else if ((job.status === RunnerJobStatus.Claimed ||
                 job.status === RunnerJobStatus.Running) &&
                 job.leaseExpiresAt &&
                 now > job.leaseExpiresAt) {
-                return this.timeoutJob({
-                    jobId: data.jobId,
-                    reason: "The runbook agent stopped responding while this step was running. The agent may have crashed or lost its network connection — check that it is still online, then try running the runbook again.",
-                });
+                // Claimed/Running with lease elapsed -> agent went silent.
+                timeoutKind = "lease_expired";
+            }
+            else if (now > overallDeadline) {
+                timeoutKind = "overall";
             }
-            if (now > overallDeadline) {
+            if (timeoutKind) {
                 return this.timeoutJob({
                     jobId: data.jobId,
-                    reason: "This step ran longer than the allowed execution window. Increase the timeout on the step or make the script complete faster.",
+                    reason: describeRunnerJobTimeout({
+                        kind: timeoutKind,
+                        wasClaimed: job.status !== RunnerJobStatus.Pending,
+                        origin: job.origin,
+                        stepType: job.stepType,
+                        claimTimeoutInMs: data.claimTimeoutInMs,
+                        executionTimeoutInMs: data.executionTimeoutInMs,
+                    }),
                 });
             }
             await Sleep.sleep(POLL_INTERVAL_MS);
@@ -482,6 +1087,12 @@ export class Service extends DatabaseService {
             RunnerJobStatus.TimedOut,
             RunnerJobStatus.Cancelled,
         ]);
+        /*
+         * The claim facts come back with the row, so a caller can tell whether
+         * any Runner ever picked the job up (claimedAt / assignedAgentId) or
+         * started it (startedAt) — "nothing ran" versus "unknown" — without a
+         * second read.
+         */
         const updated = await this.findOneById({
             id: data.jobId,
             select: {
@@ -491,6 +1102,9 @@ export class Service extends DatabaseService {
                 exitCode: true,
                 errorMessage: true,
                 completedAt: true,
+                claimedAt: true,
+                startedAt: true,
+                assignedAgentId: true,
             },
             props: { isRoot: true },
         });
@@ -535,6 +1149,12 @@ __decorate([
     __metadata("design:paramtypes", [Object]),
     __metadata("design:returntype", Promise)
 ], Service.prototype, "enqueueAiCommand", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "enqueueAiKubectlCommand", null);
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),