@oneuptime/common 14.0.1 → 14.0.2

package/build/dist/Server/Utils/AutoRemediation/CommandPlanExecutor.js

@@ -14,15 +14,24 @@ import { IncidentFeedEventType } from "../../../Models/DatabaseModels/IncidentFe
 import AutoRemediationSuggestionStatus from "../../../Types/AutoRemediation/AutoRemediationSuggestionStatus";
 import AutoRemediationSuggestionType from "../../../Types/AutoRemediation/AutoRemediationSuggestionType";
 import AutoRemediationVerificationStatus from "../../../Types/AutoRemediation/AutoRemediationVerificationStatus";
-import { AiRemediationCommandExecutionStatus, AiRemediationCommandPlanUtil, AiRemediationPlanExecutionStatus, AiRemediationRollbackStatus, } from "../../../Types/AutoRemediation/AiRemediationCommandPlan";
+import { AiRemediationCommandExecutionStatus, AiRemediationCommandPlanUtil, AiRemediationCommandPolicyVerdict, AiRemediationPlanExecutionStatus, AiRemediationRollbackStatus, getForwardCommandStepId, getRollbackCommandStepId, isSettledCommandExecutionStatus, } from "../../../Types/AutoRemediation/AiRemediationCommandPlan";
+import SortOrder from "../../../Types/BaseDatabase/SortOrder";
 import CommandPolicy from "../../../Utils/AiRemediation/CommandPolicy";
+import KubectlPolicy from "../../../Utils/AiRemediation/KubectlPolicy";
+import { KubectlCommandTier, KubernetesAiRemediationMode, } from "../../../Types/Kubernetes/KubernetesClusterAiAccess";
+import RunbookStepType from "../../../Types/Runbook/RunbookStepType";
+import RunnerJobOrigin from "../../../Types/Runbook/RunnerJobOrigin";
 import { Green500, Red500 } from "../../../Types/BrandColors";
 import AlertFeedService from "../../Services/AlertFeedService";
 import AutoRemediationSuggestionService from "../../Services/AutoRemediationSuggestionService";
 import IncidentFeedService from "../../Services/IncidentFeedService";
+import KubernetesClusterAiAccessService from "../../Services/KubernetesClusterAiAccessService";
 import RunnerJobService, { isTerminalAgentJobStatus, } from "../../Services/RunnerJobService";
 import RunnerJobStatus from "../../../Types/Runbook/RunnerJobStatus";
+import Semaphore from "../../Infrastructure/Semaphore";
 import ToolResultSerializer from "../AI/Toolbox/Serializer";
+import KubectlJobRunner, { KubectlRunState, } from "../AI/ClusterAccess/KubectlJobRunner";
+import RemediationCommandToolkit from "../AI/Remediation/RemediationCommandTools";
 import logger from "../Logger";
 import CaptureSpan from "../Telemetry/CaptureSpan";
 /*
@@ -40,10 +49,66 @@ import CaptureSpan from "../Telemetry/CaptureSpan";
  * executed command that carried a rollbackCommand is undone, in reverse
  * order. Rollback commands were visible on the approval card (or, for
  * FullAuto, passed the same policy the forward command did) and are
- * re-checked against the hard denylist here anyway.
+ * re-checked against the hard denylist here anyway. A command that may
+ * have run without saying so — its record still Pending, or a kubectl
+ * command recorded as failed whose job a Runner took with no result back
+ * (KubectlJobRunner's Unknown run state) — is never treated as never run:
+ * its undo is not run blind either, a human is told to check and undo it.
+ *
+ * Kubectl commands — forward and rollback — are additionally re-checked
+ * against the cluster's AI page AS IT IS NOW, right before they are
+ * enqueued: an approval can be clicked hours after the plan was composed,
+ * and a rollback fires at the end of the verification window. The
+ * operator's current choice (remediation off, a different Runner bound, a
+ * mode that no longer bypasses approvals) wins over the frozen plan.
+ *
+ * The two arms can overlap: verification fails a plan that overran its
+ * window while the executor is still waiting on a command. Three rules keep
+ * them from undoing each other:
+ *   - the executor starts each command (its stand-down check, the enqueue,
+ *     and the write that names the job) under a per-plan lock, and the
+ *     rollback arm reads the plan under the same lock — so every forward
+ *     job that will ever exist is either on the record the rollback arm
+ *     reads, or its command stands down;
+ *   - the rollback arm WAITS for a forward job still in flight and rolls it
+ *     back once it succeeds, instead of skipping it;
+ *   - both write the plan read-modify-write under that lock, each writing
+ *     only its own fields (the executor: forward executions and the plan's
+ *     execution lifecycle; the rollback arm: rollbacks and rollbackStatus),
+ *     so neither erases the other's record.
+ *
+ * A Worker restart mid-rollback leaves the plan's rollbackStatus unset with
+ * a heartbeat that goes stale; the verifier's recovery sweep resumes it
+ * (executeRollback settles rollbacks already handed to a Runner from their
+ * jobs, and never re-runs one that ran). The resumed rollback's outcome is
+ * the WHOLE plan's: an undo the interrupted attempt already settled as
+ * failed, or left for a human, keeps the rollback Failed — the verifier
+ * forces the follow-up round to ask first on exactly that.
  */
 const AI_COMMAND_CLAIM_TIMEOUT_MS = 60000;
 const MAX_STORED_OUTPUT_CHARS = 6000;
+const MAX_FEED_REASON_CHARS = 400;
+/*
+ * The per-plan lock covers database writes and one enqueue — never a wait
+ * on a Runner. It expires on its own if the pod dies holding it.
+ */
+const PLAN_LOCK_NAMESPACE = "AutoRemediationCommandPlan";
+const PLAN_LOCK_TIMEOUT_MS = 30000;
+const PLAN_LOCK_ACQUIRE_TIMEOUT_MS = 15000;
+// What the rollback arm needs from a RunnerJob to settle its command.
+const RECONCILE_JOB_SELECT = {
+    _id: true,
+    status: true,
+    output: true,
+    exitCode: true,
+    errorMessage: true,
+};
+/*
+ * What the rollback arm needs from a kubectl RunnerJob to tell whether
+ * kubectl ran (KubectlJobRunner.getRunStateOfJobRow): the outcome, and
+ * whether a Runner ever claimed or started the job.
+ */
+const RUN_STATE_JOB_SELECT = Object.assign(Object.assign({}, RECONCILE_JOB_SELECT), { claimedAt: true, startedAt: true, assignedAgentId: true });
 export default class CommandPlanExecutor {
     /*
      * Execute an Approved plan end-to-end. Detached-callable (the approve API
@@ -87,8 +152,12 @@ export default class CommandPlanExecutor {
             }
             plan.executionStatus = AiRemediationPlanExecutionStatus.Running;
             plan.executionStartedAt = OneUptimeDate.getCurrentDate().toISOString();
-            await this.persistPlan(suggestion, plan);
+            await this.persistPlan(suggestion, plan, "executor");
             let failed = false;
+            // What stopped the plan, for the feed: the first non-Succeeded outcome.
+            let firstFailure = null;
+            // Verification settled the remediation while the plan was executing.
+            let settledMidPlan = false;
             for (const command of plan.commands) {
                 // FullAuto-executed entries never appear in approved Suggest plans.
                 if (command.execution) {
@@ -102,51 +171,90 @@ export default class CommandPlanExecutor {
                     continue;
                 }
                 /*
-                 * Abort check between commands. Verification runs on its own sweep
-                 * and can fail this remediation while the plan is still executing
-                 * (a slow Runner, a long command); once it does, it rolls the
-                 * executed commands back. Continuing to push NEW forward commands
-                 * into a rollback that is already under way is how a plan and its
-                 * undo end up fighting each other, so the executor stops the moment
-                 * verification is no longer Pending or a human moved the
-                 * suggestion off Approved.
+                 * Start the command under the plan lock: the stand-down check, the
+                 * Pending marker, the enqueue and the write that names the job. The
+                 * rollback arm reads the plan under the same lock, so it either sees
+                 * this command's job (and waits for it) or this command sees the
+                 * settled verification and never starts.
                  */
-                if (!(await this.isStillExecutable(suggestion.id))) {
-                    command.execution = {
-                        status: AiRemediationCommandExecutionStatus.Skipped,
-                        errorMessage: "Skipped: this remediation was settled (verified, failed, or actioned by a human) before the command ran.",
-                    };
-                    failed = true;
+                const mutex = await this.lockPlan(suggestion.id);
+                let started = null;
+                try {
+                    /*
+                     * Abort check between commands. Verification runs on its own
+                     * sweep and can fail this remediation while the plan is still
+                     * executing (a slow Runner, a long command); once it does, it
+                     * rolls the executed commands back. Continuing to push NEW
+                     * forward commands into a rollback that is already under way is
+                     * how a plan and its undo end up fighting each other, so the
+                     * executor stops the moment verification is no longer Pending or
+                     * a human moved the suggestion off Approved.
+                     */
+                    if (!(await this.isStillExecutable(suggestion.id))) {
+                        command.execution = {
+                            status: AiRemediationCommandExecutionStatus.Skipped,
+                            errorMessage: "Skipped: this remediation was settled (verified, failed, or actioned by a human) before the command ran.",
+                        };
+                        failed = true;
+                        settledMidPlan = true;
+                    }
+                    else {
+                        /*
+                         * Persist the Pending marker BEFORE the side effect: a pod death
+                         * mid-command must leave a record that this command may have
+                         * run.
+                         */
+                        command.execution = {
+                            status: AiRemediationCommandExecutionStatus.Pending,
+                            startedAt: OneUptimeDate.getCurrentDate().toISOString(),
+                        };
+                        await this.persistPlan(suggestion, plan, "executor", {
+                            planLocked: true,
+                        });
+                        started = await this.startOneCommand({ suggestion, plan, command });
+                    }
+                }
+                finally {
+                    await this.unlockPlan(mutex);
+                }
+                if (!started) {
                     continue;
                 }
-                /*
-                 * Persist the Pending marker BEFORE the side effect: a pod death
-                 * mid-command must leave a record that this command may have run.
-                 */
-                command.execution = {
-                    status: AiRemediationCommandExecutionStatus.Pending,
-                    startedAt: OneUptimeDate.getCurrentDate().toISOString(),
-                };
-                await this.persistPlan(suggestion, plan);
-                const state = await this.executeOneCommand({ suggestion, command });
+                const state = await this.finishOneCommand({ command, started });
                 command.execution = state;
-                await this.persistPlan(suggestion, plan);
+                await this.persistPlan(suggestion, plan, "executor");
                 if (state.status !== AiRemediationCommandExecutionStatus.Succeeded) {
                     failed = true;
+                    firstFailure =
+                        firstFailure ||
+                            `command ${command.sequence} ${state.errorMessage
+                                ? `failed: ${this.capForFeed(state.errorMessage)}`
+                                : "failed"}`;
                 }
             }
             plan.executionStatus = failed
                 ? AiRemediationPlanExecutionStatus.Failed
                 : AiRemediationPlanExecutionStatus.Completed;
             plan.executionCompletedAt = OneUptimeDate.getCurrentDate().toISOString();
-            await this.persistPlan(suggestion, plan);
+            await this.persistPlan(suggestion, plan, "executor");
+            /*
+             * The last command can still be running when verification concludes
+             * (a plan that overran its window). "Verification is watching" would
+             * then contradict the verification and rollback notes already on the
+             * feed.
+             */
+            if (!settledMidPlan && !(await this.isStillExecutable(suggestion.id))) {
+                settledMidPlan = true;
+            }
             await this.postFeedItem({
                 suggestion,
-                markdown: failed
-                    ? `⚠️ **Approved AI command plan did not complete** — a command failed; the remaining commands were skipped. Review the per-command output on the suggestion. Verification will judge (and roll back) what ran.`
-                    : `⚡ **Approved AI command plan executed** (${plan.commands.length} command(s)). Verification is watching the monitors for recovery.`,
-                pingWorkspace: failed,
-                displayColor: failed ? Red500 : Green500,
+                markdown: settledMidPlan
+                    ? `⚠️ **Approved AI command plan finished after its verification had already concluded**${firstFailure ? ` — ${firstFailure}` : ""}. Commands that had not started yet were skipped; the verification and rollback notes on this suggestion say what was judged and what was undone. Review the per-command record.`
+                    : failed
+                        ? `⚠️ **Approved AI command plan did not complete** — ${firstFailure || "a command failed"}; the remaining commands were skipped. Review the per-command output on the suggestion. Verification will judge (and roll back) what ran.`
+                        : `⚡ **Approved AI command plan executed** (${plan.commands.length} command(s)). Verification is watching the monitors for recovery.`,
+                pingWorkspace: failed || settledMidPlan,
+                displayColor: failed || settledMidPlan ? Red500 : Green500,
             });
         }
         catch (error) {
@@ -156,29 +264,115 @@ export default class CommandPlanExecutor {
     /*
      * Undo a failed remediation: run the rollbackCommand of every command
      * that executed successfully, in REVERSE order. Called by the verifier
-     * after it wins the Failed verification transition — single caller by
-     * construction. Never throws.
+     * after it wins the Failed verification transition, and by its recovery
+     * sweep for a rollback a Worker restart interrupted. Never throws.
      */
     static async executeRollback(data) {
         var _a, _b;
         const { suggestion } = data;
+        let plan = null;
         try {
-            const plan = AiRemediationCommandPlanUtil.parse(suggestion.commandPlan);
+            /*
+             * Read the plan as stored NOW, under the plan lock — not the copy the
+             * verifier read before it won the verification transition. An
+             * approved plan's executor may have enqueued a command since; the
+             * lock guarantees its job is on this read, or that the command stands
+             * down and never starts.
+             */
+            plan = await this.readPlanForRollback(suggestion);
             if (!plan) {
-                return;
+                return this.describeRollbackOutcome(null);
             }
             if (plan.rollbackStatus &&
                 plan.rollbackStatus !== AiRemediationRollbackStatus.NotAttempted) {
-                return; // Already rolled back (or judged not applicable).
+                // Already rolled back (or judged not applicable).
+                return this.describeRollbackOutcome(plan);
             }
+            plan.rollbackHeartbeatAt = OneUptimeDate.getCurrentDate().toISOString();
+            await this.persistPlan(suggestion, plan, "rollback");
             /*
              * A command whose record still says Pending may nonetheless have run:
              * the executor persists Pending, enqueues, and can die before writing
-             * the outcome. The RunnerJob row is the authority on what actually
-             * happened, so resolve those before deciding what to undo — otherwise a
-             * pod death silently exempts a real state change from rollback.
+             * the outcome — or it is simply still waiting on the job. The
+             * RunnerJob row is the authority on what actually happened, so
+             * resolve those first, WAITING for a job still in flight: skipping it
+             * would leave a change that is about to land exempt from its undo.
+             */
+            await this.reconcilePendingExecutions(suggestion, plan, {
+                waitForInFlight: true,
+            });
+            /*
+             * A resumed rollback: undos an interrupted attempt already handed to
+             * a Runner are settled from their jobs, never run twice.
+             */
+            await this.reconcilePendingRollbacks(suggestion, plan);
+            let anyFailed = false;
+            // Rollbacks nobody may run unattended now — a human undoes them.
+            const leftForHuman = [];
+            /*
+             * A resumed rollback reports the WHOLE plan, not just what this
+             * attempt does: an undo the interrupted attempt already settled as
+             * Failed — or that reconcilePendingRollbacks just settled Failed from
+             * its job — keeps the rollback Failed, and one it left for a human is
+             * named again on the feed. Otherwise a resumed rollback whose earlier
+             * undo failed would report "completed" (or "nothing to roll back"),
+             * and the follow-up round would run unattended on top of a change
+             * that is still applied.
              */
-            await this.reconcilePendingExecutions(suggestion, plan);
+            for (const command of plan.commands) {
+                const earlier = command.rollbackExecution;
+                if ((earlier === null || earlier === void 0 ? void 0 : earlier.status) === AiRemediationCommandExecutionStatus.Failed) {
+                    anyFailed = true;
+                }
+                else if ((earlier === null || earlier === void 0 ? void 0 : earlier.status) === AiRemediationCommandExecutionStatus.Skipped) {
+                    anyFailed = true;
+                    leftForHuman.push(command);
+                }
+            }
+            /*
+             * A forward command whose job could not be resolved even after
+             * waiting may or may not have run. Its undo is not run blind; a human
+             * is told to check.
+             */
+            for (const command of plan.commands) {
+                const execution = command.execution;
+                if (execution &&
+                    !isSettledCommandExecutionStatus(execution.status) &&
+                    execution.runnerJobId &&
+                    command.rollbackCommand &&
+                    !command.rollbackExecution) {
+                    command.rollbackExecution = {
+                        status: AiRemediationCommandExecutionStatus.Skipped,
+                        errorMessage: `Rollback not run: whether the command ran could not be confirmed from its job. If it ran, undo it manually: ${command.rollbackCommand}`,
+                    };
+                    leftForHuman.push(command);
+                    anyFailed = true;
+                }
+            }
+            /*
+             * A kubectl command recorded as failed may still have changed the
+             * cluster: a Runner took it and no result came back, the wait for it
+             * broke while the job ran on, or kubectl was stopped before it
+             * finished. Its job is read again: one that succeeded after all is
+             * rolled back like any other, and one that may have changed the
+             * cluster gets the same answer as an unresolved one above — its undo
+             * is not run blind, a human is told to check.
+             */
+            for (const command of await this.reconcileFailedKubectlExecutions(suggestion, plan)) {
+                command.rollbackExecution = {
+                    status: AiRemediationCommandExecutionStatus.Skipped,
+                    errorMessage: `Rollback not run: the command reached the cluster's Runner but kubectl did not report a finished result, so whether it changed the cluster is unknown. If it did, undo it manually: ${command.rollbackCommand}`,
+                };
+                leftForHuman.push(command);
+                anyFailed = true;
+            }
+            if (plan.commands.some((command) => {
+                return (command.rollbackExecution !== undefined &&
+                    !isSettledCommandExecutionStatus(command.rollbackExecution.status));
+            })) {
+                // A rollback of an interrupted attempt whose outcome is unknown.
+                anyFailed = true;
+            }
             const rollbackTargets = plan.commands
                 .filter((command) => {
                 var _a;
@@ -188,60 +382,87 @@ export default class CommandPlanExecutor {
                     !command.rollbackExecution);
             })
                 .reverse();
-            if (rollbackTargets.length === 0) {
-                plan.rollbackStatus = AiRemediationRollbackStatus.NotApplicable;
-                await this.persistPlan(suggestion, plan);
-                return;
+            /*
+             * Nothing to run and nothing that went wrong, now or in an
+             * interrupted attempt: Completed when some undo ran, NotApplicable
+             * when none was ever needed. Anything else goes through the end of
+             * this method, which settles the status from the whole plan and says
+             * so on the feed.
+             */
+            if (rollbackTargets.length === 0 && !anyFailed) {
+                plan.rollbackStatus = plan.commands.some((command) => {
+                    var _a;
+                    return (((_a = command.rollbackExecution) === null || _a === void 0 ? void 0 : _a.status) ===
+                        AiRemediationCommandExecutionStatus.Succeeded);
+                })
+                    ? AiRemediationRollbackStatus.Completed
+                    : AiRemediationRollbackStatus.NotApplicable;
+                await this.persistPlan(suggestion, plan, "rollback");
+                return this.describeRollbackOutcome(plan);
             }
-            let anyFailed = false;
             for (const command of rollbackTargets) {
                 const rollbackCommand = command.rollbackCommand;
-                const denyReason = CommandPolicy.getDenyReason(rollbackCommand);
+                const denyReason = this.getDenyReason({
+                    stepType: command.stepType,
+                    command: rollbackCommand,
+                });
                 if (denyReason) {
                     command.rollbackExecution = {
                         status: AiRemediationCommandExecutionStatus.Failed,
                         errorMessage: `Rollback refused by the command policy: ${denyReason}.`,
                     };
                     anyFailed = true;
-                    await this.persistPlan(suggestion, plan);
+                    await this.persistPlan(suggestion, plan, "rollback");
+                    continue;
+                }
+                /*
+                 * A rollback runs with nobody watching, so it has to be allowed
+                 * unattended by the cluster as it is configured NOW — not as it was
+                 * when the plan accepted the rollback. A RiskyWrite undo accepted
+                 * while the operator bypassed approvals must not run once they
+                 * stopped bypassing them; a cluster whose remediation was turned
+                 * off must not be changed at all. Left for a human, and said so.
+                 */
+                const clusterRefusal = await this.getClusterExecutionRefusal({
+                    suggestion,
+                    command,
+                    commandText: rollbackCommand,
+                    purpose: "rollback",
+                });
+                if (clusterRefusal) {
+                    command.rollbackExecution = {
+                        status: AiRemediationCommandExecutionStatus.Skipped,
+                        errorMessage: `Rollback not run: ${clusterRefusal}. Undo it manually: ${rollbackCommand}`,
+                    };
+                    leftForHuman.push(command);
+                    anyFailed = true;
+                    await this.persistPlan(suggestion, plan, "rollback");
                     continue;
                 }
                 command.rollbackExecution = {
                     status: AiRemediationCommandExecutionStatus.Pending,
                     startedAt: OneUptimeDate.getCurrentDate().toISOString(),
                 };
-                await this.persistPlan(suggestion, plan);
+                plan.rollbackHeartbeatAt = OneUptimeDate.getCurrentDate().toISOString();
+                await this.persistPlan(suggestion, plan, "rollback");
                 try {
-                    const job = await RunnerJobService.enqueueAiCommand({
-                        projectId: suggestion.projectId,
-                        aiRunId: suggestion.aiRunId,
-                        autoRemediationSuggestionId: suggestion.id,
-                        stepId: `ai-rollback-${command.sequence}`,
-                        stepType: command.stepType,
-                        targetAgentId: new ObjectID(command.runnerId),
-                        command: rollbackCommand,
-                        credentialId: command.credentialId,
-                        timeoutInMs: command.timeoutInMs,
-                        claimTimeoutInMs: AI_COMMAND_CLAIM_TIMEOUT_MS,
+                    const job = await this.enqueue({
+                        suggestion,
+                        command,
+                        commandText: rollbackCommand,
+                        stepId: getRollbackCommandStepId(command),
                     });
+                    // The job id lands before the wait — see startOneCommand.
                     command.rollbackExecution.runnerJobId = (_a = job.id) === null || _a === void 0 ? void 0 : _a.toString();
+                    await this.persistPlan(suggestion, plan, "rollback");
                     const terminalJob = await RunnerJobService.pollUntilTerminal({
                         jobId: job.id,
                         claimTimeoutInMs: AI_COMMAND_CLAIM_TIMEOUT_MS,
                         executionTimeoutInMs: command.timeoutInMs,
                     });
-                    const succeeded = terminalJob.status === RunnerJobStatus.Succeeded;
-                    command.rollbackExecution.status = succeeded
-                        ? AiRemediationCommandExecutionStatus.Succeeded
-                        : AiRemediationCommandExecutionStatus.Failed;
-                    command.rollbackExecution.completedAt =
-                        OneUptimeDate.getCurrentDate().toISOString();
-                    command.rollbackExecution.exitCode = terminalJob.exitCode;
-                    command.rollbackExecution.output = this.capOutput(terminalJob.output || "");
-                    if (!succeeded) {
-                        command.rollbackExecution.errorMessage =
-                            terminalJob.errorMessage ||
-                                `Rollback ended with status ${terminalJob.status}.`;
+                    this.settleFromJob(command.rollbackExecution, terminalJob, "Rollback");
+                    if (command.rollbackExecution.status !==
+                        AiRemediationCommandExecutionStatus.Succeeded) {
                         anyFailed = true;
                     }
                 }
@@ -252,38 +473,102 @@ export default class CommandPlanExecutor {
                         error instanceof Error ? error.message : String(error);
                     anyFailed = true;
                 }
-                await this.persistPlan(suggestion, plan);
+                await this.persistPlan(suggestion, plan, "rollback");
             }
             plan.rollbackStatus = anyFailed
                 ? AiRemediationRollbackStatus.Failed
                 : AiRemediationRollbackStatus.Completed;
-            await this.persistPlan(suggestion, plan);
+            await this.persistPlan(suggestion, plan, "rollback");
+            const failedUndos = plan.commands.filter((command) => {
+                var _a;
+                return (((_a = command.rollbackExecution) === null || _a === void 0 ? void 0 : _a.status) ===
+                    AiRemediationCommandExecutionStatus.Failed);
+            }).length;
             await this.postFeedItem({
                 suggestion,
-                markdown: anyFailed
-                    ? `⚠️ **Auto-remediation rollback did not fully complete** — at least one rollback command failed. Review the suggestion's per-command record; manual intervention may be needed.`
-                    : `↩️ **Auto-remediation rolled back** — the executed commands' rollback commands ran after verification failed. Escalation continues as normal.`,
+                markdown: leftForHuman.length > 0
+                    ? `⚠️ **Auto-remediation rollback was not run for ${leftForHuman.length} command(s)** — the cluster no longer allows them unattended, or whether they ran could not be confirmed, so a human has to undo them. ${leftForHuman
+                        .map((command) => {
+                        var _a;
+                        return this.capForFeed(((_a = command.rollbackExecution) === null || _a === void 0 ? void 0 : _a.errorMessage) || "");
+                    })
+                        .join(" ")}${failedUndos > 0
+                        ? ` ${failedUndos} other rollback command(s) failed.`
+                        : ""} Review the suggestion's per-command record; manual intervention is needed.`
+                    : anyFailed
+                        ? `⚠️ **Auto-remediation rollback did not fully complete** — at least one rollback command failed, or its outcome is not known. Review the suggestion's per-command record; manual intervention may be needed.`
+                        : `↩️ **Auto-remediation rolled back** — the executed commands' rollback commands ran after verification failed. Escalation continues as normal.`,
                 pingWorkspace: true,
                 displayColor: anyFailed ? Red500 : Green500,
             });
+            return this.describeRollbackOutcome(plan);
         }
         catch (error) {
             logger.error(`CommandPlanExecutor: rollback failed for suggestion ${(_b = suggestion.id) === null || _b === void 0 ? void 0 : _b.toString()}: ${error}`);
+            /*
+             * Settle it as Failed so nobody assumes it was undone — and so the
+             * recovery sweep does not retry a rollback that throws every time.
+             */
+            if (plan) {
+                plan.rollbackStatus = AiRemediationRollbackStatus.Failed;
+                await this.persistPlan(suggestion, plan, "rollback");
+                await this.postFeedItem({
+                    suggestion,
+                    markdown: `⚠️ **Auto-remediation rollback could not complete** — an unexpected error stopped it. Review the suggestion's per-command record; manual intervention may be needed.`,
+                    pingWorkspace: true,
+                    displayColor: Red500,
+                });
+            }
+            return this.describeRollbackOutcome(plan);
         }
     }
     // ------------------------------------------------------------------
-    static async executeOneCommand(data) {
+    /*
+     * Everything up to and including the enqueue: the execution-time policy
+     * floor, the cluster re-check, the enqueue, and the write that names the
+     * job. Runs under the plan lock (see executeApprovedPlan).
+     */
+    static async startOneCommand(data) {
         var _a, _b;
-        const { suggestion, command } = data;
+        const { suggestion, plan, command } = data;
         /*
          * Execution-time denylist re-check: the plan was validated when it was
          * composed, but the policy may have been tightened since.
          */
-        const denyReason = CommandPolicy.getDenyReason(command.command);
+        const denyReason = this.getDenyReason({
+            stepType: command.stepType,
+            command: command.command,
+        });
         if (denyReason) {
             return {
-                status: AiRemediationCommandExecutionStatus.Failed,
-                errorMessage: `Refused by the remediation command policy at execution time: ${denyReason}.`,
+                state: {
+                    status: AiRemediationCommandExecutionStatus.Failed,
+                    errorMessage: `Refused by the remediation command policy at execution time: ${denyReason}.`,
+                },
+                job: null,
+                recordsClusterOutcome: false,
+            };
+        }
+        /*
+         * Execution-time cluster re-check: the approve API re-checks the
+         * project switches, this re-checks the cluster's own AI page. An
+         * operator who turned remediation off, or bound a different Runner,
+         * after the plan was composed expects that to stop the plan too.
+         */
+        const clusterRefusal = await this.getClusterExecutionRefusal({
+            suggestion,
+            command,
+            commandText: command.command,
+            purpose: "command",
+        });
+        if (clusterRefusal) {
+            return {
+                state: {
+                    status: AiRemediationCommandExecutionStatus.Failed,
+                    errorMessage: `Refused at execution time: ${clusterRefusal}.`,
+                },
+                job: null,
+                recordsClusterOutcome: false,
             };
         }
         const state = {
@@ -292,99 +577,534 @@ export default class CommandPlanExecutor {
                 OneUptimeDate.getCurrentDate().toISOString(),
         };
         try {
-            const job = await RunnerJobService.enqueueAiCommand({
+            const job = await this.enqueue({
+                suggestion,
+                command,
+                commandText: command.command,
+                // Approved plans never hold inline-executed commands (skipped above).
+                stepId: getForwardCommandStepId(command),
+            });
+            /*
+             * The job id lands on the record BEFORE the wait: a pod death while
+             * polling must leave a Pending command that names its RunnerJob, so
+             * the rollback arm can resolve what really happened instead of
+             * treating the command as one that never reached the Runner.
+             */
+            state.runnerJobId = (_b = job.id) === null || _b === void 0 ? void 0 : _b.toString();
+            command.execution = state;
+            await this.persistPlan(suggestion, plan, "executor", {
+                planLocked: true,
+            });
+            return { state, job, recordsClusterOutcome: true };
+        }
+        catch (error) {
+            state.status = AiRemediationCommandExecutionStatus.Failed;
+            state.completedAt = OneUptimeDate.getCurrentDate().toISOString();
+            state.errorMessage =
+                error instanceof Error ? error.message : String(error);
+            return { state, job: null, recordsClusterOutcome: true };
+        }
+    }
+    // The wait (outside the plan lock) and the outcome.
+    static async finishOneCommand(data) {
+        const { command, started } = data;
+        const state = started.state;
+        /*
+         * What the job's end says about the cluster's ACCESS. A job that never
+         * reached a Runner (the enqueue refused it) is an access failure; a
+         * wait that threw says nothing about the cluster.
+         */
+        let facts = started.job
+            ? null
+            : { status: RunnerJobStatus.Failed, errorMessage: state.errorMessage };
+        if (started.job) {
+            try {
+                const terminalJob = await RunnerJobService.pollUntilTerminal({
+                    jobId: started.job.id,
+                    claimTimeoutInMs: AI_COMMAND_CLAIM_TIMEOUT_MS,
+                    executionTimeoutInMs: command.timeoutInMs,
+                });
+                if (!isTerminalAgentJobStatus(terminalJob.status)) {
+                    throw new Error("Command job did not reach a terminal state.");
+                }
+                this.settleFromJob(state, terminalJob, "Command");
+                facts = {
+                    status: terminalJob.status,
+                    exitCode: terminalJob.exitCode,
+                    output: terminalJob.output,
+                    errorMessage: terminalJob.errorMessage,
+                };
+            }
+            catch (error) {
+                state.status = AiRemediationCommandExecutionStatus.Failed;
+                state.completedAt = OneUptimeDate.getCurrentDate().toISOString();
+                state.errorMessage =
+                    error instanceof Error ? error.message : String(error);
+            }
+        }
+        /*
+         * The cluster's "Last error" is about its AI ACCESS: a success proves
+         * the access works, and a failure is recorded only when it is about
+         * the access (never claimed, refused, timed out, unauthorized,
+         * unreachable). A deployment the plan named wrong must neither become
+         * the cluster's last error nor clear a real one.
+         */
+        const succeeded = state.status === AiRemediationCommandExecutionStatus.Succeeded;
+        const recordsOutcome = succeeded || (facts !== null && KubectlJobRunner.isAccessFailure(facts));
+        if (started.recordsClusterOutcome &&
+            recordsOutcome &&
+            command.stepType === RunbookStepType.Kubectl &&
+            command.kubernetesClusterId &&
+            ObjectID.isValidUUID(command.kubernetesClusterId)) {
+            await KubernetesClusterAiAccessService.recordCommandOutcome({
+                clusterId: new ObjectID(command.kubernetesClusterId),
+                succeeded: state.status === AiRemediationCommandExecutionStatus.Succeeded,
+                errorMessage: state.errorMessage,
+            });
+        }
+        return state;
+    }
+    // Copy a terminal job's outcome onto an execution record.
+    static settleFromJob(state, job, label) {
+        const succeeded = job.status === RunnerJobStatus.Succeeded;
+        state.status = succeeded
+            ? AiRemediationCommandExecutionStatus.Succeeded
+            : AiRemediationCommandExecutionStatus.Failed;
+        state.completedAt = OneUptimeDate.getCurrentDate().toISOString();
+        state.exitCode = job.exitCode;
+        state.output = this.capOutput(job.output || "");
+        state.errorMessage = succeeded
+            ? undefined
+            : job.errorMessage || `${label} ended with status ${job.status}.`;
+    }
+    /*
+     * One enqueue for both lanes. Bash/SSH go through enqueueAiCommand
+     * (script or SSH payload); Kubectl goes through enqueueAiKubectlCommand
+     * (argv + cluster + optional credential), whose policy is re-run there.
+     */
+    static async enqueue(data) {
+        const { suggestion, command } = data;
+        if (command.stepType === RunbookStepType.Kubectl) {
+            if (!command.kubernetesClusterId ||
+                !ObjectID.isValidUUID(command.kubernetesClusterId)) {
+                throw new Error("The kubectl command names no cluster.");
+            }
+            return RunnerJobService.enqueueAiKubectlCommand({
                 projectId: suggestion.projectId,
                 aiRunId: suggestion.aiRunId,
+                origin: RunnerJobOrigin.AiRemediation,
                 autoRemediationSuggestionId: suggestion.id,
-                stepId: `ai-approved-${command.sequence}`,
-                stepType: command.stepType,
+                kubernetesClusterId: new ObjectID(command.kubernetesClusterId),
+                stepId: data.stepId,
                 targetAgentId: new ObjectID(command.runnerId),
-                command: command.command,
                 credentialId: command.credentialId,
+                command: data.commandText,
                 timeoutInMs: command.timeoutInMs,
                 claimTimeoutInMs: AI_COMMAND_CLAIM_TIMEOUT_MS,
             });
-            state.runnerJobId = (_b = job.id) === null || _b === void 0 ? void 0 : _b.toString();
-            const terminalJob = await RunnerJobService.pollUntilTerminal({
-                jobId: job.id,
-                claimTimeoutInMs: AI_COMMAND_CLAIM_TIMEOUT_MS,
-                executionTimeoutInMs: command.timeoutInMs,
+        }
+        return RunnerJobService.enqueueAiCommand({
+            projectId: suggestion.projectId,
+            aiRunId: suggestion.aiRunId,
+            autoRemediationSuggestionId: suggestion.id,
+            stepId: data.stepId,
+            stepType: command.stepType,
+            targetAgentId: new ObjectID(command.runnerId),
+            command: data.commandText,
+            credentialId: command.credentialId,
+            timeoutInMs: command.timeoutInMs,
+            claimTimeoutInMs: AI_COMMAND_CLAIM_TIMEOUT_MS,
+        });
+    }
+    /*
+     * Why a kubectl command may NOT run on its cluster right now, or null.
+     *
+     * The plan froze the cluster, the Runner and the credential the cluster's
+     * AI page bound when the plan was composed. Before anything is enqueued
+     * the page is read again: remediation must still be enabled and ready
+     * (Disabled means "AI never runs a change on this cluster", whatever was
+     * approved earlier), and the bound Runner and credential must still be
+     * the ones on the plan — a re-bound cluster never receives a job meant
+     * for its previous access path. A rollback must ALSO be allowed
+     * unattended under the cluster's CURRENT mode and allowlist: nothing
+     * shows a rollback to a human when it fires.
+     *
+     * Fails closed: a status that cannot be read is a refusal. Bash/SSH
+     * commands have no cluster and pass through.
+     */
+    static async getClusterExecutionRefusal(data) {
+        const { suggestion, command } = data;
+        if (command.stepType !== RunbookStepType.Kubectl) {
+            return null;
+        }
+        const clusterLabel = command.kubernetesClusterNameSnapshot ||
+            command.kubernetesClusterId ||
+            "(unknown)";
+        if (!command.kubernetesClusterId ||
+            !ObjectID.isValidUUID(command.kubernetesClusterId) ||
+            !suggestion.projectId) {
+            return `the kubectl command names no valid cluster`;
+        }
+        let status;
+        try {
+            status = await KubernetesClusterAiAccessService.getStatusForCluster({
+                clusterId: new ObjectID(command.kubernetesClusterId),
+                projectId: suggestion.projectId,
             });
-            if (!isTerminalAgentJobStatus(terminalJob.status)) {
-                throw new Error("Command job did not reach a terminal state.");
-            }
-            const succeeded = terminalJob.status === RunnerJobStatus.Succeeded;
-            state.status = succeeded
-                ? AiRemediationCommandExecutionStatus.Succeeded
-                : AiRemediationCommandExecutionStatus.Failed;
-            state.completedAt = OneUptimeDate.getCurrentDate().toISOString();
-            state.exitCode = terminalJob.exitCode;
-            state.output = this.capOutput(terminalJob.output || "");
-            if (!succeeded) {
-                state.errorMessage =
-                    terminalJob.errorMessage ||
-                        `Command ended with status ${terminalJob.status}.`;
-            }
         }
         catch (error) {
-            state.status = AiRemediationCommandExecutionStatus.Failed;
-            state.completedAt = OneUptimeDate.getCurrentDate().toISOString();
-            state.errorMessage =
-                error instanceof Error ? error.message : String(error);
+            logger.error(`CommandPlanExecutor: could not read the AI access status of cluster ${command.kubernetesClusterId} before a ${data.purpose}: ${error}`);
+            return `could not confirm that cluster "${clusterLabel}" still allows AI remediation`;
         }
-        return state;
+        if (!status) {
+            return `cluster "${clusterLabel}" no longer exists in this project`;
+        }
+        if (!status.isRemediationReady) {
+            const gap = status.gaps.find((candidate) => {
+                return candidate.blocks !== "investigation";
+            });
+            return `cluster "${status.clusterName}" no longer allows AI remediation${gap ? ` (${gap.title})` : ""}`;
+        }
+        if (!status.runner || status.runner.id !== command.runnerId) {
+            return `cluster "${status.clusterName}" is no longer reached through Runner "${command.runnerNameSnapshot}", which this plan was composed for`;
+        }
+        if ((status.credentialId || undefined) !== (command.credentialId || undefined)) {
+            return `cluster "${status.clusterName}" is no longer reached with the credential this plan was composed for`;
+        }
+        /*
+         * The Runner's write scope as it reports it now: a write it would
+         * refuse is not enqueued (or, for a rollback, is left for a human with
+         * the reason) — never sent only to be refused.
+         */
+        const scopeRefusal = RemediationCommandToolkit.getRunnerScopeRefusal({
+            cluster: status,
+            command: data.commandText,
+        });
+        if (scopeRefusal) {
+            // The callers end the sentence themselves.
+            return `the cluster's Runner would refuse it: ${scopeRefusal.endsWith(".") ? scopeRefusal.slice(0, -1) : scopeRefusal}`;
+        }
+        if (data.purpose === "rollback") {
+            const verdict = KubectlPolicy.evaluateForAutoExecution({
+                command: data.commandText,
+                allowlistPatterns: status.kubectlAllowlist,
+                bypassApproval: status.remediationMode ===
+                    KubernetesAiRemediationMode.BypassApproval,
+            });
+            if (verdict.verdict !== AiRemediationCommandPolicyVerdict.AutoApproved) {
+                return `cluster "${status.clusterName}" no longer allows this change unattended (its AI remediation mode is now ${status.remediationMode}): ${verdict.reason}`;
+            }
+        }
+        return null;
+    }
+    /*
+     * The job a command's step ran under, when its record never got the job
+     * id. The enqueue and the write that names the job are two steps; a pod
+     * death or a failed write between them leaves a Pending record for a job
+     * the Runner was nonetheless handed. (suggestion, stepId) is unique per
+     * forward command — a plan executes at most once and a retried run never
+     * re-executes — and per rollback; the newest row wins should that ever
+     * change.
+     */
+    static async findJobByStepId(data) {
+        if (!data.suggestion.id) {
+            return null;
+        }
+        const jobs = await RunnerJobService.findBy({
+            query: Object.assign(Object.assign({}, (data.suggestion.projectId
+                ? { projectId: data.suggestion.projectId }
+                : {})), { autoRemediationSuggestionId: data.suggestion.id, origin: RunnerJobOrigin.AiRemediation, stepId: data.stepId }),
+            select: RECONCILE_JOB_SELECT,
+            sort: { createdAt: SortOrder.Descending },
+            limit: 1,
+            skip: 0,
+            props: { isRoot: true },
+        });
+        return jobs[0] || null;
+    }
+    static capForFeed(text) {
+        const collapsed = text.trim().replace(/\s+/g, " ");
+        return collapsed.length > MAX_FEED_REASON_CHARS
+            ? `${collapsed.slice(0, MAX_FEED_REASON_CHARS)}…`
+            : collapsed;
+    }
+    /*
+     * Execution-time policy floor per lane: the bash denylist for Bash/SSH,
+     * the Denied tier for Kubectl. Null when the command may run.
+     */
+    static getDenyReason(data) {
+        if (data.stepType === RunbookStepType.Kubectl) {
+            const policy = KubectlPolicy.evaluateCommand(data.command);
+            return policy.tier === KubectlCommandTier.Denied ? policy.reason : null;
+        }
+        return CommandPolicy.getDenyReason(data.command);
     }
     /*
      * Settle commands left in Pending/Running by reading their RunnerJob. A
      * job that reached Succeeded really ran, so the command becomes Succeeded
-     * (and therefore rollback-eligible); any other terminal state, or a
-     * missing job, becomes Failed. Jobs still in flight are left alone.
+     * (and therefore rollback-eligible); any other terminal state becomes
+     * Failed. A job still in flight is WAITED for when `waitForInFlight` is
+     * set (the rollback arm: the change is about to land, and its undo must
+     * not be skipped); otherwise it is left alone.
+     *
+     * The job is found by the id on the record, or — when the record never
+     * got one, because the pod died or the write failed between the enqueue
+     * and the persist that names the job — by the (suggestion, stepId) pair
+     * the lane enqueued it under. A command with neither never reached the
+     * Runner and is left alone.
      */
-    static async reconcilePendingExecutions(suggestion, plan) {
-        var _a, _b, _c;
+    static async reconcilePendingExecutions(suggestion, plan, options) {
+        var _a;
         let changed = false;
         for (const command of plan.commands) {
-            const status = (_a = command.execution) === null || _a === void 0 ? void 0 : _a.status;
-            const isUnsettled = status === AiRemediationCommandExecutionStatus.Pending ||
-                status === AiRemediationCommandExecutionStatus.Running;
-            if (!isUnsettled || !((_b = command.execution) === null || _b === void 0 ? void 0 : _b.runnerJobId)) {
+            const execution = command.execution;
+            if (!execution) {
+                continue;
+            }
+            const isUnsettled = execution.status === AiRemediationCommandExecutionStatus.Pending ||
+                execution.status === AiRemediationCommandExecutionStatus.Running;
+            if (!isUnsettled) {
                 continue;
             }
             try {
-                const job = await RunnerJobService.findOneById({
-                    id: new ObjectID(command.execution.runnerJobId),
-                    select: {
-                        _id: true,
-                        status: true,
-                        output: true,
-                        exitCode: true,
-                        errorMessage: true,
-                    },
-                    props: { isRoot: true },
-                });
-                if (!job || !isTerminalAgentJobStatus(job.status)) {
+                let job = execution.runnerJobId
+                    ? await RunnerJobService.findOneById({
+                        id: new ObjectID(execution.runnerJobId),
+                        select: RECONCILE_JOB_SELECT,
+                        props: { isRoot: true },
+                    })
+                    : await this.findJobByStepId({
+                        suggestion,
+                        stepId: getForwardCommandStepId(command),
+                    });
+                if (!job) {
                     continue;
                 }
+                // Recovered by stepId: name the job so the record stays resolvable.
+                if (!execution.runnerJobId && job.id) {
+                    execution.runnerJobId = job.id.toString();
+                    changed = true;
+                }
+                if (!isTerminalAgentJobStatus(job.status)) {
+                    if (!options.waitForInFlight || !job.id) {
+                        continue;
+                    }
+                    // Say where the rollback stands before a wait that can take minutes.
+                    if (changed) {
+                        await this.persistPlan(suggestion, plan, "rollback");
+                        changed = false;
+                    }
+                    plan.rollbackHeartbeatAt =
+                        OneUptimeDate.getCurrentDate().toISOString();
+                    await this.persistPlan(suggestion, plan, "rollback");
+                    job = await RunnerJobService.pollUntilTerminal({
+                        jobId: job.id,
+                        claimTimeoutInMs: AI_COMMAND_CLAIM_TIMEOUT_MS,
+                        executionTimeoutInMs: command.timeoutInMs,
+                    });
+                    if (!isTerminalAgentJobStatus(job.status)) {
+                        continue;
+                    }
+                }
                 const succeeded = job.status === RunnerJobStatus.Succeeded;
-                command.execution.status = succeeded
+                execution.status = succeeded
                     ? AiRemediationCommandExecutionStatus.Succeeded
                     : AiRemediationCommandExecutionStatus.Failed;
-                command.execution.exitCode = job.exitCode;
-                command.execution.output = this.capOutput(job.output || "");
-                command.execution.errorMessage = succeeded
+                execution.exitCode = job.exitCode;
+                execution.output = this.capOutput(job.output || "");
+                execution.errorMessage = succeeded
                     ? undefined
                     : job.errorMessage ||
                         `Command ended with status ${job.status} (recovered from the job record).`;
                 changed = true;
             }
             catch (error) {
-                logger.error(`CommandPlanExecutor: could not reconcile command ${command.sequence} of suggestion ${(_c = suggestion.id) === null || _c === void 0 ? void 0 : _c.toString()}: ${error}`);
+                logger.error(`CommandPlanExecutor: could not reconcile command ${command.sequence} of suggestion ${(_a = suggestion.id) === null || _a === void 0 ? void 0 : _a.toString()}: ${error}`);
             }
         }
         if (changed) {
-            await this.persistPlan(suggestion, plan);
+            await this.persistPlan(suggestion, plan, "rollback");
         }
     }
+    /*
+     * A rollback resumed after a Worker restart: settle the undos the
+     * interrupted attempt had already started. One with a job — on the record
+     * or found under its rollback step id — is settled from that job
+     * (waiting for it when it is still running): it is never run a second
+     * time, because running an undo twice (a second `rollout undo`) can put
+     * the bad change right back. One that never reached a Runner is cleared,
+     * so it runs now. One whose job cannot be looked up stays unsettled, and
+     * the rollback is not reported complete.
+     */
+    static async reconcilePendingRollbacks(suggestion, plan) {
+        var _a;
+        let changed = false;
+        for (const command of plan.commands) {
+            const rollback = command.rollbackExecution;
+            if (!rollback || isSettledCommandExecutionStatus(rollback.status)) {
+                continue;
+            }
+            try {
+                let job = rollback.runnerJobId
+                    ? await RunnerJobService.findOneById({
+                        id: new ObjectID(rollback.runnerJobId),
+                        select: RECONCILE_JOB_SELECT,
+                        props: { isRoot: true },
+                    })
+                    : await this.findJobByStepId({
+                        suggestion,
+                        stepId: getRollbackCommandStepId(command),
+                    });
+                if (!job) {
+                    // Never reached a Runner: run it now.
+                    command.rollbackExecution = undefined;
+                    changed = true;
+                    continue;
+                }
+                if (!rollback.runnerJobId && job.id) {
+                    rollback.runnerJobId = job.id.toString();
+                    changed = true;
+                }
+                if (!isTerminalAgentJobStatus(job.status) && job.id) {
+                    plan.rollbackHeartbeatAt =
+                        OneUptimeDate.getCurrentDate().toISOString();
+                    await this.persistPlan(suggestion, plan, "rollback");
+                    job = await RunnerJobService.pollUntilTerminal({
+                        jobId: job.id,
+                        claimTimeoutInMs: AI_COMMAND_CLAIM_TIMEOUT_MS,
+                        executionTimeoutInMs: command.timeoutInMs,
+                    });
+                }
+                if (isTerminalAgentJobStatus(job.status)) {
+                    this.settleFromJob(rollback, job, "Rollback");
+                    changed = true;
+                }
+            }
+            catch (error) {
+                logger.error(`CommandPlanExecutor: could not reconcile the rollback of command ${command.sequence} of suggestion ${(_a = suggestion.id) === null || _a === void 0 ? void 0 : _a.toString()}: ${error}`);
+            }
+        }
+        if (changed) {
+            await this.persistPlan(suggestion, plan, "rollback");
+        }
+    }
+    /*
+     * The kubectl commands recorded as Failed that may nonetheless have
+     * changed the cluster, read from their jobs the way KubectlJobRunner
+     * reads every kubectl job (KubectlJobRunner.getRunStateOfJobRow). Only a
+     * command that carries a rollbackCommand and has no rollback yet is read
+     * — nothing else has an undo to owe.
+     *
+     *  - A job that Succeeded after all (the wait for it broke, and the
+     *    record said Failed) is settled from the job: it becomes Succeeded,
+     *    and so a rollback target like any other.
+     *  - A job whose run is Unknown — a Runner took it and no result came
+     *    back, it is still in flight, or its row cannot be read — is
+     *    returned: whether it changed the cluster is not known. So is one
+     *    where kubectl ran but never finished (the Runner killed it at its
+     *    timeout, or it left output but no exit code): it may have applied
+     *    its change before it was stopped.
+     *  - A job that certainly never ran, or whose kubectl finished and
+     *    reported a failure (an exit code), owes no undo, as before.
+     */
+    static async reconcileFailedKubectlExecutions(suggestion, plan) {
+        var _a;
+        const mayHaveRun = [];
+        let changed = false;
+        for (const command of plan.commands) {
+            const execution = command.execution;
+            if (command.stepType !== RunbookStepType.Kubectl ||
+                !execution ||
+                execution.status !== AiRemediationCommandExecutionStatus.Failed ||
+                !execution.runnerJobId ||
+                !command.rollbackCommand ||
+                command.rollbackExecution) {
+                continue;
+            }
+            let runState = KubectlRunState.Unknown;
+            let job = null;
+            try {
+                job = ObjectID.isValidUUID(execution.runnerJobId)
+                    ? await RunnerJobService.findOneById({
+                        id: new ObjectID(execution.runnerJobId),
+                        select: RUN_STATE_JOB_SELECT,
+                        props: { isRoot: true },
+                    })
+                    : null;
+                if (job) {
+                    runState = KubectlJobRunner.getRunStateOfJobRow(job);
+                }
+            }
+            catch (error) {
+                logger.error(`CommandPlanExecutor: could not read whether command ${command.sequence} of suggestion ${(_a = suggestion.id) === null || _a === void 0 ? void 0 : _a.toString()} ran: ${error}`);
+            }
+            if (job && job.status === RunnerJobStatus.Succeeded) {
+                this.settleFromJob(execution, job, "Command");
+                changed = true;
+                continue;
+            }
+            const kubectlFinished = runState === KubectlRunState.Ran && typeof (job === null || job === void 0 ? void 0 : job.exitCode) === "number";
+            if (runState !== KubectlRunState.NotRun && !kubectlFinished) {
+                mayHaveRun.push(command);
+            }
+        }
+        if (changed) {
+            await this.persistPlan(suggestion, plan, "rollback");
+        }
+        return mayHaveRun;
+    }
+    static describeRollbackOutcome(plan) {
+        var _a;
+        let rolledBack = 0;
+        let failed = 0;
+        let leftForHuman = 0;
+        for (const command of (plan === null || plan === void 0 ? void 0 : plan.commands) || []) {
+            switch ((_a = command.rollbackExecution) === null || _a === void 0 ? void 0 : _a.status) {
+                case AiRemediationCommandExecutionStatus.Succeeded:
+                    rolledBack += 1;
+                    break;
+                case AiRemediationCommandExecutionStatus.Failed:
+                    failed += 1;
+                    break;
+                case AiRemediationCommandExecutionStatus.Skipped:
+                    leftForHuman += 1;
+                    break;
+                default:
+                    break;
+            }
+        }
+        const storedStatus = (plan === null || plan === void 0 ? void 0 : plan.rollbackStatus) &&
+            plan.rollbackStatus !== AiRemediationRollbackStatus.NotAttempted
+            ? plan.rollbackStatus
+            : undefined;
+        /*
+         * A status and its counts never contradict each other: an undo that
+         * failed or was left for a human makes a settled rollback Failed, even
+         * one recorded "completed" before resumed rollbacks carried earlier
+         * failures forward.
+         */
+        const rollbackStatus = storedStatus !== undefined && failed + leftForHuman > 0
+            ? AiRemediationRollbackStatus.Failed
+            : storedStatus;
+        let summary;
+        switch (rollbackStatus) {
+            case AiRemediationRollbackStatus.Completed:
+                summary = `Rollback completed: ${rolledBack} command(s) undone.`;
+                break;
+            case AiRemediationRollbackStatus.NotApplicable:
+                summary =
+                    "Nothing was rolled back: no executed command carried a rollback command.";
+                break;
+            case AiRemediationRollbackStatus.Failed:
+                summary = `Rollback did NOT fully complete: ${rolledBack} command(s) undone, ${failed} rollback(s) failed, ${leftForHuman} left for a human to undo — the change may still be applied.`;
+                break;
+            default:
+                summary =
+                    "The rollback did not finish; whatever it did not undo may still be applied.";
+                break;
+        }
+        return { rollbackStatus, rolledBack, failed, leftForHuman, summary };
+    }
     /*
      * Whether the suggestion is still an Approved remediation whose
      * verification window is open. False means someone (the verifier, or a
@@ -429,13 +1149,70 @@ export default class CommandPlanExecutor {
             return true;
         }
     }
-    static async persistPlan(suggestion, plan) {
+    /*
+     * The plan the rollback arm works from: the stored one, read under the
+     * plan lock. Falls back to the copy the caller holds when the stored plan
+     * cannot be read.
+     */
+    static async readPlanForRollback(suggestion) {
+        const mutex = suggestion.id
+            ? await this.lockPlan(suggestion.id)
+            : null;
+        try {
+            const stored = suggestion.id
+                ? await this.readStoredPlan(suggestion.id)
+                : null;
+            return (stored || AiRemediationCommandPlanUtil.parse(suggestion.commandPlan));
+        }
+        finally {
+            await this.unlockPlan(mutex);
+        }
+    }
+    // The plan as stored right now, or null when it cannot be read.
+    static async readStoredPlan(suggestionId) {
+        try {
+            /*
+             * findOneBy, not findOneById: this is the column read for a merge,
+             * not a status re-check, and it must not be mistaken for one.
+             */
+            const stored = await AutoRemediationSuggestionService.findOneBy({
+                query: { _id: suggestionId.toString() },
+                select: { _id: true, commandPlan: true },
+                props: { isRoot: true },
+            });
+            return AiRemediationCommandPlanUtil.parse(stored === null || stored === void 0 ? void 0 : stored.commandPlan);
+        }
+        catch (error) {
+            logger.warn(`CommandPlanExecutor: could not read the stored plan of suggestion ${suggestionId.toString()} to merge into; writing this writer's copy as is: ${error}`);
+            return null;
+        }
+    }
+    /*
+     * Persist the plan without erasing what the OTHER arm wrote. The stored
+     * plan is re-read (under the plan lock) and only this writer's fields are
+     * taken from its copy: the executor owns forward executions and the
+     * plan's execution lifecycle; the rollback arm owns rollbacks,
+     * rollbackStatus and the rollback heartbeat. A forward execution the
+     * rollback arm settled from its job is kept over a stale unsettled copy
+     * either way. When the stored plan cannot be read, this writer's copy is
+     * written as is — the record must still land.
+     */
+    static async persistPlan(suggestion, plan, writer, options) {
         var _a;
+        const mutex = (options === null || options === void 0 ? void 0 : options.planLocked) || !suggestion.id
+            ? null
+            : await this.lockPlan(suggestion.id);
         try {
+            const stored = suggestion.id
+                ? await this.readStoredPlan(suggestion.id)
+                : null;
+            const toWrite = stored
+                ? this.mergePlans({ mine: plan, stored, writer })
+                : plan;
             await AutoRemediationSuggestionService.updateOneById({
                 id: suggestion.id,
                 data: {
-                    commandPlan: AiRemediationCommandPlanUtil.toJSON(plan),
+                    commandPlan: AiRemediationCommandPlanUtil.toJSON(toWrite),
                 },
                 props: { isRoot: true },
             });
@@ -443,6 +1220,85 @@ export default class CommandPlanExecutor {
         catch (error) {
             logger.error(`CommandPlanExecutor: failed to persist plan progress for suggestion ${(_a = suggestion.id) === null || _a === void 0 ? void 0 : _a.toString()}: ${error}`);
         }
+        finally {
+            await this.unlockPlan(mutex);
+        }
+    }
+    static mergePlans(data) {
+        var _a, _b;
+        const { mine, stored } = data;
+        if (data.writer === "executor") {
+            const storedBySequence = new Map(stored.commands.map((command) => {
+                return [command.sequence, command];
+            }));
+            return Object.assign(Object.assign({}, mine), { commands: mine.commands.map((command) => {
+                    const storedCommand = storedBySequence.get(command.sequence);
+                    return Object.assign(Object.assign({}, command), { execution: this.pickExecution(command.execution, storedCommand === null || storedCommand === void 0 ? void 0 : storedCommand.execution), rollbackExecution: storedCommand === null || storedCommand === void 0 ? void 0 : storedCommand.rollbackExecution });
+                }), rollbackStatus: stored.rollbackStatus, rollbackHeartbeatAt: stored.rollbackHeartbeatAt });
+        }
+        const mineBySequence = new Map(mine.commands.map((command) => {
+            return [command.sequence, command];
+        }));
+        return Object.assign(Object.assign({}, stored), { commands: stored.commands.map((storedCommand) => {
+                const command = mineBySequence.get(storedCommand.sequence);
+                if (!command) {
+                    return storedCommand;
+                }
+                return Object.assign(Object.assign({}, storedCommand), { execution: this.pickExecution(command.execution, storedCommand.execution), rollbackExecution: command.rollbackExecution });
+            }), rollbackStatus: (_a = mine.rollbackStatus) !== null && _a !== void 0 ? _a : stored.rollbackStatus, rollbackHeartbeatAt: (_b = mine.rollbackHeartbeatAt) !== null && _b !== void 0 ? _b : stored.rollbackHeartbeatAt });
+    }
+    /*
+     * The more settled of two records of one command's forward execution: a
+     * record with an outcome beats one still waiting for it; with both
+     * settled (or both not) the writer's own wins, keeping any job id either
+     * of them knows.
+     */
+    static pickExecution(mine, stored) {
+        if (!mine) {
+            return stored;
+        }
+        if (!stored) {
+            return mine;
+        }
+        if (isSettledCommandExecutionStatus(mine.status)) {
+            return mine;
+        }
+        if (isSettledCommandExecutionStatus(stored.status)) {
+            return stored;
+        }
+        return mine.runnerJobId || !stored.runnerJobId
+            ? mine
+            : Object.assign(Object.assign({}, mine), { runnerJobId: stored.runnerJobId });
+    }
+    /*
+     * The per-plan lock, or null when it cannot be had (Redis unavailable):
+     * the plan's writes then proceed unserialized rather than not at all — a
+     * remediation must never be stranded half-recorded for want of a lock.
+     */
+    static async lockPlan(suggestionId) {
+        try {
+            return await Semaphore.lock({
+                key: suggestionId.toString(),
+                namespace: PLAN_LOCK_NAMESPACE,
+                lockTimeout: PLAN_LOCK_TIMEOUT_MS,
+                acquireTimeout: PLAN_LOCK_ACQUIRE_TIMEOUT_MS,
+            });
+        }
+        catch (error) {
+            logger.warn(`CommandPlanExecutor: could not take the plan lock of suggestion ${suggestionId.toString()}; continuing unserialized: ${error}`);
+            return null;
+        }
+    }
+    static async unlockPlan(mutex) {
+        if (!mutex) {
+            return;
+        }
+        try {
+            await Semaphore.release(mutex);
+        }
+        catch (error) {
+            logger.error(`CommandPlanExecutor: failed to release the plan lock: ${error}`);
+        }
     }
     static capOutput(output) {
         const redacted = ToolResultSerializer.redact(output).text;