@oneuptime/common 14.0.0 → 14.0.2

package/Tests/Server/Utils/AutoRemediation/CommandPlanExecutor.test.ts

@@ -1,6 +1,7 @@
 import CommandPlanExecutor from "../../../../Server/Utils/AutoRemediation/CommandPlanExecutor";
 import AutoRemediationSuggestionService from "../../../../Server/Services/AutoRemediationSuggestionService";
 import IncidentFeedService from "../../../../Server/Services/IncidentFeedService";
+import KubernetesClusterAiAccessService from "../../../../Server/Services/KubernetesClusterAiAccessService";
 import RunnerJobService from "../../../../Server/Services/RunnerJobService";
 import AutoRemediationSuggestion from "../../../../Models/DatabaseModels/AutoRemediationSuggestion";
 import RunnerJob from "../../../../Models/DatabaseModels/RunnerJob";
@@ -8,6 +9,10 @@ import AutoRemediationSuggestionStatus from "../../../../Types/AutoRemediation/A
 import AutoRemediationSuggestionType from "../../../../Types/AutoRemediation/AutoRemediationSuggestionType";
 import AutoRemediationVerificationStatus from "../../../../Types/AutoRemediation/AutoRemediationVerificationStatus";
 import RunnerJobStatus from "../../../../Types/Runbook/RunnerJobStatus";
+import {
+  KubernetesAiRemediationMode,
+  KubernetesClusterAiAccessStatus,
+} from "../../../../Types/Kubernetes/KubernetesClusterAiAccess";
 import ObjectID from "../../../../Types/ObjectID";
 import logger from "../../../../Server/Utils/Logger";
 import { afterEach, beforeEach, describe, expect, it } from "@jest/globals";
@@ -19,10 +24,16 @@ import { afterEach, beforeEach, describe, expect, it } from "@jest/globals";
  * non-CommandPlan suggestion or an invalid or already-started plan; the
  * per-command Pending marker is persisted BEFORE the enqueue side effect;
  * outputs are stored redacted and capped; a failed command skips the rest
- * and pings the feed; the denylist is re-checked at execution time. The
- * rollback arm undoes only Succeeded commands that carry a rollbackCommand,
- * in REVERSE sequence order, re-checking the denylist, and settles
- * rollbackStatus exactly once.
+ * and pings the feed; the denylist is re-checked at execution time; the
+ * RunnerJob id is persisted BEFORE the wait so an interrupted command can
+ * still be reconciled. The rollback arm undoes only Succeeded commands that
+ * carry a rollbackCommand, in REVERSE sequence order, re-checking the
+ * denylist, and settles rollbackStatus exactly once.
+ *
+ * Kubectl commands are re-checked against the cluster's AI page as it is
+ * NOW before every enqueue: remediation still enabled and ready, the same
+ * Runner and credential bound — and a rollback must be allowed unattended
+ * under the cluster's CURRENT mode, or it is left for a human and said so.
  */
 
 const SUGGESTION_ID: ObjectID = new ObjectID(
@@ -324,11 +335,12 @@ describe("CommandPlanExecutor.executeApprovedPlan", () => {
     expect(secondEnqueue["stepId"]).toBe("ai-approved-2");
 
     /*
-     * Persist order: Running plan, cmd1 Pending, cmd1 result, cmd2 Pending,
-     * cmd2 result, plan Completed — and each Pending marker lands BEFORE
-     * its enqueue side effect.
+     * Persist order: Running plan, cmd1 Pending, cmd1 job id, cmd1 result,
+     * cmd2 Pending, cmd2 job id, cmd2 result, plan Completed — each Pending
+     * marker lands BEFORE its enqueue side effect, and each job id lands
+     * BEFORE its wait.
      */
-    expect(update).toHaveBeenCalledTimes(6);
+    expect(update).toHaveBeenCalledTimes(8);
     const firstPendingPlan: Record<string, unknown> = persistedPlanAt(
       update,
       1,
@@ -345,9 +357,22 @@ describe("CommandPlanExecutor.executeApprovedPlan", () => {
     expect(update.mock.invocationCallOrder[1]).toBeLessThan(
       enqueue.mock.invocationCallOrder[0]!,
     );
+    const firstJobIdPlan: Record<string, unknown> = persistedPlanAt(update, 2);
+    expect(commandInPlan(firstJobIdPlan, 0)["execution"]).toEqual(
+      expect.objectContaining({
+        status: "Pending",
+        runnerJobId: JOB_ID.toString(),
+      }),
+    );
+    expect(update.mock.invocationCallOrder[2]).toBeGreaterThan(
+      enqueue.mock.invocationCallOrder[0]!,
+    );
+    expect(update.mock.invocationCallOrder[2]).toBeLessThan(
+      poll.mock.invocationCallOrder[0]!,
+    );
     const secondPendingPlan: Record<string, unknown> = persistedPlanAt(
       update,
-      3,
+      4,
     );
     expect(
       (
@@ -357,7 +382,7 @@ describe("CommandPlanExecutor.executeApprovedPlan", () => {
         >
       )["status"],
     ).toBe("Pending");
-    expect(update.mock.invocationCallOrder[3]).toBeLessThan(
+    expect(update.mock.invocationCallOrder[4]).toBeLessThan(
       enqueue.mock.invocationCallOrder[1]!,
     );
 
@@ -989,31 +1014,233 @@ describe("CommandPlanExecutor.executeRollback", () => {
     expect(finalPlan["rollbackStatus"]).toBe("NotApplicable");
   });
 
-  it("leaves a Pending command with no runnerJobId alone — it never reached the Runner", async () => {
+  /*
+   * A Pending record with NO job id: the pod died (or the write failed)
+   * between the enqueue and the persist that names the job. The Runner was
+   * still handed the job, so the record is resolved through the
+   * (suggestion, stepId) pair the lane enqueued it under — never by
+   * assuming it did not run.
+   */
+  function jobLessPendingCommand(
+    sequence: number,
+    overrides: Partial<Record<string, unknown>> = {},
+  ): Record<string, unknown> {
+    return pendingCommand(sequence, {
+      execution: {
+        status: "Pending",
+        startedAt: "2026-01-01T00:00:00.000Z",
+      },
+      ...overrides,
+    });
+  }
+
+  function mockStepIdLookup(jobs: Array<RunnerJob>): jest.SpyInstance {
+    return jest.spyOn(RunnerJobService, "findBy").mockResolvedValue(jobs);
+  }
+
+  function stepIdQuery(lookup: jest.SpyInstance): Record<string, unknown> {
+    return (lookup.mock.calls[0]![0] as { query: Record<string, unknown> })
+      .query;
+  }
+
+  it("resolves a Pending command with no runnerJobId through its (suggestion, stepId) job, names the job, and rolls it back when it Succeeded", async () => {
+    const update: jest.SpyInstance = mockPersist();
+    const enqueue: jest.SpyInstance = mockEnqueue();
+    mockPoll();
+    mockFeed();
+    const byId: jest.SpyInstance = mockJobLookup(null);
+    const byStepId: jest.SpyInstance = mockStepIdLookup([
+      terminalJob({ status: RunnerJobStatus.Succeeded, output: "restarted" }),
+    ]);
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: fakeSuggestion({
+        commandPlan: rawPlan([jobLessPendingCommand(1)], {
+          executionStatus: "Running",
+        }),
+      }),
+    });
+
+    // Looked up by the pair the approved-plan lane enqueued it under.
+    expect(byId).not.toHaveBeenCalled();
+    expect(byStepId).toHaveBeenCalledTimes(1);
+    const query: Record<string, unknown> = stepIdQuery(byStepId);
+    expect((query["autoRemediationSuggestionId"] as ObjectID).toString()).toBe(
+      SUGGESTION_ID.toString(),
+    );
+    expect((query["projectId"] as ObjectID).toString()).toBe(
+      PROJECT_ID.toString(),
+    );
+    expect(query["stepId"]).toBe("ai-approved-1");
+    expect(query["origin"]).toBe("AiRemediation");
+    expect((byStepId.mock.calls[0]![0] as { limit: number }).limit).toBe(1);
+
+    // The command really ran, so its undo really runs.
+    expect(enqueue).toHaveBeenCalledTimes(1);
+    expect(
+      (enqueue.mock.calls[0]![0] as Record<string, unknown>)["command"],
+    ).toBe("undo-1");
+
+    const finalPlan: Record<string, unknown> = lastPersistedPlan(update);
+    const execution: Record<string, unknown> = commandInPlan(finalPlan, 0)[
+      "execution"
+    ] as Record<string, unknown>;
+    expect(execution["status"]).toBe("Succeeded");
+    expect(execution["output"]).toBe("restarted");
+    // The recovered job id now lives on the record.
+    expect(execution["runnerJobId"]).toBe(JOB_ID.toString());
+    expect(finalPlan["rollbackStatus"]).toBe("Completed");
+  });
+
+  it("looks a job-less command a FullAuto run executed inline up under the inline step id", async () => {
     const update: jest.SpyInstance = mockPersist();
     const enqueue: jest.SpyInstance = mockEnqueue();
     mockPoll();
     mockFeed();
-    const jobLookup: jest.SpyInstance = mockJobLookup(terminalJob());
+    const byStepId: jest.SpyInstance = mockStepIdLookup([
+      terminalJob({ status: RunnerJobStatus.Succeeded }),
+    ]);
 
     await CommandPlanExecutor.executeRollback({
       suggestion: fakeSuggestion({
         commandPlan: rawPlan(
-          [
-            pendingCommand(1, {
-              execution: {
-                status: "Pending",
-                startedAt: "2026-01-01T00:00:00.000Z",
-              },
-            }),
-          ],
-          { executionStatus: "Running" },
+          [jobLessPendingCommand(2, { wasAutoExecuted: true })],
+          { executionStatus: "Failed" },
         ),
       }),
     });
 
-    expect(jobLookup).not.toHaveBeenCalled();
+    expect(stepIdQuery(byStepId)["stepId"]).toBe("ai-command-2");
+    expect(enqueue).toHaveBeenCalledTimes(1);
+    expect(
+      (enqueue.mock.calls[0]![0] as Record<string, unknown>)["stepId"],
+    ).toBe("ai-rollback-2");
+    expect(lastPersistedPlan(update)["rollbackStatus"]).toBe("Completed");
+  });
+
+  it("marks a job-less Pending command Failed, and does not roll it back, when its stepId job Failed", async () => {
+    const update: jest.SpyInstance = mockPersist();
+    const enqueue: jest.SpyInstance = mockEnqueue();
+    mockPoll();
+    mockFeed();
+    mockStepIdLookup([
+      terminalJob({
+        status: RunnerJobStatus.Failed,
+        exitCode: 1,
+        errorMessage: "unit failed",
+      }),
+    ]);
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: fakeSuggestion({
+        commandPlan: rawPlan([jobLessPendingCommand(1)], {
+          executionStatus: "Running",
+        }),
+      }),
+    });
+
+    expect(enqueue).not.toHaveBeenCalled();
+    const finalPlan: Record<string, unknown> = lastPersistedPlan(update);
+    const execution: Record<string, unknown> = commandInPlan(finalPlan, 0)[
+      "execution"
+    ] as Record<string, unknown>;
+    expect(execution["status"]).toBe("Failed");
+    expect(execution["errorMessage"]).toBe("unit failed");
+    expect(execution["runnerJobId"]).toBe(JOB_ID.toString());
+    expect(finalPlan["rollbackStatus"]).toBe("NotApplicable");
+  });
+
+  it("names a job-less command's still-running stepId job on the record, waits for it, and rolls it back once it succeeded", async () => {
+    /*
+     * Changed with the rollback-vs-in-flight-command fix: the rollback arm
+     * used to leave a command whose job was still running alone — and the
+     * change it made moments later was never undone.
+     */
+    const update: jest.SpyInstance = mockPersist();
+    const enqueue: jest.SpyInstance = mockEnqueue();
+    const poll: jest.SpyInstance = mockPoll();
+    mockFeed();
+    mockStepIdLookup([terminalJob({ status: RunnerJobStatus.Running })]);
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: fakeSuggestion({
+        commandPlan: rawPlan([jobLessPendingCommand(1)], {
+          executionStatus: "Running",
+        }),
+      }),
+    });
+
+    // The first wait is on the forward job, then the rollback's own job.
+    expect(
+      (
+        (poll.mock.calls[0]![0] as { jobId: ObjectID }).jobId as ObjectID
+      ).toString(),
+    ).toBe(JOB_ID.toString());
+    expect(enqueue).toHaveBeenCalledTimes(1);
+    expect(
+      (enqueue.mock.calls[0]![0] as Record<string, unknown>)["command"],
+    ).toBe("undo-1");
+    const finalPlan: Record<string, unknown> = lastPersistedPlan(update);
+    const execution: Record<string, unknown> = commandInPlan(finalPlan, 0)[
+      "execution"
+    ] as Record<string, unknown>;
+    expect(execution["status"]).toBe("Succeeded");
+    expect(execution["runnerJobId"]).toBe(JOB_ID.toString());
+    expect(finalPlan["rollbackStatus"]).toBe("Completed");
+  });
+
+  it("leaves a Pending command with no runnerJobId alone when no job carries its stepId — it never reached the Runner", async () => {
+    const update: jest.SpyInstance = mockPersist();
+    const enqueue: jest.SpyInstance = mockEnqueue();
+    mockPoll();
+    mockFeed();
+    const byId: jest.SpyInstance = mockJobLookup(terminalJob());
+    const byStepId: jest.SpyInstance = mockStepIdLookup([]);
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: fakeSuggestion({
+        commandPlan: rawPlan([jobLessPendingCommand(1)], {
+          executionStatus: "Running",
+        }),
+      }),
+    });
+
+    expect(byId).not.toHaveBeenCalled();
+    expect(byStepId).toHaveBeenCalledTimes(1);
+    expect(enqueue).not.toHaveBeenCalled();
+    const finalPlan: Record<string, unknown> = lastPersistedPlan(update);
+    const execution: Record<string, unknown> = commandInPlan(finalPlan, 0)[
+      "execution"
+    ] as Record<string, unknown>;
+    expect(execution["status"]).toBe("Pending");
+    expect(execution["runnerJobId"]).toBeUndefined();
+    expect(finalPlan["rollbackStatus"]).toBe("NotApplicable");
+  });
+
+  it("leaves a job-less Pending command alone when the stepId lookup itself fails, and logs it", async () => {
+    const error: jest.SpyInstance = jest
+      .spyOn(logger, "error")
+      .mockImplementation((): void => {
+        return undefined;
+      });
+    const update: jest.SpyInstance = mockPersist();
+    const enqueue: jest.SpyInstance = mockEnqueue();
+    mockPoll();
+    mockFeed();
+    jest
+      .spyOn(RunnerJobService, "findBy")
+      .mockRejectedValue(new Error("db down"));
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: fakeSuggestion({
+        commandPlan: rawPlan([jobLessPendingCommand(1)], {
+          executionStatus: "Running",
+        }),
+      }),
+    });
+
     expect(enqueue).not.toHaveBeenCalled();
+    expect(error).toHaveBeenCalledWith(expect.stringContaining("db down"));
     const finalPlan: Record<string, unknown> = lastPersistedPlan(update);
     expect(
       (commandInPlan(finalPlan, 0)["execution"] as Record<string, unknown>)[
@@ -1023,11 +1250,77 @@ describe("CommandPlanExecutor.executeRollback", () => {
     expect(finalPlan["rollbackStatus"]).toBe("NotApplicable");
   });
 
-  it("leaves a Pending command alone while its job is still in flight", async () => {
+  it("prefers the job id on the record over a stepId lookup when both could answer", async () => {
     const update: jest.SpyInstance = mockPersist();
     const enqueue: jest.SpyInstance = mockEnqueue();
     mockPoll();
     mockFeed();
+    const byId: jest.SpyInstance = mockJobLookup(
+      terminalJob({ status: RunnerJobStatus.Succeeded }),
+    );
+    const byStepId: jest.SpyInstance = mockStepIdLookup([
+      terminalJob({ status: RunnerJobStatus.Failed, exitCode: 1 }),
+    ]);
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: fakeSuggestion({
+        commandPlan: rawPlan([pendingCommand(1)], {
+          executionStatus: "Running",
+        }),
+      }),
+    });
+
+    expect(byId).toHaveBeenCalledTimes(1);
+    expect(byStepId).not.toHaveBeenCalled();
+    expect(enqueue).toHaveBeenCalledTimes(1);
+    expect(lastPersistedPlan(update)["rollbackStatus"]).toBe("Completed");
+  });
+
+  it("WAITS for a Pending command whose job is still in flight, then rolls it back in reverse order with the rest", async () => {
+    /*
+     * Changed with the rollback-vs-in-flight-command fix: this used to pin
+     * "leave it alone, settle NotApplicable" — which is exactly how a
+     * command that finished a moment later (a slow drain) escaped its undo.
+     */
+    const update: jest.SpyInstance = mockPersist();
+    const enqueue: jest.SpyInstance = mockEnqueue();
+    const poll: jest.SpyInstance = mockPoll();
+    mockFeed();
+    mockJobLookup(terminalJob({ status: RunnerJobStatus.Running }));
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: fakeSuggestion({
+        commandPlan: rawPlan([executedCommand(1), pendingCommand(2)], {
+          executionStatus: "Running",
+        }),
+      }),
+    });
+
+    expect(poll).toHaveBeenCalled();
+    const enqueuedCommands: Array<string> = enqueue.mock.calls.map(
+      (call: Array<unknown>) => {
+        return (call[0] as Record<string, unknown>)["command"] as string;
+      },
+    );
+    expect(enqueuedCommands).toEqual(["undo-2", "undo-1"]);
+    const finalPlan: Record<string, unknown> = lastPersistedPlan(update);
+    expect(
+      (commandInPlan(finalPlan, 1)["execution"] as Record<string, unknown>)[
+        "status"
+      ],
+    ).toBe("Succeeded");
+    expect(finalPlan["rollbackStatus"]).toBe("Completed");
+  });
+
+  it("does not roll back a command whose in-flight job ends Failed after the wait", async () => {
+    const update: jest.SpyInstance = mockPersist();
+    const enqueue: jest.SpyInstance = mockEnqueue();
+    jest
+      .spyOn(RunnerJobService, "pollUntilTerminal")
+      .mockResolvedValue(
+        terminalJob({ status: RunnerJobStatus.Failed, errorMessage: "boom" }),
+      );
+    mockFeed();
     mockJobLookup(terminalJob({ status: RunnerJobStatus.Running }));
 
     await CommandPlanExecutor.executeRollback({
@@ -1044,7 +1337,1061 @@ describe("CommandPlanExecutor.executeRollback", () => {
       (commandInPlan(finalPlan, 0)["execution"] as Record<string, unknown>)[
         "status"
       ],
-    ).toBe("Pending");
+    ).toBe("Failed");
     expect(finalPlan["rollbackStatus"]).toBe("NotApplicable");
   });
+
+  it("leaves a command whose job disappears while waited on for a human — and never reports the rollback complete", async () => {
+    const update: jest.SpyInstance = mockPersist();
+    const enqueue: jest.SpyInstance = mockEnqueue();
+    jest
+      .spyOn(RunnerJobService, "pollUntilTerminal")
+      .mockRejectedValue(new Error("RunnerJob disappeared while waiting."));
+    const feed: jest.SpyInstance = mockFeed();
+    mockJobLookup(terminalJob({ status: RunnerJobStatus.Running }));
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: fakeSuggestion({
+        commandPlan: rawPlan([pendingCommand(1)], {
+          executionStatus: "Running",
+        }),
+      }),
+    });
+
+    expect(enqueue).not.toHaveBeenCalled();
+    const finalPlan: Record<string, unknown> = lastPersistedPlan(update);
+    const rollbackExecution: Record<string, unknown> = commandInPlan(
+      finalPlan,
+      0,
+    )["rollbackExecution"] as Record<string, unknown>;
+    expect(rollbackExecution["status"]).toBe("Skipped");
+    expect(rollbackExecution["errorMessage"]).toContain(
+      "could not be confirmed",
+    );
+    expect(rollbackExecution["errorMessage"]).toContain("undo-1");
+    expect(finalPlan["rollbackStatus"]).toBe("Failed");
+    expect(feedMarkdown(feed)).toContain("a human has to undo them");
+  });
+});
+
+/*
+ * Shared fixtures for the kubectl re-check suites below. A kubectl command
+ * freezes the cluster, the Runner and the credential its AI page bound at
+ * planning time; the executor reads that page again right before enqueueing.
+ */
+const CLUSTER_ID: ObjectID = new ObjectID(
+  "33333333-3333-4333-8333-333333333333",
+);
+const OTHER_RUNNER_ID: ObjectID = new ObjectID(
+  "dddddddd-dddd-4ddd-8ddd-dddddddddddd",
+);
+const KUBECTL_RESTART: string = "kubectl rollout restart deployment/web -n web";
+const KUBECTL_UNDO: string = "kubectl rollout undo deployment/web -n web";
+const KUBECTL_SET_IMAGE: string =
+  "kubectl set image deployment/web web=nginx:1.27 -n web";
+const KUBECTL_SET_IMAGE_BACK: string =
+  "kubectl set image deployment/web web=nginx:1.26 -n web";
+
+function rawKubectlCommand(
+  overrides: Partial<Record<string, unknown>> = {},
+): Record<string, unknown> {
+  return rawCommand({
+    stepType: "Kubectl",
+    runnerNameSnapshot: "kubernetes-agent/prod-us",
+    command: KUBECTL_RESTART,
+    kubernetesClusterId: CLUSTER_ID.toString(),
+    kubernetesClusterNameSnapshot: "prod-us",
+    kubectlTier: "SafeWrite",
+    rollbackCommand: KUBECTL_UNDO,
+    ...overrides,
+  });
+}
+
+function clusterStatus(
+  overrides: Partial<KubernetesClusterAiAccessStatus> = {},
+): KubernetesClusterAiAccessStatus {
+  return {
+    clusterId: CLUSTER_ID.toString(),
+    clusterName: "prod-us",
+    runner: {
+      id: RUNNER_ID.toString(),
+      name: "kubernetes-agent/prod-us",
+      isOnline: true,
+      canRunAiCommands: true,
+      posture: { inCluster: true, allowWrites: true },
+    },
+    accessMethod: "in_cluster",
+    kubectlAllowlist: [],
+    isInvestigationEnabled: true,
+    isInvestigationReady: true,
+    remediationMode: KubernetesAiRemediationMode.RequireApproval,
+    isRemediationReady: true,
+    gaps: [],
+    evaluatedAt: new Date().toISOString(),
+    ...overrides,
+  };
+}
+
+function disabledClusterStatus(): KubernetesClusterAiAccessStatus {
+  return clusterStatus({
+    remediationMode: KubernetesAiRemediationMode.Disabled,
+    isRemediationReady: false,
+    gaps: [
+      {
+        code: "remediation_disabled",
+        title: "AI remediation is turned off for this cluster",
+        description:
+          "OneUptime AI will diagnose but never propose or apply a fix on this cluster.",
+        nextStep: "Set AI remediation on the cluster's AI page.",
+        blocks: "remediation",
+      },
+    ],
+  });
+}
+
+function mockClusterStatus(
+  status: KubernetesClusterAiAccessStatus | null,
+): jest.SpyInstance {
+  return jest
+    .spyOn(KubernetesClusterAiAccessService, "getStatusForCluster")
+    .mockResolvedValue(status);
+}
+
+function mockKubectlEnqueue(): jest.SpyInstance {
+  return jest
+    .spyOn(RunnerJobService, "enqueueAiKubectlCommand")
+    .mockResolvedValue(terminalJob({ status: RunnerJobStatus.Pending }));
+}
+
+function mockRecordOutcome(): jest.SpyInstance {
+  return jest
+    .spyOn(KubernetesClusterAiAccessService, "recordCommandOutcome")
+    .mockResolvedValue(undefined);
+}
+
+function feedMarkdown(feed: jest.SpyInstance): string {
+  return (feed.mock.calls[0]![0] as { feedInfoInMarkdown: string })
+    .feedInfoInMarkdown;
+}
+
+describe("CommandPlanExecutor persists the RunnerJob id before waiting", () => {
+  beforeEach(() => {
+    jest.spyOn(logger, "error").mockImplementation((): void => {
+      return undefined;
+    });
+  });
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  it("records the job id on the Pending command before polling an approved command", async () => {
+    mockFetch(fakeSuggestion({ commandPlan: rawPlan([rawCommand()]) }));
+    const update: jest.SpyInstance = mockPersist();
+    const enqueue: jest.SpyInstance = mockEnqueue();
+    mockFeed();
+
+    let recordAtPollStart: Record<string, unknown> | undefined = undefined;
+    const poll: jest.SpyInstance = jest
+      .spyOn(RunnerJobService, "pollUntilTerminal")
+      .mockImplementation(async (): Promise<RunnerJob> => {
+        recordAtPollStart = commandInPlan(lastPersistedPlan(update), 0)[
+          "execution"
+        ] as Record<string, unknown>;
+        return terminalJob();
+      });
+
+    await CommandPlanExecutor.executeApprovedPlan({
+      suggestionId: SUGGESTION_ID,
+    });
+
+    expect(recordAtPollStart).toEqual(
+      expect.objectContaining({
+        status: "Pending",
+        runnerJobId: JOB_ID.toString(),
+      }),
+    );
+    // enqueue < job-id persist < poll
+    const jobIdPersistOrder: number = update.mock.invocationCallOrder[2]!;
+    expect(jobIdPersistOrder).toBeGreaterThan(
+      enqueue.mock.invocationCallOrder[0]!,
+    );
+    expect(jobIdPersistOrder).toBeLessThan(poll.mock.invocationCallOrder[0]!);
+
+    const finalExecution: Record<string, unknown> = commandInPlan(
+      lastPersistedPlan(update),
+      0,
+    )["execution"] as Record<string, unknown>;
+    expect(finalExecution["status"]).toBe("Succeeded");
+    expect(finalExecution["runnerJobId"]).toBe(JOB_ID.toString());
+  });
+
+  it("keeps the job id on an approved command whose wait throws", async () => {
+    mockFetch(fakeSuggestion({ commandPlan: rawPlan([rawCommand()]) }));
+    const update: jest.SpyInstance = mockPersist();
+    mockEnqueue();
+    jest
+      .spyOn(RunnerJobService, "pollUntilTerminal")
+      .mockRejectedValue(new Error("lease lost"));
+    mockFeed();
+
+    await CommandPlanExecutor.executeApprovedPlan({
+      suggestionId: SUGGESTION_ID,
+    });
+
+    const finalPlan: Record<string, unknown> = lastPersistedPlan(update);
+    expect(finalPlan["executionStatus"]).toBe("Failed");
+    expect(commandInPlan(finalPlan, 0)["execution"]).toEqual(
+      expect.objectContaining({
+        status: "Failed",
+        runnerJobId: JOB_ID.toString(),
+        errorMessage: "lease lost",
+      }),
+    );
+  });
+
+  it("records the job id on a Pending rollback before polling it", async () => {
+    const update: jest.SpyInstance = mockPersist();
+    const enqueue: jest.SpyInstance = mockEnqueue();
+    mockFeed();
+
+    let recordAtPollStart: Record<string, unknown> | undefined = undefined;
+    const poll: jest.SpyInstance = jest
+      .spyOn(RunnerJobService, "pollUntilTerminal")
+      .mockImplementation(async (): Promise<RunnerJob> => {
+        recordAtPollStart = commandInPlan(lastPersistedPlan(update), 0)[
+          "rollbackExecution"
+        ] as Record<string, unknown>;
+        return terminalJob();
+      });
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: fakeSuggestion({
+        commandPlan: rawPlan(
+          [
+            rawCommand({
+              rollbackCommand: "systemctl stop nginx",
+              execution: { status: "Succeeded", exitCode: 0 },
+            }),
+          ],
+          { executionStatus: "Failed" },
+        ),
+      }),
+    });
+
+    expect(enqueue).toHaveBeenCalledTimes(1);
+    expect(recordAtPollStart).toEqual(
+      expect.objectContaining({
+        status: "Pending",
+        runnerJobId: JOB_ID.toString(),
+      }),
+    );
+    expect(poll).toHaveBeenCalledTimes(1);
+    expect(lastPersistedPlan(update)["rollbackStatus"]).toBe("Completed");
+  });
+});
+
+describe("CommandPlanExecutor re-checks the cluster before an approved kubectl command runs", () => {
+  let feed: jest.SpyInstance;
+  let update: jest.SpyInstance;
+  let kubectlEnqueue: jest.SpyInstance;
+  let bashEnqueue: jest.SpyInstance;
+
+  beforeEach(() => {
+    jest.spyOn(logger, "error").mockImplementation((): void => {
+      return undefined;
+    });
+    update = mockPersist();
+    kubectlEnqueue = mockKubectlEnqueue();
+    bashEnqueue = mockEnqueue();
+    mockPoll();
+    mockRecordOutcome();
+    feed = mockFeed();
+  });
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  function expectRefused(reasonFragment: string): void {
+    expect(kubectlEnqueue).not.toHaveBeenCalled();
+    const finalPlan: Record<string, unknown> = lastPersistedPlan(update);
+    expect(finalPlan["executionStatus"]).toBe("Failed");
+    const execution: Record<string, unknown> = commandInPlan(finalPlan, 0)[
+      "execution"
+    ] as Record<string, unknown>;
+    expect(execution["status"]).toBe("Failed");
+    expect(execution["errorMessage"]).toContain("Refused at execution time");
+    expect(execution["errorMessage"]).toContain(reasonFragment);
+    // The human hears the reason on the feed, not just "a command failed".
+    expect(feedMarkdown(feed)).toContain(reasonFragment);
+    expect(feed).toHaveBeenCalledWith(
+      expect.objectContaining({
+        workspaceNotification: expect.objectContaining({
+          sendWorkspaceNotification: true,
+        }),
+      }),
+    );
+  }
+
+  it("runs the command when the cluster is still ready and bound to the plan's Runner", async () => {
+    mockFetch(fakeSuggestion({ commandPlan: rawPlan([rawKubectlCommand()]) }));
+    const status: jest.SpyInstance = mockClusterStatus(clusterStatus());
+
+    await CommandPlanExecutor.executeApprovedPlan({
+      suggestionId: SUGGESTION_ID,
+    });
+
+    expect(status).toHaveBeenCalledTimes(1);
+    const statusArgs: { clusterId: ObjectID; projectId: ObjectID } = status.mock
+      .calls[0]![0] as { clusterId: ObjectID; projectId: ObjectID };
+    expect(statusArgs.clusterId.toString()).toBe(CLUSTER_ID.toString());
+    expect(statusArgs.projectId.toString()).toBe(PROJECT_ID.toString());
+
+    expect(kubectlEnqueue).toHaveBeenCalledTimes(1);
+    const enqueueArgs: Record<string, unknown> = kubectlEnqueue.mock
+      .calls[0]![0] as Record<string, unknown>;
+    expect((enqueueArgs["kubernetesClusterId"] as ObjectID).toString()).toBe(
+      CLUSTER_ID.toString(),
+    );
+    expect((enqueueArgs["targetAgentId"] as ObjectID).toString()).toBe(
+      RUNNER_ID.toString(),
+    );
+    expect(enqueueArgs["command"]).toBe(KUBECTL_RESTART);
+    expect(bashEnqueue).not.toHaveBeenCalled();
+    expect(lastPersistedPlan(update)["executionStatus"]).toBe("Completed");
+  });
+
+  it("refuses when the cluster's AI remediation was turned off after the plan was composed", async () => {
+    mockFetch(fakeSuggestion({ commandPlan: rawPlan([rawKubectlCommand()]) }));
+    mockClusterStatus(disabledClusterStatus());
+
+    await CommandPlanExecutor.executeApprovedPlan({
+      suggestionId: SUGGESTION_ID,
+    });
+
+    expectRefused('cluster "prod-us" no longer allows AI remediation');
+    expect(
+      (
+        commandInPlan(lastPersistedPlan(update), 0)["execution"] as Record<
+          string,
+          unknown
+        >
+      )["errorMessage"],
+    ).toContain("AI remediation is turned off for this cluster");
+  });
+
+  it("refuses when the cluster is not remediation-ready for any other reason (a read-only in-cluster Runner)", async () => {
+    mockFetch(fakeSuggestion({ commandPlan: rawPlan([rawKubectlCommand()]) }));
+    mockClusterStatus(
+      clusterStatus({
+        isRemediationReady: false,
+        gaps: [
+          {
+            code: "remediation_write_access_missing",
+            title: "The in-cluster Runner is read-only",
+            description: "",
+            nextStep: "",
+            blocks: "remediation",
+          },
+        ],
+      }),
+    );
+
+    await CommandPlanExecutor.executeApprovedPlan({
+      suggestionId: SUGGESTION_ID,
+    });
+
+    expectRefused("The in-cluster Runner is read-only");
+  });
+
+  it("refuses when the cluster is now bound to a different Runner than the plan froze", async () => {
+    mockFetch(fakeSuggestion({ commandPlan: rawPlan([rawKubectlCommand()]) }));
+    mockClusterStatus(
+      clusterStatus({
+        runner: {
+          id: OTHER_RUNNER_ID.toString(),
+          name: "kubernetes-agent/prod-us-v2",
+          isOnline: true,
+          canRunAiCommands: true,
+        },
+      }),
+    );
+
+    await CommandPlanExecutor.executeApprovedPlan({
+      suggestionId: SUGGESTION_ID,
+    });
+
+    expectRefused(
+      'no longer reached through Runner "kubernetes-agent/prod-us"',
+    );
+  });
+
+  it("refuses when the cluster has no Runner bound any more", async () => {
+    mockFetch(fakeSuggestion({ commandPlan: rawPlan([rawKubectlCommand()]) }));
+    mockClusterStatus(clusterStatus({ runner: null, accessMethod: "none" }));
+
+    await CommandPlanExecutor.executeApprovedPlan({
+      suggestionId: SUGGESTION_ID,
+    });
+
+    expectRefused("no longer reached through Runner");
+  });
+
+  it("refuses when the credential the plan froze is no longer the cluster's", async () => {
+    mockFetch(
+      fakeSuggestion({
+        commandPlan: rawPlan([
+          rawKubectlCommand({
+            credentialId: "55555555-5555-4555-8555-555555555555",
+            credentialNameSnapshot: "old kubeconfig",
+          }),
+        ]),
+      }),
+    );
+    mockClusterStatus(
+      clusterStatus({
+        accessMethod: "credential",
+        credentialId: "66666666-6666-4666-8666-666666666666",
+        credentialName: "new kubeconfig",
+      }),
+    );
+
+    await CommandPlanExecutor.executeApprovedPlan({
+      suggestionId: SUGGESTION_ID,
+    });
+
+    expectRefused("no longer reached with the credential");
+  });
+
+  it("refuses when the cluster no longer exists in the project", async () => {
+    mockFetch(fakeSuggestion({ commandPlan: rawPlan([rawKubectlCommand()]) }));
+    mockClusterStatus(null);
+
+    await CommandPlanExecutor.executeApprovedPlan({
+      suggestionId: SUGGESTION_ID,
+    });
+
+    expectRefused('cluster "prod-us" no longer exists in this project');
+  });
+
+  it("fails CLOSED when the cluster status cannot be read", async () => {
+    mockFetch(fakeSuggestion({ commandPlan: rawPlan([rawKubectlCommand()]) }));
+    jest
+      .spyOn(KubernetesClusterAiAccessService, "getStatusForCluster")
+      .mockRejectedValue(new Error("db down"));
+
+    await CommandPlanExecutor.executeApprovedPlan({
+      suggestionId: SUGGESTION_ID,
+    });
+
+    expectRefused("could not confirm that cluster");
+    expect(logger.error).toHaveBeenCalledWith(
+      expect.stringContaining("db down"),
+    );
+  });
+
+  it("re-checks before EVERY kubectl command, so a change made mid-plan stops the rest", async () => {
+    mockFetch(
+      fakeSuggestion({
+        commandPlan: rawPlan([
+          rawKubectlCommand(),
+          rawKubectlCommand({ sequence: 2, command: KUBECTL_UNDO }),
+        ]),
+      }),
+    );
+    jest
+      .spyOn(KubernetesClusterAiAccessService, "getStatusForCluster")
+      .mockResolvedValueOnce(clusterStatus())
+      .mockResolvedValue(disabledClusterStatus());
+
+    await CommandPlanExecutor.executeApprovedPlan({
+      suggestionId: SUGGESTION_ID,
+    });
+
+    expect(kubectlEnqueue).toHaveBeenCalledTimes(1);
+    const finalPlan: Record<string, unknown> = lastPersistedPlan(update);
+    expect(finalPlan["executionStatus"]).toBe("Failed");
+    expect(
+      (commandInPlan(finalPlan, 0)["execution"] as Record<string, unknown>)[
+        "status"
+      ],
+    ).toBe("Succeeded");
+    expect(
+      (commandInPlan(finalPlan, 1)["execution"] as Record<string, unknown>)[
+        "errorMessage"
+      ],
+    ).toContain("no longer allows AI remediation");
+    expect(feedMarkdown(feed)).toContain("command 2 failed");
+  });
+
+  it("does not consult the cluster page for Bash commands", async () => {
+    mockFetch(fakeSuggestion({ commandPlan: rawPlan([rawCommand()]) }));
+    const status: jest.SpyInstance = mockClusterStatus(clusterStatus());
+
+    await CommandPlanExecutor.executeApprovedPlan({
+      suggestionId: SUGGESTION_ID,
+    });
+
+    expect(status).not.toHaveBeenCalled();
+    expect(bashEnqueue).toHaveBeenCalledTimes(1);
+    expect(lastPersistedPlan(update)["executionStatus"]).toBe("Completed");
+  });
+
+  it("still refuses a Denied kubectl command before reading the cluster", async () => {
+    mockFetch(
+      fakeSuggestion({
+        commandPlan: rawPlan([
+          rawKubectlCommand({
+            command: "kubectl delete namespace web",
+            rollbackCommand: undefined,
+          }),
+        ]),
+      }),
+    );
+    const status: jest.SpyInstance = mockClusterStatus(clusterStatus());
+
+    await CommandPlanExecutor.executeApprovedPlan({
+      suggestionId: SUGGESTION_ID,
+    });
+
+    expect(status).not.toHaveBeenCalled();
+    expect(kubectlEnqueue).not.toHaveBeenCalled();
+    expect(
+      (
+        commandInPlan(lastPersistedPlan(update), 0)["execution"] as Record<
+          string,
+          unknown
+        >
+      )["errorMessage"],
+    ).toContain("Refused by the remediation command policy");
+  });
+});
+
+describe("CommandPlanExecutor re-checks the cluster before a kubectl rollback runs", () => {
+  let feed: jest.SpyInstance;
+  let update: jest.SpyInstance;
+  let kubectlEnqueue: jest.SpyInstance;
+  let bashEnqueue: jest.SpyInstance;
+
+  beforeEach(() => {
+    jest.spyOn(logger, "error").mockImplementation((): void => {
+      return undefined;
+    });
+    update = mockPersist();
+    kubectlEnqueue = mockKubectlEnqueue();
+    bashEnqueue = mockEnqueue();
+    mockPoll();
+    mockRecordOutcome();
+    feed = mockFeed();
+  });
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  // A kubectl command that ran, with a RiskyWrite undo (only Bypass accepts one).
+  function executedSetImage(
+    overrides: Partial<Record<string, unknown>> = {},
+  ): Record<string, unknown> {
+    return rawKubectlCommand({
+      command: KUBECTL_SET_IMAGE,
+      kubectlTier: "RiskyWrite",
+      rollbackCommand: KUBECTL_SET_IMAGE_BACK,
+      execution: { status: "Succeeded", exitCode: 0 },
+      ...overrides,
+    });
+  }
+
+  function failedPlan(
+    commands: Array<Record<string, unknown>>,
+  ): AutoRemediationSuggestion {
+    return fakeSuggestion({
+      commandPlan: rawPlan(commands, { executionStatus: "Failed" }),
+    });
+  }
+
+  function expectSkipped(reasonFragment: string): void {
+    expect(kubectlEnqueue).not.toHaveBeenCalled();
+    const finalPlan: Record<string, unknown> = lastPersistedPlan(update);
+    expect(finalPlan["rollbackStatus"]).toBe("Failed");
+    const rollbackExecution: Record<string, unknown> = commandInPlan(
+      finalPlan,
+      0,
+    )["rollbackExecution"] as Record<string, unknown>;
+    expect(rollbackExecution["status"]).toBe("Skipped");
+    expect(rollbackExecution["errorMessage"]).toContain("Rollback not run");
+    expect(rollbackExecution["errorMessage"]).toContain(reasonFragment);
+    expect(rollbackExecution["errorMessage"]).toContain(
+      `Undo it manually: ${KUBECTL_SET_IMAGE_BACK}`,
+    );
+    // Said so on the feed, with the command a human now has to run.
+    const markdown: string = feedMarkdown(feed);
+    expect(markdown).toContain("rollback was not run for 1 command(s)");
+    expect(markdown).toContain(reasonFragment);
+    expect(markdown).toContain(KUBECTL_SET_IMAGE_BACK);
+    expect(markdown).toContain("manual intervention is needed");
+    expect(feed).toHaveBeenCalledWith(
+      expect.objectContaining({
+        workspaceNotification: expect.objectContaining({
+          sendWorkspaceNotification: true,
+        }),
+      }),
+    );
+  }
+
+  it("skips a RiskyWrite rollback accepted under BypassApproval once the cluster stopped bypassing approvals, and says so", async () => {
+    mockClusterStatus(
+      clusterStatus({
+        remediationMode: KubernetesAiRemediationMode.RequireApproval,
+      }),
+    );
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: failedPlan([executedSetImage()]),
+    });
+
+    expectSkipped("no longer allows this change unattended");
+    expect(
+      (
+        commandInPlan(lastPersistedPlan(update), 0)[
+          "rollbackExecution"
+        ] as Record<string, unknown>
+      )["errorMessage"],
+    ).toContain("its AI remediation mode is now RequireApproval");
+  });
+
+  it("skips the same rollback when the cluster was downgraded to Automatic without an allowlist match", async () => {
+    mockClusterStatus(
+      clusterStatus({
+        remediationMode: KubernetesAiRemediationMode.Automatic,
+        kubectlAllowlist: ["kubectl scale deployment/web * -n web"],
+      }),
+    );
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: failedPlan([executedSetImage()]),
+    });
+
+    expectSkipped("its AI remediation mode is now Automatic");
+  });
+
+  it("still runs the RiskyWrite rollback while the cluster bypasses approvals", async () => {
+    mockClusterStatus(
+      clusterStatus({
+        remediationMode: KubernetesAiRemediationMode.BypassApproval,
+      }),
+    );
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: failedPlan([executedSetImage()]),
+    });
+
+    expect(kubectlEnqueue).toHaveBeenCalledTimes(1);
+    const enqueueArgs: Record<string, unknown> = kubectlEnqueue.mock
+      .calls[0]![0] as Record<string, unknown>;
+    expect(enqueueArgs["command"]).toBe(KUBECTL_SET_IMAGE_BACK);
+    expect(enqueueArgs["stepId"]).toBe("ai-rollback-1");
+    const finalPlan: Record<string, unknown> = lastPersistedPlan(update);
+    expect(finalPlan["rollbackStatus"]).toBe("Completed");
+    expect(
+      (
+        commandInPlan(finalPlan, 0)["rollbackExecution"] as Record<
+          string,
+          unknown
+        >
+      )["status"],
+    ).toBe("Succeeded");
+    expect(feedMarkdown(feed)).toContain("rolled back");
+  });
+
+  it("runs a RiskyWrite rollback the cluster's allowlist names, in Automatic mode", async () => {
+    mockClusterStatus(
+      clusterStatus({
+        remediationMode: KubernetesAiRemediationMode.Automatic,
+        kubectlAllowlist: ["kubectl set image deployment/web * -n web"],
+      }),
+    );
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: failedPlan([executedSetImage()]),
+    });
+
+    expect(kubectlEnqueue).toHaveBeenCalledTimes(1);
+    expect(lastPersistedPlan(update)["rollbackStatus"]).toBe("Completed");
+  });
+
+  it("runs a SafeWrite rollback on a RequireApproval cluster — safe changes are always allowed unattended", async () => {
+    mockClusterStatus(
+      clusterStatus({
+        remediationMode: KubernetesAiRemediationMode.RequireApproval,
+      }),
+    );
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: failedPlan([
+        rawKubectlCommand({
+          execution: { status: "Succeeded", exitCode: 0 },
+        }),
+      ]),
+    });
+
+    expect(kubectlEnqueue).toHaveBeenCalledTimes(1);
+    expect(
+      (kubectlEnqueue.mock.calls[0]![0] as Record<string, unknown>)["command"],
+    ).toBe(KUBECTL_UNDO);
+    expect(lastPersistedPlan(update)["rollbackStatus"]).toBe("Completed");
+  });
+
+  it("skips every kubectl rollback once the cluster's remediation is Disabled — AI never runs a change there", async () => {
+    mockClusterStatus(disabledClusterStatus());
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: failedPlan([executedSetImage()]),
+    });
+
+    expectSkipped('cluster "prod-us" no longer allows AI remediation');
+  });
+
+  it("skips the rollback when the cluster is now bound to another Runner", async () => {
+    mockClusterStatus(
+      clusterStatus({
+        remediationMode: KubernetesAiRemediationMode.BypassApproval,
+        runner: {
+          id: OTHER_RUNNER_ID.toString(),
+          name: "kubernetes-agent/prod-us-v2",
+          isOnline: true,
+          canRunAiCommands: true,
+        },
+      }),
+    );
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: failedPlan([executedSetImage()]),
+    });
+
+    expectSkipped("no longer reached through Runner");
+  });
+
+  it("fails CLOSED when the cluster status cannot be read at rollback time", async () => {
+    jest
+      .spyOn(KubernetesClusterAiAccessService, "getStatusForCluster")
+      .mockRejectedValue(new Error("db down"));
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: failedPlan([executedSetImage()]),
+    });
+
+    expectSkipped("could not confirm that cluster");
+  });
+
+  it("skips only the rollbacks the cluster refuses and still runs the rest, reporting both", async () => {
+    // Command 1: safe undo (runs). Command 2: risky undo (skipped now that the cluster asks).
+    mockClusterStatus(
+      clusterStatus({
+        remediationMode: KubernetesAiRemediationMode.RequireApproval,
+      }),
+    );
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: failedPlan([
+        rawKubectlCommand({
+          execution: { status: "Succeeded", exitCode: 0 },
+        }),
+        executedSetImage({ sequence: 2 }),
+      ]),
+    });
+
+    expect(kubectlEnqueue).toHaveBeenCalledTimes(1);
+    expect(
+      (kubectlEnqueue.mock.calls[0]![0] as Record<string, unknown>)["command"],
+    ).toBe(KUBECTL_UNDO);
+
+    const finalPlan: Record<string, unknown> = lastPersistedPlan(update);
+    expect(finalPlan["rollbackStatus"]).toBe("Failed");
+    expect(
+      (
+        commandInPlan(finalPlan, 0)["rollbackExecution"] as Record<
+          string,
+          unknown
+        >
+      )["status"],
+    ).toBe("Succeeded");
+    expect(
+      (
+        commandInPlan(finalPlan, 1)["rollbackExecution"] as Record<
+          string,
+          unknown
+        >
+      )["status"],
+    ).toBe("Skipped");
+    expect(feedMarkdown(feed)).toContain("was not run for 1 command(s)");
+  });
+
+  it("still refuses a Denied rollback before ever reading the cluster", async () => {
+    const status: jest.SpyInstance = mockClusterStatus(
+      clusterStatus({
+        remediationMode: KubernetesAiRemediationMode.BypassApproval,
+      }),
+    );
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: failedPlan([
+        executedSetImage({ rollbackCommand: "kubectl delete namespace web" }),
+      ]),
+    });
+
+    expect(status).not.toHaveBeenCalled();
+    expect(kubectlEnqueue).not.toHaveBeenCalled();
+    const rollbackExecution: Record<string, unknown> = commandInPlan(
+      lastPersistedPlan(update),
+      0,
+    )["rollbackExecution"] as Record<string, unknown>;
+    expect(rollbackExecution["status"]).toBe("Failed");
+    expect(rollbackExecution["errorMessage"]).toContain(
+      "Rollback refused by the command policy",
+    );
+  });
+
+  it("leaves Bash rollbacks alone — there is no cluster to consult", async () => {
+    const status: jest.SpyInstance = mockClusterStatus(disabledClusterStatus());
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: failedPlan([
+        rawCommand({
+          rollbackCommand: "systemctl stop nginx",
+          execution: { status: "Succeeded", exitCode: 0 },
+        }),
+      ]),
+    });
+
+    expect(status).not.toHaveBeenCalled();
+    expect(bashEnqueue).toHaveBeenCalledTimes(1);
+    expect(lastPersistedPlan(update)["rollbackStatus"]).toBe("Completed");
+  });
+});
+
+/*
+ * The cluster's "Last error" is about its AI ACCESS (PR #3953 review, known
+ * follow-up 1): an approved kubectl command records a success, and a
+ * failure only when it is about the access — never claimed, refused, timed
+ * out, unauthorized, unreachable. A deployment the plan named wrong must
+ * neither become the cluster's last error nor clear a real one.
+ */
+describe("CommandPlanExecutor records only access failures as a cluster's last error", () => {
+  let recordOutcome: jest.SpyInstance;
+
+  beforeEach(() => {
+    jest.spyOn(logger, "error").mockImplementation((): void => {
+      return undefined;
+    });
+    mockPersist();
+    mockKubectlEnqueue();
+    mockFeed();
+    mockClusterStatus(clusterStatus());
+    recordOutcome = mockRecordOutcome();
+    mockFetch(fakeSuggestion({ commandPlan: rawPlan([rawKubectlCommand()]) }));
+  });
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  function jobEndsAs(overrides: Partial<Record<string, unknown>>): void {
+    jest
+      .spyOn(RunnerJobService, "pollUntilTerminal")
+      .mockResolvedValue(terminalJob(overrides));
+  }
+
+  it("a kubectl that ran and failed on a wrong name is not recorded", async () => {
+    jobEndsAs({
+      status: RunnerJobStatus.Failed,
+      exitCode: 1,
+      output:
+        '[stdout]\n\n[stderr]\nError from server (NotFound): deployments.apps "web" not found',
+      errorMessage: "kubectl exited with code 1",
+    });
+
+    await CommandPlanExecutor.executeApprovedPlan({
+      suggestionId: SUGGESTION_ID,
+    });
+
+    expect(recordOutcome).not.toHaveBeenCalled();
+  });
+
+  it.each([
+    [
+      "the API server refused it (Forbidden)",
+      {
+        status: RunnerJobStatus.Failed,
+        exitCode: 1,
+        output:
+          '[stdout]\n\n[stderr]\nError from server (Forbidden): deployments.apps "web" is forbidden',
+      },
+    ],
+    [
+      "the Runner refused it before spawning kubectl",
+      {
+        status: RunnerJobStatus.Failed,
+        exitCode: undefined,
+        output: "",
+        errorMessage:
+          "Refused by the Runner: the namespace is outside its scope.",
+      },
+    ],
+    [
+      "it timed out",
+      {
+        status: RunnerJobStatus.TimedOut,
+        exitCode: undefined,
+        output: "",
+        errorMessage: "No Runner picked up this step in time.",
+      },
+    ],
+  ])(
+    "an access failure is recorded: %s",
+    async (_label: string, overrides: Record<string, unknown>) => {
+      jobEndsAs(overrides);
+
+      await CommandPlanExecutor.executeApprovedPlan({
+        suggestionId: SUGGESTION_ID,
+      });
+
+      expect(recordOutcome).toHaveBeenCalledTimes(1);
+      expect(recordOutcome.mock.calls[0]![0]).toMatchObject({
+        succeeded: false,
+      });
+    },
+  );
+
+  it("negative control: a success is recorded", async () => {
+    jobEndsAs({ status: RunnerJobStatus.Succeeded, exitCode: 0 });
+
+    await CommandPlanExecutor.executeApprovedPlan({
+      suggestionId: SUGGESTION_ID,
+    });
+
+    expect(recordOutcome).toHaveBeenCalledTimes(1);
+    expect(recordOutcome.mock.calls[0]![0]).toMatchObject({
+      succeeded: true,
+    });
+  });
+});
+
+/*
+ * A kubectl write the cluster's bound Runner reports it will refuse is not
+ * enqueued (PR #3953 review, XP-6): an approved command fails at execution
+ * time with the reason, and a rollback is left for a human with it —
+ * instead of being sent to the Runner only to be refused.
+ */
+describe("CommandPlanExecutor refuses a kubectl write its cluster's Runner would refuse", () => {
+  let update: jest.SpyInstance;
+  let kubectlEnqueue: jest.SpyInstance;
+  let feed: jest.SpyInstance;
+
+  const SCOPED_TO_API: KubernetesClusterAiAccessStatus = clusterStatus({
+    remediationMode: KubernetesAiRemediationMode.BypassApproval,
+    runner: {
+      id: RUNNER_ID.toString(),
+      name: "kubernetes-agent/prod-us",
+      isOnline: true,
+      canRunAiCommands: true,
+      posture: {
+        inCluster: true,
+        allowWrites: true,
+        writeNamespaces: ["api"],
+        podNamespace: "oneuptime-agent",
+      },
+    },
+  });
+
+  beforeEach(() => {
+    jest.spyOn(logger, "error").mockImplementation((): void => {
+      return undefined;
+    });
+    update = mockPersist();
+    kubectlEnqueue = mockKubectlEnqueue();
+    mockPoll();
+    mockRecordOutcome();
+    feed = mockFeed();
+  });
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  it("fails an approved command outside the Runner's write namespaces at execution time, with the reason, and never enqueues it", async () => {
+    mockFetch(fakeSuggestion({ commandPlan: rawPlan([rawKubectlCommand()]) }));
+    mockClusterStatus(SCOPED_TO_API);
+
+    await CommandPlanExecutor.executeApprovedPlan({
+      suggestionId: SUGGESTION_ID,
+    });
+
+    expect(kubectlEnqueue).not.toHaveBeenCalled();
+    const execution: Record<string, unknown> = commandInPlan(
+      lastPersistedPlan(update),
+      0,
+    )["execution"] as Record<string, unknown>;
+    expect(execution["status"]).toBe("Failed");
+    expect(execution["errorMessage"]).toContain(
+      "Refused at execution time: the cluster's Runner would refuse it",
+    );
+    expect(execution["errorMessage"]).toContain('"web"');
+    expect(feedMarkdown(feed)).toContain(
+      "the cluster's Runner would refuse it",
+    );
+  });
+
+  it("leaves a rollback the Runner would refuse for a human, with the reason and the command to run", async () => {
+    mockClusterStatus(SCOPED_TO_API);
+
+    await CommandPlanExecutor.executeRollback({
+      suggestion: fakeSuggestion({
+        commandPlan: rawPlan(
+          [
+            rawKubectlCommand({
+              execution: { status: "Succeeded", exitCode: 0 },
+            }),
+          ],
+          { executionStatus: "Completed" },
+        ),
+      }),
+    });
+
+    expect(kubectlEnqueue).not.toHaveBeenCalled();
+    const finalPlan: Record<string, unknown> = lastPersistedPlan(update);
+    expect(finalPlan["rollbackStatus"]).toBe("Failed");
+    const rollbackExecution: Record<string, unknown> = commandInPlan(
+      finalPlan,
+      0,
+    )["rollbackExecution"] as Record<string, unknown>;
+    expect(rollbackExecution["status"]).toBe("Skipped");
+    expect(rollbackExecution["errorMessage"]).toContain(
+      "the cluster's Runner would refuse it",
+    );
+    expect(rollbackExecution["errorMessage"]).toContain(
+      `Undo it manually: ${KUBECTL_UNDO}`,
+    );
+  });
+
+  it("negative control: a command inside the Runner's scope is enqueued", async () => {
+    mockFetch(
+      fakeSuggestion({
+        commandPlan: rawPlan([
+          rawKubectlCommand({
+            command: "kubectl rollout restart deployment/api -n api",
+            rollbackCommand: "kubectl rollout undo deployment/api -n api",
+          }),
+        ]),
+      }),
+    );
+    mockClusterStatus(SCOPED_TO_API);
+
+    await CommandPlanExecutor.executeApprovedPlan({
+      suggestionId: SUGGESTION_ID,
+    });
+
+    expect(kubectlEnqueue).toHaveBeenCalledTimes(1);
+  });
 });