@moondeck/station 0.0.0-stage → 1.0.0

package/lib/task-credential-cleanup.ts

@@ -0,0 +1,752 @@
+/**
+ * ADR-154's single credential cleanup procedure.
+ *
+ * The coordinator owns ordering and failure semantics; credential consumers
+ * supply their lane-specific drain/quiesce/drop hooks. Durable inventory is
+ * removed only after an exact path is independently proven absent.
+ */
+import { spawn } from "node:child_process";
+
+import type { HostCredentialCleanupPath } from "../db/schema";
+import {
+  assertCredentialCleanupPath,
+  forgetCredentialCleanupPending,
+  forgetCredentialCleanupPath,
+  forgetCredentialCleanupRestop,
+  listCredentialCleanupRecoveryTasks,
+  listCredentialCleanupPaths,
+  listCredentialCleanupRestops,
+  rememberCredentialCleanupPending,
+  rememberCredentialCleanupRestop,
+  type CredentialCleanupLane,
+  type CredentialCleanupPendingReason,
+} from "./credential-cleanup-inventory";
+import type { CredentialCleanupRequest as LeaseCredentialCleanupRequest } from "./cloud-credential-leases";
+import type { Db } from "./db";
+import { reconstructTaskEnvironment } from "./task-environment";
+import {
+  parseTaskEnvironmentLocator,
+  type TaskEnvironmentLocator,
+  type TaskEnvironmentHandle,
+} from "./task-environment/types";
+
+const CLEANUP_TIMEOUT_MS = 10_000;
+const CLEANUP_OUTPUT_BYTES = 4 * 1024;
+
+export type TaskCredentialCleanupReason = CredentialCleanupPendingReason;
+
+export interface TaskCredentialCleanupRequest {
+  taskId: string;
+  runEpoch: number;
+  reason: TaskCredentialCleanupReason;
+  /** Absent means every credential and every recovered epoch for the task. */
+  credentialIds?: readonly string[];
+  /** The lease registry's own request this one carries out, with its exact
+   *  qualified identities (class, id, revision), reason and Files scope, for
+   *  the hooks; absent for one the coordinator starts itself (boot
+   *  recovery). The durable inventory keys on `credentialIds` alone. */
+  lease?: Readonly<LeaseCredentialCleanupRequest>;
+}
+
+export interface TaskCredentialCleanupHooks {
+  /** These two hooks must be synchronous and idempotent. */
+  refuseNewWork(request: TaskCredentialCleanupRequest): void;
+  closeGate(request: TaskCredentialCleanupRequest): void;
+  drainWriters(request: TaskCredentialCleanupRequest): Promise<void>;
+  quiesceProcesses(request: TaskCredentialCleanupRequest): Promise<void>;
+  /** Retire process-backed consumers after leased engine agents have
+   * quiesced. Other task processes may still hold Files; a strict unmount
+   * which cannot prove them gone is terminal machine retirement. Their
+   * synchronous gate-close hook has already made Files unavailable to new
+   * work. */
+  closeConsumers(request: TaskCredentialCleanupRequest): Promise<void>;
+  dropMemory(request: TaskCredentialCleanupRequest): void;
+}
+
+export interface TaskCredentialCleanupDeps {
+  db?: Db;
+  hooks: TaskCredentialCleanupHooks;
+  removePath?: (row: HostCredentialCleanupPath) => Promise<void>;
+  /** Full cleanup under the cloud's taskDown (`reason: "teardown"`) only:
+   *  destroy one exact runtime and prove it absent. */
+  destroyRuntime?: (request: {
+    taskId: string;
+    runEpoch: number;
+    runtime: TaskEnvironmentLocator;
+  }) => Promise<boolean>;
+}
+
+export class TaskCredentialCleanupError extends Error {
+  constructor(
+    readonly taskId: string,
+    readonly failedPaths: readonly string[],
+    /** Why the runtime could not be reached for them, when known (a stopped
+     *  container refused for cleanup, say): host.log's only clue. */
+    reason?: unknown,
+  ) {
+    super(
+      `credential cleanup for task ${taskId} could not prove ${String(failedPaths.length)} path(s) absent` +
+        (reason === undefined
+          ? ""
+          : `: ${(reason instanceof Error ? reason.message : String(reason)).slice(0, 300)}`),
+    );
+  }
+}
+
+async function execCleanup(
+  environment: Pick<TaskEnvironmentHandle, "exec">,
+  argv: [string, ...string[]],
+  stdin?: Uint8Array,
+): Promise<{ exitCode: number | null; stdout: string }> {
+  const result = await environment.exec({
+    argv,
+    user: "root",
+    env: {
+      LD_AUDIT: "",
+      LD_LIBRARY_PATH: "",
+      LD_PRELOAD: "",
+      PYTHONHOME: "",
+      PYTHONPATH: "",
+    },
+    projectEnvProfile: "maintenance",
+    timeoutMs: CLEANUP_TIMEOUT_MS,
+    maxOutputBytes: CLEANUP_OUTPUT_BYTES,
+    ...(stdin === undefined ? {} : { stdin }),
+  });
+  return {
+    exitCode: result.exitCode,
+    stdout: Buffer.from(result.stdout).toString("utf8"),
+  };
+}
+
+/**
+ * One root process performs traversal, unlink and the absence proof through
+ * directory file descriptors. No task-owned parent is re-resolved after it is
+ * checked, so renaming it or replacing it with a symlink cannot redirect the
+ * root deletion. The program itself is streamed over stdin; no shell parses
+ * either it or the validated path.
+ */
+export const CREDENTIAL_UNLINK_HELPER = String.raw`
+import os
+import sys
+import shutil
+import stat
+
+root, absolute = sys.argv[1:3]
+directory = len(sys.argv) == 4 and sys.argv[3] == "directory"
+prefix = root.rstrip("/") + "/"
+if not absolute.startswith(prefix):
+    raise RuntimeError("credential path escaped its root")
+parts = absolute[len(prefix):].split("/")
+if not parts or any(part in ("", ".", "..") for part in parts):
+    raise RuntimeError("credential path is not canonical")
+
+flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
+opened = []
+try:
+    try:
+        current = os.open(root, flags)
+    except FileNotFoundError:
+        sys.exit(0)
+    opened.append(current)
+    for component in parts[:-1]:
+        try:
+            current = os.open(component, flags, dir_fd=current)
+        except FileNotFoundError:
+            sys.exit(0)
+        opened.append(current)
+    leaf = parts[-1]
+    try:
+        if directory:
+            info = os.stat(leaf, dir_fd=current, follow_symlinks=False)
+            if not stat.S_ISDIR(info.st_mode) or not shutil.rmtree.avoids_symlink_attacks:
+                raise RuntimeError("credential directory cannot be traversed safely")
+            shutil.rmtree(leaf, dir_fd=current)
+        else:
+            os.unlink(leaf, dir_fd=current)
+    except FileNotFoundError:
+        pass
+    try:
+        os.stat(leaf, dir_fd=current, follow_symlinks=False)
+    except FileNotFoundError:
+        sys.exit(0)
+    raise RuntimeError("credential path still exists")
+finally:
+    for descriptor in reversed(opened):
+        os.close(descriptor)
+`;
+
+/** Host-side form of the same confined primitive. The fixed, isolated
+ * interpreter is a runner dependency (and is also baked into task machines);
+ * neither PATH, Python environment/site injection nor a shell participates. */
+export async function removeHostCredentialFile(
+  root: string,
+  path: string,
+): Promise<void> {
+  const result = await new Promise<{ exitCode: number | null }>(
+    (resolve, reject) => {
+      const child = spawn(
+        "/usr/bin/python3",
+        ["-I", "-", root, path],
+        {
+          cwd: "/",
+          env: {},
+          stdio: ["pipe", "ignore", "ignore"],
+        },
+      );
+      const timeout = setTimeout(() => {
+        child.kill("SIGKILL");
+      }, CLEANUP_TIMEOUT_MS);
+      child.once("error", (error) => {
+        clearTimeout(timeout);
+        reject(error);
+      });
+      child.once("close", (exitCode) => {
+        clearTimeout(timeout);
+        resolve({ exitCode });
+      });
+      child.stdin.on("error", () => undefined);
+      child.stdin.end(Buffer.from(CREDENTIAL_UNLINK_HELPER, "utf8"));
+    },
+  );
+  if (result.exitCode !== 0) {
+    throw new Error("station credential path could not be proven absent");
+  }
+}
+
+async function removeLocalFile(path: string): Promise<void> {
+  if (!path.startsWith("/run/uai/")) {
+    throw new Error("station credential path is outside the managed root");
+  }
+  await removeHostCredentialFile("/run/uai", path);
+}
+
+export async function removeRuntimeCredentialFile(
+  environment: Pick<TaskEnvironmentHandle, "exec">,
+  path: string,
+): Promise<void> {
+  if (!path.startsWith("/home/node/")) {
+    throw new Error("runtime credential path is outside the task home");
+  }
+  const removed = await execCleanup(
+    environment,
+    ["/usr/bin/python3", "-I", "-", "/home/node", path,
+      ...(/^\/home\/node\/\.uai-connection\/[a-f0-9]{64}$/.test(path) ? ["directory"] : [])],
+    Buffer.from(CREDENTIAL_UNLINK_HELPER, "utf8"),
+  );
+  if (removed.exitCode !== 0) {
+    throw new Error("credential path could not be proven absent");
+  }
+}
+
+/** Delete one validated file, or the confined connection home owning it. */
+export async function removeCredentialCleanupPath(
+  row: HostCredentialCleanupPath,
+): Promise<void> {
+  assertCredentialCleanupPath(
+    row.lane as CredentialCleanupLane,
+    row.path,
+  );
+  if (row.lane === "files-session") {
+    await removeLocalFile(row.path);
+    return;
+  }
+  const environment = await reconstructCleanupRuntime(row.taskId, row.runtimeLocator);
+  await removeRuntimeCredentialFile(environment, row.path);
+}
+
+async function reconstructCleanupRuntime(
+  taskId: string,
+  runtimeLocator: string,
+): Promise<TaskEnvironmentHandle<unknown>> {
+  const locator = parseTaskEnvironmentLocator(runtimeLocator);
+  const environment = await reconstructTaskEnvironment(locator, "loader");
+  if (environment.descriptor.taskId !== taskId) {
+    throw new Error("cleanup runtime belongs to a different task");
+  }
+  return environment;
+}
+
+/**
+ * The exact paths of one runtime whose removal failed, tried once more
+ * against that runtime as it is found (never teardown's: its destroy
+ * fallback is its own). Returns the rows now proven absent.
+ * - `absent`, exactly: the runtime, its writable layer with it, is gone, and
+ *   so is every path in it. Any other answer, `unknown` above all, is no
+ *   proof (a failed call never proves absence, 2026-10-20).
+ * - `stopped`, inside the boot sweep only: `docker exec` cannot reach a
+ *   stopped container, and after a reboot every one is, so the sweep failed
+ *   for good and the runner stayed in `checking` (2026-10-30). The runtime
+ *   is started for the removal alone and stopped again
+ *   (`runStoppedForCleanup`), its stop owed durably before the start
+ *   (`host_credential_cleanup_restops`, settled by `#settleOwedRestops`).
+ *   Only the boot sweep may. While it runs, nothing else starts a task's
+ *   runtime: TaskUp and Stop refuse until the runtime is operational, and
+ *   the task's other cleanups queue on the coordinator's per-task tail. A
+ *   cloud taskDown is the one that does not wait: it needs only a pinned
+ *   backend, skips this coordinator while credentials are pending, and may
+ *   remove the started container mid-cleanup. That fails closed: an exec
+ *   into a removed container proves nothing, and a container proven gone
+ *   settles its owed stop.
+ * - anything else (`running`, frozen, `unknown`, a provider that cannot
+ *   start one for cleanup): the failure stands.
+ */
+async function retryFailedRuntimePaths(
+  taskId: string,
+  runtimeLocator: string,
+  rows: readonly HostCredentialCleanupPath[],
+  startStopped: boolean,
+  oweStop: (containerId: string) => void,
+): Promise<{ proven: HostCredentialCleanupPath[]; problem?: unknown }> {
+  const environment = await reconstructCleanupRuntime(taskId, runtimeLocator);
+  const status = await environment.status();
+  if (status.state === "absent") return { proven: [...rows] };
+  if (
+    !startStopped ||
+    status.state !== "stopped" ||
+    status.frozen === true ||
+    status.instances.length === 0 ||
+    environment.runStoppedForCleanup === undefined
+  ) {
+    return { proven: [] };
+  }
+  const proven: HostCredentialCleanupPath[] = [];
+  try {
+    await environment.runStoppedForCleanup(async () => {
+      for (const row of rows) {
+        try {
+          assertCredentialCleanupPath(row.lane as CredentialCleanupLane, row.path);
+          await removeRuntimeCredentialFile(environment, row.path);
+          proven.push(row);
+        } catch {
+          // This path's failure stands; the others are still tried.
+        }
+      }
+    }, oweStop);
+  } catch (error) {
+    // Not started, or not proven stopped again. What the helper proved
+    // absent is still absent; the cleanup fails all the same.
+    return { proven, problem: error };
+  }
+  return { proven };
+}
+
+async function destroyCredentialRuntime(request: {
+  taskId: string;
+  runEpoch: number;
+  runtime: TaskEnvironmentLocator;
+}): Promise<boolean> {
+  const environment = await reconstructTaskEnvironment(request.runtime, "loader");
+  if (environment.descriptor.taskId !== request.taskId) {
+    throw new Error("cleanup runtime belongs to a different task");
+  }
+  await environment.teardown();
+  const status = await environment.status();
+  return status.state === "absent";
+}
+
+function selectRows(
+  rows: readonly HostCredentialCleanupPath[],
+  request: TaskCredentialCleanupRequest,
+): HostCredentialCleanupPath[] {
+  const throughEpoch = rows.filter((row) => row.runEpoch <= request.runEpoch);
+  if (request.credentialIds === undefined) return throughEpoch;
+  const ids = new Set(request.credentialIds);
+  if (ids.size !== request.credentialIds.length || ids.size === 0) {
+    throw new Error("selective credential cleanup ids must be unique and nonempty");
+  }
+  return throughEpoch.filter((row) => ids.has(row.credentialId));
+}
+
+function isSamePhysicalPath(
+  left: HostCredentialCleanupPath,
+  right: HostCredentialCleanupPath,
+): boolean {
+  // Files-session paths live on the host, so a recreated task runtime still
+  // points at the same /run/uai file. Every other managed path lives inside
+  // one exact runtime and is identified by that locator plus its path.
+  if (left.lane === "files-session" || right.lane === "files-session") {
+    return (
+      left.lane === "files-session" &&
+      right.lane === "files-session" &&
+      left.path === right.path
+    );
+  }
+  return (
+    left.runtimeLocator === right.runtimeLocator && left.path === right.path
+  );
+}
+
+function forgetRow(row: HostCredentialCleanupPath, db?: Db): void {
+  forgetCredentialCleanupPath(
+    {
+      taskId: row.taskId,
+      runEpoch: row.runEpoch,
+      runtime: parseTaskEnvironmentLocator(row.runtimeLocator),
+      lane: row.lane as CredentialCleanupLane,
+      path: row.path,
+    },
+    db,
+  );
+}
+
+/** Serializes cleanup per task and never reopens a gate on failure. */
+export class TaskCredentialCleanupCoordinator {
+  /** The serialized state retains the highest failed epoch until an equal or
+   * newer full cleanup succeeds. This prevents a delayed lower-epoch cleanup,
+   * or an unrelated selective success, from clearing the crash witness. */
+  readonly #tails = new Map<
+    string,
+    Promise<{ failedEpoch: number; clearThroughEpoch: number }>
+  >();
+  readonly #deps: TaskCredentialCleanupDeps;
+
+  constructor(deps: TaskCredentialCleanupDeps) {
+    this.#deps = deps;
+  }
+
+  cleanup(request: TaskCredentialCleanupRequest): Promise<void> {
+    return this.#cleanup(request, false);
+  }
+
+  /** `startStopped`: the boot sweep's own cleanups (recoverAll) alone may
+   *  start a stopped runtime to clean it (retryFailedRuntimePaths). */
+  #cleanup(request: TaskCredentialCleanupRequest, startStopped: boolean): Promise<void> {
+    if (!Number.isSafeInteger(request.runEpoch) || request.runEpoch < 1) {
+      return Promise.reject(new Error("cleanup run epoch must be positive"));
+    }
+    if (request.credentialIds !== undefined) {
+      const ids = new Set(request.credentialIds);
+      if (ids.size !== request.credentialIds.length || ids.size === 0) {
+        return Promise.reject(
+          new Error("selective credential cleanup ids must be unique and nonempty"),
+        );
+      }
+    }
+    const prior =
+      this.#tails.get(request.taskId) ??
+      Promise.resolve({ failedEpoch: 0, clearThroughEpoch: 0 });
+    let hooksFailed = false;
+    let hookError: unknown;
+    const recordFailure = (error: unknown): void => {
+      if (!hooksFailed) hookError = error;
+      hooksFailed = true;
+    };
+    try {
+      // This witness precedes even the synchronous gate hooks. A crash at any
+      // later instruction therefore makes boot recovery run the full cleanup.
+      rememberCredentialCleanupPending(request, this.#deps.db);
+    } catch (error) {
+      recordFailure(error);
+    }
+    // Both fail-closed hooks get their chance even if the other hook (or the
+    // durable write) fails. Their first failure remains the returned cause.
+    try {
+      this.#deps.hooks.refuseNewWork(request);
+    } catch (error) {
+      recordFailure(error);
+    }
+    try {
+      this.#deps.hooks.closeGate(request);
+    } catch (error) {
+      recordFailure(error);
+    }
+    const outcome = prior.then(async (state) => {
+      try {
+        if (hooksFailed) throw hookError;
+        await this.#run(request, startStopped);
+        return {
+          state: {
+            failedEpoch:
+              request.credentialIds === undefined &&
+              request.runEpoch >= state.failedEpoch
+                ? 0
+                : state.failedEpoch,
+            clearThroughEpoch: Math.max(
+              state.clearThroughEpoch,
+              request.runEpoch,
+            ),
+          },
+        };
+      } catch (error) {
+        return {
+          state: {
+            failedEpoch: Math.max(state.failedEpoch, request.runEpoch),
+            clearThroughEpoch: state.clearThroughEpoch,
+          },
+          error,
+        };
+      }
+    });
+    const tail = outcome.then((result) => result.state);
+    this.#tails.set(request.taskId, tail);
+    const run = outcome.then((result) => {
+      if (
+        this.#tails.get(request.taskId) === tail &&
+        result.state.failedEpoch === 0
+      ) {
+        forgetCredentialCleanupPending(
+          request.taskId,
+          result.state.clearThroughEpoch,
+          this.#deps.db,
+        );
+      }
+      if ("error" in result) throw result.error;
+    });
+    void tail.then((state) => {
+      if (
+        state.failedEpoch === 0 &&
+        this.#tails.get(request.taskId) === tail
+      ) {
+        this.#tails.delete(request.taskId);
+      }
+    });
+    return run;
+  }
+
+  async #run(request: TaskCredentialCleanupRequest, startStopped: boolean): Promise<void> {
+    await this.#deps.hooks.drainWriters(request);
+    await this.#deps.hooks.quiesceProcesses(request);
+    let consumersFailed = false;
+    let consumerError: unknown;
+    try {
+      await this.#deps.hooks.closeConsumers(request);
+    } catch (error) {
+      // A strict Files unmount can fail (for example, EBUSY) after every gate
+      // and writer has closed. The root-only session material must still be
+      // removed below. Keep its inventory rows and the pending-cleanup row as
+      // durable mount witnesses, then return this original proof failure so a
+      // later process retries the unmount from the recorded runtime locator.
+      consumersFailed = true;
+      consumerError = error;
+    }
+
+    // A container an earlier sweep started for cleanup and never proved
+    // stopped again (a failed stop, a crash after the start, a start that
+    // timed out yet started it) is stopped, or proven stopped or gone,
+    // before this cleanup clears anything; until then nothing is.
+    const owed = await this.#settleOwedRestops(request.taskId);
+    if (owed.unsettled) throw owed.problem;
+
+    const allRows = listCredentialCleanupPaths(request.taskId, this.#deps.db);
+    const selectedRows = selectRows(allRows, request);
+    const selected = new Set(selectedRows);
+    const rows: HostCredentialCleanupPath[] = [];
+    for (const row of selectedRows) {
+      const transferred = allRows.some(
+        (candidate) =>
+          candidate.runEpoch > row.runEpoch &&
+          !selected.has(candidate) &&
+          isSamePhysicalPath(candidate, row),
+      );
+      if (!transferred) {
+        rows.push(row);
+        continue;
+      }
+      // Gates are closed and writers/processes have drained before this
+      // serialized inventory observation. The newer row is therefore the
+      // durable ownership witness for the reused physical path: retire only
+      // the obsolete row, never unlink the newer run's file.
+      forgetRow(row, this.#deps.db);
+    }
+    let failed: HostCredentialCleanupPath[] = [];
+    // A consumer failure means some non-Files process may still hold its
+    // credential. Preserve every such path and its witness for the retry.
+    // Files is the exception: its root-only session must be revoked even
+    // when the already-hidden mount cannot yet be proven absent.
+    const removableRows = consumersFailed
+      ? rows.filter((row) => row.lane === "files-session")
+      : rows;
+    for (const row of removableRows) {
+      try {
+        await (this.#deps.removePath ?? removeCredentialCleanupPath)(row);
+        if (!consumersFailed || row.lane !== "files-session") {
+          forgetRow(row, this.#deps.db);
+        }
+      } catch {
+        failed.push(row);
+      }
+    }
+
+    // A path that could not be removed in its runtime as it is: once more,
+    // against that runtime as found (absent, or stopped in the boot sweep),
+    // never for teardown, whose destroy fallback below is its own.
+    let runtimeProblem: unknown;
+    let runtimeProblemSeen = false;
+    if (!consumersFailed && request.reason !== "teardown" && failed.length > 0) {
+      const byRuntime = new Map<string, HostCredentialCleanupPath[]>();
+      for (const row of failed) {
+        if (row.lane === "files-session") continue;
+        const group = byRuntime.get(row.runtimeLocator) ?? [];
+        group.push(row);
+        byRuntime.set(row.runtimeLocator, group);
+      }
+      const proven = new Set<HostCredentialCleanupPath>();
+      for (const [runtimeLocator, runtimeRows] of byRuntime) {
+        try {
+          const retried = await retryFailedRuntimePaths(
+            request.taskId,
+            runtimeLocator,
+            runtimeRows,
+            startStopped,
+            (containerId) =>
+              rememberCredentialCleanupRestop(
+                {
+                  taskId: request.taskId,
+                  runEpoch: request.runEpoch,
+                  runtimeLocator,
+                  containerId,
+                },
+                this.#deps.db,
+              ),
+          );
+          for (const row of retried.proven) {
+            forgetRow(row, this.#deps.db);
+            proven.add(row);
+          }
+          if ("problem" in retried && !runtimeProblemSeen) {
+            runtimeProblemSeen = true;
+            runtimeProblem = retried.problem;
+          }
+        } catch {
+          // No status, or no runtime to ask: the exact-path failure stands.
+        }
+      }
+      failed = failed.filter((row) => !proven.has(row));
+      // Each start above owed its stop before it happened; a stop not proven
+      // leaves it owed, and fails this cleanup.
+      const settled = await this.#settleOwedRestops(request.taskId);
+      if (settled.unsettled && !runtimeProblemSeen) {
+        runtimeProblemSeen = true;
+        runtimeProblem = settled.problem;
+      }
+    }
+
+    // A whole-task cleanup may prove absence by destroying the exact writable
+    // runtime instead, but ONLY inside the cloud's taskDown of this run (the
+    // durable reason `teardown` has no other source). Every other cleanup —
+    // boot recovery, a fence escalation, stop, park, assignment — defers: the
+    // gate stays closed, the pending witness and inventory rows stay, and the
+    // registry's retry tries the exact paths again later. A failed exec is
+    // not proof of absence; Docker briefly gone (an OrbStack update,
+    // 2026-10-20) once turned this fallback into task-down.sh on every live
+    // task, deleting their worktrees. Selective removal must never take this
+    // broader action either.
+    if (
+      !consumersFailed &&
+      request.reason === "teardown" &&
+      request.credentialIds === undefined &&
+      failed.length > 0
+    ) {
+      const byRuntime = new Map<string, HostCredentialCleanupPath[]>();
+      for (const row of failed) {
+        if (row.lane === "files-session") continue;
+        const group = byRuntime.get(row.runtimeLocator) ?? [];
+        group.push(row);
+        byRuntime.set(row.runtimeLocator, group);
+      }
+      const proven = new Set<HostCredentialCleanupPath>();
+      for (const [runtimeLocator, runtimeRows] of byRuntime) {
+        // A locator may have been reused by a later assignment. Destroying it
+        // for delayed cleanup would then erase the newer run's credential
+        // files. Exact deletion may still proceed; the broad fallback may not.
+        if (
+          allRows.some(
+            (row) =>
+              row.runtimeLocator === runtimeLocator &&
+              row.runEpoch > request.runEpoch,
+          )
+        ) {
+          continue;
+        }
+        try {
+          const runtime = parseTaskEnvironmentLocator(runtimeLocator);
+          const absent = await (
+            this.#deps.destroyRuntime ?? destroyCredentialRuntime
+          )({
+            taskId: request.taskId,
+            runEpoch: request.runEpoch,
+            runtime,
+          });
+          if (!absent) continue;
+          for (const row of runtimeRows) {
+            forgetRow(row, this.#deps.db);
+            proven.add(row);
+          }
+        } catch {
+          // The original exact-path failure remains authoritative.
+        }
+      }
+      failed = failed.filter((row) => !proven.has(row));
+    }
+    if (consumersFailed) throw consumerError;
+    if (failed.length > 0) {
+      throw new TaskCredentialCleanupError(
+        request.taskId,
+        failed.map((row) => row.path),
+        runtimeProblemSeen ? runtimeProblem : undefined,
+      );
+    }
+    if (runtimeProblemSeen) throw runtimeProblem;
+    this.#deps.hooks.dropMemory(request);
+  }
+
+  /** Every re-stop the task is owed, tried; each proven retired. Only proof
+   *  that the exact container is stopped or gone retires one: a removed
+   *  credential path never does. */
+  async #settleOwedRestops(
+    taskId: string,
+  ): Promise<{ unsettled: false } | { unsettled: true; problem: unknown }> {
+    let problem: { unsettled: true; problem: unknown } | null = null;
+    for (const owed of listCredentialCleanupRestops(taskId, this.#deps.db)) {
+      try {
+        const environment = await reconstructCleanupRuntime(taskId, owed.runtimeLocator);
+        if (environment.settleCleanupStart === undefined) {
+          throw new Error("the runtime cannot prove a container it started for cleanup stopped again");
+        }
+        if (!(await environment.settleCleanupStart(owed.containerId))) {
+          throw new Error(
+            `the container ${owed.containerId.slice(0, 12)} started for cleanup is owed its stop and could not be proven stopped`,
+          );
+        }
+        forgetCredentialCleanupRestop(owed, this.#deps.db);
+      } catch (error) {
+        problem ??= { unsettled: true, problem: error };
+      }
+    }
+    return problem ?? { unsettled: false };
+  }
+
+  /** Every cleanup under way, and any that registers meanwhile, settled
+   *  (process shutdown). */
+  async whenIdle(): Promise<void> {
+    for (;;) {
+      const tails = [...this.#tails.values()];
+      await Promise.allSettled(tails);
+      const now = [...this.#tails.values()];
+      if (now.length === tails.length && now.every((tail, index) => tail === tails[index])) return;
+    }
+  }
+
+  /** Boot recovery sweeps every durable task before any grant may be asked.
+   *  Every task is attempted, one failing never leaving the next unswept;
+   *  then any failure, all of them, fails the sweep. */
+  async recoverAll(runEpochForTask: (taskId: string) => number): Promise<void> {
+    const tasks = listCredentialCleanupRecoveryTasks(this.#deps.db);
+    const failures: unknown[] = [];
+    for (const task of tasks) {
+      try {
+        await this.#cleanup({
+          taskId: task.taskId,
+          runEpoch: Math.max(task.runEpoch, runEpochForTask(task.taskId)),
+          reason: "restart",
+        }, true);
+      } catch (error) {
+        failures.push(error);
+      }
+    }
+    if (failures.length > 0) {
+      throw new AggregateError(failures, `credential recovery failed for ${String(failures.length)} task(s)`);
+    }
+  }
+}