@runuai/host 0.9.150 → 0.9.152

package/lib/cloud-credential-runtime.ts

@@ -0,0 +1,1035 @@
+/**
+ * ADR-154 on the runner: the one place a task on cloud credentials is wired.
+ *
+ * - The lease registry owns each task's grant, its reads and its gate. This
+ *   composes what it installs, opens and closes: the engines' accounts, MCP
+ *   connections, the owner's GitHub sign-in and SSH key, the env replica and
+ *   the task's Files install from the registry's install; each opens with
+ *   its gate on the exact committed view; every one closes with it,
+ *   synchronously, each attempted even when another throws; and each is
+ *   cleaned up through the one cleanup procedure
+ *   (`TaskCredentialCleanupCoordinator`), whose hooks this supplies, with the
+ *   registry's own qualified identities.
+ * - Task-up's seam. `beginStart` in the short admission transaction begins
+ *   the registry's startup there (its synchronous part: the older run ended,
+ *   the gate closed) and hands back the start's token; a newer run's start
+ *   interrupts an older one's. `waitForLaunchReady` outside any lifecycle
+ *   slot, bounded, ends at that startup's cleanup and the grant installed,
+ *   its env snapshot exact and its gate open. Inside the long slot,
+ *   `tryAcquireTaskUpHold`: a counted hold on the installed grant that
+ *   task-up carries through its clone, provisioning and the runtime-ready
+ *   writes. `interruptStarts` is synchronous, for Stop and a newer run to
+ *   call before they queue behind a wedged start: the token's signal aborts
+ *   its wait and its hold. `assertCurrent` is the last word before a start
+ *   publishes its success; `abandonStart` releases the hold before it closes
+ *   the run, so the cleanup never waits on it.
+ * - Frames: v1 lease frames and the env replica's frames reach their owners
+ *   here. A `lease.deny` the registry did not ask for (ADR-133's) is left to
+ *   whoever did. After shutdown, nothing is taken.
+ * - Refreshes: an engine's or MCP connection's reactive refresh goes to the
+ *   cloud's broker through the registry, never to a local refresher.
+ */
+
+import {
+  isProjectEnvReplicaV1ApplyFrame,
+  isProjectEnvReplicaV1BeginFrame,
+  isProjectEnvReplicaV1CommitFrame,
+  isProjectEnvReplicaV1PageFrame,
+  type ProjectEnvReplicaV1AckFrame,
+} from "../src/protocol";
+import type { CredentialRefreshFrame, LeaseRequestV1Frame } from "../src/cloud-credentials";
+import { adaptLeaseCredentialCleanupRequest } from "./cloud-credential-cleanup-adapter";
+import {
+  CredentialLeaseUnavailableError,
+  TaskCredentialLeaseRegistry,
+  type CredentialCleanupReason,
+  type CredentialCleanupRequest,
+  type CredentialInstallRequest,
+  type TaskAssignment,
+  type TaskCredentialLeaseRegistryOptions,
+  type TaskCredentialView,
+} from "./cloud-credential-leases";
+import {
+  cleanupEngineCredentials,
+  closeEngineGate,
+  drainLeasedEngineWrites,
+  forgetLeasedEngines,
+  installEngineCredentials,
+  openEngineGate,
+  setEngineCredentialRefresher,
+  setEngineLaunchFenceEscalator,
+} from "./credential-consumers/engines";
+import {
+  cleanupPersonCredentials,
+  closePersonCredentialGate,
+  drainPersonCredentialWrites,
+  holdTaskGitCredentials,
+  installPersonCredentials,
+  openPersonCredentialGate,
+  type TaskGitCredentialHold,
+} from "./credential-consumers/github";
+import {
+  cleanupMcpConnections,
+  closeMcpGate,
+  drainLeasedMcpRequests,
+  forgetLeasedMcp,
+  installMcpConnections,
+  leasedMcpFor,
+  openMcpGate,
+  setMcpCredentialRefresher,
+} from "./credential-consumers/mcp";
+import type { ProjectEnvTaskReplicaManager, ProjectEnvTaskReplicaSnapshot } from "./project-env-task-replica";
+import {
+  TaskCredentialCleanupCoordinator,
+  type TaskCredentialCleanupDeps,
+  type TaskCredentialCleanupRequest,
+} from "./task-credential-cleanup";
+
+/** How long a start waits for its grant, installed and opened. */
+export const LAUNCH_READY_DEADLINE_MS = 3 * 60_000;
+/** How long an install waits for the env replica's exact snapshot. */
+export const ENV_READY_TIMEOUT_MS = 60_000;
+/** How often an install looks again for that snapshot. */
+const ENV_READY_POLL_MS = 50;
+/** How long shutdown waits for the holds it aborted and the cleanups under
+ *  way; the next process's recovery sweep finishes what is left. */
+export const SHUTDOWN_DRAIN_MS = 30_000;
+
+/** The task's Files (ADR-154 §12): installed, opened and closed with the
+ *  rest. `close` is synchronous: no new Files admission after it. */
+export interface TaskFilesConsumer {
+  install(request: CredentialInstallRequest): Promise<void>;
+  open(assignment: Readonly<TaskAssignment>, view: TaskCredentialView): void;
+  close(assignment: Readonly<TaskAssignment>): void;
+  closeConsumers(request: TaskCredentialCleanupRequest): Promise<void>;
+}
+
+export interface CloudCredentialRuntimeDeps {
+  runnerKind: "owned" | "task-machine";
+  /** To the cloud, over the authenticated bridge. False: not sent now. */
+  send(frame: LeaseRequestV1Frame | CredentialRefreshFrame | ProjectEnvReplicaV1AckFrame): boolean;
+  /** Every engine of the run (and older) proven stopped in every runtime it
+   *  ran in: the orchestrator's `quiesceLeasedEngineAgents`. */
+  quiesceProcesses(request: TaskCredentialCleanupRequest): Promise<void>;
+  /** The leased MCP connections a task serves changed: rewrite its MCP
+   *  configuration from them. */
+  mcpServedChanged?(taskId: string): void;
+  /** The env replica, once the cloud streams one. Without it, an install
+   *  waits for no env snapshot. */
+  envReplica?: ProjectEnvTaskReplicaManager | null;
+  /** The composer started or stopped serving a task's env (its gate opened
+   *  on a grant's snapshot, or closed): whoever follows the task's env
+   *  (the proxy's trust delivery) looks again. */
+  envServedChanged?(taskId: string): void;
+  files?: TaskFilesConsumer | null;
+  launchDeadlineMs?: number;
+  envReadyTimeoutMs?: number;
+  shutdownDrainMs?: number;
+  /** The first retry's delay for a recovered start whose startup failed. */
+  recoveryRetryBaseMs?: number;
+  registry?: Partial<
+    Pick<TaskCredentialLeaseRegistryOptions, "createSessionReader" | "now" | "setTimer" | "clearTimer" | "randomId" | "onError">
+  >;
+  cleanup?: Pick<TaskCredentialCleanupDeps, "db" | "removePath" | "destroyRuntime">;
+}
+
+/**
+ * Why a start was interrupted, as its signal's reason: Stop, teardown or a
+ * newer run took it over, or the runner is shutting down. The first
+ * interruption sets it and nothing changes it after, so a start Stop
+ * interrupted stays Stop's even when shutdown begins while it unwinds.
+ */
+export class StartInterruptedError extends CredentialLeaseUnavailableError {
+  constructor(readonly by: "takeover" | "shutdown") {
+    super(by === "shutdown" ? "the runner is shutting down" : "this start was interrupted");
+  }
+}
+
+/** One start of a run on cloud credentials, from its admission to its
+ *  outcome: what Stop, a newer run and shutdown interrupt. */
+export interface TaskCredentialStart {
+  readonly assignment: Readonly<TaskAssignment>;
+  /** Aborted when the start is interrupted: its wait and its hold end. */
+  readonly signal: AbortSignal;
+  /** Throws unless the start is still this run's and uninterrupted: called
+   *  right before it publishes its success. */
+  assertCurrent(): void;
+}
+
+/** A start's hold on its installed grant (task-up's clone, provisioning and
+ *  runtime-ready writes). */
+export interface TaskUpHold {
+  readonly assignment: Readonly<TaskAssignment>;
+  /** The committed view the start begins on. */
+  readonly view: TaskCredentialView;
+  /** The owner's GitHub sign-in and SSH key for the clone. */
+  readonly git: TaskGitCredentialHold;
+  /** Aborted when the gate closes for the run, a cleanup reaches it or the
+   *  start is interrupted: stop at once, then release. */
+  readonly signal: AbortSignal;
+  /** The start no longer uses the grant. Idempotent. */
+  release(): void;
+}
+
+/** A recovered run (resumeRecovered). */
+interface RecoveredRun {
+  readonly runEpoch: number;
+  /** Aborted when the recovery ends: Stop, teardown, a newer run, shutdown. */
+  readonly controller: AbortController;
+  /** Its runtime's setup, until its gate first opens. */
+  setup: ((signal: AbortSignal) => Promise<void> | void) | null;
+  /** The setup under way. */
+  running: Promise<void> | null;
+  /** A startup retry pending. */
+  retry: ReturnType<typeof setTimeout> | null;
+  attempts: number;
+  /** An ordinary start of the run took its startup over. */
+  yielded: boolean;
+}
+
+/** A recovered start whose startup cleanup failed is tried again after
+ *  this, doubling to the most. */
+const RECOVERY_RETRY_BASE_MS = 1_000;
+const RECOVERY_RETRY_MAX_MS = 30_000;
+
+interface StartEntry {
+  start: TaskCredentialStart;
+  controller: AbortController;
+  /** The registry's startup for the run: its predecessor's cleanup, then
+   *  its request. */
+  started: Promise<void>;
+}
+
+interface HoldEntry {
+  runEpoch: number;
+  controller: AbortController;
+  released: Promise<void>;
+}
+
+const CLEANUP_CLASSES = ["engine", "mcp", "github", "ssh"] as const;
+
+/** What a consumer's cleanup reads: the registry's own request, exact. One
+ *  the coordinator starts itself (boot recovery) is whole-task; were one
+ *  ever selective, it would take every revision of the ids it names. */
+function consumerCleanupRequest(request: TaskCredentialCleanupRequest): CredentialCleanupRequest {
+  if (request.lease) return request.lease;
+  const assignment = { taskId: request.taskId, runEpoch: request.runEpoch };
+  if (request.credentialIds !== undefined) {
+    return {
+      assignment,
+      mode: "selective",
+      reason: "lease-replaced",
+      removed: request.credentialIds.flatMap((credentialId) =>
+        CLEANUP_CLASSES.map((kind) => ({ class: kind, credentialId, revision: null })),
+      ),
+      removeFilesSession: false,
+    };
+  }
+  const reason: CredentialCleanupReason =
+    request.reason === "teardown" ? "teardown" : request.reason === "stop" ? "stopped" : request.reason === "park" ? "parked" : "recovery";
+  return { assignment, mode: "full", reason, removed: [], removeFilesSession: false };
+}
+
+/** Every outcome settled, then the first failure. */
+async function allSettledThenFirstFailure(work: readonly Promise<unknown>[]): Promise<void> {
+  const results = await Promise.allSettled(work);
+  const failure = results.find((result): result is PromiseRejectedResult => result.status === "rejected");
+  if (failure) throw failure.reason;
+}
+
+export class CloudCredentialRuntime {
+  readonly registry: TaskCredentialLeaseRegistry;
+  readonly coordinator: TaskCredentialCleanupCoordinator;
+  readonly #deps: CloudCredentialRuntimeDeps;
+  /** The run each task's gate is open for: what is served, held and read. */
+  readonly #open = new Map<string, number>();
+  /** Running tasks that survived the restart, being recovered (§5, §6). */
+  readonly #recovered = new Map<string, RecoveredRun>();
+  /** The env replica's reader generation each committed view was installed
+   *  with, and the one each open task serves from: a snapshot another
+   *  grant's reader produced is never served for it. */
+  readonly #generations = new WeakMap<TaskCredentialView, number>();
+  readonly #served = new Map<string, number>();
+  /** Each task's run begun here and not yet closed. */
+  readonly #assignments = new Map<string, TaskAssignment>();
+  /** The newest run of each task ever begun: an older one never begins. */
+  readonly #floors = new Map<string, number>();
+  /** Each task's starts not yet ended, and its newest start: the one start
+   *  with any authority, while it is uninterrupted and not ended. */
+  readonly #starts = new Map<string, Set<StartEntry>>();
+  readonly #newest = new Map<string, StartEntry>();
+  readonly #holds = new Map<string, Set<HoldEntry>>();
+  /** Launch waits under way, aborted at shutdown. */
+  readonly #waits = new Set<AbortController>();
+  /** Closes under way, by run and reason: a second asks for the same one. */
+  readonly #closing = new Map<string, Promise<void>>();
+  /** Work accepted and not yet over (startups, frames, installs, holds not
+   *  yet released, closes): what shutdown joins. */
+  readonly #work = new Set<Promise<unknown>>();
+  readonly #setTimer: typeof setTimeout;
+  readonly #clearTimer: typeof clearTimeout;
+  #shutDown = false;
+  #shutdown: Promise<void> | null = null;
+
+  constructor(deps: CloudCredentialRuntimeDeps) {
+    this.#deps = deps;
+    this.#setTimer = deps.registry?.setTimer ?? setTimeout;
+    this.#clearTimer = deps.registry?.clearTimer ?? clearTimeout;
+    this.coordinator = new TaskCredentialCleanupCoordinator({
+      ...deps.cleanup,
+      hooks: {
+        // The registry closes its gate before any cleanup it asks for, a
+        // selective one included; closing again is idempotent, and is what
+        // closes it for one the coordinator starts itself (boot recovery).
+        refuseNewWork: (request) => this.#close({ taskId: request.taskId, runEpoch: request.runEpoch }),
+        closeGate: (request) => this.#close({ taskId: request.taskId, runEpoch: request.runEpoch }),
+        drainWriters: (request) => this.#drainWriters(request),
+        closeConsumers: (request) => this.#closeConsumers(request),
+        quiesceProcesses: (request) => deps.quiesceProcesses(request),
+        dropMemory: (request) => this.#dropMemory(request),
+      },
+    });
+    this.registry = new TaskCredentialLeaseRegistry({
+      ...deps.registry,
+      runnerKind: deps.runnerKind,
+      send: (frame) => deps.send(frame),
+      closeTaskGate: (assignment) => this.#close(assignment),
+      openTaskGate: (assignment, view) => this.#openGate(assignment, view),
+      installExternal: (request) => this.#track(this.#install(request)),
+      // From a grant's acceptance until its own reader is active, the env
+      // replica refuses the run's frames for the cloud to retry: none is
+      // read through the reader the grant replaces.
+      grantAccepted: (assignment) => {
+        const replica = deps.envReplica;
+        if (!replica || this.#shutDown) return;
+        const token = replica.expectActivation(assignment.taskId, assignment.runEpoch);
+        return () => replica.settleActivation(assignment.taskId, token);
+      },
+      // The registry's own request rides through the coordinator, exact.
+      cleanupExternal: (request) =>
+        this.coordinator.cleanup({ ...adaptLeaseCredentialCleanupRequest(request, deps.runnerKind), lease: request }),
+    });
+    // A reactive refresh is the cloud broker's, through the task's lease.
+    setEngineCredentialRefresher((assignment, credentialId, revision) =>
+      this.registry.requestCredentialRefresh(assignment, credentialId, revision),
+    );
+    // A launch slot that could not be retired in its runtime: the run's full
+    // cleanup, its gate closed first (ADR-154 §6).
+    setEngineLaunchFenceEscalator((assignment) => this.registry.escalateLaunchFence(assignment));
+    setMcpCredentialRefresher((assignment, credentialId, revision) =>
+      this.registry.requestCredentialRefresh(assignment, credentialId, revision),
+    );
+  }
+
+  // -- task-up's seam -----------------------------------------------------------
+
+  /**
+   * Phase one, synchronously, in the short admission transaction: the exact
+   * run's startup begins (an older run of the task ended, the gate closed,
+   * the task on cloud credentials from here) and its start's token comes
+   * back. Every earlier start of the task, of this run or an older one, is
+   * interrupted: one start has authority. Throws for a run older than the
+   * task's newest, or after shutdown.
+   */
+  beginStart(assignment: TaskAssignment, options: { recovery?: boolean } = {}): TaskCredentialStart {
+    if (this.#shutDown) throw new StartInterruptedError("shutdown");
+    if ((this.#floors.get(assignment.taskId) ?? 0) > assignment.runEpoch) {
+      throw new CredentialLeaseUnavailableError("a newer run of this task has begun");
+    }
+    this.#floors.set(assignment.taskId, assignment.runEpoch);
+    this.#interruptTokens(assignment.taskId, {});
+    // A newer run's start ends an older run's recovery; an ordinary start of
+    // the run being recovered takes its startup over.
+    this.#endRecovery(assignment.taskId, assignment.runEpoch - 1);
+    if (!options.recovery) this.#yieldRecovery(assignment.taskId, assignment.runEpoch);
+    const controller = new AbortController();
+    const frozen = Object.freeze({ ...assignment });
+    const entry = { controller, started: Promise.resolve() } as StartEntry;
+    entry.start = {
+      assignment: frozen,
+      signal: controller.signal,
+      assertCurrent: () => {
+        if (!this.#live(entry.start)) throw new CredentialLeaseUnavailableError("this start was interrupted");
+        if (this.#open.get(frozen.taskId) !== frozen.runEpoch) {
+          throw new CredentialLeaseUnavailableError("the task's credentials are no longer open for this run");
+        }
+      },
+    };
+    const entries = this.#starts.get(assignment.taskId) ?? new Set<StartEntry>();
+    entries.add(entry);
+    this.#starts.set(assignment.taskId, entries);
+    this.#newest.set(assignment.taskId, entry);
+    this.#assignments.set(assignment.taskId, { ...assignment });
+    // The registry's synchronous part runs here, in the slot; its cleanup
+    // and request are awaited outside it.
+    entry.started = this.#track(this.registry.startAssignment(assignment, options));
+    entry.started.catch(() => {});
+    return entry.start;
+  }
+
+  /**
+   * Synchronously interrupt starts of the task: `exact` the one run (a
+   * manual Stop or teardown), `below` every older run, or all. Their waits
+   * reject and their holds abort. Callers interrupt before they queue for a
+   * lifecycle slot, so a wedged start holding it ends instead of making them
+   * wait. An idle Stop does not interrupt: a start vetoes it.
+   */
+  interruptStarts(taskId: string, which: { exact?: number; below?: number } = {}): void {
+    this.#interruptTokens(taskId, which);
+    // A recovery of the run ends with it, ended start token or not.
+    const recovered = this.#recovered.get(taskId);
+    if (
+      recovered &&
+      (which.exact === undefined || recovered.runEpoch === which.exact) &&
+      (which.below === undefined || recovered.runEpoch < which.below)
+    ) {
+      this.#endRecovery(taskId, recovered.runEpoch);
+    }
+  }
+
+  #interruptTokens(taskId: string, which: { exact?: number; below?: number }): void {
+    for (const entry of this.#starts.get(taskId) ?? []) {
+      const runEpoch = entry.start.assignment.runEpoch;
+      if (which.exact !== undefined && runEpoch !== which.exact) continue;
+      if (which.below !== undefined && runEpoch >= which.below) continue;
+      entry.controller.abort(new StartInterruptedError("takeover"));
+    }
+  }
+
+  /** A local Stop: the task's recovery, whatever its run, ends now. One
+   *  only exists for the run this runner owned at its start. */
+  interruptRecovery(taskId: string): void {
+    const recovered = this.#recovered.get(taskId);
+    if (recovered) this.#endRecovery(taskId, recovered.runEpoch);
+  }
+
+  /** The task's newest start, while it has authority: what a local Stop
+   *  snapshots to interrupt later with `interrupt`. */
+  currentStart(taskId: string): TaskCredentialStart | null {
+    const newest = this.#newest.get(taskId);
+    return newest && this.#live(newest.start) ? newest.start : null;
+  }
+
+  /** Interrupt exactly this start, never one that replaced it. */
+  interrupt(start: TaskCredentialStart): void {
+    for (const entry of this.#starts.get(start.assignment.taskId) ?? []) {
+      if (entry.start === start) entry.controller.abort(new StartInterruptedError("takeover"));
+    }
+  }
+
+  /**
+   * Phase two, outside any lifecycle slot: the run's startup done, its
+   * grant installed, its env snapshot exact and its gate open, or a failure.
+   * Bounded: a lease refused or never answered rejects with
+   * `CredentialLeaseUnavailableError` at the deadline, as do an interrupted,
+   * replaced or ended start and shutdown. Holds nothing.
+   */
+  async waitForLaunchReady(start: TaskCredentialStart): Promise<TaskCredentialView> {
+    const entry = this.#live(start);
+    if (!entry) throw new CredentialLeaseUnavailableError("this start was interrupted");
+    const controller = new AbortController();
+    this.#waits.add(controller);
+    const timer = this.#setTimer(
+      () => controller.abort(new CredentialLeaseUnavailableError("the task's credentials did not arrive in time")),
+      this.#deps.launchDeadlineMs ?? LAUNCH_READY_DEADLINE_MS,
+    );
+    const onInterrupt = (): void => controller.abort(start.signal.reason);
+    if (start.signal.aborted) onInterrupt();
+    else start.signal.addEventListener("abort", onInterrupt, { once: true });
+    try {
+      await Promise.race([
+        entry.started,
+        new Promise<never>((_, reject) => {
+          const fail = (): void => reject(controller.signal.reason);
+          if (controller.signal.aborted) fail();
+          else controller.signal.addEventListener("abort", fail, { once: true });
+        }),
+      ]);
+      const view = await this.registry.waitUntilUsable(start.assignment, controller.signal);
+      if (!this.#live(start)) throw new CredentialLeaseUnavailableError("this start was interrupted");
+      return view;
+    } finally {
+      this.#clearTimer(timer);
+      start.signal.removeEventListener("abort", onInterrupt);
+      this.#waits.delete(controller);
+    }
+  }
+
+  /**
+   * Phase three, synchronously, inside the long lifecycle slot after the run
+   * is revalidated: a counted hold on the installed grant, or null unless
+   * the start is the task's newest, uninterrupted and not ended, and the
+   * gate is open for its exact run. Throws `CredentialLeaseUnavailableError`
+   * for a GitHub sign-in or SSH key the manifest names but the lease could
+   * not read: never "none".
+   */
+  tryAcquireTaskUpHold(start: TaskCredentialStart): TaskUpHold | null {
+    if (!this.#live(start)) return null;
+    const assignment = start.assignment;
+    if (this.#open.get(assignment.taskId) !== assignment.runEpoch) return null;
+    const view = this.registry.currentView(assignment);
+    if (!view) return null;
+    const git = holdTaskGitCredentials(assignment);
+    const controller = new AbortController();
+    let settle!: () => void;
+    const entry: HoldEntry = {
+      runEpoch: assignment.runEpoch,
+      controller,
+      released: new Promise<void>((resolve) => {
+        settle = resolve;
+      }),
+    };
+    void this.#track(entry.released);
+    const holds = this.#holds.get(assignment.taskId) ?? new Set<HoldEntry>();
+    holds.add(entry);
+    this.#holds.set(assignment.taskId, holds);
+    let released = false;
+    return {
+      assignment,
+      view,
+      git,
+      signal: AbortSignal.any([controller.signal, git.signal, start.signal]),
+      release: () => {
+        if (released) return;
+        released = true;
+        git.release();
+        holds.delete(entry);
+        if (holds.size === 0 && this.#holds.get(assignment.taskId) === holds) this.#holds.delete(assignment.taskId);
+        settle();
+      },
+    };
+  }
+
+  /** The start is over, successful or not: its token has no authority left,
+   *  and it is no longer the task's newest. */
+  endStart(start: TaskCredentialStart): void {
+    const taskId = start.assignment.taskId;
+    if (this.#newest.get(taskId)?.start === start) this.#newest.delete(taskId);
+    const entries = this.#starts.get(taskId);
+    if (!entries) return;
+    for (const entry of entries) {
+      if (entry.start === start) entries.delete(entry);
+    }
+    if (entries.size === 0 && this.#starts.get(taskId) === entries) this.#starts.delete(taskId);
+  }
+
+  /** A start that failed after its assignment began: its hold released
+   *  first (the cleanup waits for it), then the exact run closed and every
+   *  credential it installed cleaned up. Only a start not yet ended that is
+   *  still the task's newest closes: an older start's delayed failure, or
+   *  one reported after the start ended, never closes what is running. */
+  async abandonStart(start: TaskCredentialStart, hold?: TaskUpHold | null): Promise<void> {
+    hold?.release();
+    const taskId = start.assignment.taskId;
+    const newest = this.#newest.get(taskId);
+    const closes = newest?.start === start && (this.#starts.get(taskId)?.has(newest) ?? false);
+    this.endStart(start);
+    if (closes) await this.closeAssignment(start.assignment, "stop");
+  }
+
+  /** The run ended (stop, park, teardown) or was superseded: close it and
+   *  clean up everything it installed. After shutdown, no cleanup starts:
+   *  one already under way for the run and reason is joined. */
+  closeAssignment(
+    assignment: TaskAssignment,
+    reason: "stop" | "park" | "teardown" | "superseded" = "stop",
+  ): Promise<void> {
+    this.#endRecovery(assignment.taskId, assignment.runEpoch);
+    const key = `${assignment.taskId}\u0000${String(assignment.runEpoch)}\u0000${reason}`;
+    const under = this.#closing.get(key);
+    if (under) return under;
+    if (this.#shutDown) return Promise.resolve();
+    const current = this.#assignments.get(assignment.taskId);
+    if (current && current.runEpoch === assignment.runEpoch) this.#assignments.delete(assignment.taskId);
+    const closing = this.#track(this.registry.closeAssignment(assignment, reason)).finally(() => {
+      if (this.#closing.get(key) === closing) this.#closing.delete(key);
+    });
+    this.#closing.set(key, closing);
+    return closing;
+  }
+
+  /** What the task's env serves for this exact run and project set, while
+   *  its gate is open, from the snapshot its open grant's own reader
+   *  produced; null otherwise (a candidate grant's snapshot included, until
+   *  that grant is the open one). */
+  envSnapshot(assignment: TaskAssignment, projectIds: readonly string[]): ProjectEnvTaskReplicaSnapshot | null {
+    if (this.#open.get(assignment.taskId) !== assignment.runEpoch) return null;
+    const generation = this.#served.get(assignment.taskId);
+    if (generation === undefined) return null;
+    return this.#deps.envReplica?.readViewAt(assignment.taskId, assignment.runEpoch, projectIds, generation) ?? null;
+  }
+
+  /**
+   * Boot recovery (ADR-154 §5, §6): every task with cleanup left from the
+   * last process (a crash, a cleanup that failed) cleaned up, at the newer
+   * of its recorded and its admitted run, before any grant is asked for.
+   * Every task is attempted; then any failure fails the sweep.
+   */
+  recover(runEpochForTask: (taskId: string) => number): Promise<void> {
+    if (this.#shutDown) return Promise.reject(new StartInterruptedError("shutdown"));
+    return this.#track(this.coordinator.recoverAll(runEpochForTask));
+  }
+
+  /**
+   * A running task on cloud credentials that survived the runner's restart:
+   * its run's credentials asked for again, after its startup cleanup, as a
+   * start of its own; a startup cleanup that fails is tried again, with
+   * backoff. Its runtime's `setup` runs once its gate first opens, however
+   * late the grant comes, told by its signal when to stop. Stop, teardown,
+   * a newer run and shutdown end the recovery: no retry, no setup to come,
+   * a setup running told to stop and drained by the run's cleanup.
+   */
+  resumeRecovered(assignment: TaskAssignment, setup?: (signal: AbortSignal) => Promise<void> | void): void {
+    if (this.#shutDown) return;
+    const previous = this.#recovered.get(assignment.taskId);
+    if (previous) this.#endRecovery(assignment.taskId, previous.runEpoch);
+    const recovered: RecoveredRun = {
+      runEpoch: assignment.runEpoch,
+      controller: new AbortController(),
+      setup: setup ?? null,
+      running: null,
+      retry: null,
+      attempts: 0,
+      yielded: false,
+    };
+    this.#recovered.set(assignment.taskId, recovered);
+    this.#attemptRecovery({ ...assignment }, recovered);
+  }
+
+  #attemptRecovery(assignment: TaskAssignment, recovered: RecoveredRun): void {
+    if (!this.#recovering(assignment.taskId, recovered)) return;
+    let start: TaskCredentialStart;
+    try {
+      start = this.beginStart(assignment, { recovery: true });
+    } catch (error) {
+      console.warn(
+        `[credentials] task ${assignment.taskId} run ${String(assignment.runEpoch)}: no recovered start: ${
+          error instanceof Error ? error.message : String(error)
+        }`,
+      );
+      return;
+    }
+    const started = this.#newest.get(assignment.taskId)!.started;
+    void this.#track(
+      started.then(
+        // Ended once usable or its bounded wait gives up: the registry keeps
+        // asking for a grant it has had no answer to.
+        () => this.waitForLaunchReady(start).then(
+          () => this.endStart(start),
+          () => this.endStart(start),
+        ),
+        // Its startup cleanup failed: again, with backoff, while this run's
+        // recovery goes on.
+        () => {
+          this.endStart(start);
+          if (start.signal.aborted || !this.#recovering(assignment.taskId, recovered)) return;
+          const base = this.#deps.recoveryRetryBaseMs ?? RECOVERY_RETRY_BASE_MS;
+          const delay = Math.min(RECOVERY_RETRY_MAX_MS, base * 2 ** recovered.attempts);
+          recovered.attempts += 1;
+          recovered.retry = this.#setTimer(() => {
+            recovered.retry = null;
+            this.#attemptRecovery(assignment, recovered);
+          }, delay);
+        },
+      ),
+    );
+  }
+
+  /** Whether the recovery itself still drives its run's startup. */
+  #recovering(taskId: string, recovered: RecoveredRun): boolean {
+    return (
+      !this.#shutDown &&
+      !recovered.controller.signal.aborted &&
+      !recovered.yielded &&
+      this.#recovered.get(taskId) === recovered
+    );
+  }
+
+  /** An ordinary start of the run being recovered (task-up driven again,
+   *  a lost response's retry): its own startup does the cleanup and asks
+   *  for the grant from here, so the recovery makes no attempt again and
+   *  a retry pending is cancelled. Its runtime's setup still runs when the
+   *  gate first opens. */
+  #yieldRecovery(taskId: string, runEpoch: number): void {
+    const recovered = this.#recovered.get(taskId);
+    if (!recovered || recovered.runEpoch !== runEpoch) return;
+    recovered.yielded = true;
+    if (recovered.retry !== null) this.#clearTimer(recovered.retry);
+    recovered.retry = null;
+  }
+
+  /** The task's recovery of a run through `throughRunEpoch` ends: no retry,
+   *  no setup to come, a running one told to stop (the run's cleanup drains
+   *  it). Never the recovery's own startup cleanup's doing. */
+  #endRecovery(taskId: string, throughRunEpoch: number): void {
+    const recovered = this.#recovered.get(taskId);
+    if (!recovered || recovered.runEpoch > throughRunEpoch) return;
+    recovered.controller.abort(new StartInterruptedError("takeover"));
+    if (recovered.retry !== null) this.#clearTimer(recovered.retry);
+    recovered.retry = null;
+    recovered.setup = null;
+    if (recovered.running === null) this.#recovered.delete(taskId);
+  }
+
+  /** A recovered run's gate opened: its recovery is done, and its setup
+   *  runs, once, in the background, its failure its own. */
+  #runRecoveredSetup(assignment: Readonly<TaskAssignment>): void {
+    const recovered = this.#recovered.get(assignment.taskId);
+    if (!recovered || recovered.runEpoch !== assignment.runEpoch || recovered.running !== null) return;
+    if (recovered.controller.signal.aborted) return;
+    if (recovered.retry !== null) this.#clearTimer(recovered.retry);
+    recovered.retry = null;
+    const setup = recovered.setup;
+    recovered.setup = null;
+    if (setup === null) {
+      this.#recovered.delete(assignment.taskId);
+      return;
+    }
+    const signal = recovered.controller.signal;
+    recovered.running = this.#track(
+      Promise.resolve()
+        .then(() => setup(signal))
+        .catch((error: unknown) =>
+          console.warn(
+            `[credentials] task ${assignment.taskId} run ${String(assignment.runEpoch)}: recovered runtime setup failed: ${
+              error instanceof Error ? error.message : String(error)
+            }`,
+          ),
+        )
+        .finally(() => {
+          recovered.running = null;
+          if (this.#recovered.get(assignment.taskId) === recovered) this.#recovered.delete(assignment.taskId);
+        }),
+    );
+  }
+
+  /** Every frame, startup, install and close under way now, settled. */
+  async settled(): Promise<void> {
+    await Promise.allSettled([...this.#work]);
+  }
+
+  // -- frames -------------------------------------------------------------------
+
+  /**
+   * A frame from the cloud that is this runtime's: true when handled. A
+   * `lease.deny` is offered to the registry, which ignores one it did not
+   * ask for, and reported unhandled so ADR-133's lease manager sees it too.
+   * A lease frame's handling goes on after this returns: a grant's install
+   * waits for the env frames that follow it, so a dispatcher that awaits each
+   * frame before the next must not wait on it. After shutdown, every v1
+   * frame is taken and ignored.
+   */
+  async handleFrame(frame: { kind: string }): Promise<boolean> {
+    const replica = this.#deps.envReplica;
+    const isReplicaFrame =
+      isProjectEnvReplicaV1BeginFrame(frame) ||
+      isProjectEnvReplicaV1PageFrame(frame) ||
+      isProjectEnvReplicaV1ApplyFrame(frame) ||
+      isProjectEnvReplicaV1CommitFrame(frame);
+    if (this.#shutDown) return frame.kind !== "lease.deny" && (V1_FRAME_KINDS.has(frame.kind) || isReplicaFrame);
+    switch (frame.kind) {
+      case "lease.grant.v1":
+        this.#background(this.registry.handleGrant(frame));
+        return true;
+      case "lease.unavailable":
+        this.registry.handleUnavailable(frame);
+        return true;
+      case "lease.deny":
+        this.#background(this.registry.handleDenied(frame));
+        return false;
+      case "lease.stale":
+        this.registry.handleStale(frame);
+        return true;
+      case "credential.changed":
+        this.#background(this.registry.handleCredentialChanged(frame));
+        return true;
+      case "credential.refresh.result":
+        this.#background(this.registry.handleRefreshResult(frame));
+        return true;
+    }
+    if (replica && isReplicaFrame) {
+      const ack = await this.#track(replica.handleFrame(frame));
+      if (!this.#shutDown) this.#deps.send(ack);
+      return true;
+    }
+    return false;
+  }
+
+  /** Same-process authenticated reconnect: keep safe bundles, ask again. */
+  bridgeAuthenticated(): void {
+    if (!this.#shutDown) this.registry.bridgeAuthenticated();
+  }
+
+  /**
+   * Process shutdown. Nothing is taken after it: no frame, start, open,
+   * install or close. Every assignment ends first, its gate closed (env,
+   * engines, MCP, the owner's Git credentials and Files), so that anything
+   * the aborts below set off finds no run to clean up; then every start,
+   * wait and hold is aborted, and the env replica's values leave memory.
+   * What was accepted before is joined, bounded: startups and the cleanups
+   * they began, frames, installs, holds being released, closes, and any
+   * cleanup those register meanwhile. None is started: agents that survive
+   * the restart keep what they hold (ADR-154 §5), and the next process's
+   * recovery sweep cleans up before it reattaches them.
+   */
+  shutdown(): Promise<void> {
+    this.#shutdown ??= this.#shutDownNow();
+    return this.#shutdown;
+  }
+
+  async #shutDownNow(): Promise<void> {
+    this.#shutDown = true;
+    const ended = this.registry.shutdown();
+    // A run the registry no longer tracks (its close under way) closes too.
+    for (const assignment of this.#assignments.values()) {
+      try {
+        this.#close(assignment);
+      } catch {
+        // Each fence was attempted; the process is ending.
+      }
+    }
+    this.#assignments.clear();
+    const reason = new StartInterruptedError("shutdown");
+    for (const controller of this.#waits) controller.abort(reason);
+    for (const entries of this.#starts.values()) for (const entry of entries) entry.controller.abort(reason);
+    for (const holds of this.#holds.values()) for (const entry of holds) entry.controller.abort(reason);
+    this.#scrub();
+    let bound: ReturnType<typeof setTimeout> | undefined;
+    await Promise.race([
+      this.#drain(ended),
+      new Promise<void>((resolve) => {
+        bound = this.#setTimer(resolve, this.#deps.shutdownDrainMs ?? SHUTDOWN_DRAIN_MS);
+      }),
+    ]);
+    this.#clearTimer(bound);
+    // Whatever the drain finished leaves memory too.
+    this.#scrub();
+  }
+
+  /** Until no accepted work is left, no consumer's write or request is
+   *  under way (one already admitted would otherwise land after the process
+   *  exits, behind the next process's recovery cleanup), and no cleanup is
+   *  under way. */
+  async #drain(ended: Promise<void>): Promise<void> {
+    await ended;
+    for (;;) {
+      await Promise.allSettled([
+        ...this.#work,
+        drainLeasedEngineWrites(),
+        drainPersonCredentialWrites(),
+        drainLeasedMcpRequests(),
+      ]);
+      await this.coordinator.whenIdle();
+      if (this.#work.size === 0) return;
+    }
+  }
+
+  #scrub(): void {
+    for (const [taskId, recovered] of [...this.#recovered]) this.#endRecovery(taskId, recovered.runEpoch);
+    this.#deps.envReplica?.dropAll();
+    setEngineCredentialRefresher(null);
+    setEngineLaunchFenceEscalator(null);
+    setMcpCredentialRefresher(null);
+  }
+
+  // -- the registry's gate and install --------------------------------------------
+
+  /** The start's entry, while it is the task's newest, uninterrupted and not
+   *  ended; null for any other token, a forged one included. */
+  #live(start: TaskCredentialStart): StartEntry | null {
+    if (this.#shutDown) return null;
+    const entry = this.#newest.get(start.assignment.taskId);
+    if (!entry || entry.start !== start || entry.controller.signal.aborted) return null;
+    return this.#starts.get(start.assignment.taskId)?.has(entry) ? entry : null;
+  }
+
+  #track<T>(work: Promise<T>): Promise<T> {
+    this.#work.add(work);
+    const forget = (): void => void this.#work.delete(work);
+    work.then(forget, forget);
+    return work;
+  }
+
+  /** Work no caller awaits: its owner reports its own failures. */
+  #background(work: Promise<unknown>): void {
+    this.#track(work).catch(() => {});
+  }
+
+  #openGate(assignment: Readonly<TaskAssignment>, view: TaskCredentialView): void {
+    if (this.#shutDown) throw new CredentialLeaseUnavailableError("the runner is shutting down");
+    openEngineGate(assignment, view);
+    openMcpGate(assignment, view);
+    openPersonCredentialGate(assignment, view);
+    this.#deps.files?.open(assignment, view);
+    this.#open.set(assignment.taskId, assignment.runEpoch);
+    const generation = this.#generations.get(view);
+    if (generation === undefined) this.#served.delete(assignment.taskId);
+    else this.#served.set(assignment.taskId, generation);
+    this.#deps.mcpServedChanged?.(assignment.taskId);
+    this.#envServedChanged(assignment.taskId);
+    this.#runRecoveredSetup(assignment);
+  }
+
+  #envServedChanged(taskId: string): void {
+    try {
+      this.#deps.envServedChanged?.(taskId);
+    } catch {
+      // Serving already changed; a follower's failure is its own.
+    }
+  }
+
+  /**
+   * Synchronous and idempotent: every consumer of this run (and older)
+   * stops being served, env and Files included, and every start holding its
+   * grant is aborted. Each fence is attempted even when another throws;
+   * then the first failure is thrown, or all of them together.
+   */
+  #close(assignment: Readonly<TaskAssignment>): void {
+    const failures: unknown[] = [];
+    const attempt = (fence: () => void): void => {
+      try {
+        fence();
+      } catch (error) {
+        failures.push(error);
+      }
+    };
+    // Env serving closes first, and cannot throw.
+    const open = this.#open.get(assignment.taskId);
+    if (open !== undefined && open <= assignment.runEpoch) {
+      this.#open.delete(assignment.taskId);
+      this.#served.delete(assignment.taskId);
+      this.#envServedChanged(assignment.taskId);
+    }
+    for (const entry of this.#holds.get(assignment.taskId) ?? []) {
+      if (entry.runEpoch <= assignment.runEpoch) {
+        attempt(() => entry.controller.abort(new CredentialLeaseUnavailableError("the task's credentials were withdrawn")));
+      }
+    }
+    attempt(() => closeEngineGate(assignment));
+    attempt(() => {
+      const mcp = leasedMcpFor(assignment.taskId);
+      if (mcp && mcp.assignment.runEpoch <= assignment.runEpoch) closeMcpGate(mcp.assignment);
+    });
+    attempt(() => closePersonCredentialGate(assignment));
+    attempt(() => this.#deps.files?.close(assignment));
+    if (failures.length === 1) throw failures[0];
+    if (failures.length > 1) throw new AggregateError(failures, "the task's credentials could not all be closed");
+  }
+
+  /**
+   * The registry's install of a grant or credential change, fenced after
+   * every step: once the runner is shutting down, or the grant can no longer
+   * commit, nothing more is installed.
+   */
+  async #install(request: CredentialInstallRequest): Promise<void> {
+    this.#assertInstallCurrent(request);
+    const replica = this.#deps.envReplica;
+    if (replica && request.reason === "grant") {
+      const { taskId, runEpoch } = request.assignment;
+      // The grant's reader first: the cloud's env frames for it are taken
+      // from here.
+      replica.activateAssignment({ taskId, runEpoch, orgId: request.owner.orgId, reader: request.projectValues });
+      const generation = replica.readerGeneration(taskId, runEpoch);
+      if (generation === null) throw new CredentialLeaseUnavailableError("the task's env replica is not active");
+      this.#generations.set(request.next, generation);
+      // Every grant waits for a snapshot of its exact project set read
+      // entirely through its own reader: never one another grant read.
+      await this.#awaitEnvSnapshot(request, generation);
+      this.#assertInstallCurrent(request);
+    } else if (request.reason === "credential-change" && request.previous) {
+      // The same grant, the same reader: its view serves the same snapshot.
+      const generation = this.#generations.get(request.previous);
+      if (generation !== undefined) this.#generations.set(request.next, generation);
+    }
+    await installEngineCredentials(request);
+    this.#assertInstallCurrent(request);
+    installMcpConnections(request);
+    installPersonCredentials(request);
+    await this.#deps.files?.install(request);
+    this.#assertInstallCurrent(request);
+  }
+
+  #assertInstallCurrent(request: CredentialInstallRequest): void {
+    if (this.#shutDown) throw new CredentialLeaseUnavailableError("the runner is shutting down");
+    if (!request.isCurrent()) {
+      throw new CredentialLeaseUnavailableError("the grant was withdrawn while it was being installed");
+    }
+  }
+
+  /** Until the replica serves the exact project set this grant names, from
+   *  a snapshot read through this grant's own reader: never an older ready
+   *  snapshot of another set, nor one another grant's reader produced.
+   *  Bounded, and given up as soon as the grant can no longer commit. */
+  async #awaitEnvSnapshot(request: CredentialInstallRequest, generation: number): Promise<void> {
+    const replica = this.#deps.envReplica!;
+    const { taskId, runEpoch } = request.assignment;
+    let expired = false;
+    const deadline = this.#setTimer(() => {
+      expired = true;
+    }, this.#deps.envReadyTimeoutMs ?? ENV_READY_TIMEOUT_MS);
+    try {
+      while (replica.readViewAt(taskId, runEpoch, request.next.projectIds, generation) === null) {
+        if (!request.isCurrent() || this.#shutDown) {
+          throw new CredentialLeaseUnavailableError("the grant was withdrawn while its env values were awaited");
+        }
+        if (expired) throw new CredentialLeaseUnavailableError("the task's env values did not arrive in time");
+        await new Promise<void>((resolve) => this.#setTimer(resolve, ENV_READY_POLL_MS));
+      }
+    } finally {
+      this.#clearTimer(deadline);
+    }
+  }
+
+  // -- the cleanup coordinator's hooks --------------------------------------------
+
+  /** Every start holding the run's grant (or an older one's) ends, and every
+   *  consumer's writes and requests under way end, all of them settled,
+   *  before anything is deleted. */
+  async #drainWriters(request: TaskCredentialCleanupRequest): Promise<void> {
+    const holds = [...(this.#holds.get(request.taskId) ?? [])].filter((entry) => entry.runEpoch <= request.runEpoch);
+    for (const entry of holds) {
+      entry.controller.abort(new CredentialLeaseUnavailableError("the task's credentials are being cleaned up"));
+    }
+    // A recovered run's setup under way (root writes in its runtime) stops
+    // and is joined by its run's full cleanup. Its own startup cleanup comes
+    // before its gate opens, so before any setup runs.
+    const recovered = this.#recovered.get(request.taskId);
+    const setups: Promise<void>[] = [];
+    if (request.credentialIds === undefined && recovered && recovered.runEpoch <= request.runEpoch && recovered.running) {
+      recovered.controller.abort(new StartInterruptedError("takeover"));
+      setups.push(recovered.running);
+    }
+    const consumer = consumerCleanupRequest(request);
+    await allSettledThenFirstFailure([
+      ...setups,
+      ...holds.map((entry) => entry.released),
+      cleanupEngineCredentials(consumer),
+      cleanupPersonCredentials(consumer),
+      cleanupMcpConnections(consumer),
+    ]);
+  }
+
+  async #closeConsumers(request: TaskCredentialCleanupRequest): Promise<void> {
+    await this.#deps.files?.closeConsumers(request);
+  }
+
+  /** What the cleanup deleted leaves memory too. A whole-task cleanup drops
+   *  the env replica's values; a teardown forgets the task entirely. */
+  #dropMemory(request: TaskCredentialCleanupRequest): void {
+    if (request.credentialIds !== undefined) return;
+    this.#deps.envReplica?.dropAssignment(request.taskId, request.runEpoch);
+    if (request.reason !== "teardown") return;
+    forgetLeasedEngines({ taskId: request.taskId, runEpoch: request.runEpoch });
+    const mcp = leasedMcpFor(request.taskId);
+    if (mcp && mcp.assignment.runEpoch <= request.runEpoch) forgetLeasedMcp(request.taskId);
+  }
+}
+
+/** The v1 lease frames this runtime owns. */
+const V1_FRAME_KINDS = new Set([
+  "lease.grant.v1",
+  "lease.unavailable",
+  "lease.stale",
+  "credential.changed",
+  "credential.refresh.result",
+]);