@coreplane/switchboard 0.0.0 → 1.18.0

package/dist/assets/src/execution/residentReadonly.ts

@@ -0,0 +1,72 @@
+/** The read-only attach decision of the resident Worker's `attachThread`
+ *  (deploy/cloudflare-resident/worker.ts), kept pure and dependency-free so it
+ *  is unit-testable from src/ and imported across packages by the resident
+ *  Worker (like residentDetach / shellQuote) — the tested code IS the shipped
+ *  code.
+ *
+ *  Background: a review run — read-only "by convention" — held a per-attach
+ *  GitHub credential file and an `origin` pointing at GitHub, fetched another
+ *  PR's branch because the PR body referenced it, and its verdict landed on
+ *  the wrong PR (the reviewed-head guard on the post step is the other half
+ *  of the fix). This removes the capability: a read-only attach gets no
+ *  credential file and an
+ *  `origin` it cannot fetch from, so "read-only" is enforced by the worktree,
+ *  not requested of the model.
+ *
+ *  Mode is sticky per attach, not per thread: the binding records the mode
+ *  the tree was last built for, and an attach in the OTHER mode recreates the
+ *  tree (a credential-less, mirror-origin tree must never be handed to a
+ *  writable run, and a writable tree's credential file must never survive
+ *  into a read-only run). Recreating is the simple safe option — the only
+ *  state a tree carries between attaches is scratch files, and a mode switch
+ *  on one thread is rare (a thread is normally one agent for its whole life). */
+
+export type ParsedReadonly = { readonly: boolean } | { error: string };
+
+/** `/attach` body field `readonly`: absent → false (older bots never send it);
+ *  a boolean → itself; anything else → a 400-shaped error. */
+export function parseReadonly(value: unknown): ParsedReadonly {
+  if (value === undefined) return { readonly: false };
+  if (typeof value === "boolean") return { readonly: value };
+  return { error: "readonly must be a boolean when present" };
+}
+
+export interface ReadonlyAttachPlan {
+  /** The mode this attach builds/reuses the tree for (recorded on the binding). */
+  readonly: boolean;
+  /** True when the existing tree was built for the other mode and must be wiped
+   *  before reuse — evaluated before the ordinary dirty/stale checks. */
+  modeSwitch: boolean;
+  /** Mint a repo-scoped token for the tree and write the credential file (writable only). A read-only attach never puts a token in the tree; the mirror's own recovery fetch (root, outside the tree) may still mint one. */
+  credentialFile: boolean;
+  /** Remove any credential file / helper config from the tree (read-only, every
+   *  attach — a reused tree may predate this rule). */
+  scrubCredentials: boolean;
+  /** What the worktree's `origin` points at after clone. */
+  originUrl: string;
+}
+
+export function planReadonlyAttach(input: {
+  readonly: boolean;
+  /** The thread's existing binding, if any. `evicted` trees are gone from disk
+   *  and are recreated anyway, so their recorded mode is irrelevant. */
+  prior?: { readonly?: boolean; evicted?: boolean };
+  /** `owner/name` of the repo — the writable origin. */
+  slug: string;
+  /** The resident's bare mirror path — the read-only origin. Thread users are
+   *  denied traversal into it (root:worker1 750), so fetch/push fail while
+   *  the clone-time remote-tracking refs (`origin/<default>`) stay usable for
+   *  `git diff origin/main...HEAD`. */
+  mirrorDir: string;
+}): ReadonlyAttachPlan {
+  const { readonly, prior, slug, mirrorDir } = input;
+  const priorLive = prior !== undefined && !prior.evicted;
+  const modeSwitch = priorLive && (prior.readonly ?? false) !== readonly;
+  return {
+    readonly,
+    modeSwitch,
+    credentialFile: !readonly,
+    scrubCredentials: readonly,
+    originUrl: readonly ? mirrorDir : `https://github.com/${slug}.git`,
+  };
+}

package/dist/assets/src/execution/residentRefresh.ts

@@ -0,0 +1,429 @@
+/** The refresh cycle's rebuild decision for the resident Worker's
+ *  `onRefreshAlarm` (deploy/cloudflare-resident/worker.ts), kept pure and
+ *  dependency-free so it is unit-testable from src/ and imported across
+ *  packages by the resident Worker (like residentDetach) — the tested code IS
+ *  the shipped code.
+ *
+ *  Background: the refresh used to `git clean -fdx` + `npm install` on
+ *  EVERY default-branch advance (a refresh over two minutes long), even
+ *  though the committed lockfile — the dependency cache key — rarely
+ *  moves. A refresh longer than the bot's 60 s attach wait sends runs cold.
+ *  The `-x` clean itself is load-bearing: attached, sha-pinned thread
+ *  worktrees hardlink the checkout's dep/build FILE inodes, so a rebuild must
+ *  allocate fresh inodes rather than write through shared ones (review 1b).
+ *  Skipping install therefore keeps `node_modules` — inodes nobody is about
+ *  to write through — while still cleaning every other gitignored path (build
+ *  output), so the build allocates fresh inodes exactly as before.
+ *
+ *  The disk facts come from two root-only markers the cycle writes as it
+ *  materializes (deps key after install, built sha after build) plus the
+ *  checkout's actual HEAD. A worker-code deploy resets the DO mid-cycle but
+ *  leaves the container disk alone, so the next alarm can pick up where the
+ *  interrupted one stopped instead of redoing the whole rebuild. */
+
+import { DISK_FULL_FREE_KIB, diskFullReason, isDiskFullMessage } from "./residentDisk.js";
+
+export interface RefreshDisk {
+  /** `git rev-parse HEAD` of the warm checkout; null when unreadable. */
+  head: string | null;
+  /** Lockfile key whose install fully completed into the checkout's node_modules; null when absent. */
+  installedKey: string | null;
+  /** Lockfile key an install was STARTED for and never completed (the marker is
+   *  written before `install` and removed after the deps key lands); null when absent. */
+  installingKey: string | null;
+  /** Sha whose build fully completed in the checkout; null when absent. */
+  builtSha: string | null;
+}
+
+/** What the `-x` clean may remove: everything untracked, or everything except `node_modules`. */
+export type CleanScope = "all" | "keep-deps";
+
+export type RefreshPlan =
+  /** The default branch did not move — nothing to rebuild or snapshot. */
+  | { action: "unchanged" }
+  /** Checkout, deps and build already match the target (an interrupted cycle
+   *  got this far): skip straight to the snapshot. */
+  | { action: "reuse"; why: string }
+  /** Update the checkout; `install` only when the committed lockfile moved. */
+  | { action: "rebuild"; install: boolean; clean: CleanScope; why: string };
+
+export function planRefresh(input: {
+  /** Mirror sha of the default branch after the fetch. */
+  sha: string;
+  /** The sha the recorded facts (and the last snapshot) are at. */
+  factsSha: string;
+  /** Committed-lockfile key at `sha` (pure function of the commit). */
+  lockfileKey: string;
+  disk: RefreshDisk;
+}): RefreshPlan {
+  const { sha, factsSha, lockfileKey, disk } = input;
+  if (sha === factsSha) return { action: "unchanged" };
+  const depsMatch = disk.installedKey !== null && disk.installedKey === lockfileKey;
+  if (depsMatch && disk.head === sha && disk.builtSha === sha) {
+    return { action: "reuse", why: `checkout, deps and build already materialized for ${sha.slice(0, 8)}` };
+  }
+  if (depsMatch) {
+    return { action: "rebuild", install: false, clean: "keep-deps", why: "lockfile unchanged — deps kept, build only" };
+  }
+  if (disk.installedKey === null && disk.installingKey !== null && disk.installingKey === lockfileKey) {
+    // A previous cycle started this very install and ended before the deps
+    // key landed (its step timed out, or its DO isolate died; the pre-step
+    // sweep has killed any writer it left). npm reconciles a partial tree to
+    // the lockfile, so resuming converges where wipe-and-restart cannot: a
+    // repo whose cold install outruns one step budget still lands over cycles
+    // (wipe-and-restart runs full installs back to back, none finishing).
+    // Safe for the hardlink invariant (review 1b): threads only link deps
+    // whose key the deps marker vouches for, and no marker vouched for these.
+    return {
+      action: "rebuild",
+      install: true,
+      clean: "keep-deps",
+      why: "install resumes — a previous attempt for this lockfile ended before the deps marker; npm reconciles the partial tree",
+    };
+  }
+  const why = disk.installedKey === null ? "no deps marker on disk — full install" : "lockfile changed — full install";
+  return { action: "rebuild", install: true, clean: "all", why };
+}
+
+/** Tool caches that BUILDS (not installs) write inside node_modules — and
+ *  typically open+truncate in place: babel-loader/eslint/webpack under
+ *  `.cache`, vite/vitest under `.vite`. Kept deps are hardlinked into attached
+ *  worktrees, so these must go before a keep-deps build or it would write
+ *  through shared inodes (review 1b). Pruned at any depth (workspaces). */
+export const NODE_MODULES_CACHE_DIRS = [".cache", ".vite"] as const;
+
+/** The checkout-update shell for the build user (runs inside the checkout):
+ *  fetch from the local mirror, hard-reset to `sha`, then the `-x` clean.
+ *  `-e node_modules` is a git exclude pattern (matches at any depth, so
+ *  workspace packages keep theirs too) that survives `-x`; everything else
+ *  gitignored — build output above all — is still removed so the build
+ *  allocates fresh inodes (review 1b). Keep-deps additionally sweeps the
+ *  build-written caches inside node_modules (NODE_MODULES_CACHE_DIRS). */
+export function checkoutUpdateCommand(sha: string, clean: CleanScope): string {
+  const base = `git fetch --quiet origin && git reset --hard --quiet ${sha}`;
+  if (clean === "all") return `${base} && git clean -fdx`;
+  const names = NODE_MODULES_CACHE_DIRS.map((d) => `-name ${d}`).join(" -o ");
+  const sweep = `find . -path '*/node_modules/*' -type d \\( ${names} \\) -prune -exec rm -rf {} +`;
+  return `${base} && git clean -fdx -e node_modules && ${sweep}`;
+}
+
+// -- interruption vs. failure ------------------------------------------------
+
+/** How a refresh-cycle step failed. `interrupted` is the one outcome that says
+ *  NOTHING about the repository: the step was killed from outside because the
+ *  CONTAINER was replaced under it — an image-changing deploy or an explicit
+ *  container stop/restart. (A Worker-only deploy swaps the DO isolate but
+ *  leaves the container and its processes running, so it cannot interrupt a
+ *  step at all.) The kill surfaces either as the shell's
+ *  own death (SIGTERM, exit 143, "Session terminated") or, past the shell, as
+ *  the SDK's replacement errors (stale process handle, closed supervisor).
+ *  Everything else is the repo's own build failing. */
+export interface RefreshFailure {
+  /** The `degraded` reason to record. Interruptions are prefixed
+   *  `refresh-interrupted:` so the park-streak gate can exclude them by prefix,
+   *  exactly like the watchdog's stamps; a full disk is `disk-full:`
+   *  (`residentDisk.ts` — the cycle's entry gate and the recycle decision key
+   *  on it); real failures keep `<step>-failed:`. */
+  reason: string;
+  interrupted: boolean;
+  diskFull: boolean;
+}
+
+/** Root argv that kills every process the build user still owns and waits
+ *  (bounded, 5 s) until none is left. Runs before EVERY build-user step.
+ *
+ *  Why: an `npm install` that outlives its budget and the SDK's output grace
+ *  is recorded as a timeout while npm keeps extracting into
+ *  CHECKOUT_DIR/node_modules. The next cycle's `git clean -fdx` races it —
+ *  `warning: failed to remove node_modules/<pkg>: Directory not empty` on
+ *  exactly the packages being written — and the resident spirals between
+ *  `checkout-update-failed` and install timeouts (each cycle's install now
+ *  sharing 1 vCPU with the last one's orphan) for as long as the default
+ *  branch keeps moving. A Worker-only deploy is the other way to orphan a
+ *  step: the DO isolate resets, the container keeps running.
+ *
+ *  Scoped to the TREE the step is about to touch (`dir`): a process counts as
+ *  stale when its cwd is `dir` or below it. Steps on one tree are strictly
+ *  sequential, so a live build-user process inside that tree at step start
+ *  is by definition a leftover — while the same user's installs into OTHER
+ *  trees (the deps store runs distinct keys in parallel, item 59) are live
+ *  work this sweep must not touch. Survivors are NAMED on stdout (the Worker
+ *  log shows what was still running) before SIGKILL: SIGTERM would let npm
+ *  keep writing while the clean runs. Nothing matching is the happy path; a
+ *  process that survives SIGKILL for 5 s fails the step — nothing may start
+ *  beside it. */
+export function killStaleBuildProcessesCommand(user: string, dir: string): string[] {
+  const inTree = `case "$(readlink /proc/$p/cwd 2>/dev/null)" in ${dir}|${dir}/*) `;
+  const script =
+    `stale=""; for p in $(pgrep -u ${user}); do ${inTree}stale="$stale$p ";; esac; done; ` +
+    `if [ -n "$stale" ]; then ` +
+    `echo "killing stale ${user} processes under ${dir}:"; ps -o pid=,args= -p $(echo $stale | tr ' ' ',') 2>/dev/null; ` +
+    `kill -KILL $stale 2>/dev/null; ` +
+    `i=0; while :; do alive=""; for p in $stale; do kill -0 $p 2>/dev/null && alive="$alive$p "; done; ` +
+    `[ -z "$alive" ] && break; i=$((i+1)); ` +
+    `if [ $i -ge 50 ]; then echo "stale ${user} processes survived SIGKILL for 5s: $alive" >&2; exit 1; fi; ` +
+    `sleep 0.1; done; ` +
+    `fi`;
+  return ["sh", "-c", script];
+}
+
+/** Signature of a step killed from OUTSIDE its own budget: the exit status of
+ *  SIGTERM (128 + 15), bash's "Session terminated" on a killed login shell, or
+ *  a tool naming the signal. A bare "killed" is NOT enough — compilers and
+ *  OOM messages say it too — and a step the cycle itself timed out is a real
+ *  failure however it died. */
+const INTERRUPTION_SIGNATURE = /\bexit 143\b|Session terminated|SIGTERM/;
+
+/** Message wording of the Sandbox SDK's runtime-replacement error family — the
+ *  container went away UNDER a live SDK call, so the failure never reaches the
+ *  shell-kill signature above: a step that dies this way surfaces as
+ *  `StaleProcessHandleError` ("previous runtime incarnation"),
+ *  `ProcessSpawnFailedError` ("Process supervisor is closed"), one of the SDK's
+ *  interruption messages, or — when a deploy ROLLS the container out from under
+ *  the run rather than just swapping the DO isolate — the raw workerd binding
+ *  refusal "The container is not running, consider calling start()". That
+ *  last one is a spawn-phase refusal: workerd rejected the process start because
+ *  the container was not running at all, so nothing launched (safe to re-attach
+ *  and let the model re-check). It is NOT a typed SDK error (the SDK's own
+ *  auto-start path re-throws it raw when a roll outlasts its port-ready bound)
+ *  and NOT the `container_stopped` `OperationInterruptedError` reason (that is a
+ *  stop UNDER an in-flight op, already covered by the typed check), so the
+ *  message wording is the only signal — deliberately anchored to the full
+ *  "consider calling start" phrase so it can never match a genuine container
+ *  crash ("container exited with unexpected exit code") or the readiness probe
+ *  ("the container is not listening"), which must stay ordinary failures.
+ *  Shared with the resident Worker's `isRuntimeReplacement`
+ *  (deploy/cloudflare-resident/worker.ts) as its message-level fallback, so the
+ *  exec path and the refresh classifier agree on one wording list (a container
+ *  stop mid-snapshot produces "Process supervisor is closed"; classified as the
+ *  repo's own `snapshot-failed` it would re-arm at the full cadence). */
+export const RUNTIME_REPLACEMENT_WORDING =
+  /previous runtime incarnation|interrupted because the runtime changed|runtime identity is no longer active|sandbox lifetime is no longer current|platform was updating the sandbox runtime|no longer identifies pid|process supervisor is closed|container is not running, consider calling start/i;
+
+/** `freeKiB` is the `df` probe's answer (`parseDfFreeKiB`), taken AFTER the
+ *  step failed and only consulted when the message itself carries no errno:
+ *  a message saying ENOSPC is disk-full outright (the disk is the actionable
+ *  fact, whatever else the message says); a kill signature without it is an
+ *  interruption whatever the disk holds (the kill ended the step); otherwise a
+ *  probe below the floor names the disk, and no probe (`undefined`/`null`)
+ *  leaves the step's own failure — unknown is never full. */
+export function classifyRefreshFailure(input: {
+  step: string;
+  message: string;
+  freeKiB?: number | null;
+}): RefreshFailure {
+  const { step, message } = input;
+  if (isDiskFullMessage(message)) {
+    return {
+      interrupted: false,
+      diskFull: true,
+      reason: diskFullReason({ step, message, freeKiB: input.freeKiB ?? null }),
+    };
+  }
+  const timedOut = /\(timed out\)/.test(message);
+  if (!timedOut && (INTERRUPTION_SIGNATURE.test(message) || RUNTIME_REPLACEMENT_WORDING.test(message))) {
+    return { interrupted: true, diskFull: false, reason: `refresh-interrupted: ${step} ${message}` };
+  }
+  if (input.freeKiB !== undefined && input.freeKiB !== null && input.freeKiB < DISK_FULL_FREE_KIB) {
+    return { interrupted: false, diskFull: true, reason: diskFullReason({ step, message, freeKiB: input.freeKiB }) };
+  }
+  return { interrupted: false, diskFull: false, reason: `${step}-failed: ${message}` };
+}
+
+/** Re-arm delay after a cycle that did not run to completion for a reason
+ *  outside the repo — an interrupted step, or an image-stale container stop.
+ *  45 s: comfortably longer than a container restart plus rehydration
+ *  (~10–20 s observed), so the retry finds a live runtime, and an order of
+ *  magnitude under the 600 s cadence that previously left the resident
+ *  `degraded` (every run falling back cold) until the next regular alarm. */
+export const INTERRUPTED_REARM_S = 45;
+
+export type RefreshOutcome =
+  /** Cycle ran (warm, or a real failure): regular cadence. */
+  | "normal"
+  /** A step was killed from outside (see classifyRefreshFailure). */
+  | "interrupted"
+  /** `reconcileImage` stopped the container so it restarts on the new image. */
+  | "image-stale-restart"
+  /** The disk-full recovery stopped the container so it restarts on an empty
+   *  disk and the next alarm restores from R2 (`residentDisk.ts`). */
+  | "disk-full-restart"
+  /** Idle gate parked the resident. */
+  | "idle";
+
+/** How many CONSECUTIVE interrupted cycles still re-arm short. A real deploy
+ *  interrupts once, maybe twice (a deploy train); a step whose own output
+ *  happens to carry the kill signature every cycle (a test supervisor printing
+ *  `signal SIGTERM`) would otherwise retry at 45 s forever — never parking
+ *  (it is excluded from the streak) AND at 13× the cadence. Past the cap the
+ *  regular interval returns; the classification (and the streak exclusion)
+ *  stand, matching the watchdog's accepted "never parks, but at cadence". */
+export const INTERRUPTED_REARM_MAX_CONSECUTIVE = 3;
+
+export function nextRefreshDelayS(input: {
+  outcome: RefreshOutcome;
+  intervalS: number;
+  idleIntervalS: number;
+  /** Consecutive cycles (this one included) that ended `interrupted`; omitted = 1. */
+  consecutiveInterrupted?: number;
+}): number {
+  switch (input.outcome) {
+    case "idle":
+      return input.idleIntervalS;
+    case "interrupted":
+      return (input.consecutiveInterrupted ?? 1) > INTERRUPTED_REARM_MAX_CONSECUTIVE
+        ? input.intervalS
+        : INTERRUPTED_REARM_S;
+    case "image-stale-restart":
+    case "disk-full-restart":
+      return INTERRUPTED_REARM_S;
+    default:
+      return input.intervalS;
+  }
+}
+
+/** Bound a promise that offers no timeout of its own (the
+ *  Sandbox SDK's createBackup/restoreBackup take neither a timeout nor an
+ *  AbortSignal). On expiry, rejects with an error naming `what` and the
+ *  budget, so a hung R2 transfer fails the refresh cycle into its existing
+ *  degrade/goDown handling instead of stranding `refreshing`/`restoring`
+ *  until the 30-min watchdog. The losing promise keeps running (nothing can
+ *  cancel it) — its eventual rejection is swallowed so it never surfaces as
+ *  an unhandled rejection. */
+// -- restore progress ---------------------------------------------------------------
+
+/** How often the wake path samples a restore's target directory. */
+export const RESTORE_POLL_MS = 15_000;
+/** A restore whose target has not grown for this long is stalled. Generous
+ *  against R2's own hiccups, tight against a hung SDK operation: a healthy
+ *  restore writes continuously, even when the same ~2 GiB checkout snapshot
+ *  takes anywhere from under a minute to eight minutes. */
+export const RESTORE_STALL_MS = 120_000;
+/** Absolute cap for ONE HYDRATE — the wait for a previous attempt's restore,
+ *  the mirror restore and the checkout restore share it (one deadline, see
+ *  `deadlineMs`) — under the watchdog's 30-min stale-mid-flight window so a
+ *  runaway wake is still the wake path's own verdict, not the watchdog's. */
+export const RESTORE_MAX_MS = 25 * 60_000;
+
+/** Below this much of the hydrate deadline left, the wake does not start a
+ *  deps materialization at all: a download or install that cannot finish is
+ *  worse than the next refresh cycle's repair (`no deps marker on disk`). */
+export const WAKE_DEPS_MIN_MS = 60_000;
+
+export type WakeDepsBudget =
+  | { action: "skip"; remainingMs: number }
+  | { action: "materialize"; installBudgetMs: number; restoreDeadlineMs: number; remainingMs: number };
+
+/** The wake path's deps materialization (item 61 PR B: restore the warm
+ *  key's entry, or install it) lives INSIDE the hydrate's one deadline, so
+ *  the worst-case `restoring` span is still RESTORE_MAX_MS — the invariant
+ *  the watchdog's 30-min stale-mid-flight window rests on. The restore is
+ *  judged against the hydrate deadline itself; the installer's budget is the
+ *  smaller of its own and what remains; under WAKE_DEPS_MIN_MS nothing starts. */
+export function planWakeDepsBudget(input: {
+  nowMs: number;
+  deadlineMs: number;
+  installBudgetMs: number;
+}): WakeDepsBudget {
+  const remainingMs = Math.max(0, input.deadlineMs - input.nowMs);
+  if (remainingMs < WAKE_DEPS_MIN_MS) return { action: "skip", remainingMs };
+  return {
+    action: "materialize",
+    installBudgetMs: Math.min(input.installBudgetMs, remainingMs),
+    restoreDeadlineMs: input.deadlineMs,
+    remainingMs,
+  };
+}
+
+/** Where the Sandbox SDK stages a backup archive inside the container while it
+ *  downloads (`BACKUP_CONTAINER_DIR` in @cloudflare/sandbox): the restore
+ *  writes `<dir>/<backupId>.sqsh` in full FIRST and extracts into the target
+ *  only afterwards, so a restore's progress lives here during the download and
+ *  in the target during the extraction. Pinned by a test that reads the
+ *  installed SDK's constant, so an SDK bump that moves it fails the build. */
+export const SDK_BACKUP_ARCHIVE_DIR = "/var/backups";
+
+export function restoreArchivePath(backupId: string): string {
+  return `${SDK_BACKUP_ARCHIVE_DIR}/${backupId}.sqsh`;
+}
+
+export interface RestoreSample {
+  atMs: number;
+  /** `du -xsk <dir>` at that moment; null when du could not answer. */
+  kiB: number | null;
+}
+
+export type RestoreVerdict = { verdict: "wait" } | { verdict: "stalled" | "capped"; detail: string };
+
+/** Judge a running restore by its bytes, not by a clock. The Sandbox SDK's
+ *  restoreBackup accepts no timeout, progress callback or AbortSignal, and a
+ *  promise abandoned by a fixed budget keeps writing (a checkout restore that
+ *  outlives a fixed 300 s budget has already sent the resident
+ *  `down(r2-restore-failed)`, and the next hydrate runs `rm -rf` over the tree
+ *  the first one is still filling). So the wake path polls the
+ *  target directory: while bytes keep arriving it waits — a slow transfer is
+ *  a slow transfer — and it gives up only when nothing has been written for
+ *  RESTORE_STALL_MS (the clock runs from the start until the first byte) or
+ *  the whole thing exceeds RESTORE_MAX_MS. A sample du could not take is no
+ *  evidence either way: it neither counts as growth nor resets the clock. */
+export function judgeRestoreProgress(input: {
+  startedMs: number;
+  nowMs: number;
+  samples: readonly RestoreSample[];
+  stallMs?: number;
+  /** Absolute cap shared by the WHOLE hydrate — the wait for a previous
+   *  attempt's restore, the mirror restore and the checkout restore all judge
+   *  against the same instant, so the sum stays under the watchdog's window.
+   *  Defaults to this restore's start + RESTORE_MAX_MS. */
+  deadlineMs?: number;
+}): RestoreVerdict {
+  const stallMs = input.stallMs ?? RESTORE_STALL_MS;
+  const deadlineMs = input.deadlineMs ?? input.startedMs + RESTORE_MAX_MS;
+  const elapsedMs = input.nowMs - input.startedMs;
+  let highKiB = 0;
+  let lastGrowthMs = input.startedMs;
+  for (const s of input.samples) {
+    if (s.kiB !== null && s.kiB > highKiB) {
+      highKiB = s.kiB;
+      lastGrowthMs = s.atMs;
+    }
+  }
+  const gib = (kiB: number) => `${(kiB / 1_048_576).toFixed(2)} GiB`;
+  const secs = (ms: number) => `${Math.round(ms / 1000)} s`;
+  if (input.nowMs > deadlineMs) {
+    return {
+      verdict: "capped",
+      detail: `still restoring after ${secs(elapsedMs)} (${gib(highKiB)} written) — the hydrate's ${secs(RESTORE_MAX_MS)} cap passed`,
+    };
+  }
+  // Idle runs against NOW from the last observed growth (or the start): a
+  // sample du could not take proves nothing, so it neither resets nor pauses
+  // the clock — a stall window without evidence of progress is a stall.
+  const idleMs = input.nowMs - lastGrowthMs;
+  if (idleMs > stallMs) {
+    return {
+      verdict: "stalled",
+      detail: `no bytes written for ${secs(idleMs)} (${gib(highKiB)} after ${secs(elapsedMs)})`,
+    };
+  }
+  return { verdict: "wait" };
+}
+
+export function withTimeout<T>(promise: Promise<T>, ms: number, what: string): Promise<T> {
+  return new Promise<T>((resolve, reject) => {
+    const timer = setTimeout(() => {
+      promise.catch(() => {});
+      reject(new Error(`${what} timed out after ${ms}ms`));
+    }, ms);
+    promise.then(
+      (v) => {
+        clearTimeout(timer);
+        resolve(v);
+      },
+      (e: unknown) => {
+        clearTimeout(timer);
+        reject(e as Error);
+      },
+    );
+  });
+}

package/dist/assets/src/execution/residentRestoreExtract.ts

@@ -0,0 +1,130 @@
+/** Extracting an R2 restore onto the resident's own disk (docs/reference/specs/resident-repos.md
+ *  item 61), kept pure and dependency-free so it is unit-testable from
+ *  src/ and imported by the resident Worker — the tested code IS the shipped
+ *  code.
+ *
+ *  Background: in presigned mode the Sandbox SDK's restore does
+ *  not extract the archive, it MOUNTS it — squashfuse on the `.sqsh` under
+ *  `/var/backups/mounts/<id>_<ts>_<rand>/lower`, then fuse-overlayfs at the
+ *  handle's `dir` with a writable upper next to it — and leaves the `.sqsh` in
+ *  `/var/backups`. (Extraction with `unsquashfs` is the SDK's LOCAL-DEV path.)
+ *  A mount point breaks code that expects a directory on one ext4 filesystem:
+ *  `rm -rf /workspace/mirror` → "Device or resource busy" (a resident loops
+ *  `degraded` on its wake path), `chown -R` → a copy-up of every inode (past
+ *  any step budget), `du -x` → the
+ *  mirror and checkout measured as ~1 MiB, and every hardlink or rename
+ *  between the checkout and the deps store crossed devices.
+ *
+ *  So the resident restores INTO A STAGING MOUNT (a sibling of the target,
+ *  still under /workspace where the SDK allows it), extracts from it onto the
+ *  real disk — `unsquashfs` straight from the downloaded `.sqsh` when the image
+ *  has squashfs-tools (the Dockerfile installs it), `cp -a` out of the mount
+ *  while an older image lacks it — unmounts the staging mount and its lower,
+ *  removes the archive, and only then renames the extracted tree into place.
+ *  After a wake the disk looks exactly as it did before presigned mode. Every
+ *  clean step first unmounts whatever a previous incarnation left behind. */
+
+import { shellQuote } from "./shellQuote.js";
+
+/** Where the SDK keeps a mounted restore's lower/upper/work dirs. */
+export const RESTORE_MOUNT_ROOT = "/var/backups/mounts";
+
+/** The staging mount for restoring `targetDir`: a sibling, unique per attempt.
+ *  Must stay under /workspace (or the SDK's other allowed roots) because the
+ *  SDK validates the handle's `dir`. */
+export function restoreMountDir(targetDir: string, attempt: string): string {
+  return `${targetDir}.restore-${attempt}`;
+}
+
+/** A backup id as the SDK mints it (a UUID: hex and hyphens, at least one hex
+ *  digit) — it becomes a glob under the mount root, so nothing else may. */
+const BACKUP_ID_RE = /^(?=.*[0-9a-fA-F])[0-9a-fA-F-]{8,64}$/;
+
+/** Unmount ONE staging mount and what hangs off it, as root, tolerating a
+ *  path that is not (or no longer) mounted: the fuse-overlayfs at `mountDir`
+ *  first, then every squashfuse lower under the SDK's `<backupId>_<ts>_<rand>`
+ *  dirs, then those dirs. The lower is found by the backup id, not by the
+ *  overlay's options: fuse-overlayfs exposes no `lowerdir=` in /proc/mounts
+ *  (an unmount keyed on the overlay's options leaves the squashfuse lowers
+ *  mounted, each pinning its unlinked `.sqsh`).
+ *  `fusermount3 -u` is the FUSE way; `umount -l` the fallback for a busy
+ *  mount. */
+export function unmountRestoreScript(input: { mountDir: string; backupId: string }): string {
+  if (!BACKUP_ID_RE.test(input.backupId))
+    throw new Error(`restore: not a backup id: ${JSON.stringify(input.backupId)}`);
+  const m = shellQuote(input.mountDir);
+  const lowers = `${RESTORE_MOUNT_ROOT}/${input.backupId}_*/lower`;
+  return [
+    `_m=${m}`,
+    `if awk -v m="$_m" '$2==m {f=1} END {exit !f}' /proc/mounts 2>/dev/null; then fusermount3 -u "$_m" 2>/dev/null || umount -l "$_m" 2>/dev/null || true; fi`,
+    `for _l in ${lowers}; do`,
+    `  if awk -v m="$_l" '$2==m {f=1} END {exit !f}' /proc/mounts 2>/dev/null; then fusermount3 -u "$_l" 2>/dev/null || umount -l "$_l" 2>/dev/null || true; fi`,
+    `done`,
+    // The removals are best effort: after a LAZY unmount a still-open file
+    // can keep the dir alive for a moment, and this script also runs under
+    // `set -e` inside extractRestoreScript — a successful extraction must
+    // never be failed by its cleanup. The unmount-all clean step reclaims
+    // whatever is left. The glob takes EVERY mount dir of this backup id, not
+    // only this attempt's: the SDK serializes backup operations on one queue,
+    // so no other restore of the same archive can be mid-flight here, and an
+    // earlier attempt's dir for the id is exactly the debris to remove.
+    `rm -rf ${RESTORE_MOUNT_ROOT}/${input.backupId}_* 2>/dev/null || true`,
+    `rmdir "$_m" 2>/dev/null || rm -rf "$_m" 2>/dev/null || true`,
+  ].join("\n");
+}
+
+/** Turn a mounted restore into a plain directory tree, as root:
+ *   1. extract — `unsquashfs` from the `.sqsh` the SDK downloaded when the
+ *      image has it (multi-threaded, no FUSE in the read path; `-no-xattrs`
+ *      because squashfs xattrs are not worth a failed restore), else `cp -a`
+ *      out of the mount (ownership and modes preserved either way — the
+ *      archive was taken from a worker1-owned tree, so no chown follows);
+ *   2. unmount the staging mount and its lower, remove the archive;
+ *   3. rename the extracted tree to the target — the target appears LAST, so
+ *      a failure anywhere above leaves no half-populated target.
+ *  Prints `extract: unsquashfs` or `extract: cp` so the log says which. */
+export function extractRestoreScript(input: {
+  mountDir: string;
+  backupId: string;
+  archivePath: string;
+  targetDir: string;
+}): string {
+  const attempt = input.mountDir.slice(input.mountDir.lastIndexOf(".restore-") + ".restore-".length);
+  const tmp = shellQuote(`${input.targetDir}.extract-${attempt}`);
+  const archive = shellQuote(input.archivePath);
+  const target = shellQuote(input.targetDir);
+  return [
+    `set -e`,
+    `rm -rf ${tmp}`,
+    `if command -v unsquashfs >/dev/null 2>&1 && test -f ${archive}; then`,
+    `  unsquashfs -n -no-xattrs -d ${tmp} ${archive} >/dev/null`,
+    `  echo "extract: unsquashfs"`,
+    `else`,
+    `  mkdir ${tmp}`,
+    `  cp -a ${shellQuote(`${input.mountDir}/.`)} ${tmp}/`,
+    `  echo "extract: cp"`,
+    `fi`,
+    unmountRestoreScript({ mountDir: input.mountDir, backupId: input.backupId }),
+    `rm -f ${archive}`,
+    `rm -rf ${target}`,
+    `mv ${tmp} ${target}`,
+  ].join("\n");
+}
+
+/** For the clean steps (`clean-before-restore`, provisioning's
+ *  `clean-workspace`): unmount every fuse mount a previous incarnation left
+ *  under /workspace or under the SDK's mount root — deepest first, so an
+ *  overlay goes before the lower it sits on — then remove the SDK's mount dirs,
+ *  any `.sqsh` still on disk, and the staging (`*.restore-*`) and extraction
+ *  (`*.extract-*`) siblings an attempt that died mid-way left beside the
+ *  mirror or checkout (a multi-GiB partial tree on a disk-budgeted resident).
+ *  Idempotent; nothing to do is exit 0. */
+export function unmountAllRestoresScript(): string {
+  return [
+    `awk '($3 ~ /^fuse/) && ($2 ~ /^\\/workspace\\// || $2 ~ /^${RESTORE_MOUNT_ROOT.replace(/\//g, "\\/")}\\//) {print length($2), $2}' /proc/mounts 2>/dev/null | sort -rn | cut -d' ' -f2- | while read -r _m; do fusermount3 -u "$_m" 2>/dev/null || umount -l "$_m" 2>/dev/null || true; done`,
+    `rm -rf ${RESTORE_MOUNT_ROOT}/* 2>/dev/null || true`,
+    `rm -f /var/backups/*.sqsh 2>/dev/null || true`,
+    `rm -rf /workspace/*.restore-* /workspace/*.extract-* 2>/dev/null || true`,
+    `true`,
+  ].join("\n");
+}