@coreplane/switchboard 1.251.0 → 1.252.0

package/dist/assets/deploy/cloudflare-resident/worker.ts

@@ -180,8 +180,11 @@ import {
   type StoredTestOverrides,
 } from "./gc";
 import {
+  CHECKOUT_FETCH_COMMAND,
   checkoutUpdateCommand,
   classifyRefreshFailure,
+  stageCheckoutScript,
+  swapCheckoutScript,
   restoreFailureDisposition,
   isRuntimeUnreachableSignal,
   killStaleBuildProcessesCommand,
@@ -304,6 +307,7 @@ import {
   CHECKOUT_SNAPSHOT_EXCLUDES,
   DEPS_BACKUP_KEY_PREFIX,
   DEPS_BACKUP_TTL_S,
+  DEPS_ENTRY_BACKUP_EXCLUDES,
   DEPS_STORE_DIR,
   depsBackupStorageKey,
   depsBackupsToDrop,
@@ -330,6 +334,14 @@ import {
 import { buildId, injectedBuildStamp } from "../../src/deploy/buildStamp.js";
 import { createRefreshInstance, createRefreshInstanceNow, type RefreshInstanceParams } from "./refresh";
 import { drainRefusal, liveDrain, parseDrainRequest, type DrainRecord } from "./drain";
+import {
+  CGROUP_READ_ARGV,
+  MEMORY_PRESSURE_REASON,
+  MemoryGuard,
+  MemorySampleUnavailable,
+  type MemoryGateRoute,
+  type MemoryReading,
+} from "./memoryGuard";
 import {
   ControlResetError,
   RuntimeReplacedError,
@@ -485,6 +497,18 @@ const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve,
  *  DO storage is). Thread worktrees hang off the same mirror; keep these paths stable. */
 const MIRROR_DIR = "/workspace/mirror"; // bare mirror, owned by root
 const CHECKOUT_DIR = "/workspace/checkout"; // default-branch working tree + deps + build, owned by BUILD_USER
+/** The staged rebuild's side trees (resident-repos.md item 47): the refresh
+ *  builds into `staging` OUTSIDE the mirror lock and swaps it into
+ *  CHECKOUT_DIR under it (two renames); `retired` holds the replaced checkout
+ *  between the swap and its off-lock removal. Same filesystem as the
+ *  checkout, so both moves are renames and `cp -al` hardlinks. */
+const STAGING_CHECKOUT_DIR = "/workspace/checkout.staging";
+const RETIRED_CHECKOUT_DIR = "/workspace/checkout.retired";
+const CHECKOUT_SWAP_DIRS = {
+  checkout: CHECKOUT_DIR,
+  staging: STAGING_CHECKOUT_DIR,
+  retired: RETIRED_CHECKOUT_DIR,
+} as const;
 const RESIDENT_STATE_DIR = "/workspace/.resident"; // mode 700 root:root — worker users cannot traverse
 const CRED_FILE = `${RESIDENT_STATE_DIR}/git-credentials`; // one-shot token file, deleted after each git command
 const READY_MARKER = `${RESIDENT_STATE_DIR}/ready`; // holds the sha the disk was hydrated to
@@ -607,6 +631,11 @@ const DISK_FULL_RECYCLE_KEY = "resident:diskFullRecycleAt";
  *  admission projects a new tree's cost from its parts (its free-space term
  *  is a live `df` of its own). */
 const DISK_KEY = "resident:disk";
+/** The last memory reading (docs/reference/specs/resident-repos.md item 70; `memoryGuard.ts`):
+ *  one cgroup v2 read at every /exec and /attach start and on the measure
+ *  tick, persisted so the gauges (`/status`, `/residents`, the watchdog line)
+ *  read storage only — the watchdog never touches the container. */
+const MEMORY_KEY = "resident:memory";
 /** The lifecycle row (docs/reference/specs/resident-repos.md item 7): `workflow`,
  *  the one scheduler. Kept from the flagged rollout so `/status` and `/debug
  *  info` can say so; an `alarm` value a flip left behind reads `workflow`
@@ -1893,6 +1922,12 @@ export class ResidentDO extends Sandbox<Env> {
    *  deadline names the fallback. */
   private restoreWaiters = new RestoreWaiters();
 
+  /** Item 70: the memory guard — one cgroup reading per /exec and /attach
+   *  start (and per measure tick), the two-threshold gate over the last one.
+   *  In-memory like the waiter ledger: a fresh isolate re-samples at the next
+   *  route start, and the persisted copy (MEMORY_KEY) serves the gauges. */
+  private memoryGuard = new MemoryGuard({ read: () => this.readCgroup() }, (line) => console.log(line));
+
   private nextHolder(): string {
     return `${this.incarnation}:${++this.leaseSeq}`;
   }
@@ -2209,18 +2244,24 @@ export class ResidentDO extends Sandbox<Env> {
   }
 
   /** Run one command-table entry as the unprivileged build user in the warm
-   *  checkout. Never root. Never a GitHub token — the env
-   *  is whatever `su` grants the target user, nothing injected. */
-  private async buildUserRun(command: string, step: ResidentStepLabelKey, timeoutMs: number): Promise<string> {
-    // Steps on the checkout are sequential, so a live build-user process
-    // INSIDE it here is a leftover (a step whose wait was abandoned, or one
+   *  checkout (or the staged rebuild's tree, item 47). Never root. Never a
+   *  GitHub token — the env is whatever `su` grants the target user, nothing
+   *  injected. */
+  private async buildUserRun(
+    command: string,
+    step: ResidentStepLabelKey,
+    timeoutMs: number,
+    dir: string = CHECKOUT_DIR,
+  ): Promise<string> {
+    // Steps on one tree are sequential, so a live build-user process INSIDE
+    // it here is a leftover (a step whose wait was abandoned, or one
     // orphaned by a Worker-only deploy resetting the DO) — and it is writing
-    // into the tree this step is about to clean or build. Scoped to the
-    // checkout: the same user's deps-store installs run in their own scratch
+    // into the tree this step is about to clean or build. Scoped to that
+    // tree: the same user's deps-store installs run in their own scratch
     // trees, in parallel, and are live work (killStaleBuildProcessesCommand).
-    const swept = await this.runOk(killStaleBuildProcessesCommand(BUILD_USER, CHECKOUT_DIR), `${step}-stale-sweep`);
+    const swept = await this.runOk(killStaleBuildProcessesCommand(BUILD_USER, dir), `${step}-stale-sweep`);
     if (swept.trim()) console.log(`${step}: ${swept.trim()}`);
-    return this.runOk(["su", "-s", "/bin/bash", BUILD_USER, "-c", `cd ${CHECKOUT_DIR} && ${command}`], step, {
+    return this.runOk(["su", "-s", "/bin/bash", BUILD_USER, "-c", `cd ${dir} && ${command}`], step, {
       timeoutMs,
     });
   }
@@ -2563,9 +2604,10 @@ export class ResidentDO extends Sandbox<Env> {
     }
   }
 
-  /** Bring the checkout to `sha` and build it, under the mirror mutex. The
-   *  disk markers decide (planBuild over the refresh planner): a checkout
-   *  whose HEAD, deps and build markers all name the target is done. */
+  /** Bring the checkout to `sha` and build it — staged outside the mirror
+   *  mutex, swapped in under it (item 47). The disk markers decide (planBuild
+   *  over the refresh planner): a checkout whose HEAD, deps and build markers
+   *  all name the target is done. */
   async runBuild(input: {
     sha: string;
     factsSha: string;
@@ -2577,47 +2619,116 @@ export class ResidentDO extends Sandbox<Env> {
     const { sha, lockfileKey } = input;
     const plan = planBuild({ sha, factsSha: input.factsSha, lockfileKey, disk: await this.readRefreshDisk() });
     if (plan.action === "done") return { done: true, why: plan.why };
-    // Serialize the CHECKOUT_DIR mutation on the mirror mutex:
-    // materializeThreadDeps reads CHECKOUT_DIR via `cp -al` under the same
-    // lock, so an attach/op dep-copy can never hardlink a half-rebuilt
-    // checkout into a thread tree (torn cache → false ❌ from `repo test`).
-    // No wait timeout, exactly like the fetch lock: the background refresh
-    // queues behind an in-flight attach instead of flipping to degraded on
+    // The mirror mutex is held only to STAGE and later to SWAP — never for
+    // the build (item 47): a heavy build under the lock pushed concurrent
+    // attaches past ATTACH_MUTEX_WAIT_MS into mirror-busy 503s and cold
+    // fallbacks. The stage lock covers the two reads the mutex actually
+    // protects: a consistent hardlink copy of the warm checkout (exactly
+    // materializeThreadDeps' torn-cache guarantee, which the `cp -al` under
+    // the same lock keeps — CHECKOUT_DIR now changes only by rename under
+    // this mutex) and the fetch from the mirror into the staging tree (an
+    // attach's `fetch --prune` could delete a ref under it). No wait
+    // timeout, exactly like the fetch lock: the background refresh queues
+    // behind an in-flight attach instead of flipping to degraded on
     // transient lock contention. Token-free: repo code runs during the build.
+    // Off the lock, before the stage lease: a failed or interrupted off-lock
+    // build leaves a full staging tree — deps view plus partial build output,
+    // hundreds of thousands of inodes — and the retry 30 s later would
+    // otherwise hold the mirror lock through its removal, the contention
+    // class the staged rebuild exists to remove. The tree is private to this
+    // rebuild, so its stale writers are killed and the tree dropped without
+    // the lock; the retired tree's heal-then-remove stays in the staged
+    // script under the lease, where the checkout path's consistency needs it.
+    const swept = await this.runOk(
+      killStaleBuildProcessesCommand(BUILD_USER, STAGING_CHECKOUT_DIR),
+      "checkout-stage-stale-sweep",
+    );
+    if (swept.trim()) console.log(`checkout-stage: ${swept.trim()}`);
+    await this.runOk(["rm", "-rf", STAGING_CHECKOUT_DIR], "checkout-stage-clear", {
+      timeoutMs: GIT_NETWORK_TIMEOUT_MS,
+    });
+    await this.withMirrorLock(
+      async () => {
+        await this.runOk(["sh", "-c", stageCheckoutScript(CHECKOUT_SWAP_DIRS)], "checkout-stage", {
+          timeoutMs: GIT_NETWORK_TIMEOUT_MS,
+        });
+        await this.buildUserRun(
+          CHECKOUT_FETCH_COMMAND,
+          "checkout-update",
+          GIT_NETWORK_TIMEOUT_MS,
+          STAGING_CHECKOUT_DIR,
+        );
+      },
+      0,
+      // The lease is the section's own budget (resident-repos.md item 22) —
+      // the backstop by which a taker judges this holder dead. The section
+      // runs two commands each budgeted GIT_NETWORK_TIMEOUT_MS (the stage
+      // script and the mirror fetch), so the lease is their sum: a lease of
+      // one budget let a taker steal the lock mid-stage and race its
+      // `fetch --prune` against the staging fetch's negotiation.
+      { step: "checkout-stage", budgetMs: 2 * GIT_NETWORK_TIMEOUT_MS },
+    );
+    // Outside the lock, in the staging tree. Isolation invariant (review 1b):
+    // attached, sha-pinned thread worktrees hold hardlinks to the store
+    // entry's FILE inodes, and so does the staged copy. A build that writes
+    // THROUGH an existing inode — many bundlers do (e.g. .next incremental
+    // manifests open+truncate rather than recreate) — would mutate every
+    // consumer's pinned artifacts. The `-x` clean removes the staged copy's
+    // build output so the build allocates FRESH inodes; the entry's own files
+    // are owner-read-only (deps-harden), so a write through them fails loudly
+    // instead of silently reaching the store; the tool caches inside
+    // node_modules are the checkout's private copies (item 18).
+    //
+    // Install gate: when the committed lockfile key is unchanged, node_modules
+    // (the view) is excluded from the clean and no deps work happens; a
+    // changed key re-links the view to the new entry — which is also what
+    // drops deps the new lockfile no longer has.
+    await this.buildUserRun(
+      checkoutUpdateCommand(sha, plan.clean),
+      "checkout-update",
+      GIT_NETWORK_TIMEOUT_MS,
+      STAGING_CHECKOUT_DIR,
+    );
+    if (plan.install && input.depsEntry) {
+      // The old view (a resumed install's keep-deps clean leaves it in
+      // place, item 57) makes way for the new entry's: hardlinks only,
+      // the entry's inodes are untouched.
+      await this.runOk(["rm", "-rf", `${STAGING_CHECKOUT_DIR}/node_modules`], "unlink-deps-view");
+      await this.linkDepsView(
+        `${input.depsEntry}/node_modules`,
+        STAGING_CHECKOUT_DIR,
+        BUILD_USER,
+        STAGING_CHECKOUT_DIR,
+      );
+    }
+    await this.buildUserRun(input.buildCmd, "build", REFRESH_BUILD_TIMEOUT_MS, STAGING_CHECKOUT_DIR);
+    // The swap: two renames — milliseconds — and the markers written with the
+    // tree they vouch for, under one short lease.
     await this.withMirrorLock(
       async () => {
-        // Isolation invariant (review 1b): attached, sha-pinned thread
-        // worktrees hold hardlinks to the store entry's FILE inodes, and so
-        // does the checkout. A build that writes THROUGH an existing inode —
-        // many bundlers do (e.g. .next incremental manifests open+truncate
-        // rather than recreate) — would mutate every consumer's pinned
-        // artifacts. The `-x` clean removes the checkout's build output so
-        // the build allocates FRESH inodes; the entry's own files are
-        // owner-read-only (deps-harden), so a write through them fails
-        // loudly instead of silently reaching the store; the tool caches
-        // inside node_modules are the checkout's private copies (item 18).
-        //
-        // Install gate: when the committed lockfile key is unchanged,
-        // node_modules (the view) is excluded from the clean and no deps
-        // work happens; a changed key re-links the view to the new entry —
-        // which is also what drops deps the new lockfile no longer has.
-        await this.buildUserRun(checkoutUpdateCommand(sha, plan.clean), "checkout-update", GIT_NETWORK_TIMEOUT_MS);
+        // A leftover writer inside the old checkout would keep writing into
+        // the retired tree while it is removed.
+        const swept = await this.runOk(
+          killStaleBuildProcessesCommand(BUILD_USER, CHECKOUT_DIR),
+          "checkout-swap-stale-sweep",
+        );
+        if (swept.trim()) console.log(`checkout-swap: ${swept.trim()}`);
+        await this.runOk(["sh", "-c", swapCheckoutScript(CHECKOUT_SWAP_DIRS)], "checkout-swap", {
+          timeoutMs: GIT_NETWORK_TIMEOUT_MS,
+        });
         if (plan.install) {
-          // The old view (a resumed install's keep-deps clean leaves it in
-          // place, item 57) makes way for the new entry's: hardlinks only,
-          // the entry's inodes are untouched.
-          if (input.depsEntry) {
-            await this.runOk(["rm", "-rf", `${CHECKOUT_DIR}/node_modules`], "unlink-deps-view");
-            await this.linkDepsView(`${input.depsEntry}/node_modules`, CHECKOUT_DIR, BUILD_USER);
-          }
           await this.writeDiskMarkers({ depsKey: lockfileKey });
           await this.runOk(["rm", "-f", INSTALLING_MARKER], "clear-installing-marker");
         }
-        await this.buildUserRun(input.buildCmd, "build", REFRESH_BUILD_TIMEOUT_MS);
         await this.writeDiskMarkers({ builtSha: sha });
       },
       0,
-      { step: "build", budgetMs: GIT_NETWORK_TIMEOUT_MS + REFRESH_BUILD_TIMEOUT_MS },
+      { step: "checkout-swap", budgetMs: GIT_NETWORK_TIMEOUT_MS },
+    );
+    // Off the lock: the replaced tree's removal (hardlinks — mostly link
+    // drops). A failure here is a log line; the next stage's seed sweeps it.
+    await this.runOk(["rm", "-rf", RETIRED_CHECKOUT_DIR], "checkout-retire").catch((err: unknown) =>
+      console.log(`checkout-retire: ${errMsg(err)} — the next staged rebuild sweeps it`),
     );
     return { done: false, why: plan.why };
   }
@@ -2709,7 +2820,10 @@ export class ResidentDO extends Sandbox<Env> {
     // (item 61: the SDK's presigned restore mounts) — `rm -rf` on a mount
     // point is "Device or resource busy". Unmount first, every time.
     await this.runOk(["sh", "-c", unmountAllRestoresScript()], "unmount-restores");
-    await this.runOk(["rm", "-rf", MIRROR_DIR, CHECKOUT_DIR, ...DISK_MARKERS], "clean-before-restore");
+    await this.runOk(
+      ["rm", "-rf", MIRROR_DIR, CHECKOUT_DIR, STAGING_CHECKOUT_DIR, RETIRED_CHECKOUT_DIR, ...DISK_MARKERS],
+      "clean-before-restore",
+    );
     try {
       // The restore pair IS the cold-wake critical path. Sequential on purpose:
       // the SDK serializes backup operations anyway (one queue), so a
@@ -2962,7 +3076,10 @@ export class ResidentDO extends Sandbox<Env> {
         });
       }
       await this.runOk(["sh", "-c", unmountAllRestoresScript()], "unmount-restores");
-      await this.runOk(["rm", "-rf", MIRROR_DIR, CHECKOUT_DIR, ...DISK_MARKERS], "clean-workspace");
+      await this.runOk(
+        ["rm", "-rf", MIRROR_DIR, CHECKOUT_DIR, STAGING_CHECKOUT_DIR, RETIRED_CHECKOUT_DIR, ...DISK_MARKERS],
+        "clean-workspace",
+      );
       await this.ensureGitSetup();
       await this.withMirrorLock(() =>
         this.gitWithCred(
@@ -4234,7 +4351,13 @@ export class ResidentDO extends Sandbox<Env> {
       cycle.count();
       // Same gate as the sweep: a silent control port cannot answer a `df`.
       if (await this.runtimeUnreachableRow()) return { status: "stopped", why: "runtime-unreachable" };
-      return this.housekeeping("measure", async () => ({ measured: (await this.measureDisk()) !== null }));
+      return this.housekeeping("measure", async () => {
+        // Item 70: the tick's memory line rides the measure step's cadence —
+        // the instance the watchdog cron creates, never the watchdog's own
+        // pass (which must not touch the container). No polling loop.
+        await this.sampleMemory();
+        return { measured: (await this.measureDisk()) !== null };
+      });
     });
   }
 
@@ -4424,6 +4547,76 @@ export class ResidentDO extends Sandbox<Env> {
     return sample;
   }
 
+  // -- memory guard (docs/reference/specs/resident-repos.md item 70) ------------
+
+  /** The one real `CgroupReader`: `CGROUP_READ_ARGV` through the exec choke
+   *  point (`run()`), one exec per sample. A throw there (busy, replaced,
+   *  reset — the container's moment, not the cgroup) is the transient
+   *  `MemorySampleUnavailable`, which never disables the gate — and a typed
+   *  runtime replacement `invalidates` the last reading: the container it
+   *  measured is gone, and gating on its numbers would answer
+   *  `memory-pressure` where the route's own gate owes `runtime-replaced`.
+   *  A non-zero exit is the unreadable cgroup that logs once and disables it. */
+  private async readCgroup(): Promise<string> {
+    let r: { stdout: string; stderr: string; exitCode: number };
+    try {
+      r = await this.run([...CGROUP_READ_ARGV]);
+    } catch (err) {
+      throw new MemorySampleUnavailable(errMsg(err), err instanceof RuntimeReplacedError);
+    }
+    if (r.exitCode !== 0) throw new Error(`cgroup read exit ${r.exitCode}: ${(r.stderr || r.stdout).trim()}`);
+    return r.stdout;
+  }
+
+  /** One sample, one structured log line (the guard's). Never on an inactive
+   *  runtime — a sleeping container is not woken to be measured (item 55's
+   *  rule) — and persisted for the gauges when taken. */
+  private async sampleMemory(): Promise<MemoryReading | null> {
+    if (!(await this.isRuntimeActive().catch(() => false))) {
+      // An inactive container holds no memory: whatever the guard last read
+      // came from a container that no longer runs, and a route hitting the
+      // gate before anything wakes the runtime (a fresh hydration memo skips
+      // the restore) must not be refused on a dead container's percent.
+      this.memoryGuard.invalidate();
+      return null;
+    }
+    const reading = await this.memoryGuard.sample(new Date(systemClock()).toISOString());
+    if (reading) await this.ctx.storage.put(MEMORY_KEY, reading);
+    return reading;
+  }
+
+  /** The last reading, for `/status`, `/residents` and the watchdog line —
+   *  this incarnation's when it has one, else the persisted one (storage
+   *  only: the watchdog never touches the container). */
+  async memoryGauge(): Promise<MemoryReading | null> {
+    return this.memoryGuard.lastReading ?? (await this.ctx.storage.get<MemoryReading>(MEMORY_KEY)) ?? null;
+  }
+
+  /** The route gate (item 70): one fresh sample, then the pure verdict over
+   *  the last reading. A refusal is the same 503 shape as `mirror-busy`, so
+   *  the bot falls back or waits legibly — and a command already running is
+   *  never touched: the gate sits at the route's start and kills nothing. */
+  private async memoryGate(route: MemoryGateRoute, exemptRegisteredRun = false): Promise<ThreadErr | null> {
+    await this.sampleMemory();
+    // A run already in flight — registered from attach to release (item 44) —
+    // is what the gate holds the resident open FOR: its re-attach (a rolled
+    // container, an evicted worktree, a resume) passes as it passes the drain,
+    // sampled but never refused — refusing it would strand the warm worktree
+    // and lose the very work the gate protects.
+    if (exemptRegisteredRun) return null;
+    const refusal = this.memoryGuard.gate(route);
+    if (!refusal) return null;
+    const s = await this.getStatus();
+    return {
+      error: refusal.message,
+      status: 503,
+      state: s.state,
+      stateReason: s.reason,
+      reason: MEMORY_PRESSURE_REASON,
+      cause: "system",
+    };
+  }
+
   /** `{usedKiB, totalKiB, at}` of the last sample for the watchdog line (storage
    *  only — the watchdog never touches the container). */
   private async diskGauge(): Promise<{ usedKiB: number; totalKiB: number; freeKiB: number; at: string } | null> {
@@ -4683,11 +4876,17 @@ export class ResidentDO extends Sandbox<Env> {
     action: "none" | "provision-timed-out" | "auto-rebuilt";
     /** Item 55: the last disk sample's gauge, for the watchdog's status line. */
     disk: { usedKiB: number; totalKiB: number; freeKiB: number; at: string } | null;
+    /** Item 70: the last memory reading, storage only — the tick's gauge. */
+    memory: MemoryReading | null;
     /** Item 7: what the cron's instance-creation decision reads, after the check above settled the state. */
     refresh: RefreshRow;
   }> {
-    const [check, disk] = await Promise.all([this.watchdogCheckLifecycle(), this.diskGauge()]);
-    return { ...check, disk, refresh: await this.refreshRow() };
+    const [check, disk, memory] = await Promise.all([
+      this.watchdogCheckLifecycle(),
+      this.diskGauge(),
+      this.memoryGauge(),
+    ]);
+    return { ...check, disk, memory, refresh: await this.refreshRow() };
   }
 
   private async watchdogCheckLifecycle(): Promise<{
@@ -4981,10 +5180,18 @@ export class ResidentDO extends Sandbox<Env> {
       // closed on the run it is closed for. Read before the image reconcile so
       // a refused attach never restarts a container.
       const drain = await this.fleetDrain();
-      if (drain && !(await this.ctx.storage.get(runRegKey(threadKey)))) {
+      const registered = (await this.ctx.storage.get(runRegKey(threadKey))) !== undefined;
+      if (drain && !registered) {
         const refusal: ThreadErr & { draining: DrainRecord } = drainRefusal(drain);
         return refusal;
       }
+      // Item 70: above the soft memory threshold a NEW attach is refused like
+      // `mirror-busy` (the bot falls back or waits, the card says why) — after
+      // the drain (storage only, cheaper) and before the image reconcile, so a
+      // refused attach never restarts a container. A registered run's
+      // re-attach passes for the same reason it passes the drain above.
+      const memory = await this.memoryGate("attach", registered);
+      if (memory) return memory;
       const resourceId = (await this.ctx.storage.get<string>(RESOURCE_KEY)) ?? "";
       if (await this.reconcileImage("attach")) {
         return {
@@ -5975,10 +6182,14 @@ export class ResidentDO extends Sandbox<Env> {
       const resource = (await this.ctx.storage.get<string>(RESOURCE_KEY)) ?? "";
       const backup = await withTimeout(
         this.createBackup({
-          dir: `${depsEntryPath(key)}/node_modules`,
+          // The entry dir itself, not only its top-level node_modules: nested
+          // workspace node_modules are entry content too. The markers are
+          // excluded so a restore's own commit writes `.complete` LAST.
+          dir: depsEntryPath(key),
           localBucket: transfer.localBucket,
           ttl: DEPS_BACKUP_TTL_S,
           name: `${resource} deps ${key.slice(0, 8)}`,
+          excludes: [...DEPS_ENTRY_BACKUP_EXCLUDES],
         }),
         R2_TRANSFER_TIMEOUT_MS,
         "deps backup",
@@ -6020,9 +6231,16 @@ export class ResidentDO extends Sandbox<Env> {
       await this.ensureDepsStoreDir();
       console.log(`deps: restoring ${key.slice(0, 8)} from backup ${record.backup.id.slice(0, 8)} into ${scratch}`);
       await this.runOk(["mkdir", "-p", scratch], "deps-restore-scratch");
+      // The archive is the entry dir (top-level node_modules PLUS nested
+      // workspace ones at their tree-relative paths), so it extracts into the
+      // scratch root and the commit script moves each node_modules like an
+      // install's. An archive from before nested coverage held only the
+      // node_modules CONTENTS: its extract leaves no scratch/node_modules, the
+      // commit fails, the record is dropped and the installer runs — the
+      // documented fall-through, never a stranded key.
       await this.restoreExtracted(
         record.backup,
-        `${scratch}/node_modules`,
+        scratch,
         `deps restore ${key.slice(0, 8)}`,
         "deps-restore-extract",
         deadlineMs,
@@ -6104,6 +6322,15 @@ export class ResidentDO extends Sandbox<Env> {
           parsed.mutableListing,
           "deps-seed-swap",
         );
+        for (const rel of parsed.nested) {
+          await this.swapMutableCaches(
+            `${depsEntryPath(seed)}/${rel}`,
+            `${scratch}/${rel}`,
+            BUILD_USER,
+            parsed.mutableListing,
+            "deps-seed-swap",
+          );
+        }
         console.log(`deps: ${key.slice(0, 8)} seeded from ${seed.slice(0, 8)} (${parsed.mech})`);
       }
       // Unprivileged and token-free, in a tree only this attempt
@@ -6194,25 +6421,46 @@ export class ResidentDO extends Sandbox<Env> {
     return mech;
   }
 
-  /** The view itself, for callers already holding the mirror lock (the
-   *  refresh rebuild and provisioning link the CHECKOUT this way). */
+  /** The view itself, lock-agnostic: provisioning links the CHECKOUT under
+   *  the mirror lock it already holds; the staged rebuild links its staging
+   *  tree OFF the lock — safe because the tree is private to the rebuild and
+   *  the store entry's inodes are immutable (deps-harden). */
   private async linkDepsView(
     entryNodeModules: string | null,
     tree: string,
     user: string,
+    /** The build dirs' source. The staged rebuild passes its own tree (its
+     *  clean just removed them — nothing to copy; the WARM checkout's build
+     *  output must never seed the tree that is about to rebuild it). */
+    from: string = CHECKOUT_DIR,
   ): Promise<DepCacheMaterialization | "none"> {
     const parsed = await this.runDepScript(
-      depCacheScript(CHECKOUT_DIR, tree, user, entryNodeModules ? { nodeModulesSrc: entryNodeModules } : {}),
+      depCacheScript(from, tree, user, entryNodeModules ? { nodeModulesSrc: entryNodeModules } : {}),
       "deps-materialize",
       REFRESH_BUILD_TIMEOUT_MS,
     );
     await this.swapMutableCaches(
-      entryNodeModules ?? `${CHECKOUT_DIR}/node_modules`,
+      entryNodeModules ?? `${from}/node_modules`,
       `${tree}/node_modules`,
       user,
       parsed.mutableListing,
       "deps-mutable-swap",
     );
+    // Nested workspace node_modules the script hardlinked from the entry get
+    // the same tool-cache swap, each scoped to its own root (mutableCachePaths
+    // keeps only the listing under that root).
+    if (entryNodeModules) {
+      const entryRoot = entryNodeModules.replace(/\/node_modules$/, "");
+      for (const rel of parsed.nested) {
+        await this.swapMutableCaches(
+          `${entryRoot}/${rel}`,
+          `${tree}/${rel}`,
+          user,
+          parsed.mutableListing,
+          "deps-mutable-swap",
+        );
+      }
+    }
     return parsed.mech;
   }
 
@@ -6553,6 +6801,10 @@ export class ResidentDO extends Sandbox<Env> {
     timeoutMs: number,
     env?: Record<string, string>,
   ): Promise<{ stdout: string; stderr: string; exitCode: number; truncated: boolean } | ThreadErr> {
+    // Item 70: a new command is refused by name above the hard memory
+    // threshold — before the preflight, so nothing of it starts.
+    const memory = await this.memoryGate("exec");
+    if (memory) return memory;
     const pre = await this.threadPreflight(threadKey);
     if ("error" in pre) return pre;
     const { binding } = pre;
@@ -7568,6 +7820,7 @@ export class ResidentDO extends Sandbox<Env> {
       RUNTIME_UNREACHABLE_KEY,
       AUTO_REBUILDS_KEY,
       INFRA_STREAK_KEY,
+      MEMORY_KEY,
     ]);
     const facts = map.get(FACTS_KEY) as RepoFacts | undefined;
     const snap = map.get(SNAPSHOT_KEY) as SnapshotRecord | undefined;
@@ -7670,6 +7923,9 @@ export class ResidentDO extends Sandbox<Env> {
       // Item 55: the last disk sample (`residentDiskBudget.ts` DiskSample), or
       // null before the first measurement of this incarnation.
       disk,
+      // Item 70: the last memory reading (used/cap bytes, percent), or null
+      // before the first sample of this incarnation reaches storage.
+      memory: (map.get(MEMORY_KEY) as MemoryReading | undefined) ?? null,
       // Item 64: consecutive connects the control port did not answer, with
       // the rung that count is on; null while the port answers.
       runtimeUnreachable: unreachable ? { ...unreachable, rung: runtimeUnreachableRung(unreachable.count) } : null,
@@ -8804,12 +9060,13 @@ async function handleStatus(env: Env, url: URL): Promise<Response> {
   // deploy gate reads /residents. The registry check rides in the same flight
   // (its 404 is judged first, the probes' results discarded then).
   const stub = residentStub(env, resource.resource);
-  const [record, status, inFlight, refresh, snapshot] = await Promise.all([
+  const [record, status, inFlight, refresh, snapshot, memory] = await Promise.all([
     registryStub(env).getRecord(resource.resource),
     stub.getStatus(),
     stub.getInFlightCount(),
     stub.getRefreshView(),
     stub.snapshotHandle(),
+    stub.memoryGauge(),
   ]);
   if (!record) return json({ error: `${resource.resource} is not onboarded` }, 404);
   // Item 7: which scheduler drives the refresh cycle and, on the Workflow
@@ -8822,6 +9079,9 @@ async function handleStatus(env: Env, url: URL): Promise<Response> {
     lifecycle: refresh.lifecycle,
     refresh: { instance: refresh.instance, skipped: refresh.skipped },
     snapshot,
+    // Item 70: the last memory reading, so the bot and the residents page can
+    // show what the resident's own gate is reading.
+    memory,
   });
 }
 
@@ -9312,6 +9572,7 @@ async function runWatchdog(env: Env, parent?: TraceSpan): Promise<WatchdogSummar
           reason: s.value.reason,
           action: s.value.action,
           disk: s.value.disk,
+          memory: s.value.memory,
           instance: s.value.instance,
         }
       : { resource: record.resource, error: errMsg(s.reason) };

package/dist/assets/deploy/cloudflare-sandbox/Dockerfile

@@ -207,5 +207,7 @@ RUN npm install -g --allow-scripts=@opencode/cli @opencode/cli@2.0.3 \
 # stays PID 1 — it reaps orphans and forwards the platform's signals — and
 # runs the supervisor, which starts the server again when it exits.
 # src/deploy/sandboxRuntimeSupervisor.test.ts holds this pair of lines.
-COPY --chmod=0755 runtime-supervisor.sh /usr/local/bin/sandbox-runtime-supervisor
-ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/sandbox-runtime-supervisor"]
+# The .mjs extension keeps Node reading it as ESM; its `#!/usr/bin/env node`
+# resolves to the Node copied in above.
+COPY --chmod=0755 runtime-supervisor.mjs /usr/local/bin/sandbox-runtime-supervisor.mjs
+ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/sandbox-runtime-supervisor.mjs"]

package/dist/assets/deploy/cloudflare-sandbox/runtime-supervisor.d.mts

@@ -0,0 +1,31 @@
+// Types for runtime-supervisor.mjs (plain-Node ESM: the sandbox image copies
+// the file as-is, no build step). The seams exist so the tests can drive the
+// loop with a fake child, a fake clock and a fake signal source.
+export const DEFAULT_RUNTIME: "/container-server/sandbox";
+export const QUICK_EXIT_SECS: 10;
+export const QUICK_EXIT_LIMIT: 5;
+export const RESTART_PAUSE_MS: 1000;
+
+/** The least a spawned child must offer the loop. */
+export type SupervisedChild = {
+  kill(signal: "SIGTERM"): unknown;
+  once(event: "exit", listener: (code: number | null, signal: string | null) => void): unknown;
+  once(event: "error", listener: (err: unknown) => void): unknown;
+};
+
+export type SignalSource = {
+  on(event: "SIGTERM" | "SIGINT", listener: () => void): unknown;
+  removeListener(event: "SIGTERM" | "SIGINT", listener: () => void): unknown;
+};
+
+export function endCode(code: number | null, signal: string | null): number;
+
+export function supervise(deps: {
+  runtime: string;
+  args: string[];
+  spawn: (runtime: string, args: string[]) => SupervisedChild;
+  now: () => number;
+  sleep: (ms: number) => Promise<unknown>;
+  log: (line: string) => void;
+  signals: SignalSource;
+}): Promise<number>;