@coreplane/switchboard 1.236.0 → 1.238.0

package/dist/assets/deploy/cloudflare-resident/worker.ts

@@ -227,6 +227,7 @@ import {
   type FetchRecord,
   type SnapshotStamp,
 } from "../../src/execution/residentStepPlan.js";
+import { backupIdsOf, RETIRED_SNAPSHOT_KEY, rotateSnapshots } from "../../src/execution/snapshotRetention.js";
 import {
   DF_FREE_ARGV,
   DISK_FULL_FREE_KIB,
@@ -252,7 +253,13 @@ import {
   type DiskSample,
   type ThreadCostKind,
 } from "../../src/execution/residentDiskBudget.js";
-import { attachTarget, mirrorNeedsFetch, parseWantSha, wantShaForBinding } from "../../src/execution/residentHead.js";
+import {
+  attachTarget,
+  type FetchReason,
+  mirrorFetchReason,
+  parseWantSha,
+  wantShaForBinding,
+} from "../../src/execution/residentHead.js";
 import { residentText, sanitizeResidentBody } from "../../src/execution/residentText.js";
 import {
   abandonedWaitStepResult,
@@ -297,6 +304,9 @@ import {
   parseDepsStoreListing,
   planDepsEviction,
   planDepsMaterialization,
+  DEPS_STORE_MAX_UNREFERENCED_UNDER_PRESSURE,
+  type DepsEvictionPlan,
+  type DepsStoreListing,
 } from "../../src/execution/residentDepsStore.js";
 import { buildId, injectedBuildStamp } from "../../src/deploy/buildStamp.js";
 import { createRefreshInstance, createRefreshInstanceNow, type RefreshInstanceParams } from "./refresh";
@@ -1002,10 +1012,11 @@ interface ResidentRecord {
 interface ThreadBinding {
   threadKey: string;
   ref: string;
-  /** How `ref` was chosen when the binding was made: the repo default for want
-   *  of a named branch (item 30), or a branch someone named. Absent on a
-   *  binding made before the field: read as `default` iff `ref` is the default
-   *  branch. Only a `default` binding may ever move. */
+  /** How `ref` was chosen: the repo default for want of a named branch — when
+   *  the binding was made (item 30), or when a return put it there because the
+   *  branch it was on is gone (`returned`) — or a branch someone named. Absent
+   *  on a binding made before the field: read as `default` iff `ref` is the
+   *  default branch. Only a `default` binding may ever move onto a branch. */
   boundBy?: BoundBy;
   /** The move a binding may make (item 16): from the default it was bound to,
    *  onto the branch the thread's own run opened a pull request on. Set, the
@@ -1013,6 +1024,12 @@ interface ThreadBinding {
    *  branch was gone from the mirror and the binding went back to the
    *  default), and then for the next own pull request. */
   rebound?: Rebound;
+  /** The binding's last move back to the default (item 16's second movement):
+   *  the branch it was on — one a rebind moved it onto, or one this thread's
+   *  own runs pushed (`ownBranches`) and bound by name, a ship unit's — was
+   *  gone from the mirror at an attach. From where, to where, which pull
+   *  request's branch it was, when. */
+  returned?: Returned;
   /** The branches the thread's own runs pushed and the pull requests they
    *  head, as each run's release handed them over (`/detach` `pushed`, item
    *  16a) — written before any eviction decision, so the fact outlives the
@@ -1122,9 +1139,10 @@ interface AttachOk {
    *  and why — so the bot can say where the follow-up runs and why. */
   rebindRefused?: RebindRefused;
   /** This attach moved the binding BACK to the default branch (item 16's
-   *  second movement): the branch a rebind had moved it onto is gone from the
-   *  mirror, so the tree was provisioned on the default instead of the attach
-   *  failing `unknown-ref`. From where, to where, which PR's branch it was. */
+   *  second movement): the branch it was on — a rebind's, or one this thread
+   *  itself pushed — is gone from the mirror, so the tree was provisioned on
+   *  the default instead of the attach failing `unknown-ref`. From where, to
+   *  where, which PR's branch it was. */
   returned?: Returned;
 }
 
@@ -2094,13 +2112,20 @@ export class ResidentDO extends Sandbox<Env> {
     throw new StepError("cat-file", `git cat-file exited ${r.exitCode}: ${r.stderr.trim() || "(no output)"}`);
   }
 
-  /** Attach's fetch decision (item 51) over the mirror's actual state: the ref
-   *  is missing, or `wantSha` names a commit the ref's tip is not at. The
-   *  decision itself is the pure, tested `mirrorNeedsFetch`. */
-  private async mirrorNeedsFetchFor(ref: string, wantSha: string | null): Promise<boolean> {
+  /** Attach's fetch decision (item 51) over the mirror's actual state — why
+   *  the mirror is fetched before the tree is cloned, or null: the ref is
+   *  missing, `wantSha` names a commit the ref's tip is not at, or the ref is
+   *  one the binding could return from (item 16's second movement) and is
+   *  verified against the origin before the mirror's word on it is trusted.
+   *  The decision itself is the pure, tested `mirrorFetchReason`. */
+  private async mirrorFetchReasonFor(
+    ref: string,
+    wantSha: string | null,
+    returnable: boolean,
+  ): Promise<FetchReason | null> {
     const refExists = await this.refExists(ref);
     const mirrorSha = refExists && wantSha !== null ? await this.readMirrorSha(ref) : undefined;
-    return mirrorNeedsFetch({ refExists, mirrorSha, wantSha });
+    return mirrorFetchReason({ refExists, mirrorSha, wantSha, returnable });
   }
 
   /** Dependency/build cache key: sha256 over the ls-tree lines (mode,
@@ -2597,7 +2622,7 @@ export class ResidentDO extends Sandbox<Env> {
       [UPDATED_KEY]: new Date(systemClock()).toISOString(),
       [DEADLINE_AT_KEY]: systemClock() + provisioningTimeoutMs,
     });
-    await this.ctx.storage.delete([FACTS_KEY, SNAPSHOT_KEY]); // defensive: no stale facts from a past life
+    await this.ctx.storage.delete([FACTS_KEY, SNAPSHOT_KEY, RETIRED_SNAPSHOT_KEY]); // defensive: no stale facts from a past life
     try {
       this.deleteSchedules(PROVISIONING_CALLBACK);
       this.deleteSchedules(PROVISION_RUN_CALLBACK);
@@ -3139,18 +3164,40 @@ export class ResidentDO extends Sandbox<Env> {
 
   /** The cycle's snapshot phase: the stamped pair to R2 and, when this cycle's
    *  record stands (not superseded by another writer's), the ready marker and
-   *  the replaced snapshot's objects swept. Answers whether the record committed. */
+   *  the rotation — the replaced pair retired, the pair retired before it
+   *  swept (docs/reference/specs/resident-repos.md item 7: two generations, so a
+   *  handle a seeded restore read from `/status` still resolves for a cycle).
+   *  Answers whether the record committed. */
   private async refreshSnapshot(resource: string, stamp: SnapshotStamp): Promise<boolean> {
     const snapped = await this.snapshot({ resource, stamp });
     const committed = snapped.done || !snapped.superseded;
     if (committed) {
       await this.writeDiskMarkers({ ready: stamp.sha });
-      const previous = !snapped.done && !snapped.superseded ? snapped.previous : undefined;
-      if (previous) await this.deleteBackupObjects([previous.mirror.id, previous.checkout.id]).catch(() => {});
+      const replaced = !snapped.done && !snapped.superseded ? snapped.previous : undefined;
+      if (replaced) await this.retireSnapshot(replaced);
     }
     return committed;
   }
 
+  /** The rotation's bookkeeping: the retired generation recorded (the storage
+   *  write first, so a deletion that fails leaves the record true and the
+   *  1-year R2 TTL as the backstop), then the objects the rotation frees. */
+  private async retireSnapshot(replaced: SnapshotRecord): Promise<void> {
+    const rotation = rotateSnapshots({ replaced, retired: await this.retiredSnapshot() });
+    if (rotation.retired) await this.ctx.storage.put(RETIRED_SNAPSHOT_KEY, rotation.retired);
+    if (rotation.deleteIds.length > 0) await this.deleteBackupObjects(rotation.deleteIds).catch(() => {});
+  }
+
+  private retiredSnapshot(): Promise<SnapshotRecord | undefined> {
+    return this.ctx.storage.get<SnapshotRecord>(RETIRED_SNAPSHOT_KEY);
+  }
+
+  /** The backup ids of every recorded generation — the current pair and the
+   *  retired one — for the paths that delete or itemize them all. */
+  private async recordedBackupIds(current: SnapshotRecord | undefined): Promise<string[]> {
+    return backupIdsOf([current, await this.retiredSnapshot()]);
+  }
+
   /** The cycle's completion: the facts to the stamp (the snapshot step moved
    *  them in the same write as the record — a wake never sees a half-updated
    *  pair; this is the cycle's own bookkeeping on a fresh read, and a
@@ -4067,12 +4114,36 @@ export class ResidentDO extends Sandbox<Env> {
         committedKiB: this.diskCommittedKiB,
         diskBudgetMb: input.record.diskBudgetMb,
         kind,
+        // A deps-store entry backup stages its archive from the deps (item
+        // 61); the reserve holds their share only while one runs.
+        depsBackupInFlight: this.depsBackupsInFlight.size > 0,
       });
     let rawFree = df.freeKiB;
     let verdict = decide(rawFree);
     const evicted: Array<{ threadKey: string; freedKiB: number | null }> = [];
     // Item 62: keep decisions travel as tokens; the key stays for the log line.
     const kept: Array<{ threadKey: string; why: DiskKeepWhy }> = [];
+    // The deps store's spares go first (item 55): a warm cache no live tree
+    // references, worth minutes on a future attach, against a tree refused
+    // now. Their bytes come back before any idle tree is considered.
+    let spares: Array<{ key: string; freedKiB: number }> = [];
+    if (!verdict.fits) {
+      spares = await this.evictDepsSparesUnderPressure().catch((err) => {
+        console.log(`disk-pressure: deps spare eviction failed: ${errMsg(err)}`);
+        return [];
+      });
+      if (spares.length > 0) {
+        console.log(
+          `disk-pressure: evicted ${spares.length} deps-store spare(s) (${formatGiB(spares.reduce((a, e) => a + e.freedKiB, 0))} back) to make room for ${input.threadKey}`,
+        );
+        rawFree = rawFreeAfterEviction(
+          rawFree,
+          await this.dfSample(),
+          spares.reduce((a, e) => a + e.freedKiB, 0),
+        );
+        verdict = decide(rawFree);
+      }
+    }
     if (!verdict.fits) {
       const live = await this.liveBindings();
       const candidates: DiskEvictionCandidate[] = live.map((b) => ({
@@ -4119,7 +4190,7 @@ export class ResidentDO extends Sandbox<Env> {
     // truthiness does not narrow a boolean-literal discriminant.
     const final = verdict;
     if (final.fits === false) {
-      const reason = diskPressureReason({ verdict: final, evicted, kept });
+      const reason = diskPressureReason({ verdict: final, evicted, kept, spares });
       console.log(`attach ${input.threadKey}: ${reason}`);
       const s = await this.getStatus();
       return { error: reason, status: 503, state: s.state, reason: DISK_PRESSURE_REASON };
@@ -4127,7 +4198,7 @@ export class ResidentDO extends Sandbox<Env> {
     const m = final.math;
     console.log(
       `attach ${input.threadKey}: disk admitted — ${kind} ${formatGiB(m.projectedKiB)} projected, ${formatGiB(m.freeKiB)} free, ` +
-        `reserve ${formatGiB(m.reserve.totalKiB)}, headroom ${formatGiB(m.headroomKiB)}${evicted.length > 0 ? `, evicted ${evicted.map((e) => e.threadKey).join(", ")}` : ""}`,
+        `reserve ${formatGiB(m.reserve.totalKiB)}, headroom ${formatGiB(m.headroomKiB)}${spares.length > 0 ? `, evicted ${spares.length} deps spare(s)` : ""}${evicted.length > 0 ? `, evicted ${evicted.map((e) => e.threadKey).join(", ")}` : ""}`,
     );
     const committedKiB = m.projectedKiB ?? 0;
     this.diskCommittedKiB += committedKiB;
@@ -4461,12 +4532,13 @@ export class ResidentDO extends Sandbox<Env> {
       lastAttachAt: now,
       evicted: false,
       // An evicted binding's own history: how its ref was chosen (absent on
-      // one made before the field — never overwritten by this attach's flag)
-      // and the one move it may have made.
+      // one made before the field — never overwritten by this attach's flag),
+      // the one move it may have made and its last move back.
       ...(existing
         ? {
             ...(existing.boundBy !== undefined ? { boundBy: existing.boundBy } : {}),
             ...(existing.rebound !== undefined ? { rebound: existing.rebound } : {}),
+            ...(existing.returned !== undefined ? { returned: existing.returned } : {}),
             ...(existing.ownBranches !== undefined ? { ownBranches: existing.ownBranches } : {}),
           }
         : { boundBy }),
@@ -4819,27 +4891,27 @@ export class ResidentDO extends Sandbox<Env> {
 
   /** Item 16's second movement, decided where the fact is established — under
    *  the mirror mutex, after the attach's fetch found the bound ref gone: a
-   *  binding a rebind moved onto its own pull request's branch, that branch
-   *  now deleted (its pull request merged), goes back to the default it was
-   *  bound to, so the run starts clean there (item 17) instead of the thread
-   *  failing `unknown-ref` for the rest of its life. The row as it stands is
-   *  re-read and re-judged (`canReturnToDefault`: a ref a person named never
-   *  returns — that branch is theirs to sort out), the move stamped returned
-   *  so the thread may follow its next pull request, and the answer carries
-   *  the move back for the card. A row that no longer qualifies — another
-   *  attach moved it meanwhile — is answered as it stands, nothing written. */
+   *  binding on a branch that is now deleted (its pull request merged) goes
+   *  back to the default, so the run starts clean there (item 17) instead of
+   *  the thread failing `unknown-ref` for the rest of its life — when the
+   *  branch is one a rebind moved it onto, or one this thread's own runs
+   *  pushed as the binding remembers it (`ownBranches`): a ship unit's coding
+   *  child binds its unit branch by name and pushes it, and the merge deletes
+   *  it. The row as it stands is re-read and re-judged (the pure
+   *  `returnToDefault`, which writes nothing for a row `canReturnToDefault`
+   *  does not admit: a ref a person named that the thread never pushed never
+   *  returns — that branch is theirs to sort out), the move recorded so the
+   *  thread may follow its next pull request, and the answer carries the move
+   *  back for the card. A row that no longer qualifies — another attach moved
+   *  it meanwhile — is answered as it stands, nothing written. */
   private async returnBindingToDefault(
     binding: ThreadBinding,
     defaultRef: string,
   ): Promise<{ binding: ThreadBinding; returned?: Returned }> {
     const key = threadBindingKey(binding.threadKey);
     const current = (await this.ctx.storage.get<ThreadBinding>(key)) ?? binding;
-    if (current.rebound === undefined || !canReturnToDefault(current, defaultRef)) return { binding: current };
-    const back = returnToDefault(
-      { ...current, rebound: current.rebound },
-      defaultRef,
-      new Date(systemClock()).toISOString(),
-    );
+    const back = returnToDefault(current, defaultRef, new Date(systemClock()).toISOString());
+    if (back === undefined) return { binding: current };
     await this.ctx.storage.put(key, back.binding);
     console.log(
       `attach ${binding.threadKey}: ${back.returned.from} is gone from the mirror (the thread's own pull request #${back.returned.pr}) — returned to ${back.returned.to}; the tree is provisioned there`,
@@ -4921,9 +4993,22 @@ export class ResidentDO extends Sandbox<Env> {
     // The expected head applies to the ref it was resolved for; a sticky
     // binding on another branch drops it (item 51) rather than fetching on
     // every attach of a thread that can never be at that commit.
+    // A ref the binding could return from — a branch a rebind moved it onto,
+    // or one its own runs pushed (item 16's second movement) — is fetched for
+    // whether or not the mirror holds it: the branch dies when its pull
+    // request merges, the deletion reaches the mirror only through a
+    // `fetch --prune`, and a mirror trusted for still holding the ref would
+    // provision a tree at the deleted branch's stale tip until the refresh
+    // cycle's prune caught up. One predicate, read once here, drives the mint
+    // pre-check, the fetch under the lock and the return gate alike.
     let want = wantShaForBinding({ boundRef: binding.ref, refHint, wantSha });
+    const returnable = canReturnToDefault(binding, facts.defaultRef);
     let fetchToken: string | null = token;
-    if (!fetchToken && githubAppConfigured(this.env) && (await this.mirrorNeedsFetchFor(binding.ref, want))) {
+    if (
+      !fetchToken &&
+      githubAppConfigured(this.env) &&
+      (await this.mirrorFetchReasonFor(binding.ref, want, returnable)) !== null
+    ) {
       fetchToken = (await mintRepoScopedToken(this.env, slug).catch(() => null))?.token ?? null;
     }
 
@@ -4931,14 +5016,27 @@ export class ResidentDO extends Sandbox<Env> {
     try {
       locked = await this.withMirrorLock(async () => {
         await this.ensureGitSetup();
-        const fetched = await this.mirrorNeedsFetchFor(binding.ref, want);
-        if (fetched) {
-          await this.gitWithCred(
-            fetchToken,
-            ["-C", MIRROR_DIR, "fetch", "--prune", "origin"],
-            "fetch",
-            GIT_NETWORK_TIMEOUT_MS,
-          );
+        const why = await this.mirrorFetchReasonFor(binding.ref, want, returnable);
+        if (why !== null) {
+          try {
+            await this.gitWithCred(
+              fetchToken,
+              ["-C", MIRROR_DIR, "fetch", "--prune", "origin"],
+              "fetch",
+              GIT_NETWORK_TIMEOUT_MS,
+            );
+          } catch (err) {
+            // A fetch for a missing or stale ref fails the attach as it always
+            // did. One that only verifies a returnable ref the mirror holds is
+            // not the attach's failure — the origin unreachable, no token for
+            // a private repo — so the attach goes on with the mirror's ref, as
+            // it did before the verification existed; the return is only
+            // ever decided on a fetch that ran.
+            if (why !== "returnable-ref") throw err;
+            console.log(
+              `attach ${threadKey}: the fetch verifying ${binding.ref} at the origin failed (${errMsg(err)}) — the attach goes on with the mirror's ref`,
+            );
+          }
         }
         // What to check out, now that the mirror is as fresh as it will get
         // (item 51, the pure `attachTarget`): the ref's tip when the ref is
@@ -4947,18 +5045,21 @@ export class ResidentDO extends Sandbox<Env> {
         // `refs/pull/N/head` still names its head, and the caller told us that
         // head — so a review of a merged PR stays on the warm resident instead
         // of falling back to a cold sandbox that clones the same commit itself.
-        // No fetch ran ⇒ the ref existed at the check: `mirrorNeedsFetch` fetches
-        // for every missing ref, so only a fetched mirror needs the re-read.
-        // (Under the mirror mutex — the mirror cannot change in between.)
-        const refExists = !fetched || (await this.refExists(binding.ref));
+        // No fetch was due ⇒ the ref existed at the check: `mirrorFetchReason`
+        // fetches for every missing ref, so only a fetched mirror needs the
+        // re-read — and a returnable ref is re-read for real, the fact the
+        // second movement is decided on. (Under the mirror mutex — the mirror
+        // cannot change in between.)
+        const refExists = why === null || (await this.refExists(binding.ref));
         let target = attachTarget({
           refExists,
           wantSha: want,
           commitInMirror: !refExists && want !== null && (await this.commitInMirror(want)),
         });
-        if (target.kind === "unknown-ref" && canReturnToDefault(binding, facts.defaultRef)) {
-          // The branch a rebind moved this thread onto is gone even after the
-          // fetch — deleted once its pull request merged. The binding goes
+        if (target.kind === "unknown-ref" && returnable) {
+          // The thread's branch — one a rebind moved it onto, or one its own
+          // runs pushed and it was bound to by name — is gone even after the
+          // fetch: deleted once its pull request merged. The binding goes
           // back to the default (item 16's second movement) and the attach
           // goes on there: the default is always in the mirror, and the
           // worktree step below provisions the tree at its tip.
@@ -5713,27 +5814,24 @@ export class ResidentDO extends Sandbox<Env> {
    *  in flight, and remove what `planDepsEviction` names — debris first, then
    *  the coldest spares beyond DEPS_STORE_MAX_UNREFERENCED. Housekeeping:
    *  a failure is a log line, never a lifecycle flip. */
-  private async sweepDepsStore(): Promise<void> {
-    const listed = await this.run(["sh", "-c", depsStoreListScript()], { timeoutMs: DU_TIMEOUT_MS });
-    if (listed.exitCode !== 0) return;
-    const listing = parseDepsStoreListing(listed.stdout);
-    if (listing.entries.length === 0 && listing.leftovers.length === 0) return;
+  /** The store entries no eviction may touch: the checkout's key, every live
+   *  binding's, every install and every entry backup in flight. */
+  private async depsProtectedKeys(): Promise<Set<string>> {
     const facts = await this.ctx.storage.get<RepoFacts>(FACTS_KEY);
     const protectedKeys = new Set<string>([...this.depsInFlight.keys(), ...this.depsBackupsInFlight]);
     if (facts?.lockfileHash) protectedKeys.add(facts.lockfileHash);
     for (const b of await this.liveBindings()) if (b.depsKey) protectedKeys.add(b.depsKey);
-    // A scratch/staging dir of an install still running in this incarnation
-    // is live work, not debris; the listing cannot tell them apart, so
-    // leftovers wait for a sweep with nothing in flight.
-    const leftovers = this.depsInFlight.size > 0 ? [] : listing.leftovers;
-    const plan = planDepsEviction({ entries: listing.entries, leftovers, protectedKeys });
-    if (plan.remove.length === 0) return;
+    return protectedKeys;
+  }
+
+  /** Remove the store entries a plan names and drop their entry backups with
+   *  them (item 61 PR B: a spare nothing references on disk is a spare nothing
+   *  will wake into). Shared by the sweep and the attach's pressure path. */
+  private async applyDepsEviction(plan: DepsEvictionPlan, listing: DepsStoreListing, who: string): Promise<void> {
     await this.runOk(["rm", "-rf", ...plan.remove], "deps-evict", { timeoutMs: DU_TIMEOUT_MS });
     console.log(
-      `deps: evicted ${plan.remove.length} path(s) — ${plan.remove.map((p) => p.slice(DEPS_STORE_DIR.length + 1, DEPS_STORE_DIR.length + 9)).join(", ")}; kept ${plan.keep.map((k) => k.slice(0, 8)).join(", ")}`,
+      `deps: ${who} evicted ${plan.remove.length} path(s) — ${plan.remove.map((p) => p.slice(DEPS_STORE_DIR.length + 1, DEPS_STORE_DIR.length + 9)).join(", ")}; kept ${plan.keep.map((k) => k.slice(0, 8)).join(", ")}`,
     );
-    // The evicted entries' archives go with them (item 61 PR B): a spare
-    // nothing references on disk is a spare nothing will wake into.
     const kept = new Set(plan.keep);
     const evictedKeys = listing.entries.map((e) => e.key).filter((k) => !kept.has(k));
     const drop = depsBackupsToDrop({
@@ -5748,6 +5846,44 @@ export class ResidentDO extends Sandbox<Env> {
     }
   }
 
+  /** Under disk pressure (item 55) the store keeps no spare: every complete
+   *  entry no live tree references goes, coldest first, before any idle tree
+   *  is considered — a spare is a cache worth minutes on a future attach, a
+   *  refused tree is a run falling to a cold sandbox now. Leftovers are the
+   *  sweep's business (they may be live installs' scratch). Answers what it
+   *  removed with the bytes each measured at; a listing that fails answers
+   *  nothing and the tree path decides as before. */
+  private async evictDepsSparesUnderPressure(): Promise<Array<{ key: string; freedKiB: number }>> {
+    const listed = await this.run(["sh", "-c", depsStoreListScript()], { timeoutMs: DU_TIMEOUT_MS });
+    if (listed.exitCode !== 0) return [];
+    const listing = parseDepsStoreListing(listed.stdout);
+    if (listing.entries.length === 0) return [];
+    const plan = planDepsEviction({
+      entries: listing.entries,
+      leftovers: [],
+      protectedKeys: await this.depsProtectedKeys(),
+      maxUnreferenced: DEPS_STORE_MAX_UNREFERENCED_UNDER_PRESSURE,
+    });
+    if (plan.remove.length === 0) return [];
+    await this.applyDepsEviction(plan, listing, "disk-pressure");
+    return plan.evicted.map((e) => ({ key: e.key, freedKiB: e.kib }));
+  }
+
+  private async sweepDepsStore(): Promise<void> {
+    const listed = await this.run(["sh", "-c", depsStoreListScript()], { timeoutMs: DU_TIMEOUT_MS });
+    if (listed.exitCode !== 0) return;
+    const listing = parseDepsStoreListing(listed.stdout);
+    if (listing.entries.length === 0 && listing.leftovers.length === 0) return;
+    const protectedKeys = await this.depsProtectedKeys();
+    // A scratch/staging dir of an install still running in this incarnation
+    // is live work, not debris; the listing cannot tell them apart, so
+    // leftovers wait for a sweep with nothing in flight.
+    const leftovers = this.depsInFlight.size > 0 ? [] : listing.leftovers;
+    const plan = planDepsEviction({ entries: listing.entries, leftovers, protectedKeys });
+    if (plan.remove.length === 0) return;
+    await this.applyDepsEviction(plan, listing, "sweep");
+  }
+
   /** The per-user 700 staging dir (`install -d` is idempotent), memoized per
    *  incarnation — the write/credential paths used to fork for it on every
    *  call. Cleared with the other memos; a fresh disk simply re-creates it. */
@@ -7049,6 +7185,7 @@ export class ResidentDO extends Sandbox<Env> {
       };
     }
     const snap = await this.ctx.storage.get<SnapshotRecord>(SNAPSHOT_KEY);
+    const recorded = await this.recordedBackupIds(snap);
     const bindings = await this.ctx.storage.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX });
     const plan = {
       resource,
@@ -7065,7 +7202,7 @@ export class ResidentDO extends Sandbox<Env> {
               checkoutBackupId: snap.checkout.id,
             }
           : null,
-        backupObjects: snap ? await this.countBackupObjects([snap.mirror.id, snap.checkout.id]) : 0,
+        backupObjects: await this.countBackupObjects(recorded),
       },
       reprovision: { defaultRef, provisioningTimeoutMs },
       keeps: { registryRecord: true as const, threadBindings: bindings.size },
@@ -7076,9 +7213,9 @@ export class ResidentDO extends Sandbox<Env> {
     // and backups/<id>/ lives outside the resident/<resource>/ prefix — this
     // is the only path that can still reach them (same ordering as teardown).
     let backupObjectsDeleted = 0;
-    if (snap) {
+    if (recorded.length > 0) {
       try {
-        backupObjectsDeleted = await this.deleteBackupObjects([snap.mirror.id, snap.checkout.id]);
+        backupObjectsDeleted = await this.deleteBackupObjects(recorded);
       } catch {
         // best effort — the 1-year R2 TTL is the leak backstop
       }
@@ -7114,8 +7251,7 @@ export class ResidentDO extends Sandbox<Env> {
       this.listSchedules(PROVISIONING_CALLBACK),
       this.listSchedules(PROVISION_RUN_CALLBACK),
     ]);
-    const snap = await this.ctx.storage.get<SnapshotRecord>(SNAPSHOT_KEY);
-    const ids = snap ? [snap.mirror.id, snap.checkout.id] : [];
+    const ids = await this.recordedBackupIds(await this.ctx.storage.get<SnapshotRecord>(SNAPSHOT_KEY));
     const bindings = await this.ctx.storage.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX });
     return {
       ...status,
@@ -7143,10 +7279,10 @@ export class ResidentDO extends Sandbox<Env> {
     let backupObjectsDeleted = 0;
     this.deleteSchedules(PROVISIONING_CALLBACK);
     this.deleteSchedules(PROVISION_RUN_CALLBACK);
-    const snap = await this.ctx.storage.get<SnapshotRecord>(SNAPSHOT_KEY);
-    if (snap) {
+    const recorded = await this.recordedBackupIds(await this.ctx.storage.get<SnapshotRecord>(SNAPSHOT_KEY));
+    if (recorded.length > 0) {
       try {
-        backupObjectsDeleted = await this.deleteBackupObjects([snap.mirror.id, snap.checkout.id]);
+        backupObjectsDeleted = await this.deleteBackupObjects(recorded);
       } catch (err) {
         errors.push(`backup object deletion failed: ${errMsg(err)}`);
       }

package/dist/assets/deploy/cloudflare-sandbox/worker.ts

@@ -41,6 +41,9 @@ import {
   SANDBOX_SLEEP_AFTER,
   isRecycleError,
   recycledMidCommandMessage,
+  DETACH_HINT,
+  OUTPUT_AFTER_EXIT_MS,
+  heldOutputNote,
 } from "../../src/execution/sandboxLifecycle.js";
 import { envFromRequest } from "../../src/execution/sandboxEnv.js";
 import { IdleGuard, type IdleGuardHost } from "../../src/execution/sandboxIdle.js";
@@ -88,10 +91,17 @@ const EXEC_TIMEOUT_SECS = 280;
  *  the shell-level 124 is always the deadline a command meets first. */
 const SDK_BACKSTOP_MARGIN_MS = 40_000;
 
-/** How long the Durable Object waits for the process's exit past the SDK's
- *  own limit before it kills the process itself and reports the timeout: the
- *  runtime should have ended it at the limit; a starved container ends it late. */
-const OUTPUT_WAIT_MARGIN_MS = 30_000;
+/** How long the Durable Object waits for the process's output to END past
+ *  the command's own deadline before it asks the runtime what became of the
+ *  process (docs/reference/specs/execution.md item 24). The runtime ends the
+ *  output stream only when every holder of the command's stdout and stderr is
+ *  gone; a process that exited while a detached child holds them is answered
+ *  with its exit code and a note, a process still running is killed and
+ *  answered as the shell-level timeout. The wait is `OUTPUT_AFTER_EXIT_MS`
+ *  (20 s), inside the executor's per-send margin, so this answer always
+ *  reaches the bot before it gives the command up — the SDK's own limit
+ *  (`SDK_BACKSTOP_MARGIN_MS`) stays above it as the last resort. */
+const OUTPUT_WAIT_AFTER_DEADLINE_MS = OUTPUT_AFTER_EXIT_MS;
 
 /** The interruption reasons that mean the container's runtime is not the one
  *  the command started on: the process, if it started, is gone with its
@@ -303,7 +313,10 @@ export class SwitchboardSandbox extends Sandbox<Env> {
       return this.execFailure(err, startedAt);
     }
     try {
-      const out = await proc.output({ encoding: "utf8", timeout: backstopMs + OUTPUT_WAIT_MARGIN_MS });
+      const out = await proc.output({
+        encoding: "utf8",
+        timeout: execTimeoutSecs * 1000 + OUTPUT_WAIT_AFTER_DEADLINE_MS,
+      });
       // coreutils `timeout` exits 124 when the deadline killed the command
       // (137 when the follow-up SIGKILL had to); the SDK's own limit, if it
       // ever wins, reports `timedOut`. All three are the one story.
@@ -321,9 +334,21 @@ export class SwitchboardSandbox extends Sandbox<Env> {
       };
     } catch (err) {
       if (err instanceof ProcessWaitTimeoutError) {
-        // The runtime should have ended the process at the SDK's limit and
-        // did not; abandoning a live process would leave it running in the
-        // workspace. End it, then report the shell-level timeout it exceeded.
+        // The output never ended inside the command's deadline plus the
+        // margin. Two stories, told apart by asking the runtime (item 24):
+        // the process EXITED and a detached child holds its output open —
+        // the answer is its exit code and the note, the child runs on; or
+        // the process is still RUNNING past its own `timeout` — end it, and
+        // report the shell-level timeout it exceeded.
+        const status = await proc.status().catch(() => null);
+        if (status?.state === "exited") {
+          return {
+            stdout: "",
+            stderr: heldOutputNote(status.exit.code),
+            exitCode: status.exit.code,
+            durationMs: systemClock() - startedAt,
+          };
+        }
         await proc.kill(9).catch(() => {});
         return {
           stdout: "",
@@ -449,11 +474,12 @@ export class SwitchboardSandbox extends Sandbox<Env> {
 /** The stderr line an exit 124 carries: the limit, the knob, and the way to
  *  outlive a command (`setsid -f`: every /exec runs under `timeout … bash -c`,
  *  whose process group is reaped when the command returns, so a plain
- *  background job dies with it). */
+ *  background job dies with it — and with the wrapper's stdio redirected, so
+ *  the command's output ends when the command does, item 24). */
 function timeoutNote(execTimeoutSecs: number): string {
   return (
     `command timed out in the sandbox after ${execTimeoutSecs}s (pass the bash tool's timeoutMs for longer commands, max ${BASH_TIMEOUT_MAX_MS} ms); ` +
-    "re-run as smaller/faster steps, or start it detached with `setsid -f sh -c '<command> > /tmp/job.log 2>&1'` and poll the log on later calls"
+    `re-run as smaller/faster steps, or start it detached with \`${DETACH_HINT}\` and poll the log on later calls`
   );
 }
 

package/dist/assets/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "switchboard",
-  "version": "1.236.0",
+  "version": "1.238.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "switchboard",
-      "version": "1.236.0",
+      "version": "1.238.0",
       "license": "Apache-2.0",
       "workspaces": [
         "web",
@@ -20241,7 +20241,7 @@
     },
     "packages/switchboard": {
       "name": "@coreplane/switchboard",
-      "version": "1.236.0",
+      "version": "1.238.0",
       "license": "Apache-2.0",
       "dependencies": {
         "@earendil-works/pi-ai": "0.85.1",

package/dist/assets/package.json

@@ -1,6 +1,6 @@
 {
   "name": "switchboard",
-  "version": "1.236.0",
+  "version": "1.238.0",
   "private": true,
   "description": "Mention it in Slack and an agent reviews the PR, ships the fix, or answers the question — on the model you choose, with its tools running where you decide.",
   "license": "Apache-2.0",

package/dist/assets/source.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.236.0",
-  "commit": "cf9308dd5f28ac788804260c2978403ce5b59c81",
-  "builtAt": "2026-09-16T20:27:06.883Z"
+  "version": "1.238.0",
+  "commit": "9990b225865690781f1d97b7128228af23e39500",
+  "builtAt": "2026-09-16T22:38:02.661Z"
 }