@coreplane/switchboard 1.233.0 → 1.235.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/dist/assets/deploy/cloudflare-resident/gc.ts +6 -9
  2. package/dist/assets/deploy/cloudflare-resident/worker.ts +310 -277
  3. package/dist/assets/deploy/cloudflare-sandbox/worker.ts +82 -13
  4. package/dist/assets/package-lock.json +3 -3
  5. package/dist/assets/package.json +1 -1
  6. package/dist/assets/source.json +3 -3
  7. package/dist/assets/src/agents/registry.ts +13 -7
  8. package/dist/assets/src/core/provider.ts +9 -6
  9. package/dist/assets/src/core/runEvents.ts +25 -0
  10. package/dist/assets/src/execution/residentCleanliness.ts +69 -5
  11. package/dist/assets/src/execution/residentDiskBudget.ts +7 -6
  12. package/dist/assets/src/execution/residentRebind.ts +92 -77
  13. package/dist/assets/src/execution/residentReuse.ts +7 -18
  14. package/dist/assets/src/execution/residentSteps.ts +0 -1
  15. package/dist/assets/src/execution/sandboxErrors.ts +5 -0
  16. package/dist/assets/src/execution/sandboxIdle.ts +220 -0
  17. package/dist/assets/web/dist/.vite/manifest.json +68 -28
  18. package/dist/assets/web/dist/assets/{ResidentDetailPage-V4K35SYX.js → ResidentDetailPage-DSpsye5e.js} +1 -1
  19. package/dist/assets/web/dist/assets/{ResidentsIndexPage-3HuYaDwD.js → ResidentsIndexPage-CT8fI1M3.js} +1 -1
  20. package/dist/assets/web/dist/assets/RunFoldRow-Cqh8pTXC.js +9 -0
  21. package/dist/assets/web/dist/assets/RunRoutePage-oN9GkVEA.js +6 -0
  22. package/dist/assets/web/dist/assets/RunsIndexPage-sjXr8iCV.js +1 -0
  23. package/dist/assets/web/dist/assets/{ScheduledPage-dfrjQ5E9.js → ScheduledPage-Tz__zLzi.js} +1 -1
  24. package/dist/assets/web/dist/assets/{StatusDot-DexKtvUC.js → StatusDot-D14iJMy7.js} +1 -1
  25. package/dist/assets/web/dist/assets/{Tooltip-DGQ2gu9M.js → Tooltip-B0Ob5MQ4.js} +1 -1
  26. package/dist/assets/web/dist/assets/UnitRoutePage-CbKCL58v.js +1 -0
  27. package/dist/assets/web/dist/assets/{dist-boTdDLF4.js → dist-pOdUzj6D.js} +1 -1
  28. package/dist/assets/web/dist/assets/favicon-BQsePYv5.js +1 -0
  29. package/dist/assets/web/dist/assets/indexRow-Bnj883ii.js +1 -0
  30. package/dist/assets/web/dist/assets/{main-BHhLVMQh.js → main-D3lG-yQ9.js} +2 -2
  31. package/dist/assets/web/dist/assets/main-Dm11o0hc.css +1 -0
  32. package/dist/cli.js +1231 -586
  33. package/package.json +1 -1
  34. package/dist/assets/web/dist/assets/RunRoutePage-Dr867-Ha.js +0 -13
  35. package/dist/assets/web/dist/assets/RunsIndexPage-D_lpXsr8.js +0 -1
  36. package/dist/assets/web/dist/assets/favicon-CWPcvWvp.js +0 -1
  37. package/dist/assets/web/dist/assets/main-CAVqMbiX.css +0 -1
@@ -87,6 +87,7 @@ import { parseReadonly, planReadonlyAttach } from "../../src/execution/residentR
87
87
  import { decideWorktree, parseReuse, type WorktreeFacts } from "../../src/execution/residentReuse.js";
88
88
  import {
89
89
  boundByFor,
90
+ canReturnToDefault,
90
91
  parseOwnPr,
91
92
  parsePushed,
92
93
  parseRefByDefault,
@@ -94,10 +95,12 @@ import {
94
95
  rebindRefused,
95
96
  rebindVerdict,
96
97
  rememberOwnBranches,
98
+ returnToDefault,
97
99
  type BoundBy,
98
100
  type OwnBranch,
99
101
  type OwnPr,
100
102
  type PushedBranch,
103
+ type Returned,
101
104
  type Rebound,
102
105
  type RebindRefused,
103
106
  type RebindTreeFacts,
@@ -112,7 +115,15 @@ import {
112
115
  type DepCacheMaterialization,
113
116
  type ThreadDepsMechanism,
114
117
  } from "../../src/execution/residentDepCache.js";
115
- import { parseWorktreeCleanliness, worktreeCleanlinessScript } from "../../src/execution/residentCleanliness.js";
118
+ import {
119
+ evictedTreeOf,
120
+ evictedTreeSentence,
121
+ parseWorktreeCleanliness,
122
+ worktreeCleanlinessScript,
123
+ type EvictedTree,
124
+ type LeftBehind,
125
+ type WorktreeCleanliness,
126
+ } from "../../src/execution/residentCleanliness.js";
116
127
  import {
117
128
  base64LengthOf,
118
129
  chunkPlan,
@@ -994,15 +1005,17 @@ interface ThreadBinding {
994
1005
  * binding made before the field: read as `default` iff `ref` is the default
995
1006
  * branch. Only a `default` binding may ever move. */
996
1007
  boundBy?: BoundBy;
997
- /** The one move a binding may make (item 16): from the default it was bound
998
- * to, onto the branch the thread's own run opened a pull request on. Set,
999
- * the binding never moves again. */
1008
+ /** The move a binding may make (item 16): from the default it was bound to,
1009
+ * onto the branch the thread's own run opened a pull request on. Set, the
1010
+ * binding moves again only after the move is returned (`returnedAt`: the
1011
+ * branch was gone from the mirror and the binding went back to the
1012
+ * default), and then for the next own pull request. */
1000
1013
  rebound?: Rebound;
1001
1014
  /** The branches the thread's own runs pushed and the pull requests they
1002
1015
  * head, as each run's release handed them over (`/detach` `pushed`, item
1003
1016
  * 16a) — written before any eviction decision, so the fact outlives the
1004
- * tree: a clean tree is released the moment a run ends, and this is what
1005
- * lets the follow-up's attach move onto the branch and recreate the tree
1017
+ * tree: the tree is released the moment its run ends, and this is what
1018
+ * lets the follow-up's attach move onto the branch and provision the tree
1006
1019
  * there. Newest last; the oldest fall off past `OWN_BRANCHES_MAX`. */
1007
1020
  ownBranches?: OwnBranch[];
1008
1021
  /** Allocated OS user (worker2..worker17); "" once evicted (pool released). */
@@ -1013,8 +1026,17 @@ interface ThreadBinding {
1013
1026
  evicted?: boolean;
1014
1027
  evictedAt?: string;
1015
1028
  /** Why the last eviction happened (the audit trail): `ttl`, `clean-idle`,
1016
- * `detach`, or a reclamation fate — `merged #N` / `closed #N` / `gone`. */
1029
+ * `detach`, `disk-pressure`, or a reclamation fate — `merged #N` /
1030
+ * `closed #N` / `gone`. */
1017
1031
  evictedWhy?: string;
1032
+ /** What the tree held when that eviction removed it (item 17: never a
1033
+ * reason to keep the tree, never discarded silently): the tracked changes
1034
+ * and unpushed commits measured as the thread user. Absent when the tree
1035
+ * was clean, already gone with the disk, or not measured (a force detach). */
1036
+ evictedLeftBehind?: LeftBehind;
1037
+ /** Why the tree could not be measured before that eviction (git could not
1038
+ * read it) — so an unreadable tree is never recorded as clean. */
1039
+ evictedUnmeasured?: string;
1018
1040
  /** How deps were last materialized (evidence that the per-branch reconciliation ran). */
1019
1041
  deps?: ThreadDepsMechanism;
1020
1042
  /** Commit the worktree was last attached at (the ref's tip in the mirror
@@ -1088,6 +1110,20 @@ interface AttachOk {
1088
1110
  /** The caller asked for that move and the binding stood: the branch, the PR
1089
1111
  * and why — so the bot can say where the follow-up runs and why. */
1090
1112
  rebindRefused?: RebindRefused;
1113
+ /** This attach moved the binding BACK to the default branch (item 16's
1114
+ * second movement): the branch a rebind had moved it onto is gone from the
1115
+ * mirror, so the tree was provisioned on the default instead of the attach
1116
+ * failing `unknown-ref`. From where, to where, which PR's branch it was. */
1117
+ returned?: Returned;
1118
+ }
1119
+
1120
+ /** What `POST /detach` answers: whether the pool user went back, why not, and
1121
+ * — on a release — what the tree still held (item 16a), now gone with it. */
1122
+ interface DetachAnswer {
1123
+ released: boolean;
1124
+ reason?: string;
1125
+ user?: string;
1126
+ leftBehind?: LeftBehind;
1091
1127
  }
1092
1128
 
1093
1129
  /** Why the caller's `refHint` is what it is (item 16), beside the hint itself:
@@ -1105,17 +1141,11 @@ const NO_REF_HINT_REASON: RefHintReason = { ownPr: null, refByDefault: false };
1105
1141
 
1106
1142
  /** What the rebind decided under the mutex (item 16): nothing to move (the
1107
1143
  * row is already on the branch), a named refusal, or the move — written on
1108
- * the row, whether a checkout in the live tree or a ref the attach will
1109
- * recreate the tree at. */
1144
+ * the row; the attach that follows provisions the tree at the moved ref. */
1110
1145
  type RebindOutcome =
1111
1146
  | { kind: "none"; binding: ThreadBinding }
1112
1147
  | { kind: "refuse"; refused: RebindRefused }
1113
- | { kind: "rebound"; moved: ThreadBinding; rebound: Rebound; keepTree: false }
1114
- /** The move was the record alone — a dirty tree already on the branch, no
1115
- * checkout run — with the verdict's `note` for the log line. The attach
1116
- * that follows must keep that tree (`keepTree`, the worktree step's word
1117
- * for it): the promise the note makes is the attach's to keep. */
1118
- | { kind: "rebound"; moved: ThreadBinding; rebound: Rebound; keepTree: true; note: string };
1148
+ | { kind: "rebound"; moved: ThreadBinding; rebound: Rebound };
1119
1149
 
1120
1150
  /** Result of one /op test/build execution. `ok` is the command's
1121
1151
  * verdict — a failing test run is a RESULT with ok:false, never an error. */
@@ -3768,15 +3798,16 @@ export class ResidentDO extends Sandbox<Env> {
3768
3798
  });
3769
3799
  }
3770
3800
 
3771
- /** Idle = no live binding attached within IDLE_AFTER_S AND (when the
3772
- * runtime is up) no live tree is dirty. Bindings are storage; dirtiness
3773
- * needs the container — if it is already asleep there is nothing to lose. */
3801
+ /** Idle = no live binding attached within IDLE_AFTER_S and nothing in
3802
+ * flight. What a live tree holds is not asked (item 17): a dirty tree no
3803
+ * run is using protects nothing — the next attach wipes it — so it is
3804
+ * never a reason to keep the container awake; the sweep's release records
3805
+ * what it held. Bindings are storage, so this needs no container. */
3774
3806
  private async isIdle(): Promise<boolean> {
3775
3807
  const live = await this.liveBindings();
3776
3808
  const recent = systemClock() - IDLE_AFTER_S * 1000;
3777
3809
  if (live.some((b) => Date.parse(b.lastAttachAt) >= recent)) return false;
3778
- if (this.inFlightCount() > 0) return false;
3779
- return this.liveTreesClean(live);
3810
+ return this.inFlightCount() === 0;
3780
3811
  }
3781
3812
 
3782
3813
  /** Bindings that hold a pool user and a tree on disk (not evicted). */
@@ -3785,14 +3816,13 @@ export class ResidentDO extends Sandbox<Env> {
3785
3816
  return [...all.values()].filter((b) => !b.evicted && b.user);
3786
3817
  }
3787
3818
 
3788
- /** True when no live tree holds work a lost disk would destroy: every one
3789
- * is clean (as its thread user), or the runtime is already down — a sleep
3790
- * has destroyed the disk, so there is nothing left to lose. */
3819
+ /** The disk-full recycle's tree guard (item 54), its one reader: true when
3820
+ * every live tree is clean as its thread user, or the runtime is already
3821
+ * down — a sleep has destroyed the disk, so there is nothing left to lose. */
3791
3822
  private async liveTreesClean(live: ThreadBinding[]): Promise<boolean> {
3792
3823
  if (!(await this.isRuntimeActive().catch(() => false))) return true;
3793
3824
  // Concurrent: each check touches only its own (disjoint) tree, and one
3794
- // spawn each — the idle gate does not pay a serial
3795
- // 3-probe round-trip per live binding.
3825
+ // spawn each — no serial 3-probe round-trip per live binding.
3796
3826
  const checks = await Promise.all(live.map((b) => this.worktreeCleanliness(b)));
3797
3827
  return checks.every((c) => c.clean); // any dirty or unknown → not clean
3798
3828
  }
@@ -3930,10 +3960,11 @@ export class ResidentDO extends Sandbox<Env> {
3930
3960
  * already on disk is `reuse` (0 — a dirty/stale recreate frees the old one
3931
3961
  * first). A ref not yet in the mirror (fetched under the lock, moments later)
3932
3962
  * is projected as `hardlink`, the common case. When it does not fit, the
3933
- * coldest clean idle trees go first (`orderEvictionCandidates`: never the
3963
+ * coldest idle trees go first (`orderEvictionCandidates`: never the
3934
3964
  * requesting thread, a busy tree, the default branch, or one attached within
3935
- * DISK_EVICT_MIN_IDLE_MS; cleanliness checked as the thread user, dirty or
3936
- * unreadable kept — the sweep's rules), `df` re-probed after each; still
3965
+ * DISK_EVICT_MIN_IDLE_MS), whatever they hold — item 17: dirt never keeps a
3966
+ * tree; each is measured as its thread user for the eviction's record —
3967
+ * `df` re-probed after each; still
3937
3968
  * short → `503 {reason:"disk-pressure"}` with the whole math in `error`, the
3938
3969
  * same shape as `mirror-busy`, so the bot falls back cold legibly. Never a
3939
3970
  * lifecycle flip: the checkout is intact and every existing tree keeps
@@ -4003,12 +4034,10 @@ export class ResidentDO extends Sandbox<Env> {
4003
4034
  if (verdict.fits) break;
4004
4035
  const binding = live.find((b) => b.threadKey === c.threadKey);
4005
4036
  if (!binding) continue;
4006
- const clean = await this.worktreeCleanliness(binding);
4007
- if (!clean.clean) {
4008
- kept.push({ threadKey: c.threadKey, why: "dirty" });
4009
- continue;
4010
- }
4011
- // Same re-read guards as the sweep: the clean check awaited.
4037
+ // What the tree holds goes on the eviction's record (item 17), never
4038
+ // into a keep; the runtime is up — this attach is running on it.
4039
+ const tree = await this.measureTreeBeforeEviction(binding);
4040
+ // Same re-read guards as the sweep: the measurement awaited.
4012
4041
  const current = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(c.threadKey));
4013
4042
  if (
4014
4043
  !current ||
@@ -4019,7 +4048,7 @@ export class ResidentDO extends Sandbox<Env> {
4019
4048
  kept.push({ threadKey: c.threadKey, why: "busy" });
4020
4049
  continue;
4021
4050
  }
4022
- if (!(await this.evictBinding(current, true, "disk-pressure", DISK_PRESSURE_REASON))) {
4051
+ if (!(await this.evictBinding(current, true, "disk-pressure", DISK_PRESSURE_REASON, tree))) {
4023
4052
  kept.push({ threadKey: c.threadKey, why: "other" });
4024
4053
  continue;
4025
4054
  }
@@ -4514,10 +4543,12 @@ export class ResidentDO extends Sandbox<Env> {
4514
4543
 
4515
4544
  // The binding's ref wins for the thread's whole life, with one exception
4516
4545
  // (item 16): a thread bound to the repo default for want of a named branch
4517
- // moves, once and in place, onto the branch its own run opened a pull
4518
- // request on — when that branch is a local branch of this thread's tree and
4519
- // the tree is clean. Decided here, before the ref is chosen, so the rest of
4520
- // the attach (fetch, stale check, deps, credentials) runs on the moved ref.
4546
+ // moves, once per pull request, onto the branch its own run opened a pull
4547
+ // request on — when that branch is the thread's own and the mirror holds
4548
+ // it. A decision about the binding alone, made here, before the ref is
4549
+ // chosen, so the rest of the attach (fetch, the worktree step that
4550
+ // provisions the tree clean at the moved ref, deps, credentials) runs on
4551
+ // the moved ref.
4521
4552
  const rebind = await this.rebindToOwnPr(
4522
4553
  stored.get(threadBindingKey(threadKey)) as ThreadBinding | undefined,
4523
4554
  reason.ownPr,
@@ -4573,7 +4604,6 @@ export class ResidentDO extends Sandbox<Env> {
4573
4604
  refHint,
4574
4605
  wantSha,
4575
4606
  reuse,
4576
- keepTree: rebind.keepTree === true,
4577
4607
  slug,
4578
4608
  t0,
4579
4609
  facts,
@@ -4591,54 +4621,36 @@ export class ResidentDO extends Sandbox<Env> {
4591
4621
 
4592
4622
  /** Item 16's one exception to the sticky binding: a thread bound to the repo
4593
4623
  * default for want of a named branch moves onto the branch its own run
4594
- * opened a pull request on (`ownPr`), once, in place. The decision is the
4595
- * pure `rebindPlan` / `rebindVerdict` of src/execution/residentRebind.ts;
4596
- * this method measures the tree as the thread user and runs the verdict: a
4597
- * `git checkout` inside the existing worktree — same path, same pool user,
4598
- * no clone, no `rm -rf`, deps and snapshot lineage untouched — under the
4599
- * mirror mutex so no other attach of the thread touches the tree meanwhile.
4600
- * The rest of the attach then runs on the moved ref: the mirror fetch for
4601
- * the PR head (item 51) and item 17's stale check, which keeps a tree whose
4602
- * HEAD is the ref's tip or a descendant of it. When the verdict moved the
4603
- * record alone — a dirty tree with the branch already checked out — the
4604
- * answer says so (`keepTree`) and the worktree step keeps that tree, dirt
4605
- * included, instead of item 17's dirty wipe: the promise not to touch it is
4606
- * the whole attach's. The binding records the move (`rebound`); the answer
4607
- * carries it, or the named refusal, so the bot can say where the follow-up
4608
- * runs and why. A refusal never fails the attach. */
4624
+ * opened a pull request on (`ownPr`), once per pull request. A decision
4625
+ * about the BINDING alone — the pure `rebindPlan` / `rebindVerdict` of
4626
+ * src/execution/residentRebind.ts: the branch must be the thread's own,
4627
+ * remembered from a release (`ownBranches`) or, when nothing was
4628
+ * remembered, a local branch of the thread's surviving tree, measured as
4629
+ * the thread user — and the mirror must hold it (`moveOntoOwnBranch`), since
4630
+ * the attach that follows provisions the tree at the moved ref from the
4631
+ * mirror as it provisions any tree (item 17: clean at the ref's tip, a tree
4632
+ * left dirty or on the old branch recreated, the binding's path kept). Under
4633
+ * the mirror mutex, on the row as it stands there. The binding records the
4634
+ * move (`rebound`); the answer carries it, or the named refusal, so the bot
4635
+ * can say where the follow-up runs and why. A refusal never fails the attach. */
4609
4636
  private async rebindToOwnPr(
4610
4637
  prior: ThreadBinding | undefined,
4611
4638
  ownPr: OwnPr | null,
4612
4639
  reuse: boolean,
4613
4640
  defaultRef: string,
4614
4641
  slug: string,
4615
- ): Promise<
4616
- | {
4617
- binding: ThreadBinding | undefined;
4618
- rebound?: Rebound;
4619
- rebindRefused?: RebindRefused;
4620
- /** The move was the record alone — the tree is dirty with the branch
4621
- * already checked out — and the attach must keep that tree as it
4622
- * stands: the worktree step's item 17 wipe of a dirty tree would
4623
- * destroy the very work the rebind declined to touch. */
4624
- keepTree?: true;
4625
- }
4626
- | ThreadErr
4627
- > {
4642
+ ): Promise<{ binding: ThreadBinding | undefined; rebound?: Rebound; rebindRefused?: RebindRefused } | ThreadErr> {
4628
4643
  const plan = rebindPlan({ ownPr, reuse, binding: prior, defaultRef });
4629
4644
  if (plan.kind === "none" || prior === undefined) return { binding: prior };
4630
4645
  if (plan.kind === "refuse") return { binding: prior, rebindRefused: plan.refused };
4631
4646
  const key = threadBindingKey(prior.threadKey);
4632
- // A tree that is gone is recreated at the branch by the attach (item 16):
4633
- // the mirror must hold the branch first, and a branch pushed since the
4634
- // last refresh cycle is fetched for — with a token minted here, before
4635
- // the mutex, like the attach's own recovery fetch. Only when the binding
4636
- // allows a recreate at all: an evicted tree, or a live one whose runs
4637
- // pushed the branch (the tree may turn out missing).
4638
- const fetchToken =
4639
- (plan.kind === "recreate" || plan.own) && githubAppConfigured(this.env)
4640
- ? ((await mintRepoScopedToken(this.env, slug).catch(() => null))?.token ?? null)
4641
- : null;
4647
+ // The tree is provisioned at the branch from the mirror, so the mirror
4648
+ // must hold it first: a branch pushed since the last refresh cycle is
4649
+ // fetched for — with a token minted here, before the mutex, like the
4650
+ // attach's own recovery fetch.
4651
+ const fetchToken = githubAppConfigured(this.env)
4652
+ ? ((await mintRepoScopedToken(this.env, slug).catch(() => null))?.token ?? null)
4653
+ : null;
4642
4654
  let outcome: RebindOutcome;
4643
4655
  try {
4644
4656
  outcome = (
@@ -4653,79 +4665,25 @@ export class ResidentDO extends Sandbox<Env> {
4653
4665
  const again = rebindPlan({ ownPr, reuse, binding: current, defaultRef });
4654
4666
  if (again.kind === "none") return { kind: "none", binding: current };
4655
4667
  if (again.kind === "refuse") return again;
4656
- if (again.kind === "recreate") return this.recreateAtOwnBranch(current, again, fetchToken);
4657
- const wt = current.worktreePath;
4658
- // The facts, as the thread user in the thread's own tree: is the
4659
- // branch a local branch here (the physical trace of this thread's
4660
- // run having made it), and are the tracked files clean — judged
4661
- // only off a status git could read.
4662
- const tree: RebindTreeFacts = { exists: (await this.run(["test", "-d", `${wt}/.git`])).exitCode === 0 };
4663
- if (tree.exists) {
4664
- const branch = await this.threadRun(
4665
- current.user,
4666
- wt,
4667
- `git rev-parse --verify --quiet ${shellQuote(`refs/heads/${again.to}`)}`,
4668
- DEFAULT_EXEC_TIMEOUT_MS,
4669
- );
4670
- tree.branchExists = branch.exitCode === 0;
4671
- const status = await this.threadRun(
4672
- current.user,
4673
- wt,
4674
- "git status --porcelain -uno",
4675
- DEFAULT_EXEC_TIMEOUT_MS,
4676
- );
4677
- tree.readable = status.exitCode === 0;
4678
- if (tree.readable) tree.dirty = status.stdout.trim() !== "";
4679
- if (tree.dirty) {
4680
- // A dirty tree is refused unless its HEAD is already the branch
4681
- // (the run made it here and left an edit after pushing): the
4682
- // one fact that tells the two apart, measured only when it
4683
- // decides anything.
4684
- const head = await this.threadRun(
4668
+ if (again.kind === "measure" && !again.own) {
4669
+ // Nothing remembered from a release: the branch must be a local
4670
+ // branch of the thread's surviving tree — the physical trace of
4671
+ // this thread's run having made it — measured as the thread user.
4672
+ const wt = current.worktreePath;
4673
+ const tree: RebindTreeFacts = { exists: (await this.run(["test", "-d", `${wt}/.git`])).exitCode === 0 };
4674
+ if (tree.exists) {
4675
+ const branch = await this.threadRun(
4685
4676
  current.user,
4686
4677
  wt,
4687
- "git rev-parse --abbrev-ref HEAD",
4678
+ `git rev-parse --verify --quiet ${shellQuote(`refs/heads/${again.to}`)}`,
4688
4679
  DEFAULT_EXEC_TIMEOUT_MS,
4689
4680
  );
4690
- if (head.exitCode === 0) tree.head = head.stdout.trim();
4691
- }
4692
- }
4693
- // Judged on the re-read plan: it carries whether the branch is the
4694
- // thread's own, which the pre-lock plan lacks when it was a recreate.
4695
- const verdict = rebindVerdict(again, tree);
4696
- if (verdict.kind === "recreate") return this.recreateAtOwnBranch(current, again, fetchToken);
4697
- if (verdict.kind !== "rebind") return verdict;
4698
- if (verdict.checkout) {
4699
- const startedAt = systemClock();
4700
- const checkout = await this.threadRun(
4701
- current.user,
4702
- wt,
4703
- `git checkout --quiet ${shellQuote(again.to)}`,
4704
- DEFAULT_EXEC_TIMEOUT_MS,
4705
- );
4706
- this.stepTrace.getStore()?.record("rebind-checkout", {
4707
- startedAt,
4708
- endedAt: systemClock(),
4709
- exitCode: checkout.exitCode,
4710
- timedOut: checkout.timedOut,
4711
- });
4712
- if (checkout.exitCode !== 0) {
4713
- const detail =
4714
- checkout.stderr.trim().split("\n")[0]?.slice(0, 200) || `git checkout exited ${checkout.exitCode}`;
4715
- return { kind: "refuse", refused: rebindRefused(again, "checkout-failed", detail) };
4681
+ tree.branchExists = branch.exitCode === 0;
4716
4682
  }
4683
+ const verdict = rebindVerdict(again, tree);
4684
+ if (verdict.kind === "refuse") return verdict;
4717
4685
  }
4718
- const rebound: Rebound = {
4719
- from: current.ref,
4720
- to: again.to,
4721
- pr: again.pr,
4722
- at: new Date(systemClock()).toISOString(),
4723
- };
4724
- const moved: ThreadBinding = { ...current, ref: again.to, rebound };
4725
- await this.ctx.storage.put(key, moved);
4726
- return verdict.checkout
4727
- ? { kind: "rebound", moved, rebound, keepTree: false }
4728
- : { kind: "rebound", moved, rebound, keepTree: true, note: verdict.note };
4686
+ return this.moveOntoOwnBranch(current, again, fetchToken);
4729
4687
  }, ATTACH_MUTEX_WAIT_MS)
4730
4688
  ).value;
4731
4689
  } catch (err) {
@@ -4743,28 +4701,22 @@ export class ResidentDO extends Sandbox<Env> {
4743
4701
  return { binding: prior, rebindRefused: outcome.refused };
4744
4702
  }
4745
4703
  const { moved, rebound } = outcome;
4746
- if (!outcome.keepTree) {
4747
- console.log(
4748
- `attach ${prior.threadKey}: rebound ${rebound.from} → ${rebound.to} (the thread's own pull request #${rebound.pr})`,
4749
- );
4750
- return { binding: moved, rebound };
4751
- }
4752
4704
  console.log(
4753
- `attach ${prior.threadKey}: rebound ${rebound.from} → ${rebound.to} (the thread's own pull request #${rebound.pr}) — ${outcome.note}`,
4705
+ `attach ${prior.threadKey}: rebound ${rebound.from} → ${rebound.to} (the thread's own pull request #${rebound.pr}); the tree is provisioned at it`,
4754
4706
  );
4755
- return { binding: moved, rebound, keepTree: true };
4756
- }
4757
-
4758
- /** The move when the thread's tree is gone (item 16): the binding's ref
4759
- * becomes the branch its own run pushed, and the attach that follows
4760
- * recreates the tree at it — `ensureThreadWorktree` finds no tree and
4761
- * clones the branch from the mirror. So the mirror must hold the branch:
4762
- * one pushed since the last refresh cycle is fetched for here, under the
4763
- * mutex; a branch still missing after the fetch (deleted after a merge, or
4764
- * never pushed) is a `branch-absent` refusal and the binding stands — the
4765
- * attach then goes on at the ref it had, never `unknown-ref`. Nothing on
4766
- * disk is touched: no checkout (there is no tree), no clone (the attach's). */
4767
- private async recreateAtOwnBranch(
4707
+ return { binding: moved, rebound };
4708
+ }
4709
+
4710
+ /** The move itself (item 16): the binding's ref becomes the branch, and the
4711
+ * attach that follows provisions the tree at it — `ensureThreadWorktree`
4712
+ * clones the branch from the mirror when the tree is gone, dirty or on the
4713
+ * old ref. So the mirror must hold the branch: one pushed since the last
4714
+ * refresh cycle is fetched for here, under the mutex; a branch still
4715
+ * missing after the fetch (deleted after a merge, or never pushed) is a
4716
+ * `branch-absent` refusal and the binding stands — the attach then goes on
4717
+ * at the ref it had, never `unknown-ref`. Nothing on disk is touched here:
4718
+ * the row is written, the tree is the attach's. */
4719
+ private async moveOntoOwnBranch(
4768
4720
  current: ThreadBinding,
4769
4721
  plan: { to: string; pr: number },
4770
4722
  fetchToken: string | null,
@@ -4795,7 +4747,7 @@ export class ResidentDO extends Sandbox<Env> {
4795
4747
  refused: rebindRefused(
4796
4748
  plan,
4797
4749
  "branch-absent",
4798
- `the thread's worktree is gone and the mirror does not hold ${JSON.stringify(plan.to)} (deleted after a merge, or never pushed); the tree cannot be recreated at it`,
4750
+ `the mirror does not hold ${JSON.stringify(plan.to)} even after a fetch (deleted after a merge, or never pushed); the tree cannot be provisioned at it`,
4799
4751
  ),
4800
4752
  };
4801
4753
  }
@@ -4807,13 +4759,43 @@ export class ResidentDO extends Sandbox<Env> {
4807
4759
  };
4808
4760
  const moved: ThreadBinding = { ...current, ref: plan.to, rebound };
4809
4761
  await this.ctx.storage.put(threadBindingKey(current.threadKey), moved);
4810
- return { kind: "rebound", moved, rebound, keepTree: false };
4762
+ return { kind: "rebound", moved, rebound };
4763
+ }
4764
+
4765
+ /** Item 16's second movement, decided where the fact is established — under
4766
+ * the mirror mutex, after the attach's fetch found the bound ref gone: a
4767
+ * binding a rebind moved onto its own pull request's branch, that branch
4768
+ * now deleted (its pull request merged), goes back to the default it was
4769
+ * bound to, so the run starts clean there (item 17) instead of the thread
4770
+ * failing `unknown-ref` for the rest of its life. The row as it stands is
4771
+ * re-read and re-judged (`canReturnToDefault`: a ref a person named never
4772
+ * returns — that branch is theirs to sort out), the move stamped returned
4773
+ * so the thread may follow its next pull request, and the answer carries
4774
+ * the move back for the card. A row that no longer qualifies — another
4775
+ * attach moved it meanwhile — is answered as it stands, nothing written. */
4776
+ private async returnBindingToDefault(
4777
+ binding: ThreadBinding,
4778
+ defaultRef: string,
4779
+ ): Promise<{ binding: ThreadBinding; returned?: Returned }> {
4780
+ const key = threadBindingKey(binding.threadKey);
4781
+ const current = (await this.ctx.storage.get<ThreadBinding>(key)) ?? binding;
4782
+ if (current.rebound === undefined || !canReturnToDefault(current, defaultRef)) return { binding: current };
4783
+ const back = returnToDefault(
4784
+ { ...current, rebound: current.rebound },
4785
+ defaultRef,
4786
+ new Date(systemClock()).toISOString(),
4787
+ );
4788
+ await this.ctx.storage.put(key, back.binding);
4789
+ console.log(
4790
+ `attach ${binding.threadKey}: ${back.returned.from} is gone from the mirror (the thread's own pull request #${back.returned.pr}) — returned to ${back.returned.to}; the tree is provisioned there`,
4791
+ );
4792
+ return back;
4811
4793
  }
4812
4794
 
4813
4795
  /** The release's word on what the run pushed (`/detach` `pushed`, item
4814
- * 16a), remembered on the binding BEFORE the detach decides anything: a
4815
- * clean tree is evicted at once, a dirty one kept, an already-evicted
4816
- * binding answered as such — and in every case the fact must outlive the
4796
+ * 16a), remembered on the binding BEFORE the detach decides anything: the
4797
+ * tree evicted, a busy one kept, an already-evicted binding answered as
4798
+ * such — and in every case the fact must outlive the
4817
4799
  * tree, since it is what a follow-up's rebind onto the thread's own pull
4818
4800
  * request branch reads once the tree is gone. No binding → nothing to
4819
4801
  * remember on (the detach answers its 404 next). */
@@ -4835,10 +4817,6 @@ export class ResidentDO extends Sandbox<Env> {
4835
4817
  refHint: string | null;
4836
4818
  wantSha: string | null;
4837
4819
  reuse: boolean;
4838
- /** The rebind moved the record alone over a dirty tree already on the
4839
- * branch (item 16) and promised not to touch it: the worktree step keeps
4840
- * the tree as it stands instead of item 17's dirty wipe. */
4841
- keepTree: boolean;
4842
4820
  slug: string;
4843
4821
  t0: number;
4844
4822
  facts: RepoFacts;
@@ -4850,8 +4828,14 @@ export class ResidentDO extends Sandbox<Env> {
4850
4828
  rebound?: Rebound;
4851
4829
  rebindRefused?: RebindRefused;
4852
4830
  }): Promise<AttachOk | ThreadErr> {
4853
- const { threadKey, refHint, wantSha, reuse, keepTree, slug, t0, facts, record, binding, mode, rollback } = input;
4831
+ const { threadKey, refHint, wantSha, reuse, slug, t0, facts, record, mode, rollback } = input;
4854
4832
  const { rebound, rebindRefused } = input;
4833
+ // Reassigned once, under the lock, when the bound ref turns out gone from
4834
+ // the mirror and the binding goes back to the default (item 16's second
4835
+ // movement): everything after the lock — deps, credentials, the row's
4836
+ // final write, the answer — then speaks of the returned binding.
4837
+ let binding = input.binding;
4838
+ let returned: Returned | undefined;
4855
4839
 
4856
4840
  // Command-level token mint — before the lock so mint latency
4857
4841
  // never holds the mutex, and failure never blocks the attach.
@@ -4882,7 +4866,7 @@ export class ResidentDO extends Sandbox<Env> {
4882
4866
  // The expected head applies to the ref it was resolved for; a sticky
4883
4867
  // binding on another branch drops it (item 51) rather than fetching on
4884
4868
  // every attach of a thread that can never be at that commit.
4885
- const want = wantShaForBinding({ boundRef: binding.ref, refHint, wantSha });
4869
+ let want = wantShaForBinding({ boundRef: binding.ref, refHint, wantSha });
4886
4870
  let fetchToken: string | null = token;
4887
4871
  if (!fetchToken && githubAppConfigured(this.env) && (await this.mirrorNeedsFetchFor(binding.ref, want))) {
4888
4872
  fetchToken = (await mintRepoScopedToken(this.env, slug).catch(() => null))?.token ?? null;
@@ -4912,11 +4896,27 @@ export class ResidentDO extends Sandbox<Env> {
4912
4896
  // for every missing ref, so only a fetched mirror needs the re-read.
4913
4897
  // (Under the mirror mutex — the mirror cannot change in between.)
4914
4898
  const refExists = !fetched || (await this.refExists(binding.ref));
4915
- const target = attachTarget({
4899
+ let target = attachTarget({
4916
4900
  refExists,
4917
4901
  wantSha: want,
4918
4902
  commitInMirror: !refExists && want !== null && (await this.commitInMirror(want)),
4919
4903
  });
4904
+ if (target.kind === "unknown-ref" && canReturnToDefault(binding, facts.defaultRef)) {
4905
+ // The branch a rebind moved this thread onto is gone even after the
4906
+ // fetch — deleted once its pull request merged. The binding goes
4907
+ // back to the default (item 16's second movement) and the attach
4908
+ // goes on there: the default is always in the mirror, and the
4909
+ // worktree step below provisions the tree at its tip.
4910
+ const back = await this.returnBindingToDefault(binding, facts.defaultRef);
4911
+ binding = back.binding;
4912
+ returned = back.returned;
4913
+ want = wantShaForBinding({ boundRef: binding.ref, refHint, wantSha });
4914
+ target = attachTarget({
4915
+ refExists: await this.refExists(binding.ref),
4916
+ wantSha: want,
4917
+ commitInMirror: false,
4918
+ });
4919
+ }
4920
4920
  if (target.kind === "unknown-ref") {
4921
4921
  throw new StepError(
4922
4922
  "unknown-ref",
@@ -4929,7 +4929,6 @@ export class ResidentDO extends Sandbox<Env> {
4929
4929
  const recreated = await this.ensureThreadWorktree(binding, sha, mode.originUrl, mode.modeSwitch, {
4930
4930
  detached: target.kind === "sha",
4931
4931
  reuse,
4932
- keepTree,
4933
4932
  });
4934
4933
  return { sha, threadLockKey, recreated };
4935
4934
  }, ATTACH_MUTEX_WAIT_MS);
@@ -5027,20 +5026,20 @@ export class ResidentDO extends Sandbox<Env> {
5027
5026
  trace: this.currentSteps(),
5028
5027
  ...(rebound !== undefined ? { rebound } : {}),
5029
5028
  ...(rebindRefused !== undefined ? { rebindRefused } : {}),
5029
+ ...(returned !== undefined ? { returned } : {}),
5030
5030
  };
5031
5031
  }
5032
5032
 
5033
5033
  /** Ensure the worktree exists. A provisioning attach wipes dirty/stale
5034
- * trees; a reusing attach (`opts.reuse`, item 66) keeps a readable tree
5035
- * exactly as it stands (the dirt and the HEAD are the resumed run's own
5036
- * work) and throws `ReuseRefusedError` for one it cannot keep, touching
5037
- * nothing; a provisioning attach whose own rebind moved the record alone
5038
- * over a dirty tree already on the branch (`opts.keepTree`, item 16) keeps
5039
- * that tree as it stands too — the dirt is why the rebind declined to touch
5040
- * it — and provisions one that turns out not to be there. The decision is
5041
- * the pure `decideWorktree`; this method measures the facts and runs the
5042
- * verdict. Returns true when the tree was (re)created. MUST be called
5043
- * holding the mirror mutex: the clone reads the mirror.
5034
+ * trees — a run starts from a clean tree at the bound ref's tip (item 17),
5035
+ * and a rebind that moved the binding leaves a tree on the old branch that
5036
+ * this discipline recreates like any stale one; a reusing attach
5037
+ * (`opts.reuse`, item 66) keeps a readable tree exactly as it stands (the
5038
+ * dirt and the HEAD are the resumed run's own work) and throws
5039
+ * `ReuseRefusedError` for one it cannot keep, touching nothing. The
5040
+ * decision is the pure `decideWorktree`; this method measures the facts
5041
+ * and runs the verdict. Returns true when the tree was (re)created. MUST be
5042
+ * called holding the mirror mutex: the clone reads the mirror.
5044
5043
  *
5045
5044
  * "Dirty" is tracked-file dirt (`status --porcelain -uno`): untracked
5046
5045
  * scratch files are the thread's own state and survive re-attach.
@@ -5063,11 +5062,8 @@ export class ResidentDO extends Sandbox<Env> {
5063
5062
  /** `detached`: the bound ref is gone from the mirror and `sha` is the
5064
5063
  * expected commit it still holds (item 51) — the tree is checked out at
5065
5064
  * that commit, detached, instead of at a branch. `reuse`: a resumed
5066
- * run's attach (item 66): keep the tree as it stands, or refuse.
5067
- * `keepTree`: this attach's rebind moved the record alone over a dirty
5068
- * tree already on the branch (item 16): keep a readable tree as it
5069
- * stands, provision one that is not there. */
5070
- opts: { detached: boolean; reuse: boolean; keepTree: boolean } = { detached: false, reuse: false, keepTree: false },
5065
+ * run's attach (item 66): keep the tree as it stands, or refuse. */
5066
+ opts: { detached: boolean; reuse: boolean } = { detached: false, reuse: false },
5071
5067
  ): Promise<boolean> {
5072
5068
  const wt = binding.worktreePath;
5073
5069
  const threadDir = parentDir(wt);
@@ -5104,14 +5100,7 @@ export class ResidentDO extends Sandbox<Env> {
5104
5100
  }
5105
5101
  }
5106
5102
  }
5107
- const decision = decideWorktree({
5108
- reuse: opts.reuse,
5109
- keepTree: opts.keepTree,
5110
- modeSwitch,
5111
- sha,
5112
- worktreePath: wt,
5113
- facts,
5114
- });
5103
+ const decision = decideWorktree({ reuse: opts.reuse, modeSwitch, sha, worktreePath: wt, facts });
5115
5104
  if (decision.kind === "refuse") throw new ReuseRefusedError(decision.why);
5116
5105
  if (decision.kind === "reuse") return false;
5117
5106
 
@@ -6044,13 +6033,21 @@ export class ResidentDO extends Sandbox<Env> {
6044
6033
 
6045
6034
  /** Remove a thread's worktree (when the runtime is up — a slept container
6046
6035
  * already lost it) and release its pool user; the binding is KEPT, marked
6047
- * evicted, so the ref stays sticky and the next attach recreates the tree
6048
- * with it. Shared by the inactivity sweep and /detach. */
6036
+ * evicted with the cause (`evictedWhy`) and with what the tree held
6037
+ * (`evictedLeftBehind`, or `evictedUnmeasured` when git could not read it
6038
+ * — item 17: never a reason to keep the tree, never discarded silently), so
6039
+ * the ref stays sticky and the next attach recreates the tree with it.
6040
+ * Shared by the inactivity sweep, the reclamation pass, the disk-pressure
6041
+ * path and /detach. */
6049
6042
  private async evictBinding(
6050
6043
  binding: ThreadBinding,
6051
6044
  runtimeActive: boolean,
6052
6045
  logCtx: string,
6053
6046
  why: string,
6047
+ /** What the caller measured in the tree first (`measureTreeBeforeEviction`,
6048
+ * as the thread user with the runtime up); absent when the tree was clean,
6049
+ * already gone with the disk, or not measured (a force detach). */
6050
+ tree?: EvictedTree,
6054
6051
  ): Promise<boolean> {
6055
6052
  const threadDir = parentDir(binding.worktreePath);
6056
6053
  if (runtimeActive && threadDir.startsWith(`${THREADS_DIR}/`)) {
@@ -6086,25 +6083,30 @@ export class ResidentDO extends Sandbox<Env> {
6086
6083
  evicted: true,
6087
6084
  evictedAt: new Date(systemClock()).toISOString(),
6088
6085
  evictedWhy: why,
6086
+ evictedLeftBehind: tree && "leftBehind" in tree ? tree.leftBehind : undefined,
6087
+ evictedUnmeasured: tree && "unmeasured" in tree ? tree.unmeasured : undefined,
6089
6088
  } satisfies ThreadBinding);
6089
+ if (tree) console.log(`${logCtx}: ${binding.threadKey} evicted (${why}) — ${evictedTreeSentence(tree)}`);
6090
6090
  return true;
6091
6091
  }
6092
6092
 
6093
6093
  /** POST /detach: a run has ended — give the thread's pool user back now
6094
6094
  * instead of holding it until the TTL sweep (the pool is sized for
6095
- * simultaneous runs). `force` releases unconditionally (read-only agents,
6096
- * hard stops): an op still in flight is KILLED first (the bot has
6097
- * already dropped its fetch, the command would otherwise run on and hold
6098
- * the user until the sweep); otherwise a busy thread, or a worktree with
6099
- * uncommitted or unpushed work, is KEPT and the caller learns why. No
6100
- * binding → 404-shaped error; already evicted → a no-op success. Never
6101
- * flips lifecycle state. */
6095
+ * simultaneous runs). A run starts from a clean tree (item 17), so the tree
6096
+ * goes whatever it holds: what it held — uncommitted changes, unpushed
6097
+ * commits — is measured once, as the thread user, and named in the answer
6098
+ * (`leftBehind`) so the loss is never silent. The one thing that keeps a
6099
+ * tree is an op still in flight in it (`busy`) — `force` (read-only agents,
6100
+ * hard stops) KILLS that op first (the bot has already dropped its fetch,
6101
+ * the command would otherwise run on and hold the user until the sweep)
6102
+ * and measures nothing. No binding → 404-shaped error; already evicted → a
6103
+ * no-op success. Never flips lifecycle state. */
6102
6104
  async detachThread(
6103
6105
  threadKey: string,
6104
6106
  force: boolean,
6105
6107
  /** What the ending run pushed (item 16a): remembered on the binding first, whatever the detach then decides. */
6106
6108
  pushed: readonly PushedBranch[] = [],
6107
- ): Promise<{ released: boolean; reason?: string; user?: string } | ThreadErr> {
6109
+ ): Promise<DetachAnswer | ThreadErr> {
6108
6110
  if (pushed.length > 0) await this.rememberOwnBranches(threadKey, pushed);
6109
6111
  const binding = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
6110
6112
  if (!binding) return { error: `no-binding: ${threadKey} has never attached to this resident`, status: 404 };
@@ -6125,11 +6127,14 @@ export class ResidentDO extends Sandbox<Env> {
6125
6127
  if (left > 0) return { released: false, reason: busyAfterKillReason(left), user: binding.user };
6126
6128
  }
6127
6129
  const active = await this.isRuntimeActive().catch(() => false);
6128
- if (!force && active) {
6129
- const c = await this.worktreeCleanliness(binding);
6130
- if (!c.clean) return { released: false, reason: `${c.reason} — kept`, user: binding.user };
6131
- }
6132
- // Re-check right before removal: the clean check above awaited (the DO
6130
+ // What the tree still holds, for the answer and the eviction's record —
6131
+ // never a reason to keep it. Not measured on a force release: a read-only
6132
+ // tree holds nothing, and a hard stop's tree is whatever the killed
6133
+ // command left. A probe that fails names nothing in the answer (never a
6134
+ // guess); the record and the log say it could not be measured.
6135
+ let tree: EvictedTree | undefined;
6136
+ if (!force && active) tree = await this.measureTreeBeforeEviction(binding);
6137
+ // Re-check right before removal: the measurement above awaited (the DO
6133
6138
  // yields at each await), so an exec that arrived mid-detach would otherwise
6134
6139
  // have its tree removed under it.
6135
6140
  const busyNow = this.threadOpsInFlight.get(threadKey) ?? 0;
@@ -6139,22 +6144,23 @@ export class ResidentDO extends Sandbox<Env> {
6139
6144
  reason: `busy: ${busyNow} operation(s) started during detach — kept`,
6140
6145
  user: binding.user,
6141
6146
  };
6142
- // Same re-read as the sweep: a re-attach during the clean check means a
6147
+ // Same re-read as the sweep: a re-attach during the measurement means a
6143
6148
  // fresh tree we must not remove from a stale snapshot.
6144
6149
  const current = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
6145
6150
  if (!current || current.evicted) return { released: false, reason: "already-evicted" };
6146
6151
  if (current.lastAttachAt !== binding.lastAttachAt)
6147
- return { released: false, reason: "re-attached during the clean check — kept", user: current.user };
6152
+ return { released: false, reason: "re-attached during the detach — kept", user: current.user };
6148
6153
  const user = current.user;
6149
- // Same as the sweep: `active` was read before the clean check's awaits; a
6154
+ // Same as the sweep: `active` was read before the measurement's awaits; a
6150
6155
  // container that woke meanwhile must get the rm, not an orphaned tree.
6151
6156
  const activeNow = await this.isRuntimeActive().catch(() => false);
6152
- if (!(await this.evictBinding(current, activeNow, `detach`, "detach"))) {
6157
+ if (!(await this.evictBinding(current, activeNow, `detach`, "detach", tree))) {
6153
6158
  return { released: false, reason: "re-attached during eviction — kept", user };
6154
6159
  }
6155
6160
  // Item 55: the tree is gone; the gauge catches up at the next refresh
6156
6161
  // instance's `measure` step, and the admission's `df` sees the space now.
6157
- return { released: true, user };
6162
+ const leftBehind = tree && "leftBehind" in tree ? tree.leftBehind : undefined;
6163
+ return { released: true, user, ...(leftBehind !== undefined ? { leftBehind } : {}) };
6158
6164
  }
6159
6165
 
6160
6166
  /** Force-detach's kill: end every process owned by the pool user —
@@ -6191,20 +6197,22 @@ export class ResidentDO extends Sandbox<Env> {
6191
6197
  }
6192
6198
  }
6193
6199
 
6194
- /** Is this thread's tree safe to destroy? The git probes run AS THE THREAD
6195
- * USER (su), never as root: the worktree is thread-owned, so root git in
6196
- * it would be refused by safe.directory and would be the exact repo-local-
6197
- * config execution vector safe.directory exists to block. Unknown (git
6198
- * failed) counts as NOT clean — never destroy work on a guess. A tree
6199
- * that no longer exists (disk recycled by a sleep/wake) has nothing to
6200
- * preserve: releasable, so a post-wake binding does not hold a pool user
6201
- * for 7 days on behalf of files that are already gone.
6200
+ /** What this thread's tree holds: the tracked changes and unpushed commits,
6201
+ * or that git could not read it. The git probes run AS THE THREAD USER
6202
+ * (su), never as root: the worktree is thread-owned, so root git in it
6203
+ * would be refused by safe.directory and would be the exact repo-local-
6204
+ * config execution vector safe.directory exists to block. Read by
6205
+ * `measureTreeBeforeEviction` for the eviction's record — never a reason
6206
+ * to keep a tree (item 17) — and by the disk-full recycle's guard (item
6207
+ * 54), where unknown (git failed) counts as NOT clean. A tree that no
6208
+ * longer exists (disk recycled by a sleep/wake) is clean with nothing
6209
+ * measured: there was nothing to discard.
6202
6210
  *
6203
6211
  * ONE spawn: the presence test and both git probes fold
6204
6212
  * into the pure `worktreeCleanlinessScript` (test -d as root, both git
6205
6213
  * commands inside a single privilege-dropped `su`, tagged lines out);
6206
6214
  * `parseWorktreeCleanliness` encodes the exact decision table above. */
6207
- private async worktreeCleanliness(binding: ThreadBinding): Promise<{ clean: boolean; reason?: string }> {
6215
+ private async worktreeCleanliness(binding: ThreadBinding): Promise<WorktreeCleanliness> {
6208
6216
  const injected = { GIT_TERMINAL_PROMPT: "0" }; // same injection as threadRun — fail fast, never prompt
6209
6217
  validateEnvNames(injected);
6210
6218
  const r = await this.run(["sh", "-c", worktreeCleanlinessScript(binding.worktreePath, binding.user)], {
@@ -6214,6 +6222,14 @@ export class ResidentDO extends Sandbox<Env> {
6214
6222
  return parseWorktreeCleanliness(r);
6215
6223
  }
6216
6224
 
6225
+ /** What a tree holds right before its eviction, for the record and the log
6226
+ * (item 17): the one-spawn probe as the thread user, read once into
6227
+ * `evictedTreeOf` — never into a keep decision. Callers skip it when the
6228
+ * runtime is down: the disk, and the tree with it, is already gone. */
6229
+ private async measureTreeBeforeEviction(binding: ThreadBinding): Promise<EvictedTree | undefined> {
6230
+ return evictedTreeOf(await this.worktreeCleanliness(binding));
6231
+ }
6232
+
6217
6233
  /** Anything that must not be interrupted by a container stop or counted
6218
6234
  * as idle: thread exec/read/write, disposable /op runs, and attaches past
6219
6235
  * their own image check (mid clone/install under the mirror lock). */
@@ -6254,35 +6270,29 @@ export class ResidentDO extends Sandbox<Env> {
6254
6270
  const ttlDays = record?.worktreeTtlDays ?? WORKTREE_TTL_DAYS_DEFAULT;
6255
6271
  const cutoff = systemClock() - ttlDays * 86_400_000;
6256
6272
  const all = await this.ctx.storage.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX });
6257
- const active = await this.isRuntimeActive().catch(() => false);
6258
6273
  const idleCutoff = systemClock() - CLEAN_IDLE_RELEASE_S * 1000;
6259
6274
  for (const binding of all.values()) {
6260
6275
  if (binding.evicted || !binding.user) continue;
6261
6276
  const last = Date.parse(binding.lastAttachAt);
6262
6277
  if (last >= cutoff) {
6263
- // Not past the TTL. Still release it if it has been idle for an hour,
6264
- // nothing is running on it, and the tree is provably clean — the run
6265
- // that used it is over and there is nothing to preserve. A slept
6266
- // container has NO tree any more (sleep destroys the disk), so an
6267
- // idle binding on an inactive runtime is releasable outright: there is
6268
- // nothing left to protect, only a pool user to give back. (Keeping
6269
- // them would leave idle bindings on a sleeping resident until the
6270
- // 7-day TTL.)
6278
+ // Not past the TTL. Still release it if it has been idle for an hour
6279
+ // and nothing is running on it: the run that used it is over, and
6280
+ // whatever its tree holds has no future — the next attach provisions
6281
+ // a clean tree (item 17) — so there is nothing to keep it for. This is
6282
+ // what drains the bindings of runs whose release never came (a
6283
+ // resident that was sick at the run's end, a run older than
6284
+ // `/detach`). A live run is protected by its op in flight, not by its
6285
+ // dirt. A slept container has no tree any more anyway (sleep destroys
6286
+ // the disk).
6271
6287
  const busy = this.threadOpsInFlight.get(binding.threadKey) ?? 0;
6272
- const cleanIdle =
6273
- last < idleCutoff && busy === 0 && (!active || (await this.worktreeCleanliness(binding)).clean);
6274
- // Re-read right before removal: the clean check awaited (the DO
6275
- // yields), so an exec that arrived meanwhile would otherwise have
6276
- // its tree removed under it — same guard as detachThread.
6277
- const busyNow = this.threadOpsInFlight.get(binding.threadKey) ?? 0;
6278
- if (!cleanIdle || busyNow > 0) {
6288
+ if (last >= idleCutoff || busy > 0) {
6279
6289
  kept++;
6280
6290
  continue;
6281
6291
  }
6282
6292
  }
6283
- // Re-read the binding too: a re-attach that completed inside the
6284
- // clean-check await bumped lastAttachAt and rebuilt the tree — evicting
6285
- // from this loop's stale snapshot would rm the fresh tree.
6293
+ // Re-read the binding: earlier iterations awaited (the DO yields), so a
6294
+ // re-attach that completed meanwhile bumped lastAttachAt and rebuilt the
6295
+ // tree — evicting from this loop's stale snapshot would rm the fresh tree.
6286
6296
  const current = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(binding.threadKey));
6287
6297
  if (!current || current.evicted || current.lastAttachAt !== binding.lastAttachAt) {
6288
6298
  kept++;
@@ -6292,8 +6302,17 @@ export class ResidentDO extends Sandbox<Env> {
6292
6302
  // attach), and an eviction decided on a stale "inactive" would skip the
6293
6303
  // rm and orphan a real tree.
6294
6304
  const activeNow = await this.isRuntimeActive().catch(() => false);
6305
+ // What the tree holds goes on the eviction's record (item 17), measured
6306
+ // while the runtime is up; a slept container has no tree to measure.
6307
+ const tree = activeNow ? await this.measureTreeBeforeEviction(current) : undefined;
6295
6308
  if (
6296
- await this.evictBinding(current, activeNow, `worktree-sweep ${resource}`, last >= cutoff ? "clean-idle" : "ttl")
6309
+ await this.evictBinding(
6310
+ current,
6311
+ activeNow,
6312
+ `worktree-sweep ${resource}`,
6313
+ last >= cutoff ? "clean-idle" : "ttl",
6314
+ tree,
6315
+ )
6297
6316
  )
6298
6317
  evicted.push(binding.threadKey);
6299
6318
  else kept++;
@@ -6551,9 +6570,11 @@ export class ResidentDO extends Sandbox<Env> {
6551
6570
  * mirror (the refresh cycle's `fetch --prune` just ran) or its PR was
6552
6571
  * merged/closed. Runs inside the refresh cycle — a poll on the existing
6553
6572
  * cadence, since the GitHub App has webhooks off — and via /debug
6554
- * reclaim-now. Never touches the default branch, a busy thread, or a dirty
6555
- * tree (reclaimDecision); every keep is named. The eviction itself is the
6556
- * sweep's `evictBinding` with the same re-read guards. */
6573
+ * reclaim-now. Never touches the default branch or a busy thread
6574
+ * (reclaimDecision); every keep is named. What the tree holds is never a
6575
+ * reason to keep it (item 17) — it is measured once as the thread user and
6576
+ * recorded by the eviction, the sweep's `evictBinding` with the same
6577
+ * re-read guards. */
6557
6578
  async reclaimFinishedRefs(
6558
6579
  resource: string,
6559
6580
  defaultRef: string,
@@ -6613,21 +6634,16 @@ export class ResidentDO extends Sandbox<Env> {
6613
6634
  // by name), so its fate is never looked up.
6614
6635
  const { fate, detail } = (!isDefaultRef && fates.get(binding.ref)) || { fate: "unknown" as RefFate, detail: "" };
6615
6636
  const busy = this.threadOpsInFlight.get(binding.threadKey) ?? 0;
6616
- // The clean check runs as the thread user and only when it can decide
6617
- // anything: a finished ref, nothing running, runtime up (down → the tree
6618
- // is already gone with the disk → null).
6619
- const finished = fate === "gone" || fate === "merged" || fate === "closed";
6620
- const clean = !active
6621
- ? null
6622
- : finished && !isDefaultRef && busy === 0
6623
- ? (await this.worktreeCleanliness(binding)).clean
6624
- : null;
6625
- const decision = reclaimDecision({ fate, isDefaultRef, busy, clean });
6637
+ const decision = reclaimDecision({ fate, isDefaultRef, busy });
6626
6638
  if (!decision.reclaim) {
6627
6639
  kept.push({ threadKey: binding.threadKey, ref: binding.ref, why: decision.why });
6628
6640
  continue;
6629
6641
  }
6630
- // Same guards as the sweep: the clean check awaited, so re-read the
6642
+ // What the tree holds goes on the eviction's record (item 17), never
6643
+ // into the decision: measured as the thread user while the runtime is
6644
+ // up; down → the tree is already gone with the disk.
6645
+ const tree = active ? await this.measureTreeBeforeEviction(binding) : undefined;
6646
+ // Same guards as the sweep: the measurement awaited, so re-read the
6631
6647
  // binding (a re-attach means a fresh tree) and the op counter.
6632
6648
  const busyNow = this.threadOpsInFlight.get(binding.threadKey) ?? 0;
6633
6649
  const current = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(binding.threadKey));
@@ -6641,7 +6657,7 @@ export class ResidentDO extends Sandbox<Env> {
6641
6657
  }
6642
6658
  const activeNow = await this.isRuntimeActive().catch(() => false);
6643
6659
  const why = `${decision.why}${detail}`;
6644
- if (await this.evictBinding(current, activeNow, `reclaim ${resource}`, why)) {
6660
+ if (await this.evictBinding(current, activeNow, `reclaim ${resource}`, why, tree)) {
6645
6661
  reclaimed.push({ threadKey: binding.threadKey, ref: binding.ref, why });
6646
6662
  console.log(`reclaim ${resource}: evicted ${binding.threadKey} on ${binding.ref} — ${why}`);
6647
6663
  } else kept.push({ threadKey: binding.threadKey, ref: binding.ref, why: "re-attached" });
@@ -6752,18 +6768,35 @@ export class ResidentDO extends Sandbox<Env> {
6752
6768
  // left out; nothing here is secret (credential files are never persisted).
6753
6769
  const threads = [...bindings.values()]
6754
6770
  .sort((a, b) => b.lastAttachAt.localeCompare(a.lastAttachAt))
6755
- .map(({ threadKey, ref, sha, user, deps, boundAt, lastAttachAt, evicted, evictedAt, evictedWhy }) => ({
6756
- threadKey,
6757
- ref,
6758
- sha: sha ?? null,
6759
- user,
6760
- deps: deps ?? null,
6761
- boundAt,
6762
- lastAttachAt,
6763
- evicted: evicted ?? false,
6764
- evictedAt: evictedAt ?? null,
6765
- evictedWhy: evictedWhy ?? null,
6766
- }));
6771
+ .map(
6772
+ ({
6773
+ threadKey,
6774
+ ref,
6775
+ sha,
6776
+ user,
6777
+ deps,
6778
+ boundAt,
6779
+ lastAttachAt,
6780
+ evicted,
6781
+ evictedAt,
6782
+ evictedWhy,
6783
+ evictedLeftBehind,
6784
+ evictedUnmeasured,
6785
+ }) => ({
6786
+ threadKey,
6787
+ ref,
6788
+ sha: sha ?? null,
6789
+ user,
6790
+ deps: deps ?? null,
6791
+ boundAt,
6792
+ lastAttachAt,
6793
+ evicted: evicted ?? false,
6794
+ evictedAt: evictedAt ?? null,
6795
+ evictedWhy: evictedWhy ?? null,
6796
+ evictedLeftBehind: evictedLeftBehind ?? null,
6797
+ evictedUnmeasured: evictedUnmeasured ?? null,
6798
+ }),
6799
+ );
6767
6800
  return {
6768
6801
  resource: map.get(RESOURCE_KEY) ?? null,
6769
6802
  state: map.get(STATE_KEY) ?? "down",