@coreplane/switchboard 1.267.2 → 1.268.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/dist/assets/deploy/cloudflare-memory/testDiagnostics.ts +44 -3
  2. package/dist/assets/deploy/cloudflare-memory/testDiagnosticsProtocol.ts +7 -0
  3. package/dist/assets/deploy/cloudflare-memory/testDiagnosticsReporter.ts +22 -0
  4. package/dist/assets/deploy/cloudflare-memory/testFetch.ts +21 -0
  5. package/dist/assets/deploy/cloudflare-memory/worker.ts +66 -1
  6. package/dist/assets/deploy/cloudflare-resident/imageReconcileState.ts +28 -0
  7. package/dist/assets/deploy/cloudflare-resident/runRegistration.ts +68 -0
  8. package/dist/assets/deploy/cloudflare-resident/shared.ts +2 -3
  9. package/dist/assets/deploy/cloudflare-resident/worker.ts +541 -192
  10. package/dist/assets/deploy/cloudflare-sandbox/worker.ts +17 -2
  11. package/dist/assets/package-lock.json +3 -3
  12. package/dist/assets/package.json +1 -1
  13. package/dist/assets/source.json +3 -3
  14. package/dist/assets/src/agents/registry.ts +2 -2
  15. package/dist/assets/src/core/budgets.ts +6 -0
  16. package/dist/assets/src/core/coordinator/contract.ts +14 -8
  17. package/dist/assets/src/core/costs.ts +2 -0
  18. package/dist/assets/src/core/runEvents.ts +18 -1
  19. package/dist/assets/src/core/runLedger/types.ts +2 -0
  20. package/dist/assets/src/core/runRecord.ts +27 -1
  21. package/dist/assets/src/core/runUsage.ts +53 -0
  22. package/dist/assets/src/core/ship/coordinator.ts +4 -1
  23. package/dist/assets/src/core/shipSpend.ts +52 -0
  24. package/dist/assets/src/core/trace/attrs.ts +22 -0
  25. package/dist/assets/src/core/trace/tracer.ts +10 -2
  26. package/dist/assets/src/core/trace/types.ts +12 -1
  27. package/dist/assets/src/core/trace/workerTrace.ts +2 -2
  28. package/dist/assets/src/execution/residentPoolSpends.ts +9 -0
  29. package/dist/assets/src/execution/residentRecreateAdmission.ts +28 -0
  30. package/dist/assets/src/execution/seedPlan.ts +13 -4
  31. package/dist/assets/web/dist/.vite/manifest.json +31 -31
  32. package/dist/assets/web/dist/assets/CostsPage-6P9rJ7k_.js +1 -0
  33. package/dist/assets/web/dist/assets/{HomePage-BQmun9kS.js → HomePage-B4LGwXHO.js} +1 -1
  34. package/dist/assets/web/dist/assets/ResidentDetailPage-jpGfsR5n.js +1 -0
  35. package/dist/assets/web/dist/assets/ResidentsIndexPage-C4-5nG9M.js +1 -0
  36. package/dist/assets/web/dist/assets/{RunFoldRow-C1w-mwch.js → RunFoldRow-u4FTh57c.js} +1 -1
  37. package/dist/assets/web/dist/assets/{RunRoutePage-zWMHTzla.js → RunRoutePage-CxOKb7oC.js} +4 -4
  38. package/dist/assets/web/dist/assets/{RunsIndexPage-DF16D2Gg.js → RunsIndexPage-CULSlv_d.js} +1 -1
  39. package/dist/assets/web/dist/assets/{ScheduledPage-CyRs0fXN.js → ScheduledPage-CDjkZuqM.js} +1 -1
  40. package/dist/assets/web/dist/assets/{SettingsPage-BRuMCHSj.js → SettingsPage-BMSomotV.js} +1 -1
  41. package/dist/assets/web/dist/assets/{StatusDot-BIAQN7iD.js → StatusDot-XxLkmRaK.js} +1 -1
  42. package/dist/assets/web/dist/assets/{Tooltip-CxiQ7luL.js → Tooltip-qB73RL4b.js} +1 -1
  43. package/dist/assets/web/dist/assets/{UnitRoutePage-BQB5mAAl.js → UnitRoutePage-DFRWAdOT.js} +1 -1
  44. package/dist/assets/web/dist/assets/{dist-I2OEmPqz.js → dist-BsLBAkQJ.js} +1 -1
  45. package/dist/assets/web/dist/assets/{main-DYwKyLY5.js → main-9ynpo780.js} +10 -10
  46. package/dist/assets/web/dist/assets/main-DSiJ9sHU.css +1 -0
  47. package/dist/cli.js +1178 -431
  48. package/package.json +1 -1
  49. package/dist/assets/web/dist/assets/CostsPage-RRggVvIx.js +0 -1
  50. package/dist/assets/web/dist/assets/ResidentDetailPage-CT6v9INM.js +0 -1
  51. package/dist/assets/web/dist/assets/ResidentsIndexPage-CWqPFf40.js +0 -1
  52. package/dist/assets/web/dist/assets/main-DPB0UO1-.css +0 -1
@@ -89,6 +89,18 @@ import {
89
89
  } from "./legacyCredentials.js";
90
90
  import { hasUnexpectedOwnedThreadDir } from "./orphanThreadUsers.js";
91
91
  import { KeyedAsyncLock } from "./keyedAsyncLock.js";
92
+ import {
93
+ nextDeployImageReconcile,
94
+ replacementContainerStarted,
95
+ type DeployImageReconcileState,
96
+ } from "./imageReconcileState.js";
97
+ import {
98
+ registeredRunAllowsClaim,
99
+ registeredRunAllowsReattach,
100
+ registeredRunNeedsProtection,
101
+ registeredRunOwnsRelease,
102
+ } from "./runRegistration.js";
103
+ import { DAY_MS, RUN_REGISTRATION_GRACE_MS } from "../../src/core/budgets.js";
92
104
  import { BASH_TIMEOUT_MAX_MS, clampBashTimeout } from "../../src/execution/bashTimeout.js";
93
105
  import { RESIDENT_MIRROR_PERMISSIONS, verifyGithubMintScope } from "../../src/execution/githubMintScope.js";
94
106
  import { selectBindingsToPurge } from "../../src/execution/bindingPurge.js";
@@ -172,15 +184,17 @@ import { shellQuote } from "../../src/execution/shellQuote.js";
172
184
  import { envFromRequest } from "../../src/execution/sandboxEnv.js";
173
185
  import { CREDENTIAL_EXPIRY_MARGIN_MS } from "../../src/execution/residentCredentials.js";
174
186
  import { destroyWithPersistentFence } from "../../src/execution/residentDestroyGate.js";
175
- import { ResidentRecreateAdmission } from "../../src/execution/residentRecreateAdmission.js";
187
+ import { ResidentRecreateAdmission, idleForPoolRecycle } from "../../src/execution/residentRecreateAdmission.js";
176
188
  import {
177
189
  claimPoolBinding,
178
190
  mayRunAsPoolUser,
191
+ ownedPoolUsers,
179
192
  parsePoolBindings,
180
193
  parseSpentPoolUsers,
181
194
  rebuildPoolBindingIndex,
182
195
  releasePoolBinding,
183
196
  spendPoolUser,
197
+ unavailablePoolUsers,
184
198
  } from "../../src/execution/residentPoolSpends.js";
185
199
  import {
186
200
  recordFiring,
@@ -584,16 +598,10 @@ const THREADS_DIR = "/workspace/threads";
584
598
  * at the latest (disk is cache). */
585
599
  const OPS_DIR = "/workspace/ops";
586
600
 
587
- /** The thread-user pool. worker1 is the engine's build user; each
588
- * attach allocates one of these to the thread (persisted in the binding)
589
- * and every /exec /read /write for that thread runs privilege-dropped as
590
- * that user. The pool is released by the inactivity sweep. */
591
- /** Pool of OS users for thread worktrees (worker1 is the build user). Sized
592
- * for SIMULTANEOUS runs, not for every thread ever seen: a run returns its
593
- * user via /detach when it ends, so the pool only fills when 16 runs on one
594
- * repo are genuinely concurrent. Memory, not this list, is the real ceiling
595
- * — see the instance_type note in wrangler.jsonc. Must match the useradd loop
596
- * in the Dockerfile. */
601
+ /** Pool of OS users for thread worktrees and disposable ops. A UID stays with
602
+ * its first owner for this VM generation; detach releases the live binding,
603
+ * not the UID spend. Exhaustion may recycle a proven-idle VM. Must match the
604
+ * useradd loop in the Dockerfile. */
597
605
  const THREAD_USERS = Array.from({ length: THREAD_POOL_SIZE }, (_, i) => `worker${i + 2}`);
598
606
 
599
607
  /** Force-detach: after killing the thread user's processes, how long
@@ -1194,6 +1202,8 @@ interface ThreadBinding {
1194
1202
  lastAttachAt: string;
1195
1203
  evicted?: boolean;
1196
1204
  evictedAt?: string;
1205
+ /** Owner of the registration removed with this tree, for a delayed push receipt. */
1206
+ lastRunOwner?: { runId?: string; ownerGen?: string; ownerFence?: number };
1197
1207
  /** Why the last eviction happened (the audit trail): `ttl`, `clean-idle`,
1198
1208
  * `detach`, `disk-pressure`, or a reclamation fate — `merged #N` /
1199
1209
  * `closed #N` / `gone`. */
@@ -1287,7 +1297,7 @@ interface AttachOk {
1287
1297
  returned?: Returned;
1288
1298
  }
1289
1299
 
1290
- /** What `POST /detach` answers: whether the pool user went back, why not, and
1300
+ /** What `POST /detach` answers: whether the live binding was released, why not, and
1291
1301
  * — on a release — what the tree still held (item 16a), now gone with it. */
1292
1302
  interface DetachAnswer {
1293
1303
  released: boolean;
@@ -1469,6 +1479,7 @@ const DRAIN_KEY = "drain";
1469
1479
  * verify this resident's container on the new image; the next reconcile that
1470
1480
  * finds it current reports to the registry and clears it. */
1471
1481
  const IMAGE_REPORT_PENDING_KEY = "imageReportPending";
1482
+ const IMAGE_RECONCILE_KEY = "imageReconcile";
1472
1483
 
1473
1484
  type OnboardResult = { ok: true; record: ResidentRecord } | { ok: false; status: number; error: string };
1474
1485
 
@@ -1764,9 +1775,17 @@ const parentDir = (p: string): string => p.slice(0, p.lastIndexOf("/"));
1764
1775
  * a fresh isolate still counts the process that survived in the container. */
1765
1776
  const RUN_REG_KEY_PREFIX = "runReg:";
1766
1777
  const runRegKey = (threadKey: string) => `${RUN_REG_KEY_PREFIX}${threadKey}`;
1778
+ // A thread's highest accepted ledger claim survives registration release and
1779
+ // binding purge: a delayed older /attach must never reopen an evicted tree.
1780
+ const RUN_FENCE_KEY_PREFIX = "runFence:";
1781
+ const runFenceKey = (threadKey: string) => `${RUN_FENCE_KEY_PREFIX}${threadKey}`;
1767
1782
  interface RunRegistration {
1768
1783
  threadKey: string;
1769
1784
  registeredAt: string;
1785
+ deadlineAt?: number;
1786
+ runId?: string;
1787
+ ownerGen?: string;
1788
+ ownerFence?: number;
1770
1789
  }
1771
1790
 
1772
1791
  /** Deterministic per-thread+ref worktree path. Slugs replace anything outside
@@ -3893,8 +3912,8 @@ export class ResidentDO extends Sandbox<Env> {
3893
3912
  await this.ctx.storage.delete(INFRA_STREAK_KEY);
3894
3913
  await this.setResidentState("warm");
3895
3914
  // Event-triggered reclamation: the prune above already told the
3896
- // mirror which branches died; finished refs give their worktree and
3897
- // pool user back now, not at the idle TTL. Housekeeping, never a
3915
+ // mirror which branches died; finished refs release their worktree and
3916
+ // live binding now, not at the idle TTL. Housekeeping, never a
3898
3917
  // lifecycle flip — a failure here is a log line.
3899
3918
  try {
3900
3919
  const gc = await this.reclaimFinishedRefs(resource, facts.defaultRef, cycle.token);
@@ -4644,23 +4663,21 @@ export class ResidentDO extends Sandbox<Env> {
4644
4663
 
4645
4664
  /** Whether a run may be using this disk: a live binding attached to or
4646
4665
  * used (an exec bumps `lastAttachAt` too, item 21) within IDLE_AFTER_S.
4647
- * The op counter is 0 between a run's tool calls, so this floor is what
4648
- * stands for a run mid-flight. The one predicate behind "the disk may go
4649
- * away": the idle-sleep gate (item 16b — a platform sleep destroys the
4650
- * disk) and the disk-full recycle (item 54) both read it, and neither asks
4651
- * what a tree holds (item 17): a dirty tree no run is using protects
4652
- * nothing — the next attach wipes it. Bindings are storage, so this needs
4653
- * no container. */
4666
+ * The run registration protects the gap between tool calls past this floor.
4667
+ * The idle-sleep gate (item 16b) and disk-full recycle (item 54) both read
4668
+ * the floor and the bounded registrations, never what a tree holds (item
4669
+ * 17): a dirty tree no run is using protects nothing. */
4654
4670
  private recentlyUsed(live: ThreadBinding[]): boolean {
4655
4671
  const recent = systemClock() - IDLE_AFTER_S * 1000;
4656
4672
  return live.some((b) => Date.parse(b.lastAttachAt) >= recent);
4657
4673
  }
4658
4674
 
4659
- /** Idle = no live binding used within the floor and nothing in flight; the
4660
- * sweep's release records what an idle tree held. */
4675
+ /** A live run registration prevents sleep even after an hour without an op;
4676
+ * an expired one cannot keep the container awake. */
4661
4677
  private async isIdle(): Promise<boolean> {
4662
4678
  if (this.recentlyUsed(await this.liveBindings())) return false;
4663
- return this.inFlightCount() === 0;
4679
+ if (this.inFlightCount() > 0) return false;
4680
+ return (await this.registeredRunsBeyondOps()) === 0;
4664
4681
  }
4665
4682
 
4666
4683
  /** Bindings that hold a pool user and a tree on disk (not evicted). */
@@ -4698,10 +4715,10 @@ export class ResidentDO extends Sandbox<Env> {
4698
4715
  * path as a platform sleep. Only when the pure plan allows it: no recycle
4699
4716
  * within the cooldown, nothing in flight (`selfInFlight` excludes the
4700
4717
  * calling refresh cycle from the count) and no live binding used within
4701
- * the idle floor — the idle-sleep gate's own predicate (`recentlyUsed`),
4702
- * because a sleep and a recycle destroy the same disk: it may go away when
4703
- * no run is using it, never for what the trees hold (item 17). A recycle
4704
- * discards every live tree the way an eviction does, so each is measured
4718
+ * the idle floor and no live run registration — the idle-sleep gate's own
4719
+ * predicates, because a sleep and a recycle destroy the same disk: it may
4720
+ * go away when no run is using it, never for what the trees hold (item 17).
4721
+ * A recycle discards every live tree the way an eviction does, so each is measured
4705
4722
  * first, as its thread user, and its binding records what went
4706
4723
  * (`recordRecycledTree`); the plan is decided again after those awaits,
4707
4724
  * right before the stop. A refused recycle is written to
@@ -4710,21 +4727,23 @@ export class ResidentDO extends Sandbox<Env> {
4710
4727
  * whether the container was recycled. */
4711
4728
  private async recoverFromDiskFull(reason: string, selfInFlight: number): Promise<boolean> {
4712
4729
  const lastRecycleAt = await this.ctx.storage.get<number>(DISK_FULL_RECYCLE_KEY);
4713
- const plan = (live: ThreadBinding[]) =>
4714
- planDiskFullRecovery({
4730
+ const plan = async (live: ThreadBinding[]) => {
4731
+ const liveRegistrations = await this.registeredRunsBeyondOps();
4732
+ return planDiskFullRecovery({
4715
4733
  now: systemClock(),
4716
4734
  lastRecycleAt,
4717
4735
  inFlight: this.inFlightCount() - selfInFlight,
4718
- recentlyUsed: this.recentlyUsed(live),
4736
+ recentlyUsed: this.recentlyUsed(live) || liveRegistrations > 0,
4719
4737
  idleFloorS: IDLE_AFTER_S,
4720
4738
  });
4739
+ };
4721
4740
  const kept = async (why: string) => {
4722
4741
  console.log(`disk-full: container kept — ${why}`);
4723
4742
  await this.recordRefreshError(`${reason} — container kept: ${why}`);
4724
4743
  return false;
4725
4744
  };
4726
4745
  const live = await this.liveBindings();
4727
- const first = plan(live);
4746
+ const first = await plan(live);
4728
4747
  if (first.action === "wait") return kept(first.why);
4729
4748
  // What each live tree holds, for its record — never a reason to keep the
4730
4749
  // container. Concurrent: each probe touches only its own tree, one spawn
@@ -4737,7 +4756,7 @@ export class ResidentDO extends Sandbox<Env> {
4737
4756
  : live.map((binding) => [binding, undefined] as const);
4738
4757
  // The measurement awaited (the DO yields at each await): decide again over
4739
4758
  // fresh facts — an attach or an op that landed meanwhile keeps the container.
4740
- const verdict = plan(await this.liveBindings());
4759
+ const verdict = await plan(await this.liveBindings());
4741
4760
  if (verdict.action === "wait") return kept(verdict.why);
4742
4761
  console.log(
4743
4762
  `disk-full: recycling the container — the next cycle restores mirror + checkout from R2 onto an empty disk (${reason})`,
@@ -5153,8 +5172,8 @@ export class ResidentDO extends Sandbox<Env> {
5153
5172
  * run — stop the container so it restarts on the current image (state is DO
5154
5173
  * storage + R2 — the disk is a cache). A deferred restart re-checks on
5155
5174
  * every later refresh cycle until the resident is quiet; a registration
5156
- * whose release never came defers it only until the clean-idle sweep drains
5157
- * that registration. The attach path never stops the container (issue 2101,
5175
+ * whose release never came defers it only through the clean-idle window;
5176
+ * a later sweep removes its binding. The attach path never stops the container (issue 2101,
5158
5177
  * the `stale` answer below); the stop is the refresh cycle's or the
5159
5178
  * deploy's alone. Answers `restarted` when a stop was issued, else why
5160
5179
  * not. */
@@ -5233,7 +5252,12 @@ export class ResidentDO extends Sandbox<Env> {
5233
5252
  }
5234
5253
  console.log(`image-stale (${where}): ${stale} — stopping so it restarts on the current image`);
5235
5254
  this.swapIncarnation(); // deliberate incarnation swap
5236
- await this.stop().catch((err) => console.log(`image-stale: stop failed: ${errMsg(err)}`));
5255
+ try {
5256
+ await this.stop();
5257
+ } catch (err) {
5258
+ console.log(`image-stale: stop failed: ${errMsg(err)}`);
5259
+ if (where === "deploy") return "deferred";
5260
+ }
5237
5261
  // A successful stop proves only that the old process is gone. The pending
5238
5262
  // marker deliberately survives it: the replacement's own hydration reports
5239
5263
  // after reaching `warm`, so `imageReport: current` can never precede a
@@ -5289,9 +5313,66 @@ export class ResidentDO extends Sandbox<Env> {
5289
5313
  * added afterwards would wait for a report nothing sends (until the drain's
5290
5314
  * `until`). Held first, every report finds its hold. */
5291
5315
  async reconcileForDeploy(resource: string): Promise<{ result: ImageReconcileResult; verified: boolean }> {
5292
- await registryStub(this.env).holdDrainFor([resource]);
5293
- await this.ctx.storage.put(IMAGE_REPORT_PENDING_KEY, { resource });
5316
+ // The deploy's image report and an idle pool recycle both reason about the
5317
+ // same VM. Admit reconciliation as disk work before its first await, so a
5318
+ // recycle sees it as busy; a recycle already holding admission defers this
5319
+ // pass for the deploy's bounded readiness retry.
5320
+ const admitted = await this.withRecreateSafeAdmin(() => this.reconcileForDeployAdmitted(resource));
5321
+ return "error" in admitted ? { result: "deferred", verified: false } : admitted;
5322
+ }
5323
+
5324
+ private async reconcileForDeployAdmitted(
5325
+ resource: string,
5326
+ ): Promise<{ result: ImageReconcileResult; verified: boolean }> {
5327
+ const progress = await this.ctx.storage.get<DeployImageReconcileState>(IMAGE_RECONCILE_KEY);
5328
+ const pending = await this.ctx.storage.get<{ resource: string }>(IMAGE_REPORT_PENDING_KEY);
5329
+ const action = nextDeployImageReconcile(BUILD.commit, progress, pending !== undefined);
5330
+ if (action === "verified") return { result: "current", verified: true };
5331
+ if (action === "start") {
5332
+ await registryStub(this.env).holdDrainFor([resource]);
5333
+ await this.ctx.storage.put(IMAGE_REPORT_PENDING_KEY, { resource });
5334
+ await this.ctx.storage.put(IMAGE_RECONCILE_KEY, { build: BUILD.commit, cycleIssued: false });
5335
+ }
5336
+ if (action === "await-report") {
5337
+ const active = await this.isRuntimeActive().catch(() => null);
5338
+ if (active === false) await this.reportPendingImageCurrent("deploy");
5339
+ else if (active === true) {
5340
+ const [state, facts] = await Promise.all([
5341
+ this.ctx.storage.get<ResidentState>(STATE_KEY),
5342
+ this.ctx.storage.get<RepoFacts>(FACTS_KEY),
5343
+ ]);
5344
+ // A replacement can reuse the disk and leave `warm` unchanged; that
5345
+ // hydrate shortcut does not send a report. A different boot after the
5346
+ // successful stop is proof that a replacement started. Finish its
5347
+ // hydration before reporting; an old warm state alone proves nothing.
5348
+ if (replacementContainerStarted(progress?.containerBeforeCycle, await this.containerIdentity())) {
5349
+ await this.ensureHydrated();
5350
+ await this.reportPendingImageCurrent("deploy");
5351
+ } else if (
5352
+ state === "warm" &&
5353
+ facts?.lastRestore?.at &&
5354
+ Date.parse(facts.lastRestore.at) >= Date.parse(progress?.cycleStoppedAt ?? "")
5355
+ ) {
5356
+ await this.reportPendingImageCurrent("deploy");
5357
+ }
5358
+ }
5359
+ return {
5360
+ result: active === false ? "inactive" : "deferred",
5361
+ verified: (await this.ctx.storage.get(IMAGE_REPORT_PENDING_KEY)) === undefined,
5362
+ };
5363
+ }
5364
+ // Reading the boot ID runs a command and would wake a sleeping resident.
5365
+ // Preserve the inactive path's report-without-start behavior.
5366
+ const activeBeforeCycle = await this.isRuntimeActive().catch(() => null);
5367
+ const containerBeforeCycle = activeBeforeCycle === true ? await this.containerIdentity() : undefined;
5294
5368
  const result = await this.reconcileImage("deploy", true);
5369
+ if (result === "restarted")
5370
+ await this.ctx.storage.put(IMAGE_RECONCILE_KEY, {
5371
+ build: BUILD.commit,
5372
+ cycleIssued: true,
5373
+ cycleStoppedAt: new Date(systemClock()).toISOString(),
5374
+ containerBeforeCycle,
5375
+ } satisfies DeployImageReconcileState);
5295
5376
  if (result === "deferred") return { result, verified: false };
5296
5377
  // Inactivity reports synchronously; a successful stop does not. A
5297
5378
  // concurrent fresh start may also have hydrated while this reconcile
@@ -5598,7 +5679,11 @@ export class ResidentDO extends Sandbox<Env> {
5598
5679
  /** Storage bindings, active ops and inspections all reserve a pool user.
5599
5680
  * Claim inside this method, before its promise resolves: two concurrent
5600
5681
  * callers must not each read the same free UID before either can claim it. */
5601
- private async findFreePoolUser(excludeThreadKey?: string, excluded = new Set<string>()): Promise<string | undefined> {
5682
+ private async findFreePoolUser(
5683
+ excludeThreadKey?: string,
5684
+ excluded = new Set<string>(),
5685
+ preferred: readonly string[] = [],
5686
+ ): Promise<string | undefined> {
5602
5687
  const all = await this.ctx.storage.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX });
5603
5688
  const used = new Set(
5604
5689
  [...all.values()].filter((b) => !b.evicted && b.user && b.threadKey !== excludeThreadKey).map((b) => b.user),
@@ -5606,7 +5691,7 @@ export class ResidentDO extends Sandbox<Env> {
5606
5691
  for (const u of this.opUsersInUse.keys()) used.add(u);
5607
5692
  for (const u of this.poolUsersInspecting) used.add(u);
5608
5693
  for (const u of excluded) used.add(u);
5609
- const user = THREAD_USERS.find((u) => !used.has(u));
5694
+ const user = [...preferred, ...THREAD_USERS].find((u) => !used.has(u));
5610
5695
  if (user) this.poolUsersInspecting.add(user);
5611
5696
  return user;
5612
5697
  }
@@ -5643,19 +5728,82 @@ export class ResidentDO extends Sandbox<Env> {
5643
5728
  throw new Error("pool-user-stage-scan-invalid");
5644
5729
  }
5645
5730
 
5731
+ /** A full one-use pool can serve the next request by destroying its VM,
5732
+ * never by assigning another owner's UID. The admission mark closes every
5733
+ * other entry before the idle read and stays held through the restore. */
5734
+ private async recycleSpentPoolForAdmission(): Promise<true | ThreadErr> {
5735
+ try {
5736
+ const admission = await this.recreateAdmission.run(async (): Promise<true | ThreadErr> => {
5737
+ const spent = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
5738
+ if (!spent) return { error: "pool-generation-unknown: fresh VM required before UID assignment", status: 503 };
5739
+ if (spent.size < THREAD_USERS.length) return true; // another checked recreate already cleared the generation
5740
+ const [status, drain, imagePending, snapshot, registeredRuns, bindings] = await Promise.all([
5741
+ this.getStatus(),
5742
+ this.registry().getDrain(),
5743
+ this.ctx.storage.get(IMAGE_REPORT_PENDING_KEY),
5744
+ this.ctx.storage.get(SNAPSHOT_KEY),
5745
+ this.registeredRunsBeyondOps(),
5746
+ this.ctx.storage.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX }),
5747
+ ]);
5748
+ if (
5749
+ !snapshot ||
5750
+ !idleForPoolRecycle({
5751
+ state: status.state,
5752
+ draining: liveDrain(drain, systemClock()) !== null,
5753
+ imagePending: imagePending !== undefined,
5754
+ inFlight: this.inFlightCount(),
5755
+ refreshAdmissions: this.refreshAdmissionsInFlight,
5756
+ adminWork: this.adminWorkInFlight,
5757
+ hydrating: this.hydration !== null,
5758
+ registeredRuns,
5759
+ liveBindings: [...bindings.values()].filter((binding) => !binding.evicted).length,
5760
+ inspecting: this.poolUsersInspecting.size,
5761
+ })
5762
+ )
5763
+ return {
5764
+ error: "pool-recycle-required: all UIDs spent and the resident is not idle for checked VM recycle",
5765
+ status: 503,
5766
+ };
5767
+ await this.recreateContainer("pool-recycle: all UIDs spent; no other resident work owns this VM");
5768
+ await this.ensureHydrated();
5769
+ const fresh = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
5770
+ const ready = await this.getStatus();
5771
+ const imageStillPending = (await this.ctx.storage.get(IMAGE_REPORT_PENDING_KEY)) !== undefined;
5772
+ if (!fresh || fresh.size !== 0 || ready.state !== "warm" || imageStillPending)
5773
+ return { error: "pool-recycle-failed: the restored VM is not warm, current and empty", status: 503 };
5774
+ return true;
5775
+ });
5776
+ return admission.busy ? this.recreateRefusal() : admission.value;
5777
+ } catch (err) {
5778
+ console.log(`pool-recycle: checked VM recycle failed (${errMsg(err)})`);
5779
+ return { error: "pool-recycle-failed: the checked VM restore did not complete", status: 503 };
5780
+ }
5781
+ }
5782
+
5646
5783
  private async reserveSafePoolUser(owner: string, excludeThreadKey?: string): Promise<string | ThreadErr> {
5647
- const spent = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
5784
+ let spent = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
5648
5785
  if (!spent) return { error: "pool-generation-unknown: fresh VM required before UID assignment", status: 503 };
5649
- const excluded = new Set(spent.keys());
5786
+ let excluded = unavailablePoolUsers(spent, owner);
5650
5787
  let contaminated = false;
5788
+ let recycled = false;
5651
5789
  for (;;) {
5652
- const user = await this.findFreePoolUser(excludeThreadKey, excluded);
5653
- if (!user)
5790
+ const user = await this.findFreePoolUser(excludeThreadKey, excluded, ownedPoolUsers(spent, owner));
5791
+ if (!user) {
5792
+ if (!contaminated && excluded.size === THREAD_USERS.length && !recycled) {
5793
+ recycled = true;
5794
+ const result = await this.recycleSpentPoolForAdmission();
5795
+ if (result !== true) return result;
5796
+ spent = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
5797
+ if (!spent) return { error: "pool-generation-unknown: fresh VM required before UID assignment", status: 503 };
5798
+ excluded = unavailablePoolUsers(spent, owner);
5799
+ continue;
5800
+ }
5654
5801
  return contaminated
5655
5802
  ? { error: "pool-user-contaminated: no clean pool user is available", status: 503 }
5656
5803
  : excluded.size === THREAD_USERS.length
5657
5804
  ? { error: "pool-recycle-required: all UIDs spent on this VM; confirm idle and recreate it", status: 503 }
5658
5805
  : { error: "user-pool-exhausted: no safe pool user is available", status: 429 };
5806
+ }
5659
5807
  let safe = false;
5660
5808
  try {
5661
5809
  const clean = !(await this.poolUserHasOldThreadDir(user, [])) && !(await this.poolUserHasOldStageContent(user));
@@ -5798,23 +5946,37 @@ export class ResidentDO extends Sandbox<Env> {
5798
5946
  traceparent?: string,
5799
5947
  reason: RefHintReason = NO_REF_HINT_REASON,
5800
5948
  githubDoor?: { baseUrl: string; bearer: string },
5949
+ runBudgetMs?: number,
5950
+ runId?: string,
5951
+ ownerGen?: string,
5952
+ ownerFence?: number,
5801
5953
  ): Promise<AttachOk | ThreadErr> {
5802
5954
  // One step trace per attach (docs/reference/specs/tracing.md item 19): every command
5803
5955
  // the attach runs lands on it, and the answer carries it.
5804
5956
  const t0 = systemClock();
5805
5957
  const trace = createStepTrace(t0);
5806
5958
  let res: AttachOk | ThreadErr;
5807
- if (this.recreateAdmission.pending) {
5808
- res = this.recreateRefusal();
5809
- } else {
5810
- this.attachAdmissionsInFlight++;
5811
- try {
5812
- res = await this.stepTrace.run(trace, () =>
5813
- this.attachThreadTraced(threadKey, refHint, readonly, wantSha, reuse, record, t0, reason, githubDoor),
5814
- );
5815
- } finally {
5816
- this.attachAdmissionsInFlight--;
5817
- }
5959
+ this.attachAdmissionsInFlight++;
5960
+ try {
5961
+ res = await this.stepTrace.run(trace, () =>
5962
+ this.attachThreadTraced(
5963
+ threadKey,
5964
+ refHint,
5965
+ readonly,
5966
+ wantSha,
5967
+ reuse,
5968
+ record,
5969
+ t0,
5970
+ reason,
5971
+ githubDoor,
5972
+ runBudgetMs,
5973
+ runId,
5974
+ ownerGen,
5975
+ ownerFence,
5976
+ ),
5977
+ );
5978
+ } finally {
5979
+ this.attachAdmissionsInFlight--;
5818
5980
  }
5819
5981
  // The same steps as the resident's own `resident.attach` root (item 22).
5820
5982
  emitStepRoot("resident.attach", t0, trace.steps(), traceparent, "error" in res ? refusalOutcome(res) : "ok");
@@ -5832,56 +5994,80 @@ export class ResidentDO extends Sandbox<Env> {
5832
5994
  t0: number,
5833
5995
  reason: RefHintReason,
5834
5996
  githubDoor?: { baseUrl: string; bearer: string },
5997
+ runBudgetMs?: number,
5998
+ runId?: string,
5999
+ ownerGen?: string,
6000
+ ownerFence?: number,
5835
6001
  ): Promise<AttachOk | ThreadErr> {
5836
6002
  try {
5837
- if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
5838
- await this.ensureHydrated();
5839
- if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
5840
- // The fleet drain (item 69): a deploy is waiting for the runs in flight
5841
- // to end, and a NEW run's attach is refused with the record the bot
5842
- // waits on — a real 503 in the streamed document, read by the client as
5843
- // `draining`, never as the platform's transient. A run already in flight
5844
- // — registered from its attach to its release (item 44) — re-attaches
5845
- // through: a rolled container, an evicted worktree, a resumed run are
5846
- // the runs the drain waits FOR, and refusing them would hold the fleet
5847
- // closed on the run it is closed for. Read before the image reconcile so
5848
- // a refused attach never restarts a container.
5849
- const drain = await this.fleetDrain();
5850
- const registered = (await this.ctx.storage.get(runRegKey(threadKey))) !== undefined;
5851
- if (drain && !registered) {
5852
- const refusal: ThreadErr & { draining: DrainRecord } = drainRefusal(drain);
5853
- return refusal;
5854
- }
5855
- // Item 70: above the soft memory threshold a NEW attach is refused like
5856
- // `mirror-busy` (the bot falls back or waits, the card says why) — after
5857
- // the drain (storage only, cheaper) and before the image reconcile, so a
5858
- // refused attach never restarts a container. A registered run's
5859
- // re-attach passes for the same reason it passes the drain above.
5860
- const memory = await this.memoryGate("attach", registered);
5861
- if (memory) return memory;
5862
- const resourceId = (await this.ctx.storage.get<string>(RESOURCE_KEY)) ?? "";
5863
- // An attach never restarts the container (issue 2101): a `stale` verdict
5864
- // refuses the NEW run — it falls back to the seeded sandbox — while a
5865
- // registered run's re-attach passes exactly as it passes the drain and
5866
- // the memory gate; the restart itself is the refresh cycle's or the
5867
- // deploy's.
5868
- if ((await this.reconcileImage("attach")) === "stale" && !registered) {
5869
- const s = await this.getStatus();
5870
- return {
5871
- error:
5872
- "image-stale: the container predates the deploy and restarts on the next quiet refresh; new runs use the fallback sandbox until then",
5873
- status: 503,
5874
- state: s.state,
5875
- stateReason: s.reason,
5876
- reason: "image-stale",
5877
- };
5878
- }
5879
- // From here the attach may hold the mirror lock through clone/install:
5880
- // count it so a concurrent refresh-cycle reconcileImage never stops the
5881
- // container under it (and isIdle never parks the cycle mid-attach).
5882
- this.attachesInFlight++;
5883
- try {
5884
- return await this.threadAttaches.run(threadKey, async () => {
6003
+ return await this.threadAttaches.run(threadKey, async () => {
6004
+ // The attachment fence takes precedence over every 503 gate. Keep the
6005
+ // same thread lock through the gates and registration write so a newer
6006
+ // attach cannot slip between this check and a fallback-eligible refusal.
6007
+ const current = await this.ctx.storage.get<RunRegistration>(runRegKey(threadKey));
6008
+ const accepted = await this.ctx.storage.get<Pick<RunRegistration, "runId" | "ownerGen" | "ownerFence">>(
6009
+ runFenceKey(threadKey),
6010
+ );
6011
+ if (
6012
+ !registeredRunAllowsClaim(current, runId, ownerGen, ownerFence) ||
6013
+ !registeredRunAllowsClaim(accepted, runId, ownerGen, ownerFence)
6014
+ )
6015
+ return { error: "run-registration-mismatch: a newer generation owns the thread", status: 409 };
6016
+ if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
6017
+ await this.ensureHydrated();
6018
+ if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
6019
+ // The fleet drain (item 69): a deploy is waiting for the runs in flight
6020
+ // to end, and a NEW run's attach is refused with the record the bot
6021
+ // waits on — a real 503 in the streamed document, read by the client as
6022
+ // `draining`, never as the platform's transient. A run already in flight
6023
+ // — with an owned, live registration (item 44) — re-attaches
6024
+ // through: a rolled container, an evicted worktree, a resumed run are
6025
+ // the runs the drain waits FOR, and refusing them would hold the fleet
6026
+ // closed on the run it is closed for. Read before the image reconcile so
6027
+ // a refused attach never restarts a container.
6028
+ const drain = await this.fleetDrain();
6029
+ const registration = await this.ctx.storage.get<RunRegistration>(runRegKey(threadKey));
6030
+ const registered = registeredRunAllowsReattach(
6031
+ registration,
6032
+ runId,
6033
+ systemClock(),
6034
+ RUN_REGISTRATION_GRACE_MS,
6035
+ ownerGen,
6036
+ ownerFence,
6037
+ );
6038
+ if (drain && !registered) {
6039
+ const refusal: ThreadErr & { draining: DrainRecord } = drainRefusal(drain);
6040
+ return refusal;
6041
+ }
6042
+ // Item 70: above the soft memory threshold a NEW attach is refused like
6043
+ // `mirror-busy` (the bot falls back or waits, the card says why) — after
6044
+ // the drain (storage only, cheaper) and before the image reconcile, so a
6045
+ // refused attach never restarts a container. An owned, live run's
6046
+ // re-attach passes for the same reason it passes the drain above.
6047
+ const memory = await this.memoryGate("attach", registered);
6048
+ if (memory) return memory;
6049
+ const resourceId = (await this.ctx.storage.get<string>(RESOURCE_KEY)) ?? "";
6050
+ // An attach never restarts the container (issue 2101): a `stale` verdict
6051
+ // refuses the NEW run — it falls back to the seeded sandbox — while a
6052
+ // owned, live run's re-attach passes exactly as it passes the drain and
6053
+ // the memory gate; the restart itself is the refresh cycle's or the
6054
+ // deploy's.
6055
+ if ((await this.reconcileImage("attach")) === "stale" && !registered) {
6056
+ const s = await this.getStatus();
6057
+ return {
6058
+ error:
6059
+ "image-stale: the container predates the deploy and restarts on the next quiet refresh; new runs use the fallback sandbox until then",
6060
+ status: 503,
6061
+ state: s.state,
6062
+ stateReason: s.reason,
6063
+ reason: "image-stale",
6064
+ };
6065
+ }
6066
+ // From here the attach may hold the mirror lock through clone/install:
6067
+ // count it so a concurrent refresh-cycle reconcileImage never stops the
6068
+ // container under it (and isIdle never parks the cycle mid-attach).
6069
+ this.attachesInFlight++;
6070
+ try {
5885
6071
  const res = await this.attachThreadBody(
5886
6072
  threadKey,
5887
6073
  refHint,
@@ -5896,12 +6082,12 @@ export class ResidentDO extends Sandbox<Env> {
5896
6082
  );
5897
6083
  // The run this attach opens is now in flight until its release —
5898
6084
  // whatever its op counters read between the bot's calls (item 44).
5899
- if (!("error" in res)) await this.registerRun(threadKey);
6085
+ if (!("error" in res)) await this.registerRun(threadKey, runBudgetMs, runId, ownerGen, ownerFence);
5900
6086
  return res;
5901
- });
5902
- } finally {
5903
- this.attachesInFlight--;
5904
- }
6087
+ } finally {
6088
+ this.attachesInFlight--;
6089
+ }
6090
+ });
5905
6091
  } catch (err) {
5906
6092
  return catchAllErr(err, "attach-failed");
5907
6093
  }
@@ -7736,6 +7922,7 @@ export class ResidentDO extends Sandbox<Env> {
7736
7922
  user: "",
7737
7923
  evicted: true,
7738
7924
  evictedAt: new Date(systemClock()).toISOString(),
7925
+ lastRunOwner: await this.ctx.storage.get<RunRegistration>(runRegKey(binding.threadKey)),
7739
7926
  evictedWhy: why,
7740
7927
  evictedLeftBehind: tree && "leftBehind" in tree ? tree.leftBehind : undefined,
7741
7928
  evictedUnmeasured: tree && "unmeasured" in tree ? tree.unmeasured : undefined,
@@ -7747,9 +7934,9 @@ export class ResidentDO extends Sandbox<Env> {
7747
7934
  return "evicted";
7748
7935
  }
7749
7936
 
7750
- /** POST /detach: a run has ended — give the thread's pool user back now
7751
- * instead of holding it until the TTL sweep (the pool is sized for
7752
- * simultaneous runs). A run starts from a clean tree (item 17), so the tree
7937
+ /** POST /detach: a run has ended — release its live binding now instead of
7938
+ * holding it until the TTL sweep. The UID remains spent for this VM
7939
+ * generation. A run starts from a clean tree (item 17), so the tree
7753
7940
  * goes whatever it holds: what it held — uncommitted changes, unpushed
7754
7941
  * commits — is measured once, as the thread user, and named in the answer
7755
7942
  * (`leftBehind`) so the loss is never silent. The one thing that keeps a
@@ -7763,65 +7950,77 @@ export class ResidentDO extends Sandbox<Env> {
7763
7950
  force: boolean,
7764
7951
  /** What the ending run pushed (item 16a): remembered on the binding first, whatever the detach then decides. */
7765
7952
  pushed: readonly PushedBranch[] = [],
7953
+ runId?: string,
7954
+ ownerGen?: string,
7955
+ ownerFence?: number,
7766
7956
  ): Promise<DetachAnswer | ThreadErr> {
7767
- if (pushed.length > 0) await this.rememberOwnBranches(threadKey, pushed);
7768
- const binding = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
7769
- if (!binding) return { error: `no-binding: ${threadKey} has never attached to this resident`, status: 404 };
7770
- if (binding.evicted || !binding.user) return { released: false, reason: "already-evicted" };
7771
- if (!(await this.poolUserOwnerMatches(binding.user, `thread:${threadKey}`)))
7772
- return { error: "pool-owner-mismatch: detach refused for conflicting UID owner", status: 503 };
7773
- const plan = planForceDetach({
7774
- force,
7775
- inFlight: this.threadOpsInFlight.get(threadKey) ?? 0,
7776
- user: binding.user,
7777
- poolUsers: THREAD_USERS,
7778
- });
7779
- if (plan.action === "refuse") return { released: false, reason: plan.reason, user: binding.user };
7780
- if (plan.action === "kill") {
7781
- if (!(await this.killThreadUserProcesses(plan.user, threadKey)))
7782
- return { error: "pool-owner-mismatch: force detach refused before kill", status: 503 };
7783
- console.log(
7784
- `detach: force — killed ${plan.user}'s processes for ${threadKey} (${plan.inFlight} op(s) were in flight)`,
7785
- );
7786
- const left = await this.waitForThreadDrain(threadKey);
7787
- if (left > 0) return { released: false, reason: busyAfterKillReason(left), user: binding.user };
7788
- }
7789
- const active = await this.isRuntimeActive().catch(() => false);
7790
- // What the tree still holds, for the answer and the eviction's record —
7791
- // never a reason to keep it. Not measured on a force release: a read-only
7792
- // tree holds nothing, and a hard stop's tree is whatever the killed
7793
- // command left. A probe that fails names nothing in the answer (never a
7794
- // guess); the record and the log say it could not be measured.
7795
- let tree: EvictedTree | undefined;
7796
- if (!force && active) tree = await this.measureTreeBeforeEviction(binding);
7797
- // Re-check right before removal: the measurement above awaited (the DO
7798
- // yields at each await), so an exec that arrived mid-detach would otherwise
7799
- // have its tree removed under it.
7800
- const busyNow = this.threadOpsInFlight.get(threadKey) ?? 0;
7801
- if (busyNow > 0)
7802
- return {
7803
- released: false,
7804
- reason: `busy: ${busyNow} operation(s) started during detach — kept`,
7957
+ return this.threadAttaches.run(threadKey, async () => {
7958
+ const binding = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
7959
+ if (!binding) return { error: `no-binding: ${threadKey} has never attached to this resident`, status: 404 };
7960
+ if (binding.evicted || !binding.user) {
7961
+ if (pushed.length > 0 && registeredRunOwnsRelease(binding.lastRunOwner, runId, ownerGen, ownerFence))
7962
+ await this.rememberOwnBranches(threadKey, pushed);
7963
+ return { released: false, reason: "already-evicted" };
7964
+ }
7965
+ const registration = await this.ctx.storage.get<RunRegistration>(runRegKey(threadKey));
7966
+ if (!registeredRunOwnsRelease(registration, runId, ownerGen, ownerFence))
7967
+ return { released: false, reason: "run-registration-mismatch: the workspace belongs to another run" };
7968
+ if (pushed.length > 0) await this.rememberOwnBranches(threadKey, pushed);
7969
+ if (!(await this.poolUserOwnerMatches(binding.user, `thread:${threadKey}`)))
7970
+ return { error: "pool-owner-mismatch: detach refused for conflicting UID owner", status: 503 };
7971
+ const plan = planForceDetach({
7972
+ force,
7973
+ inFlight: this.threadOpsInFlight.get(threadKey) ?? 0,
7805
7974
  user: binding.user,
7806
- };
7807
- // Same re-read as the sweep: a re-attach during the measurement means a
7808
- // fresh tree we must not remove from a stale snapshot.
7809
- const current = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
7810
- if (!current || current.evicted) return { released: false, reason: "already-evicted" };
7811
- if (current.lastAttachAt !== binding.lastAttachAt)
7812
- return { released: false, reason: "re-attached during the detach — kept", user: current.user };
7813
- const user = current.user;
7814
- // Same as the sweep: `active` was read before the measurement's awaits; a
7815
- // container that woke meanwhile must get the rm, not an orphaned tree.
7816
- const activeNow = await this.isRuntimeActive().catch(() => true);
7817
- const eviction = await this.evictBinding(current, activeNow, `detach`, "detach", tree);
7818
- if (eviction === "cleanup-failed")
7819
- return { released: false, reason: "thread-cleanup-failed: pool user kept", user };
7820
- if (eviction === "changed") return { released: false, reason: "re-attached during eviction — kept", user };
7821
- // Item 55: the tree is gone; the gauge catches up at the next refresh
7822
- // instance's `measure` step, and the admission's `df` sees the space now.
7823
- const leftBehind = tree && "leftBehind" in tree ? tree.leftBehind : undefined;
7824
- return { released: true, user, ...(leftBehind !== undefined ? { leftBehind } : {}) };
7975
+ poolUsers: THREAD_USERS,
7976
+ });
7977
+ if (plan.action === "refuse") return { released: false, reason: plan.reason, user: binding.user };
7978
+ if (plan.action === "kill") {
7979
+ if (!(await this.killThreadUserProcesses(plan.user, threadKey)))
7980
+ return { error: "pool-owner-mismatch: force detach refused before kill", status: 503 };
7981
+ console.log(
7982
+ `detach: force — killed ${plan.user}'s processes for ${threadKey} (${plan.inFlight} op(s) were in flight)`,
7983
+ );
7984
+ const left = await this.waitForThreadDrain(threadKey);
7985
+ if (left > 0) return { released: false, reason: busyAfterKillReason(left), user: binding.user };
7986
+ }
7987
+ const active = await this.isRuntimeActive().catch(() => false);
7988
+ // What the tree still holds, for the answer and the eviction's record —
7989
+ // never a reason to keep it. Not measured on a force release: a read-only
7990
+ // tree holds nothing, and a hard stop's tree is whatever the killed
7991
+ // command left. A probe that fails names nothing in the answer (never a
7992
+ // guess); the record and the log say it could not be measured.
7993
+ let tree: EvictedTree | undefined;
7994
+ if (!force && active) tree = await this.measureTreeBeforeEviction(binding);
7995
+ // Re-check right before removal: the measurement above awaited (the DO
7996
+ // yields at each await), so an exec that arrived mid-detach would otherwise
7997
+ // have its tree removed under it.
7998
+ const busyNow = this.threadOpsInFlight.get(threadKey) ?? 0;
7999
+ if (busyNow > 0)
8000
+ return {
8001
+ released: false,
8002
+ reason: `busy: ${busyNow} operation(s) started during detach — kept`,
8003
+ user: binding.user,
8004
+ };
8005
+ // Same re-read as the sweep: a re-attach during the measurement means a
8006
+ // fresh tree we must not remove from a stale snapshot.
8007
+ const current = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
8008
+ if (!current || current.evicted) return { released: false, reason: "already-evicted" };
8009
+ if (current.lastAttachAt !== binding.lastAttachAt)
8010
+ return { released: false, reason: "re-attached during the detach — kept", user: current.user };
8011
+ const user = current.user;
8012
+ // Same as the sweep: `active` was read before the measurement's awaits; a
8013
+ // container that woke meanwhile must get the rm, not an orphaned tree.
8014
+ const activeNow = await this.isRuntimeActive().catch(() => true);
8015
+ const eviction = await this.evictBinding(current, activeNow, `detach`, "detach", tree);
8016
+ if (eviction === "cleanup-failed")
8017
+ return { released: false, reason: "thread-cleanup-failed: pool user kept", user };
8018
+ if (eviction === "changed") return { released: false, reason: "re-attached during eviction — kept", user };
8019
+ // Item 55: the tree is gone; the gauge catches up at the next refresh
8020
+ // instance's `measure` step, and the admission's `df` sees the space now.
8021
+ const leftBehind = tree && "leftBehind" in tree ? tree.leftBehind : undefined;
8022
+ return { released: true, user, ...(leftBehind !== undefined ? { leftBehind } : {}) };
8023
+ });
7825
8024
  }
7826
8025
 
7827
8026
  /** Force-detach's kill: end every process owned by the pool user —
@@ -7936,25 +8135,74 @@ export class ResidentDO extends Sandbox<Env> {
7936
8135
  async getInFlightCount(): Promise<number> {
7937
8136
  return this.inFlightCount() + (await this.registeredRunsBeyondOps());
7938
8137
  }
7939
- /** A run holds its worktree from `/attach` to its release (`/detach`, the
7940
- * sweep, disk pressure — every path ends in `evictBinding`, which clears
7941
- * the row). Written by the attach; idempotent, a re-attach refreshes it. */
7942
- private async registerRun(threadKey: string): Promise<void> {
7943
- await this.ctx.storage.put(runRegKey(threadKey), {
7944
- threadKey,
7945
- registeredAt: new Date(systemClock()).toISOString(),
7946
- } satisfies RunRegistration);
8138
+ /** A run's registration protects its worktree until release or the bounded
8139
+ * deadline. `/detach`, the sweep, and disk pressure end in `evictBinding`,
8140
+ * which clears the row; a re-attach refreshes it. */
8141
+ private async registerRun(
8142
+ threadKey: string,
8143
+ runBudgetMs?: number,
8144
+ runId?: string,
8145
+ ownerGen?: string,
8146
+ ownerFence?: number,
8147
+ ): Promise<void> {
8148
+ const now = systemClock();
8149
+ await this.ctx.storage.transaction(async (txn) => {
8150
+ if (ownerFence !== undefined) await txn.put(runFenceKey(threadKey), { runId, ownerGen, ownerFence });
8151
+ await txn.put(runRegKey(threadKey), {
8152
+ threadKey,
8153
+ registeredAt: new Date(now).toISOString(),
8154
+ ...(runBudgetMs !== undefined ? { deadlineAt: now + runBudgetMs } : {}),
8155
+ ...(runId !== undefined ? { runId } : {}),
8156
+ ...(ownerGen !== undefined ? { ownerGen } : {}),
8157
+ ...(ownerFence !== undefined ? { ownerFence } : {}),
8158
+ } satisfies RunRegistration);
8159
+ });
8160
+ }
8161
+ /** The harness's lease begins after attach. Tighten the provisional setup
8162
+ * bound to its actual remaining clock, only for the run that attached. */
8163
+ async updateRunDeadline(
8164
+ threadKey: string,
8165
+ runId: string,
8166
+ remainingMs: number,
8167
+ ownerGen?: string,
8168
+ ownerFence?: number,
8169
+ ): Promise<{ deadlineAt: number } | ThreadErr> {
8170
+ return this.ctx.storage.transaction(async (txn) => {
8171
+ const registration = await txn.get<RunRegistration>(runRegKey(threadKey));
8172
+ if (
8173
+ !registration ||
8174
+ registration.runId !== runId ||
8175
+ registration.ownerGen !== ownerGen ||
8176
+ registration.ownerFence !== ownerFence
8177
+ )
8178
+ return { error: "run-registration-mismatch: this run does not own the thread", status: 409 };
8179
+ const deadlineAt = systemClock() + remainingMs;
8180
+ await txn.put(runRegKey(threadKey), { ...registration, deadlineAt });
8181
+ return { deadlineAt };
8182
+ });
7947
8183
  }
7948
8184
  /** The registrations the op counters do not already see: a thread with an
7949
8185
  * op in flight is counted by `runsInFlightCount`, so its registration is
7950
- * not counted again. Deliberately NOT part of `inFlightCount()`: the
7951
- * container-lifecycle predicates (isIdle, reconcileImage) must not let a
7952
- * stale registration pin a container awake — the clean-idle sweep is what
7953
- * drains a registration whose release never came. */
8186
+ * not counted again. The idle, image-reconcile and disk-recycle gates read
8187
+ * this bounded count alongside their op counters, so a live run protects
8188
+ * its container while a stale registration cannot pin it awake. */
7954
8189
  private async registeredRunsBeyondOps(): Promise<number> {
7955
8190
  const regs = await this.ctx.storage.list<RunRegistration>({ prefix: RUN_REG_KEY_PREFIX });
8191
+ const cutoff = systemClock() - CLEAN_IDLE_RELEASE_S * 1000;
7956
8192
  let n = 0;
7957
- for (const r of regs.values()) if ((this.threadOpsInFlight.get(r.threadKey) ?? 0) === 0) n++;
8193
+ for (const r of regs.values()) {
8194
+ const binding = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(r.threadKey));
8195
+ if (
8196
+ registeredRunNeedsProtection(
8197
+ binding?.lastAttachAt,
8198
+ this.threadOpsInFlight.get(r.threadKey) ?? 0,
8199
+ cutoff,
8200
+ r.deadlineAt === undefined ? undefined : r.deadlineAt + RUN_REGISTRATION_GRACE_MS,
8201
+ systemClock(),
8202
+ )
8203
+ )
8204
+ n++;
8205
+ }
7958
8206
  return n;
7959
8207
  }
7960
8208
  private attachesInFlight = 0;
@@ -7983,6 +8231,15 @@ export class ResidentDO extends Sandbox<Env> {
7983
8231
  const idleCutoff = systemClock() - CLEAN_IDLE_RELEASE_S * 1000;
7984
8232
  for (const binding of all.values()) {
7985
8233
  if (binding.evicted || !binding.user) continue;
8234
+ const registration = await this.ctx.storage.get<RunRegistration>(runRegKey(binding.threadKey));
8235
+ if (
8236
+ registration?.deadlineAt !== undefined &&
8237
+ Number.isSafeInteger(registration.deadlineAt) &&
8238
+ systemClock() <= registration.deadlineAt + RUN_REGISTRATION_GRACE_MS
8239
+ ) {
8240
+ kept++;
8241
+ continue;
8242
+ }
7986
8243
  const last = Date.parse(binding.lastAttachAt);
7987
8244
  if (last >= cutoff) {
7988
8245
  // Not past the TTL. Still release it if it has been idle for an hour
@@ -7991,9 +8248,9 @@ export class ResidentDO extends Sandbox<Env> {
7991
8248
  // a clean tree (item 17) — so there is nothing to keep it for. This is
7992
8249
  // what drains the bindings of runs whose release never came (a
7993
8250
  // resident that was sick at the run's end, a run older than
7994
- // `/detach`). A live run is protected by its op in flight, not by its
7995
- // dirt. A slept container has no tree any more anyway (sleep destroys
7996
- // the disk).
8251
+ // `/detach`). A live run is protected by its budget above or an op
8252
+ // in flight, not by its dirt. A slept container has no tree any more
8253
+ // anyway (sleep destroys the disk).
7997
8254
  const busy = this.threadOpsInFlight.get(binding.threadKey) ?? 0;
7998
8255
  if (last >= idleCutoff || busy > 0) {
7999
8256
  kept++;
@@ -8390,13 +8647,30 @@ export class ResidentDO extends Sandbox<Env> {
8390
8647
  fates.set(ref, { fate: looked.fate, detail });
8391
8648
  }),
8392
8649
  );
8393
- // The runs registered on this resident (item 44): a run holds its tree
8394
- // from its attach to its release, and its process lives in the container
8650
+ // The runs registered on this resident (item 44): a live registration
8651
+ // holds its tree through the deadline, and its process lives in the container
8395
8652
  // between the bot's calls — the op counters read 0 while its model thinks.
8653
+ const registrations = new Map(
8654
+ [...(await this.ctx.storage.list<RunRegistration>({ prefix: RUN_REG_KEY_PREFIX })).values()].map((r) => [
8655
+ r.threadKey,
8656
+ r,
8657
+ ]),
8658
+ );
8659
+ const protectsTree = (binding: ThreadBinding, registration: RunRegistration | undefined): boolean => {
8660
+ if (!registration) return false;
8661
+ const now = systemClock();
8662
+ return registeredRunNeedsProtection(
8663
+ binding.lastAttachAt,
8664
+ 0,
8665
+ now - CLEAN_IDLE_RELEASE_S * 1000,
8666
+ registration.deadlineAt === undefined ? undefined : registration.deadlineAt + RUN_REGISTRATION_GRACE_MS,
8667
+ now,
8668
+ );
8669
+ };
8396
8670
  const registered = new Set(
8397
- [...(await this.ctx.storage.list<RunRegistration>({ prefix: RUN_REG_KEY_PREFIX })).values()].map(
8398
- (r) => r.threadKey,
8399
- ),
8671
+ live
8672
+ .filter((binding) => protectsTree(binding, registrations.get(binding.threadKey)))
8673
+ .map((binding) => binding.threadKey),
8400
8674
  );
8401
8675
  for (const binding of live) {
8402
8676
  const isDefaultRef = binding.ref === defaultRef;
@@ -8423,7 +8697,8 @@ export class ResidentDO extends Sandbox<Env> {
8423
8697
  continue;
8424
8698
  }
8425
8699
  // A run attached while the tree was measured holds it now (item 44).
8426
- if ((await this.ctx.storage.get<RunRegistration>(runRegKey(binding.threadKey))) !== undefined) {
8700
+ const currentRegistration = await this.ctx.storage.get<RunRegistration>(runRegKey(binding.threadKey));
8701
+ if (protectsTree(current ?? binding, currentRegistration)) {
8427
8702
  kept.push({ threadKey: binding.threadKey, ref: binding.ref, why: "run-held" });
8428
8703
  continue;
8429
8704
  }
@@ -8643,6 +8918,7 @@ export class ResidentDO extends Sandbox<Env> {
8643
8918
  // The runs whose process lives in the container between operator calls
8644
8919
  // (item 44): what the deploy preflight must see, added to both counts.
8645
8920
  const registeredRuns = await this.registeredRunsBeyondOps();
8921
+ const poolSpends = parseSpentPoolUsers(map.get(SPENT_POOL_USERS_KEY), THREAD_USERS);
8646
8922
  return {
8647
8923
  resource: map.get(RESOURCE_KEY) ?? null,
8648
8924
  state: this.destroying || map.get(DESTROY_UNCONFIRMED_KEY) ? "down" : (map.get(STATE_KEY) ?? "down"),
@@ -8674,7 +8950,10 @@ export class ResidentDO extends Sandbox<Env> {
8674
8950
  // running container the last deploy could not verify.
8675
8951
  imageReport: map.get(IMAGE_REPORT_PENDING_KEY) !== undefined ? "pending" : "current",
8676
8952
  recreateAdmissionHeld: this.recreateAdmission.pending || map.get(RECREATE_ADMISSION_KEY) === true,
8677
- poolUsersSpent: parseSpentPoolUsers(map.get(SPENT_POOL_USERS_KEY), THREAD_USERS)?.size ?? null,
8953
+ poolUsersSpent: poolSpends?.size ?? null,
8954
+ // Read-scoped owner receipts explain historical occupancy even after
8955
+ // every live binding and disposable op has ended.
8956
+ poolUserSpends: poolSpends ? [...poolSpends].map(([user, owner]) => ({ user, owner })) : null,
8678
8957
  // Item 22: who holds what, as the rows say — the mirror mutex and the
8679
8958
  // cycle/hydration leases, each judged against this incarnation.
8680
8959
  incarnation: this.incarnation,
@@ -9249,6 +9528,7 @@ const ROUTES: Record<string, { scope: Scope; method: string }> = {
9249
9528
  "/debug": { scope: "read", method: "POST" }, // per-op: READ_DEBUG_OPS for read scope, everything for admin
9250
9529
  "/status": { scope: "operator", method: "GET" },
9251
9530
  "/attach": { scope: "operator", method: "POST" },
9531
+ "/run-deadline": { scope: "operator", method: "POST" },
9252
9532
  "/detach": { scope: "operator", method: "POST" },
9253
9533
  "/exec": { scope: "operator", method: "POST" },
9254
9534
  "/read": { scope: "operator", method: "POST" },
@@ -9366,6 +9646,8 @@ export default {
9366
9646
  return await handleStatus(env, url);
9367
9647
  case "/attach":
9368
9648
  return await handleAttach(env, body, traceparent);
9649
+ case "/run-deadline":
9650
+ return await handleRunDeadline(env, body);
9369
9651
  case "/detach":
9370
9652
  return await handleDetach(env, body);
9371
9653
  case "/exec":
@@ -9996,6 +10278,27 @@ async function handleAttach(env: Env, body: Record<string, unknown>, traceparent
9996
10278
  if ("error" in want) return json({ error: want.error }, 400);
9997
10279
  const reuse = parseReuse(body.reuse);
9998
10280
  if ("error" in reuse) return json({ error: reuse.error }, 400);
10281
+ let runBudgetMs: number | undefined;
10282
+ if (body.runBudgetMs !== undefined) {
10283
+ const value = body.runBudgetMs;
10284
+ if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 0 || value > DAY_MS)
10285
+ return json({ error: "invalid run budget" }, 400);
10286
+ runBudgetMs = value;
10287
+ }
10288
+ const runId = body.runId;
10289
+ if (runId !== undefined && (typeof runId !== "string" || runId.length === 0 || runId.length > 128))
10290
+ return json({ error: "invalid run id" }, 400);
10291
+ const ownerGen = body.ownerGen;
10292
+ if (ownerGen !== undefined && (typeof ownerGen !== "string" || ownerGen.length === 0 || ownerGen.length > 128))
10293
+ return json({ error: "invalid run generation" }, 400);
10294
+ const ownerFence = body.ownerFence;
10295
+ if (
10296
+ ownerFence !== undefined &&
10297
+ (typeof ownerFence !== "number" || !Number.isSafeInteger(ownerFence) || ownerFence <= 0)
10298
+ )
10299
+ return json({ error: "invalid run fence" }, 400);
10300
+ if (ownerFence !== undefined && (runId === undefined || ownerGen === undefined))
10301
+ return json({ error: "run fence requires a run id and generation" }, 400);
9999
10302
  // Why the hint is what it is (item 16): the thread's own pull request and
10000
10303
  // its head branch — the branch checked against the one ref pattern like
10001
10304
  // every ref, before it can become a git argument — and the bound-by-default flag.
@@ -10050,6 +10353,10 @@ async function handleAttach(env: Env, body: Record<string, unknown>, traceparent
10050
10353
  traceparent,
10051
10354
  reason,
10052
10355
  githubDoor,
10356
+ runBudgetMs,
10357
+ runId,
10358
+ ownerGen,
10359
+ ownerFence,
10053
10360
  ),
10054
10361
  ),
10055
10362
  ({ result, levels }) => ({ ...(result as object), ...(levels ? { levels } : {}) }),
@@ -10057,6 +10364,29 @@ async function handleAttach(env: Env, body: Record<string, unknown>, traceparent
10057
10364
  );
10058
10365
  }
10059
10366
 
10367
+ async function handleRunDeadline(env: Env, body: Record<string, unknown>): Promise<Response> {
10368
+ const ctx = await resolveThreadRoute(env, body);
10369
+ if (ctx instanceof Response) return ctx;
10370
+ const runId = body.runId;
10371
+ const ownerGen = body.ownerGen;
10372
+ const ownerFence = body.ownerFence;
10373
+ const remainingMs = body.remainingMs;
10374
+ if (typeof runId !== "string" || runId.length === 0 || runId.length > 128)
10375
+ return json({ error: "invalid run id" }, 400);
10376
+ if (ownerGen !== undefined && (typeof ownerGen !== "string" || ownerGen.length === 0 || ownerGen.length > 128))
10377
+ return json({ error: "invalid run generation" }, 400);
10378
+ if (
10379
+ ownerFence !== undefined &&
10380
+ (typeof ownerFence !== "number" || !Number.isSafeInteger(ownerFence) || ownerFence <= 0)
10381
+ )
10382
+ return json({ error: "invalid run fence" }, 400);
10383
+ if (typeof remainingMs !== "number" || !Number.isSafeInteger(remainingMs) || remainingMs <= 0 || remainingMs > DAY_MS)
10384
+ return json({ error: "invalid remaining lease" }, 400);
10385
+ const result = await ctx.stub.updateRunDeadline(ctx.threadKey, runId, remainingMs, ownerGen, ownerFence);
10386
+ if ("error" in result) return threadErrResponse(result);
10387
+ return json(result);
10388
+ }
10389
+
10060
10390
  /** POST /await-restore (docs/reference/specs/execution.md item 27): the bot's
10061
10391
  * one held request while the resident restores. Held server-side by the DO's
10062
10392
  * waiter ledger; the answer streams like /attach (heartbeat whitespace then
@@ -10078,6 +10408,18 @@ async function handleAwaitRestore(env: Env, body: Record<string, unknown>): Prom
10078
10408
  async function handleDetach(env: Env, body: Record<string, unknown>): Promise<Response> {
10079
10409
  const ctx = await resolveThreadRoute(env, body);
10080
10410
  if (ctx instanceof Response) return ctx;
10411
+ const runId = body.runId;
10412
+ if (runId !== undefined && (typeof runId !== "string" || runId.length === 0 || runId.length > 128))
10413
+ return json({ error: "invalid run id" }, 400);
10414
+ const ownerGen = body.ownerGen;
10415
+ if (ownerGen !== undefined && (typeof ownerGen !== "string" || ownerGen.length === 0 || ownerGen.length > 128))
10416
+ return json({ error: "invalid run generation" }, 400);
10417
+ const ownerFence = body.ownerFence;
10418
+ if (
10419
+ ownerFence !== undefined &&
10420
+ (typeof ownerFence !== "number" || !Number.isSafeInteger(ownerFence) || ownerFence <= 0)
10421
+ )
10422
+ return json({ error: "invalid run fence" }, 400);
10081
10423
  // What the run pushed (item 16a), each ref through the one ref pattern
10082
10424
  // before it can be stored or become a git argument.
10083
10425
  const pushed = parsePushed(body.pushed);
@@ -10086,7 +10428,14 @@ async function handleDetach(env: Env, body: Record<string, unknown>): Promise<Re
10086
10428
  const ref = parseRef(entry.ref, "pushed[].ref");
10087
10429
  if ("error" in ref) return json({ error: ref.error }, 400);
10088
10430
  }
10089
- const result = await ctx.stub.detachThread(ctx.threadKey, body.force === true, pushed.pushed);
10431
+ const result = await ctx.stub.detachThread(
10432
+ ctx.threadKey,
10433
+ body.force === true,
10434
+ pushed.pushed,
10435
+ runId,
10436
+ ownerGen,
10437
+ ownerFence,
10438
+ );
10090
10439
  if ("error" in result) return threadErrResponse(result);
10091
10440
  return json(result);
10092
10441
  }