@coreplane/switchboard 1.267.0 → 1.267.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/assets/deploy/cloudflare-resident/gc.ts +1 -0
  2. package/dist/assets/deploy/cloudflare-resident/keyedAsyncLock.ts +20 -0
  3. package/dist/assets/deploy/cloudflare-resident/legacyCredentials.ts +57 -0
  4. package/dist/assets/deploy/cloudflare-resident/orphanThreadUsers.ts +14 -0
  5. package/dist/assets/deploy/cloudflare-resident/worker.ts +646 -204
  6. package/dist/assets/package-lock.json +3 -3
  7. package/dist/assets/package.json +1 -1
  8. package/dist/assets/source.json +3 -3
  9. package/dist/assets/src/core/budgets.ts +10 -0
  10. package/dist/assets/src/core/coordinator/contract.ts +29 -0
  11. package/dist/assets/src/core/coordinator/driver.ts +15 -2
  12. package/dist/assets/src/core/runRecord.ts +32 -0
  13. package/dist/assets/src/core/ship/coordinator.ts +42 -4
  14. package/dist/assets/src/execution/githubMintScope.ts +49 -0
  15. package/dist/assets/src/execution/residentCredentials.ts +9 -37
  16. package/dist/assets/src/execution/residentDestroyGate.ts +12 -0
  17. package/dist/assets/src/execution/residentPoolSpends.ts +96 -0
  18. package/dist/assets/src/execution/residentRecreateAdmission.ts +41 -0
  19. package/dist/assets/src/execution/residentSteps.ts +1 -0
  20. package/dist/assets/web/dist/.vite/manifest.json +67 -67
  21. package/dist/assets/web/dist/assets/{DeliveryPage-BUJckSuW.js → DeliveryPage-QyS6_Q0m.js} +1 -1
  22. package/dist/assets/web/dist/assets/{HomePage-BfpIrKDX.js → HomePage-Bm40r0wF.js} +1 -1
  23. package/dist/assets/web/dist/assets/{PendingTurnRow-C6hroa00.js → PendingTurnRow-BqY-EaZR.js} +1 -1
  24. package/dist/assets/web/dist/assets/{PlanePage-C2Cb1mkT.js → PlanePage-VmzjNLAz.js} +1 -1
  25. package/dist/assets/web/dist/assets/{ResidentDetailPage-Bd9_Q87l.js → ResidentDetailPage-hjX3tRCh.js} +1 -1
  26. package/dist/assets/web/dist/assets/{ResidentsIndexPage-C8eS7hxp.js → ResidentsIndexPage-BKGnlCEh.js} +1 -1
  27. package/dist/assets/web/dist/assets/{RunFoldRow-DrfFauYJ.js → RunFoldRow-BOWvSoIv.js} +1 -1
  28. package/dist/assets/web/dist/assets/{RunRoutePage-B_eXfNAF.js → RunRoutePage-D9ztf-uE.js} +3 -3
  29. package/dist/assets/web/dist/assets/{RunsIndexPage-BpQu36Dm.js → RunsIndexPage-DTYQbhtr.js} +1 -1
  30. package/dist/assets/web/dist/assets/{ScheduledPage-DGHcjcSt.js → ScheduledPage-BAYR0tuv.js} +1 -1
  31. package/dist/assets/web/dist/assets/{SettingsPage-IfQ71QjN.js → SettingsPage-BlYBmCG9.js} +1 -1
  32. package/dist/assets/web/dist/assets/{SilentTurn-D9ON15a8.js → SilentTurn-CHfzRQW8.js} +1 -1
  33. package/dist/assets/web/dist/assets/{StatusDot-DyFjGg4V.js → StatusDot-BMkUFVE8.js} +1 -1
  34. package/dist/assets/web/dist/assets/{Tooltip-BvPkV4kZ.js → Tooltip-Dfot16IA.js} +1 -1
  35. package/dist/assets/web/dist/assets/{UnitRoutePage-BNe3nBIG.js → UnitRoutePage-BdZoT-4d.js} +1 -1
  36. package/dist/assets/web/dist/assets/budgets-CJHiTZmM.js +1 -0
  37. package/dist/assets/web/dist/assets/{dist-v3Py3_QR.js → dist-XRrc0hhm.js} +1 -1
  38. package/dist/assets/web/dist/assets/{indexRow-BsWcHHVh.js → indexRow-BGjjLnN5.js} +1 -1
  39. package/dist/assets/web/dist/assets/{main-ks3kBojV.js → main-B32N1vUh.js} +2 -2
  40. package/dist/assets/web/dist/assets/{sseReplay-DuALaBPq.js → sseReplay-BsNQPLmJ.js} +2 -2
  41. package/dist/cli.js +928 -346
  42. package/package.json +1 -1
  43. package/dist/assets/web/dist/assets/budgets-BQgcCyvb.js +0 -1
@@ -82,8 +82,15 @@ import { AsyncLocalStorage } from "node:async_hooks";
82
82
  import type { DirectoryBackup, SandboxCommand } from "@cloudflare/sandbox";
83
83
  import { createExtensionProcessSandbox } from "@cloudflare/sandbox/extensions";
84
84
  import { DurableObject } from "cloudflare:workers";
85
- import { legacyCredentialScrubCommand } from "./legacyCredentials.js";
85
+ import {
86
+ legacyCredentialScrubCommand,
87
+ legacyStageContentScanCommand,
88
+ legacyStageReuseScrubCommand,
89
+ } from "./legacyCredentials.js";
90
+ import { hasUnexpectedOwnedThreadDir } from "./orphanThreadUsers.js";
91
+ import { KeyedAsyncLock } from "./keyedAsyncLock.js";
86
92
  import { BASH_TIMEOUT_MAX_MS, clampBashTimeout } from "../../src/execution/bashTimeout.js";
93
+ import { RESIDENT_MIRROR_PERMISSIONS, verifyGithubMintScope } from "../../src/execution/githubMintScope.js";
87
94
  import { selectBindingsToPurge } from "../../src/execution/bindingPurge.js";
88
95
  import { busyAfterKillReason, planForceDetach } from "../../src/execution/residentDetach.js";
89
96
  import {
@@ -164,6 +171,17 @@ import {
164
171
  import { shellQuote } from "../../src/execution/shellQuote.js";
165
172
  import { envFromRequest } from "../../src/execution/sandboxEnv.js";
166
173
  import { CREDENTIAL_EXPIRY_MARGIN_MS } from "../../src/execution/residentCredentials.js";
174
+ import { destroyWithPersistentFence } from "../../src/execution/residentDestroyGate.js";
175
+ import { ResidentRecreateAdmission } from "../../src/execution/residentRecreateAdmission.js";
176
+ import {
177
+ claimPoolBinding,
178
+ mayRunAsPoolUser,
179
+ parsePoolBindings,
180
+ parseSpentPoolUsers,
181
+ rebuildPoolBindingIndex,
182
+ releasePoolBinding,
183
+ spendPoolUser,
184
+ } from "../../src/execution/residentPoolSpends.js";
167
185
  import {
168
186
  recordFiring,
169
187
  scheduleForCron,
@@ -941,8 +959,9 @@ const tail = (s: string, n: number): string => s.trim().slice(-n);
941
959
  // — Worker/DO scope only; the private key never
942
960
  // enters the container. RS256 App JWT on WebCrypto (node:crypto is not
943
961
  // available here), then POST /app/installations/:id/access_tokens with
944
- // `repositories: [<own repo name>]` so a minted token never grants more than
945
- // the resident's one repo. Cache per slug, but only serve a cached token while
962
+ // `repositories: [<own repo name>]` and mirror-read permissions, so a minted
963
+ // token grants only the resident's root-owned clone/fetch and PR lookup. Cache
964
+ // per slug, but only serve a cached token while
946
965
  // it has more than `CREDENTIAL_EXPIRY_MARGIN_MS` of life left, so a new
947
966
  // attach never inherits a near-expiry token minted for an earlier thread.
948
967
  // ---------------------------------------------------------------------------
@@ -991,7 +1010,7 @@ export async function mintRepoScopedToken(env: Env, slug: string, opts?: { fresh
991
1010
  "content-type": "application/json",
992
1011
  "user-agent": "switchboard-resident",
993
1012
  },
994
- body: JSON.stringify({ repositories: [repoName] }),
1013
+ body: JSON.stringify({ repositories: [repoName], permissions: RESIDENT_MIRROR_PERMISSIONS }),
995
1014
  // A slow GitHub must not hang attach/refresh. The 10s abort surfaces as a
996
1015
  // command-level Error (below), never a lifecycle transition.
997
1016
  signal: AbortSignal.timeout(10_000),
@@ -1010,6 +1029,7 @@ export async function mintRepoScopedToken(env: Env, slug: string, opts?: { fresh
1010
1029
  throw new Error(`github-token-mint-failed: HTTP ${res.status} ${body.slice(0, 300)}`);
1011
1030
  }
1012
1031
  const data = (await res.json()) as { token: string; expires_at: string };
1032
+ verifyGithubMintScope(data, slug, RESIDENT_MIRROR_PERMISSIONS);
1013
1033
  const minted: MintedToken = { token: data.token, expiresAtMs: Date.parse(data.expires_at) };
1014
1034
  githubTokenCache.set(slug, minted);
1015
1035
  return minted;
@@ -1204,15 +1224,10 @@ interface ThreadBinding {
1204
1224
  * file, origin = the unreadable mirror. An attach in the other mode
1205
1225
  * recreates the tree. Absent (pre-field bindings) = writable. */
1206
1226
  readonly?: boolean;
1207
- /** Epoch ms when `.git/github-credentials` was last written (attach or the
1208
- * per-exec refresh). Absent = unknown → the next writable exec re-mints
1209
- * (`shouldRefreshThreadCredentials`); cleared by a read-only attach. */
1227
+ /** Legacy pre-door binding field. No new model credential file is written;
1228
+ * attach clears the old value after scrubbing that file. */
1210
1229
  credentialsWrittenAt?: number;
1211
- /** Epoch ms when the written token expires. The per-exec refresh keys
1212
- * on this — not `credentialsWrittenAt` — so a near-expiry token inherited
1213
- * from an earlier thread's mint is re-minted before the first writable exec.
1214
- * Absent (pre-field binding) → the next writable exec falls back to the
1215
- * `credentialsWrittenAt` file-age rule; cleared alongside it on attach. */
1230
+ /** Legacy pre-door token expiry; cleared with `credentialsWrittenAt`. */
1216
1231
  tokenExpiresAtMs?: number;
1217
1232
  /** Non-secret host of the run-bearer Git door. The bearer lives only in
1218
1233
  * the attach request and the run's per-exec environment. */
@@ -1720,6 +1735,11 @@ const AUTO_REBUILDS_KEY = "resident:autoRebuilds";
1720
1735
  const RETIRED_REBUILD_STRIKES_KEY = "resident:rebuildStrikes";
1721
1736
  /** Consecutive connects the container's control port did not answer (item 64): the ladder's count. */
1722
1737
  const RUNTIME_UNREACHABLE_KEY = "resident:runtimeUnreachable";
1738
+ /** A failed or interrupted VM destroy forbids another process on an unconfirmed disk. */
1739
+ const DESTROY_UNCONFIRMED_KEY = "resident:destroyUnconfirmed";
1740
+ const RECREATE_ADMISSION_KEY = "resident:recreateAdmission";
1741
+ /** UIDs that have run untrusted code on this VM. Missing means an unknown old generation. */
1742
+ const SPENT_POOL_USERS_KEY = "resident:spentPoolUsers";
1723
1743
 
1724
1744
  /** The row under RUNTIME_UNREACHABLE_KEY: the count and both instants, so the
1725
1745
  * fleet watch sees how long the runtime has been silent. Cleared by any exec
@@ -1733,6 +1753,8 @@ interface RuntimeUnreachableRow {
1733
1753
  /** Thread bindings live under their own prefix, keyed by threadKey. */
1734
1754
  const THREAD_KEY_PREFIX = "thread:";
1735
1755
  const threadBindingKey = (threadKey: string) => `${THREAD_KEY_PREFIX}${threadKey}`;
1756
+ const POOL_BINDING_KEY_PREFIX = "resident:poolBindings:";
1757
+ const poolBindingKey = (user: string) => `${POOL_BINDING_KEY_PREFIX}${user}`;
1736
1758
 
1737
1759
  const parentDir = (p: string): string => p.slice(0, p.lastIndexOf("/"));
1738
1760
 
@@ -1839,16 +1861,53 @@ export class ResidentDO extends Sandbox<Env> {
1839
1861
 
1840
1862
  constructor(...args: ConstructorParameters<typeof Sandbox<Env>>) {
1841
1863
  super(...args);
1842
- // The base class created `container_schedules` synchronously above; the
1843
- // rows the retired alarm chain armed are gone before this object handles
1844
- // its first event (RETIRED_SCHEDULE_CALLBACKS).
1864
+ // The SDK deletes these rows synchronously. Do it before scheduling any
1865
+ // asynchronous initialization so retired callbacks cannot fire on boot.
1845
1866
  for (const name of RETIRED_SCHEDULE_CALLBACKS) this.deleteSchedules(name);
1867
+ // Build the bounded UID view from primary rows before serving this DO
1868
+ // instance. A new instance repairs an old-image write or interrupted
1869
+ // migration; no durable "ready" flag can bless a stale index.
1870
+ void this.ctx.blockConcurrencyWhile(() => this.rebuildPoolBindingIndex());
1846
1871
  }
1847
1872
 
1848
1873
  /** Serializes concurrent hydration attempts within one DO lifetime. Never
1849
1874
  * used as a "hydrated" flag — the container can sleep while the DO object
1850
1875
  * survives, so hydration state is always probed from disk. */
1851
1876
  private hydration: Promise<void> | null = null;
1877
+ /** Concurrent lifecycle paths share one destruction attempt. */
1878
+ private destroying: Promise<void> | null = null;
1879
+ /** An operator's idle check and destroy are one admission interval. */
1880
+ private readonly recreateAdmission = new ResidentRecreateAdmission({
1881
+ mark: () => this.ctx.storage.put(RECREATE_ADMISSION_KEY, true),
1882
+ clear: async () => {
1883
+ await this.ctx.storage.delete(RECREATE_ADMISSION_KEY);
1884
+ },
1885
+ held: async () => (await this.ctx.storage.get(RECREATE_ADMISSION_KEY)) === true,
1886
+ });
1887
+
1888
+ private recreateRefusal(): ThreadErr {
1889
+ return {
1890
+ error: "recreate-in-progress: new resident work is paused during the operator's checked VM recycle",
1891
+ status: 503,
1892
+ reason: "recreate-in-progress",
1893
+ cause: "system",
1894
+ };
1895
+ }
1896
+
1897
+ /** Direct admin disk work is admitted before its first await, so the
1898
+ * operator's idle check cannot pass while it is about to mutate the VM. */
1899
+ private adminWorkInFlight = 0;
1900
+
1901
+ private async withRecreateSafeAdmin<T>(action: () => Promise<T>): Promise<T | ThreadErr> {
1902
+ if (this.recreateAdmission.pending) return this.recreateRefusal();
1903
+ this.adminWorkInFlight++;
1904
+ try {
1905
+ if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
1906
+ return await action();
1907
+ } finally {
1908
+ this.adminWorkInFlight--;
1909
+ }
1910
+ }
1852
1911
 
1853
1912
  /** Whether this resident knows the container it held is gone
1854
1913
  * (docs/reference/specs/resident-repos.md item 43): a restore under way for
@@ -2245,6 +2304,9 @@ export class ResidentDO extends Sandbox<Env> {
2245
2304
  argv: readonly string[],
2246
2305
  opts: { cwd?: string; timeoutMs?: number; env?: Record<string, string> } = {},
2247
2306
  ): Promise<{ stdout: string; stderr: string; exitCode: number; timedOut: boolean; truncated?: boolean }> {
2307
+ if (this.destroying || (await this.ctx.storage.get(DESTROY_UNCONFIRMED_KEY))) {
2308
+ throw new Error("resident-destroy-unconfirmed: container reuse refused until destruction is confirmed");
2309
+ }
2248
2310
  const timeout = opts.timeoutMs ?? DEFAULT_EXEC_TIMEOUT_MS;
2249
2311
  const launch = {
2250
2312
  ...(opts.cwd ? { cwd: opts.cwd } : {}),
@@ -2316,6 +2378,11 @@ export class ResidentDO extends Sandbox<Env> {
2316
2378
  console.log(
2317
2379
  `exec: runtime replaced before the process started (SDK says retryable) — retrying once: ${errMsg(err)}`,
2318
2380
  );
2381
+ if (this.destroying || (await this.ctx.storage.get(DESTROY_UNCONFIRMED_KEY))) {
2382
+ throw new Error("resident-destroy-unconfirmed: container reuse refused until destruction is confirmed", {
2383
+ cause: err,
2384
+ });
2385
+ }
2319
2386
  proc = await createExtensionProcessSandbox(this).exec(argv as unknown as SandboxCommand, launch);
2320
2387
  }
2321
2388
  // The spawn is the proof the control port answers: a persisted count of
@@ -3156,7 +3223,16 @@ export class ResidentDO extends Sandbox<Env> {
3156
3223
  * the provisioning work and its fail-closed deadline. Does NOT start the
3157
3224
  * container — onboard must return immediately; the container first starts
3158
3225
  * when runProvisioning's schedule fires (~1s later). */
3159
- async initResident(resource: string, provisioningTimeoutMs: number): Promise<ResidentStatus> {
3226
+ async initResident(resource: string, provisioningTimeoutMs: number, rebuilding = false): Promise<ResidentStatus> {
3227
+ if (this.destroying || (await this.ctx.storage.get(DESTROY_UNCONFIRMED_KEY))) {
3228
+ throw new Error("resident-destroy-unconfirmed: onboarding refused until destruction is confirmed");
3229
+ }
3230
+ if (!rebuilding && (await this.ctx.storage.get(RESOURCE_KEY))) {
3231
+ throw new Error("resident-old-state-present: onboarding refused until prior teardown is confirmed");
3232
+ }
3233
+ // Offboard can clear storage without replacing the DO isolate. Restore
3234
+ // claimant rows before this same isolate records a new onboard.
3235
+ await this.rebuildPoolBindingIndex();
3160
3236
  await this.ctx.storage.put({
3161
3237
  [RESOURCE_KEY]: resource,
3162
3238
  [STATE_KEY]: "onboarding" satisfies ResidentState,
@@ -3192,6 +3268,13 @@ export class ResidentDO extends Sandbox<Env> {
3192
3268
  const resource = payload || ((await this.ctx.storage.get<string>(RESOURCE_KEY)) ?? "");
3193
3269
  if ((await this.ctx.storage.get<ResidentState>(STATE_KEY)) !== "onboarding") return; // stale schedule
3194
3270
  try {
3271
+ // An older image could have deleted every DO key after a failed
3272
+ // offboard destroy. No prior row is evidence of a fresh VM; confirm
3273
+ // destruction here, after onboard returned, before any provision step.
3274
+ if (!parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS)) {
3275
+ this.swapIncarnation();
3276
+ await this.destroyConfirmed();
3277
+ }
3195
3278
  const record = await this.registry().getRecord(resource);
3196
3279
  if (!record) throw new StepError("registry", "registry record missing (offboarded mid-onboard?)");
3197
3280
  const slug = resource.slice("repo:".length);
@@ -3894,8 +3977,7 @@ export class ResidentDO extends Sandbox<Env> {
3894
3977
  // on a fresh one.
3895
3978
  const reason = infraStreakReason(row, "down");
3896
3979
  this.swapIncarnation(); // deliberate incarnation swap
3897
- await this.forgetRuntimeIdentity();
3898
- await this.destroy().catch((err) => console.log(`infra-streak: destroy failed: ${errMsg(err)}`));
3980
+ await this.destroyConfirmed();
3899
3981
  await this.ctx.storage.delete(INFRA_STREAK_KEY);
3900
3982
  await this.goDown(reason);
3901
3983
  return;
@@ -4015,9 +4097,7 @@ export class ResidentDO extends Sandbox<Env> {
4015
4097
  // (REHYDRATION_FAILURE_RE) — goDown rebuilds on the transition, item 36 —
4016
4098
  // and the VM destroyed, so the rebuild's provisioning starts on a new one.
4017
4099
  this.swapIncarnation(); // deliberate incarnation swap
4018
- await this.destroy().catch((destroyErr) =>
4019
- console.log(`runtime-unreachable: destroy failed: ${errMsg(destroyErr)}`),
4020
- );
4100
+ await this.destroyConfirmed();
4021
4101
  await this.clearInstanceLease(instance);
4022
4102
  return { status: "failed", reason: (await this.goDown(reason)).reason };
4023
4103
  }
@@ -4039,8 +4119,7 @@ export class ResidentDO extends Sandbox<Env> {
4039
4119
  `recreate: destroying the container — snapshots kept; the next exec restores from R2 (${reason.slice(0, 200)})`,
4040
4120
  );
4041
4121
  this.swapIncarnation(); // deliberate incarnation swap
4042
- await this.forgetRuntimeIdentity();
4043
- await this.destroy().catch((err) => console.log(`runtime-unreachable: destroy failed: ${errMsg(err)}`));
4122
+ await this.destroyConfirmed();
4044
4123
  await this.setResidentState("degraded", reason);
4045
4124
  }
4046
4125
 
@@ -4055,6 +4134,34 @@ export class ResidentDO extends Sandbox<Env> {
4055
4134
  await this.ctx.storage.delete(SDK_RUNTIME_RECORD_KEY);
4056
4135
  }
4057
4136
 
4137
+ /** Mark uncertainty before touching SDK runtime identity. A failed destroy
4138
+ * remains visible after an isolate restart and refuses every new exec. */
4139
+ private async destroyConfirmed(): Promise<void> {
4140
+ const pending =
4141
+ this.destroying ??
4142
+ destroyWithPersistentFence(
4143
+ {
4144
+ mark: () => this.ctx.storage.put(DESTROY_UNCONFIRMED_KEY, true),
4145
+ clear: async () => {
4146
+ await this.ctx.storage.delete(DESTROY_UNCONFIRMED_KEY);
4147
+ },
4148
+ },
4149
+ async () => {
4150
+ await this.forgetRuntimeIdentity();
4151
+ await this.destroy();
4152
+ // Only confirmed destruction creates an empty generation. If this
4153
+ // write fails, the uncertainty fence remains and no UID is reused.
4154
+ await this.ctx.storage.put(SPENT_POOL_USERS_KEY, []);
4155
+ },
4156
+ );
4157
+ this.destroying = pending;
4158
+ try {
4159
+ await pending;
4160
+ } finally {
4161
+ if (this.destroying === pending) this.destroying = null;
4162
+ }
4163
+ }
4164
+
4058
4165
  // -- the refresh cycle as a Workflow instance (item 7) --------------------------
4059
4166
  //
4060
4167
  // `ResidentRefresh` (the Workflow entrypoint, refresh.ts) calls these
@@ -4227,6 +4334,7 @@ export class ResidentDO extends Sandbox<Env> {
4227
4334
  ): Promise<InstanceStepAnswer<T>> {
4228
4335
  const startedAt = systemClock();
4229
4336
  const trace = createStepTrace(startedAt);
4337
+ this.refreshAdmissionsInFlight++;
4230
4338
  let counted = false;
4231
4339
  const count = () => {
4232
4340
  if (counted) return;
@@ -4235,6 +4343,11 @@ export class ResidentDO extends Sandbox<Env> {
4235
4343
  };
4236
4344
  let outcome = "done";
4237
4345
  try {
4346
+ if (await this.recreateAdmission.blocked()) {
4347
+ outcome = "stopped (recreate-in-progress)";
4348
+ await this.clearInstanceLease(instance);
4349
+ return { status: "stopped", why: "recreate-in-progress", startedAt, trace: trace.steps() };
4350
+ }
4238
4351
  const result = await this.stepTrace.run(trace, () => fn({ count }));
4239
4352
  if (result.status !== "done") {
4240
4353
  outcome = result.status === "stopped" ? `stopped (${result.why})` : `failed (${result.reason})`;
@@ -4297,6 +4410,7 @@ export class ResidentDO extends Sandbox<Env> {
4297
4410
  outcome = `failed (${failure.reason})`;
4298
4411
  return { status: "failed", reason: failure.reason, startedAt, trace: trace.steps() };
4299
4412
  } finally {
4413
+ this.refreshAdmissionsInFlight--;
4300
4414
  if (counted) this.refreshesInFlight--;
4301
4415
  await this.recordInstanceStep(instance, step, outcome).catch((err) =>
4302
4416
  console.log(`refresh instance ${instance}: recording ${step} failed: ${errMsg(err)}`),
@@ -4648,7 +4762,7 @@ export class ResidentDO extends Sandbox<Env> {
4648
4762
  private async recordRecycledTree(binding: ThreadBinding, tree: EvictedTree | undefined): Promise<void> {
4649
4763
  const now = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(binding.threadKey));
4650
4764
  if (!now || now.evicted || now.lastAttachAt !== binding.lastAttachAt) return;
4651
- await this.ctx.storage.put(threadBindingKey(binding.threadKey), {
4765
+ await this.putThreadBinding({
4652
4766
  ...now,
4653
4767
  recycledLeftBehind: tree && "leftBehind" in tree ? tree.leftBehind : undefined,
4654
4768
  recycledUnmeasured: tree && "unmeasured" in tree ? tree.unmeasured : undefined,
@@ -4773,7 +4887,10 @@ export class ResidentDO extends Sandbox<Env> {
4773
4887
  const at = new Date(systemClock()).toISOString();
4774
4888
  const all = await this.ctx.storage.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX });
4775
4889
  const used = new Set([...all.values()].filter((b) => !b.evicted && b.user).map((b) => b.user));
4776
- for (const u of this.opUsersInUse) used.add(u);
4890
+ const spent = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
4891
+ for (const user of spent?.keys() ?? THREAD_USERS) used.add(user);
4892
+ for (const u of this.opUsersInUse.keys()) used.add(u);
4893
+ for (const u of this.poolUsersInspecting) used.add(u);
4777
4894
  const reading = await this.memoryGauge();
4778
4895
  const generation = this.containerIdMemo ?? this.incarnation;
4779
4896
  const seat = { side: seatSide(used.size, THREAD_USERS.length), used: used.size, total: THREAD_USERS.length };
@@ -4949,7 +5066,7 @@ export class ResidentDO extends Sandbox<Env> {
4949
5066
  kept.push({ threadKey: c.threadKey, why: "busy" });
4950
5067
  continue;
4951
5068
  }
4952
- if (!(await this.evictBinding(current, true, "disk-pressure", DISK_PRESSURE_REASON, tree))) {
5069
+ if ((await this.evictBinding(current, true, "disk-pressure", DISK_PRESSURE_REASON, tree)) !== "evicted") {
4953
5070
  kept.push({ threadKey: c.threadKey, why: "other" });
4954
5071
  continue;
4955
5072
  }
@@ -5340,6 +5457,8 @@ export class ResidentDO extends Sandbox<Env> {
5340
5457
  capFiles?: { out: string; err: string },
5341
5458
  env?: Record<string, string>,
5342
5459
  ): Promise<{ stdout: string; stderr: string; exitCode: number; timedOut: boolean; truncated?: boolean }> {
5460
+ if (THREAD_USERS.includes(user) && !(await this.poolUserOwnerMatches(user)))
5461
+ throw new Error("pool-owner-mismatch: model command refused before UID ownership");
5343
5462
  // The caller's variables (an /exec body's `env`, docs/reference/specs/
5344
5463
  // harness-pi.md item 4) under the Worker's own: a caller never overrides
5345
5464
  // what the Worker injects. `su` without `-` keeps this environment for the
@@ -5396,23 +5515,192 @@ export class ResidentDO extends Sandbox<Env> {
5396
5515
  return this.assertOk(await this.threadRun(user, worktreePath, command, timeoutMs), step);
5397
5516
  }
5398
5517
 
5399
- /** The user-pool scan shared by both allocators: live bindings (optionally
5400
- * ignoring one threadKey's own binding) plus users transiently held by
5401
- * in-flight ops mark the pool as used — an attach must never share an OS
5402
- * user with a running op, and vice versa. Storage reads + set reads only,
5403
- * so callers stay atomic under the DO input gate. */
5404
- private async findFreePoolUser(excludeThreadKey?: string): Promise<string | undefined> {
5518
+ /** A candidate remains reserved across the disk inspection's awaits. */
5519
+ private poolUsersInspecting = new Set<string>();
5520
+
5521
+ /** A UID is spent before any untrusted command receives it. The storage
5522
+ * transaction prevents concurrent admissions from losing each other's
5523
+ * marks. An old resident without this row is an unknown generation. */
5524
+ private async markPoolUserSpent(user: string, owner: string): Promise<boolean> {
5525
+ return this.ctx.storage.transaction(async (txn) => {
5526
+ const current = await txn.get<unknown>(SPENT_POOL_USERS_KEY);
5527
+ const next = spendPoolUser(current, THREAD_USERS, user, owner);
5528
+ if (!next) return false;
5529
+ await txn.put(SPENT_POOL_USERS_KEY, next);
5530
+ return true;
5531
+ });
5532
+ }
5533
+
5534
+ private async rebuildPoolBindingIndex(): Promise<void> {
5535
+ await this.ctx.storage.transaction(async (txn) => {
5536
+ const bindings = await txn.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX });
5537
+ const byUser = rebuildPoolBindingIndex(bindings.values(), THREAD_USERS);
5538
+ if (!byUser) throw new Error("pool-binding-index: malformed live binding; refusing resident work");
5539
+ for (const [user, keys] of byUser) await txn.put(poolBindingKey(user), keys);
5540
+ });
5541
+ }
5542
+
5543
+ /** Primary and per-UID rows change atomically. The index keeps every live
5544
+ * claimant, including a legacy duplicate that must block shell access. */
5545
+ private async putThreadBinding(next: ThreadBinding): Promise<void> {
5546
+ const key = threadBindingKey(next.threadKey);
5547
+ await this.ctx.storage.transaction(async (txn) => {
5548
+ const prior = await txn.get<ThreadBinding>(key);
5549
+ const oldUser = prior && !prior.evicted ? prior.user : "";
5550
+ const newUser = !next.evicted ? next.user : "";
5551
+ if (oldUser && !THREAD_USERS.includes(oldUser)) throw new Error("pool-binding-index: unknown old UID");
5552
+ if (newUser && !THREAD_USERS.includes(newUser)) throw new Error("pool-binding-index: unknown new UID");
5553
+ if (oldUser !== newUser) {
5554
+ if (oldUser) {
5555
+ const keys = releasePoolBinding(await txn.get<unknown>(poolBindingKey(oldUser)), next.threadKey);
5556
+ if (!keys) throw new Error("pool-binding-index: missing old claimant");
5557
+ await txn.put(poolBindingKey(oldUser), keys);
5558
+ }
5559
+ if (newUser) {
5560
+ const keys = claimPoolBinding(await txn.get<unknown>(poolBindingKey(newUser)), next.threadKey);
5561
+ if (!keys) throw new Error("pool-binding-index: conflicting new claimant");
5562
+ await txn.put(poolBindingKey(newUser), keys);
5563
+ }
5564
+ }
5565
+ await txn.put(key, next);
5566
+ });
5567
+ }
5568
+
5569
+ private async deleteThreadBinding(threadKey: string, onlyEvicted = false): Promise<boolean> {
5570
+ return this.ctx.storage.transaction(async (txn) => {
5571
+ const key = threadBindingKey(threadKey);
5572
+ const prior = await txn.get<ThreadBinding>(key);
5573
+ if (!prior || (onlyEvicted && !prior.evicted)) return false;
5574
+ if (!prior.evicted && prior.user) {
5575
+ const keys = releasePoolBinding(await txn.get<unknown>(poolBindingKey(prior.user)), threadKey);
5576
+ if (!keys) throw new Error("pool-binding-index: missing deleted claimant");
5577
+ await txn.put(poolBindingKey(prior.user), keys);
5578
+ }
5579
+ await txn.delete(key);
5580
+ return true;
5581
+ });
5582
+ }
5583
+
5584
+ /** Every UID-scoped action checks both the durable claimant and the live
5585
+ * bindings. Two retained bindings naming one UID fail closed, even when one
5586
+ * is the ledger owner. Ops have a separate exact in-flight owner. */
5587
+ private async poolUserOwnerMatches(user: string, expectedOwner?: string): Promise<boolean> {
5588
+ if (!THREAD_USERS.includes(user)) return false;
5589
+ const [spent, bindings] = await Promise.all([
5590
+ this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY),
5591
+ this.ctx.storage.get<unknown>(poolBindingKey(user)),
5592
+ ]);
5593
+ const liveThreadKeys = parsePoolBindings(bindings);
5594
+ if (!liveThreadKeys) return false;
5595
+ return mayRunAsPoolUser(spent, THREAD_USERS, user, liveThreadKeys, this.opUsersInUse.get(user), expectedOwner);
5596
+ }
5597
+
5598
+ /** Storage bindings, active ops and inspections all reserve a pool user.
5599
+ * Claim inside this method, before its promise resolves: two concurrent
5600
+ * callers must not each read the same free UID before either can claim it. */
5601
+ private async findFreePoolUser(excludeThreadKey?: string, excluded = new Set<string>()): Promise<string | undefined> {
5405
5602
  const all = await this.ctx.storage.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX });
5406
5603
  const used = new Set(
5407
5604
  [...all.values()].filter((b) => !b.evicted && b.user && b.threadKey !== excludeThreadKey).map((b) => b.user),
5408
5605
  );
5409
- for (const u of this.opUsersInUse) used.add(u);
5410
- return THREAD_USERS.find((u) => !used.has(u));
5606
+ for (const u of this.opUsersInUse.keys()) used.add(u);
5607
+ for (const u of this.poolUsersInspecting) used.add(u);
5608
+ for (const u of excluded) used.add(u);
5609
+ const user = THREAD_USERS.find((u) => !used.has(u));
5610
+ if (user) this.poolUsersInspecting.add(user);
5611
+ return user;
5612
+ }
5613
+
5614
+ /** Root-side inspection fails closed on a missing/partial answer. A former
5615
+ * binding may have been freed by an old image while its private tree stayed
5616
+ * on disk; assigning that UID would let a new run read the old tree. */
5617
+ private async poolUserHasOldThreadDir(user: string, allowed: readonly string[]): Promise<boolean> {
5618
+ for (const dir of [THREADS_DIR, OPS_DIR]) {
5619
+ const result = await this.run([
5620
+ "sh",
5621
+ "-c",
5622
+ 'if test -L "$1"; then exit 42; fi; if test -d "$1"; then find "$1" -mindepth 1 -maxdepth 1 -type d -user "$2" -print; fi',
5623
+ "_",
5624
+ dir,
5625
+ user,
5626
+ ]);
5627
+ if (result.exitCode !== 0 || result.timedOut || result.truncated)
5628
+ throw new Error("pool-user-disk-scan-incomplete");
5629
+ if (hasUnexpectedOwnedThreadDir(result.stdout, dir, dir === THREADS_DIR ? allowed : [])) return true;
5630
+ }
5631
+ return false;
5632
+ }
5633
+
5634
+ /** A freed UID may still have a pre-fix credential or staged write in its
5635
+ * private staging directory. Inspect as root before assigning it to an op
5636
+ * (which does not run attach's credential scrub) or a fresh thread. */
5637
+ private async poolUserHasOldStageContent(user: string): Promise<boolean> {
5638
+ const result = await this.run(legacyStageContentScanCommand(`/workspace/.stage-${user}`));
5639
+ if (result.exitCode !== 0 || result.timedOut || result.truncated || result.stderr !== "")
5640
+ throw new Error("pool-user-stage-scan-incomplete");
5641
+ if (result.stdout === "stale\n") return true;
5642
+ if (result.stdout === "clean\n") return false;
5643
+ throw new Error("pool-user-stage-scan-invalid");
5644
+ }
5645
+
5646
+ private async reserveSafePoolUser(owner: string, excludeThreadKey?: string): Promise<string | ThreadErr> {
5647
+ const spent = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
5648
+ if (!spent) return { error: "pool-generation-unknown: fresh VM required before UID assignment", status: 503 };
5649
+ const excluded = new Set(spent.keys());
5650
+ let contaminated = false;
5651
+ for (;;) {
5652
+ const user = await this.findFreePoolUser(excludeThreadKey, excluded);
5653
+ if (!user)
5654
+ return contaminated
5655
+ ? { error: "pool-user-contaminated: no clean pool user is available", status: 503 }
5656
+ : excluded.size === THREAD_USERS.length
5657
+ ? { error: "pool-recycle-required: all UIDs spent on this VM; confirm idle and recreate it", status: 503 }
5658
+ : { error: "user-pool-exhausted: no safe pool user is available", status: 429 };
5659
+ let safe = false;
5660
+ try {
5661
+ const clean = !(await this.poolUserHasOldThreadDir(user, [])) && !(await this.poolUserHasOldStageContent(user));
5662
+ if (clean) {
5663
+ safe = await this.markPoolUserSpent(user, owner);
5664
+ if (safe) return user;
5665
+ } else contaminated = true;
5666
+ excluded.add(user);
5667
+ } catch {
5668
+ return { error: "pool-user-inspection-failed: allocation refused", status: 503 };
5669
+ } finally {
5670
+ if (!safe) this.poolUsersInspecting.delete(user);
5671
+ }
5672
+ // A rejected UID may still own a private tree. It stays unavailable to
5673
+ // this request, and every later allocation checks disk again.
5674
+ }
5675
+ }
5676
+
5677
+ /** The prior binding read, allocation and possible rollback are one
5678
+ * same-thread operation, even while an attach awaits disk or Git. */
5679
+ private readonly threadAttaches = new KeyedAsyncLock();
5680
+
5681
+ /** Claim an old retained binding before any user-scoped probe. A duplicate
5682
+ * legacy binding may name the same UID, but only its ledger owner proceeds. */
5683
+ private async claimRetainedThreadUser(binding: ThreadBinding): Promise<ThreadErr | null> {
5684
+ const user = binding.user;
5685
+ if (!THREAD_USERS.includes(user)) return { error: "pool-user-invalid: binding names a non-pool user", status: 409 };
5686
+ try {
5687
+ if (await this.poolUserHasOldThreadDir(user, [parentDir(binding.worktreePath)]))
5688
+ return { error: "pool-user-contaminated: another thread directory has this user", status: 409 };
5689
+ } catch {
5690
+ return { error: "pool-user-inspection-failed: allocation refused", status: 503 };
5691
+ }
5692
+ try {
5693
+ if (!(await this.markPoolUserSpent(user, `thread:${binding.threadKey}`)))
5694
+ return { error: "pool-owner-mismatch: existing binding cannot share a UID", status: 503 };
5695
+ } catch {
5696
+ return { error: "pool-generation-write-failed: existing binding cannot run", status: 503 };
5697
+ }
5698
+ if (!(await this.poolUserOwnerMatches(user, `thread:${binding.threadKey}`)))
5699
+ return { error: "pool-owner-mismatch: conflicting retained binding names this UID", status: 503 };
5700
+ return null;
5411
5701
  }
5412
5702
 
5413
- /** Storage-only allocation (atomic under the DO input gate: get → list →
5414
- * put touches nothing but this object's storage, so two concurrent
5415
- * attaches cannot both claim the same user). A hint that names no branch
5703
+ /** Allocation stays provisional until attach finishes. A hint that names no branch
5416
5704
  * keeps the sticky ref. An explicit named ref records named authority even
5417
5705
  * when it matches the sticky ref; only a changed ref replaces the path. */
5418
5706
  private async allocateThreadUser(
@@ -5429,49 +5717,54 @@ export class ResidentDO extends Sandbox<Env> {
5429
5717
  const replacingRef = existing !== undefined && replaceRef && existing.ref !== ref;
5430
5718
  const recordingNamedAuthority = existing !== undefined && replaceRef && existing.boundBy !== boundBy;
5431
5719
  const deferRefWrite = replacingRef && !existing.evicted && existing.user !== "";
5432
- if (existing && !existing.evicted && existing.user && !replacingRef && !recordingNamedAuthority)
5433
- return { binding: existing, wrote: false };
5434
- const user =
5435
- existing && !existing.evicted && existing.user ? existing.user : await this.findFreePoolUser(threadKey);
5436
- if (!user) {
5437
- return {
5438
- error: `user-pool-exhausted: all ${THREAD_USERS.length} thread users are allocated; wait for the inactivity sweep or evict a thread`,
5439
- status: 429,
5720
+ const existingUser = existing && !existing.evicted && existing.user ? existing.user : undefined;
5721
+ if (existing && existingUser) {
5722
+ if (existing.threadKey !== threadKey)
5723
+ return { error: "pool-owner-mismatch: retained binding belongs to another thread", status: 503 };
5724
+ const refusal = await this.claimRetainedThreadUser(existing);
5725
+ if (refusal) return refusal;
5726
+ }
5727
+ if (existingUser && !replacingRef && !recordingNamedAuthority) return { binding: existing!, wrote: false };
5728
+ const reserved = existingUser ?? (await this.reserveSafePoolUser(`thread:${threadKey}`, threadKey));
5729
+ if (typeof reserved !== "string") return reserved;
5730
+ const user = reserved;
5731
+ try {
5732
+ const now = new Date(systemClock()).toISOString();
5733
+ const binding: ThreadBinding = {
5734
+ threadKey,
5735
+ ref: replacingRef ? ref : (existing?.ref ?? ref),
5736
+ user,
5737
+ worktreePath: replacingRef ? worktreePath : (existing?.worktreePath ?? worktreePath),
5738
+ boundAt: existing?.boundAt ?? now,
5739
+ lastAttachAt: now,
5740
+ evicted: false,
5741
+ // A named ref starts a new binding choice, so stale move/return records
5742
+ // do not make it look default-bound. The thread's pushed-branch memory
5743
+ // remains its history across every ref choice.
5744
+ ...(existing
5745
+ ? {
5746
+ ...(replacingRef || recordingNamedAuthority
5747
+ ? { boundBy }
5748
+ : {
5749
+ ...(existing.boundBy !== undefined ? { boundBy: existing.boundBy } : {}),
5750
+ ...(existing.rebound !== undefined ? { rebound: existing.rebound } : {}),
5751
+ ...(existing.returned !== undefined ? { returned: existing.returned } : {}),
5752
+ }),
5753
+ ...(existing.ownBranches !== undefined ? { ownBranches: existing.ownBranches } : {}),
5754
+ }
5755
+ : { boundBy }),
5440
5756
  };
5757
+ // A live existing binding already reserves this user. Its named-ref
5758
+ // replacement stays provisional until deps and credentials succeed;
5759
+ // keeping the old row here lets a failed attach discard the new checkout
5760
+ // without ever publishing a binding that did not complete. An evicted row
5761
+ // has no user reservation, so its fresh allocation is written and rolled
5762
+ // back through the ordinary path.
5763
+ if (!deferRefWrite) await this.putThreadBinding(binding);
5764
+ return { binding, wrote: !deferRefWrite };
5765
+ } finally {
5766
+ if (!existingUser) this.poolUsersInspecting.delete(user);
5441
5767
  }
5442
- const now = new Date(systemClock()).toISOString();
5443
- const binding: ThreadBinding = {
5444
- threadKey,
5445
- ref: replacingRef ? ref : (existing?.ref ?? ref),
5446
- user,
5447
- worktreePath: replacingRef ? worktreePath : (existing?.worktreePath ?? worktreePath),
5448
- boundAt: existing?.boundAt ?? now,
5449
- lastAttachAt: now,
5450
- evicted: false,
5451
- // A named ref starts a new binding choice, so stale move/return records
5452
- // do not make it look default-bound. The thread's pushed-branch memory
5453
- // remains its history across every ref choice.
5454
- ...(existing
5455
- ? {
5456
- ...(replacingRef || recordingNamedAuthority
5457
- ? { boundBy }
5458
- : {
5459
- ...(existing.boundBy !== undefined ? { boundBy: existing.boundBy } : {}),
5460
- ...(existing.rebound !== undefined ? { rebound: existing.rebound } : {}),
5461
- ...(existing.returned !== undefined ? { returned: existing.returned } : {}),
5462
- }),
5463
- ...(existing.ownBranches !== undefined ? { ownBranches: existing.ownBranches } : {}),
5464
- }
5465
- : { boundBy }),
5466
- };
5467
- // A live existing binding already reserves this user. Its named-ref
5468
- // replacement stays provisional until deps and credentials succeed;
5469
- // keeping the old row here lets a failed attach discard the new checkout
5470
- // without ever publishing a binding that did not complete. An evicted row
5471
- // has no user reservation, so its fresh allocation is written and rolled
5472
- // back through the ordinary path.
5473
- if (!deferRefWrite) await this.ctx.storage.put(key, binding);
5474
- return { binding, wrote: !deferRefWrite };
5475
5768
  }
5476
5769
 
5477
5770
  /** Best-effort cleanup after a provisional named-ref replacement settles.
@@ -5510,9 +5803,19 @@ export class ResidentDO extends Sandbox<Env> {
5510
5803
  // the attach runs lands on it, and the answer carries it.
5511
5804
  const t0 = systemClock();
5512
5805
  const trace = createStepTrace(t0);
5513
- const res = await this.stepTrace.run(trace, () =>
5514
- this.attachThreadTraced(threadKey, refHint, readonly, wantSha, reuse, record, t0, reason, githubDoor),
5515
- );
5806
+ let res: AttachOk | ThreadErr;
5807
+ if (this.recreateAdmission.pending) {
5808
+ res = this.recreateRefusal();
5809
+ } else {
5810
+ this.attachAdmissionsInFlight++;
5811
+ try {
5812
+ res = await this.stepTrace.run(trace, () =>
5813
+ this.attachThreadTraced(threadKey, refHint, readonly, wantSha, reuse, record, t0, reason, githubDoor),
5814
+ );
5815
+ } finally {
5816
+ this.attachAdmissionsInFlight--;
5817
+ }
5818
+ }
5516
5819
  // The same steps as the resident's own `resident.attach` root (item 22).
5517
5820
  emitStepRoot("resident.attach", t0, trace.steps(), traceparent, "error" in res ? refusalOutcome(res) : "ok");
5518
5821
  // A refusal carries the steps that led to it; a success already does.
@@ -5531,7 +5834,9 @@ export class ResidentDO extends Sandbox<Env> {
5531
5834
  githubDoor?: { baseUrl: string; bearer: string },
5532
5835
  ): Promise<AttachOk | ThreadErr> {
5533
5836
  try {
5837
+ if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
5534
5838
  await this.ensureHydrated();
5839
+ if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
5535
5840
  // The fleet drain (item 69): a deploy is waiting for the runs in flight
5536
5841
  // to end, and a NEW run's attach is refused with the record the bot
5537
5842
  // waits on — a real 503 in the streamed document, read by the client as
@@ -5576,22 +5881,24 @@ export class ResidentDO extends Sandbox<Env> {
5576
5881
  // container under it (and isIdle never parks the cycle mid-attach).
5577
5882
  this.attachesInFlight++;
5578
5883
  try {
5579
- const res = await this.attachThreadBody(
5580
- threadKey,
5581
- refHint,
5582
- readonly,
5583
- wantSha,
5584
- reuse,
5585
- resourceId,
5586
- t0,
5587
- record,
5588
- reason,
5589
- githubDoor,
5590
- );
5591
- // The run this attach opens is now in flight until its release —
5592
- // whatever its op counters read between the bot's calls (item 44).
5593
- if (!("error" in res)) await this.registerRun(threadKey);
5594
- return res;
5884
+ return await this.threadAttaches.run(threadKey, async () => {
5885
+ const res = await this.attachThreadBody(
5886
+ threadKey,
5887
+ refHint,
5888
+ readonly,
5889
+ wantSha,
5890
+ reuse,
5891
+ resourceId,
5892
+ t0,
5893
+ record,
5894
+ reason,
5895
+ githubDoor,
5896
+ );
5897
+ // The run this attach opens is now in flight until its release —
5898
+ // whatever its op counters read between the bot's calls (item 44).
5899
+ if (!("error" in res)) await this.registerRun(threadKey);
5900
+ return res;
5901
+ });
5595
5902
  } finally {
5596
5903
  this.attachesInFlight--;
5597
5904
  }
@@ -5674,6 +5981,23 @@ export class ResidentDO extends Sandbox<Env> {
5674
5981
  if (!readonly && storedPrior?.githubDoorHost && !githubDoor)
5675
5982
  return { error: "Git door credential is required to reattach this writable tree", status: 403, cause: "request" };
5676
5983
  const namedRef = refHint !== null && !reason.refByDefault && reason.ownPr === null;
5984
+ // The own-PR rebind may run a branch probe as this UID before ordinary
5985
+ // allocation. Establish exact ownership first on a recycled VM.
5986
+ if (!namedRef && storedPrior && !storedPrior.evicted && storedPrior.user) {
5987
+ if (storedPrior.threadKey !== threadKey)
5988
+ return { error: "pool-owner-mismatch: retained binding belongs to another thread", status: 503 };
5989
+ const refusal = await this.claimRetainedThreadUser(storedPrior);
5990
+ if (refusal) return refusal;
5991
+ // The own-PR fallback may inspect a local branch as this pool user.
5992
+ // Remove any legacy App token before that first user-scoped Git probe.
5993
+ if (reason.ownPr !== null) {
5994
+ try {
5995
+ await this.scrubThreadCredentialFiles(storedPrior);
5996
+ } catch (err) {
5997
+ return this.attachFailed(err);
5998
+ }
5999
+ }
6000
+ }
5677
6001
  const rebind = namedRef
5678
6002
  ? { binding: storedPrior }
5679
6003
  : await this.rebindToOwnPr(storedPrior, reason.ownPr, reuse, facts.defaultRef, slug);
@@ -5715,10 +6039,23 @@ export class ResidentDO extends Sandbox<Env> {
5715
6039
  // bind sticky garbage nor leak a pool user.
5716
6040
  const rollback = async (): Promise<void> => {
5717
6041
  if (!alloc.wrote) return;
5718
- if (prior) await this.ctx.storage.put(threadBindingKey(threadKey), prior);
5719
- else await this.ctx.storage.delete(threadBindingKey(threadKey));
6042
+ if (prior) await this.putThreadBinding(prior);
6043
+ else await this.deleteThreadBinding(threadKey);
5720
6044
  };
5721
6045
 
6046
+ // A named ref builds beside the prior checkout and removes that sibling
6047
+ // only after the new binding commits. If cleanup later fails, the same
6048
+ // pool user can still traverse the old tree. Remove its legacy App token
6049
+ // before admitting the replacement or any of its model commands.
6050
+ if (replacingNamedRef && prior && !prior.evicted) {
6051
+ try {
6052
+ await this.scrubThreadCredentialFiles(prior);
6053
+ } catch (err) {
6054
+ await rollback();
6055
+ return this.attachFailed(err);
6056
+ }
6057
+ }
6058
+
5722
6059
  // Disk admission (item 55): before the lock, since making room takes it.
5723
6060
  // An evicted prior binding has already published its replacement to
5724
6061
  // reserve the user; keep that row provisional if admission itself throws.
@@ -5910,7 +6247,7 @@ export class ResidentDO extends Sandbox<Env> {
5910
6247
  at: new Date(systemClock()).toISOString(),
5911
6248
  };
5912
6249
  const moved: ThreadBinding = { ...current, ref: plan.to, rebound };
5913
- await this.ctx.storage.put(threadBindingKey(current.threadKey), moved);
6250
+ await this.putThreadBinding(moved);
5914
6251
  return { kind: "rebound", moved, rebound };
5915
6252
  }
5916
6253
 
@@ -5937,7 +6274,7 @@ export class ResidentDO extends Sandbox<Env> {
5937
6274
  const current = (await this.ctx.storage.get<ThreadBinding>(key)) ?? binding;
5938
6275
  const back = returnToDefault(current, defaultRef, new Date(systemClock()).toISOString());
5939
6276
  if (back === undefined) return { binding: current };
5940
- await this.ctx.storage.put(key, back.binding);
6277
+ await this.putThreadBinding(back.binding);
5941
6278
  console.log(
5942
6279
  `attach ${binding.threadKey}: ${back.returned.from} is gone from the mirror (the thread's own pull request #${back.returned.pr}) — returned to ${back.returned.to}; the tree is provisioned there`,
5943
6280
  );
@@ -5954,7 +6291,7 @@ export class ResidentDO extends Sandbox<Env> {
5954
6291
  private async rememberOwnBranches(threadKey: string, pushed: readonly PushedBranch[]): Promise<void> {
5955
6292
  const binding = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
5956
6293
  if (!binding) return;
5957
- await this.ctx.storage.put(threadBindingKey(threadKey), {
6294
+ await this.putThreadBinding({
5958
6295
  ...binding,
5959
6296
  ownBranches: rememberOwnBranches(binding.ownBranches, pushed, new Date(systemClock()).toISOString()),
5960
6297
  } satisfies ThreadBinding);
@@ -6219,7 +6556,7 @@ export class ResidentDO extends Sandbox<Env> {
6219
6556
  githubDoorHost: _priorDoor,
6220
6557
  ...bindingSansCred
6221
6558
  } = binding;
6222
- await this.ctx.storage.put(threadBindingKey(threadKey), {
6559
+ await this.putThreadBinding({
6223
6560
  ...bindingSansCred,
6224
6561
  lastAttachAt: new Date(systemClock()).toISOString(),
6225
6562
  deps: deps.deps,
@@ -7028,6 +7365,7 @@ export class ResidentDO extends Sandbox<Env> {
7028
7365
  * binding, worktree actually on disk (the container may have slept since
7029
7366
  * the last attach — disk is cache, re-attach recreates). */
7030
7367
  private async threadPreflight(threadKey: string): Promise<{ binding: ThreadBinding } | ThreadErr> {
7368
+ if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
7031
7369
  try {
7032
7370
  await this.ensureHydrated();
7033
7371
  } catch (err) {
@@ -7056,6 +7394,9 @@ export class ResidentDO extends Sandbox<Env> {
7056
7394
  needs: "attach",
7057
7395
  };
7058
7396
  }
7397
+ if (!(await this.poolUserOwnerMatches(binding.user, `thread:${threadKey}`))) {
7398
+ return { error: "pool-owner-mismatch: model command refused before UID ownership", status: 503 };
7399
+ }
7059
7400
  if ((await this.run(["test", "-d", `${binding.worktreePath}/.git`])).exitCode !== 0) {
7060
7401
  return {
7061
7402
  error: "worktree-missing: the container disk was recycled since the last attach — POST /attach to recreate",
@@ -7155,7 +7496,7 @@ export class ResidentDO extends Sandbox<Env> {
7155
7496
  !/^sbr_[A-Za-z0-9_-]{1,64}\.[A-Za-z0-9_-]{20,128}$/.test(env?.GH_ENTERPRISE_TOKEN ?? ""))
7156
7497
  )
7157
7498
  return { error: "Git door run bearer is missing from this command", status: 403, cause: "request" };
7158
- await this.ctx.storage.put(threadBindingKey(threadKey), {
7499
+ await this.putThreadBinding({
7159
7500
  ...binding,
7160
7501
  lastAttachAt: new Date(systemClock()).toISOString(), // exec counts as activity for the sweep
7161
7502
  } satisfies ThreadBinding);
@@ -7281,9 +7622,15 @@ export class ResidentDO extends Sandbox<Env> {
7281
7622
  path: string,
7282
7623
  content: string,
7283
7624
  ): Promise<{ ok: true; bytes: number } | ThreadErr> {
7284
- return this.withThreadBusy(threadKey, () => this.writeThreadFileImpl(threadKey, path, content));
7625
+ return this.withThreadBusy(threadKey, () =>
7626
+ this.threadWrites.run(threadKey, () => this.writeThreadFileImpl(threadKey, path, content)),
7627
+ );
7285
7628
  }
7286
7629
 
7630
+ /** All writes for a thread share its stage file; keep staging and cleanup
7631
+ * in one operation so a failed write cannot remove another write's data. */
7632
+ private readonly threadWrites = new KeyedAsyncLock();
7633
+
7287
7634
  private async writeThreadFileImpl(
7288
7635
  threadKey: string,
7289
7636
  path: string,
@@ -7312,6 +7659,12 @@ export class ResidentDO extends Sandbox<Env> {
7312
7659
  DEFAULT_EXEC_TIMEOUT_MS,
7313
7660
  );
7314
7661
  } catch (err) {
7662
+ try {
7663
+ await this.runOk(legacyStageReuseScrubCommand(stageDir), "stage-write-cleanup");
7664
+ } catch (cleanupErr) {
7665
+ // The next UID reservation still refuses nonempty staging content.
7666
+ console.log(`write: stage cleanup failed for ${threadKey}: ${errMsg(cleanupErr)}`);
7667
+ }
7315
7668
  if (err instanceof ControlResetError) return controlResetErr(err);
7316
7669
  if (err instanceof RuntimeReplacedError) return runtimeReplacedErr(err);
7317
7670
  if (err instanceof SandboxRuntimeBusyError) return runtimeBusyErr(err);
@@ -7338,20 +7691,31 @@ export class ResidentDO extends Sandbox<Env> {
7338
7691
  * as the thread user with the runtime up); absent when the tree was clean,
7339
7692
  * already gone with the disk, or not measured (a force detach). */
7340
7693
  tree?: EvictedTree,
7341
- ): Promise<boolean> {
7694
+ ): Promise<"evicted" | "changed" | "cleanup-failed"> {
7695
+ if (!(await this.poolUserOwnerMatches(binding.user, `thread:${binding.threadKey}`))) return "cleanup-failed";
7342
7696
  const threadDir = parentDir(binding.worktreePath);
7343
- if (runtimeActive && threadDir.startsWith(`${THREADS_DIR}/`)) {
7697
+ if (runtimeActive && !threadDir.startsWith(`${THREADS_DIR}/`)) return "cleanup-failed";
7698
+ if (runtimeActive) {
7344
7699
  try {
7345
7700
  // Worktree removal counts as a mirror-adjacent mutation — same mutex.
7346
- await this.withMirrorLock(() => this.runOk(["rm", "-rf", threadDir], "evict"));
7701
+ await this.withMirrorLock(async () => {
7702
+ if (!(await this.poolUserOwnerMatches(binding.user, `thread:${binding.threadKey}`)))
7703
+ throw new Error("pool-owner-mismatch: eviction refused before directory removal");
7704
+ await this.runOk(["rm", "-rf", threadDir], "evict");
7705
+ });
7347
7706
  } catch (err) {
7348
7707
  console.log(`${logCtx}: rm failed for ${binding.threadKey}: ${errMsg(err)}`);
7708
+ // The directory may still contain a credential file readable by this
7709
+ // UID. Keep the binding live so the allocator cannot give it to a new
7710
+ // thread; a later sweep or detach may retry the removal.
7711
+ return "cleanup-failed";
7349
7712
  }
7350
7713
  // Item 55: what an `install` thread's package manager left OUTSIDE the
7351
7714
  // tree — its pnpm store (the tree's hardlink source: 0 unique bytes while
7352
7715
  // the tree lived, all of them now), npm/yarn/bun caches — goes with it.
7353
7716
  // Pool users only, never the build user (its store backs the warm checkout).
7354
7717
  if ((THREAD_USERS as readonly string[]).includes(binding.user)) {
7718
+ if (!(await this.poolUserOwnerMatches(binding.user, `thread:${binding.threadKey}`))) return "cleanup-failed";
7355
7719
  await this.run(threadUserCacheCleanArgv(`/home/${binding.user}`)).catch((err) =>
7356
7720
  console.log(`${logCtx}: home cache rm failed for ${binding.user}: ${errMsg(err)}`),
7357
7721
  );
@@ -7365,9 +7729,9 @@ export class ResidentDO extends Sandbox<Env> {
7365
7729
  if (!now || now.evicted || now.lastAttachAt !== binding.lastAttachAt) {
7366
7730
  const why = !now ? "binding deleted" : now.evicted ? "already evicted (concurrent eviction)" : "re-attached";
7367
7731
  console.log(`${logCtx}: ${binding.threadKey} ${why} during eviction — binding left as is`);
7368
- return false;
7732
+ return "changed";
7369
7733
  }
7370
- await this.ctx.storage.put(threadBindingKey(binding.threadKey), {
7734
+ await this.putThreadBinding({
7371
7735
  ...now,
7372
7736
  user: "",
7373
7737
  evicted: true,
@@ -7380,7 +7744,7 @@ export class ResidentDO extends Sandbox<Env> {
7380
7744
  // never on a give-way above — a re-attach has re-registered it anyway.
7381
7745
  await this.ctx.storage.delete(runRegKey(binding.threadKey));
7382
7746
  if (tree) console.log(`${logCtx}: ${binding.threadKey} evicted (${why}) — ${evictedTreeSentence(tree)}`);
7383
- return true;
7747
+ return "evicted";
7384
7748
  }
7385
7749
 
7386
7750
  /** POST /detach: a run has ended — give the thread's pool user back now
@@ -7404,6 +7768,8 @@ export class ResidentDO extends Sandbox<Env> {
7404
7768
  const binding = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
7405
7769
  if (!binding) return { error: `no-binding: ${threadKey} has never attached to this resident`, status: 404 };
7406
7770
  if (binding.evicted || !binding.user) return { released: false, reason: "already-evicted" };
7771
+ if (!(await this.poolUserOwnerMatches(binding.user, `thread:${threadKey}`)))
7772
+ return { error: "pool-owner-mismatch: detach refused for conflicting UID owner", status: 503 };
7407
7773
  const plan = planForceDetach({
7408
7774
  force,
7409
7775
  inFlight: this.threadOpsInFlight.get(threadKey) ?? 0,
@@ -7412,7 +7778,8 @@ export class ResidentDO extends Sandbox<Env> {
7412
7778
  });
7413
7779
  if (plan.action === "refuse") return { released: false, reason: plan.reason, user: binding.user };
7414
7780
  if (plan.action === "kill") {
7415
- await this.killThreadUserProcesses(plan.user);
7781
+ if (!(await this.killThreadUserProcesses(plan.user, threadKey)))
7782
+ return { error: "pool-owner-mismatch: force detach refused before kill", status: 503 };
7416
7783
  console.log(
7417
7784
  `detach: force — killed ${plan.user}'s processes for ${threadKey} (${plan.inFlight} op(s) were in flight)`,
7418
7785
  );
@@ -7446,10 +7813,11 @@ export class ResidentDO extends Sandbox<Env> {
7446
7813
  const user = current.user;
7447
7814
  // Same as the sweep: `active` was read before the measurement's awaits; a
7448
7815
  // container that woke meanwhile must get the rm, not an orphaned tree.
7449
- const activeNow = await this.isRuntimeActive().catch(() => false);
7450
- if (!(await this.evictBinding(current, activeNow, `detach`, "detach", tree))) {
7451
- return { released: false, reason: "re-attached during eviction — kept", user };
7452
- }
7816
+ const activeNow = await this.isRuntimeActive().catch(() => true);
7817
+ const eviction = await this.evictBinding(current, activeNow, `detach`, "detach", tree);
7818
+ if (eviction === "cleanup-failed")
7819
+ return { released: false, reason: "thread-cleanup-failed: pool user kept", user };
7820
+ if (eviction === "changed") return { released: false, reason: "re-attached during eviction — kept", user };
7453
7821
  // Item 55: the tree is gone; the gauge catches up at the next refresh
7454
7822
  // instance's `measure` step, and the admission's `df` sees the space now.
7455
7823
  const leftBehind = tree && "leftBehind" in tree ? tree.leftBehind : undefined;
@@ -7464,12 +7832,14 @@ export class ResidentDO extends Sandbox<Env> {
7464
7832
  * (runtime replaced mid-kill) is fine as well: the drain wait after it is
7465
7833
  * what decides, and it is bounded. Only ever called with a plan from
7466
7834
  * `planForceDetach`, which refuses anything but a `THREAD_USERS` member. */
7467
- private async killThreadUserProcesses(user: string): Promise<void> {
7835
+ private async killThreadUserProcesses(user: string, threadKey: string): Promise<boolean> {
7836
+ if (!(await this.poolUserOwnerMatches(user, `thread:${threadKey}`))) return false;
7468
7837
  try {
7469
7838
  await this.run(["su", "-s", "/bin/bash", user, "-c", "kill -9 -1"], { timeoutMs: FORCE_DETACH_KILL_TIMEOUT_MS });
7470
7839
  } catch (err) {
7471
7840
  console.log(`detach: force — kill as ${user} threw (continuing to the drain wait): ${errMsg(err)}`);
7472
7841
  }
7842
+ return true;
7473
7843
  }
7474
7844
 
7475
7845
  /** Wait (bounded, see FORCE_DETACH_DRAIN_MS) for this thread's in-flight op
@@ -7505,6 +7875,8 @@ export class ResidentDO extends Sandbox<Env> {
7505
7875
  * commands inside a single privilege-dropped `su`, tagged lines out);
7506
7876
  * `parseWorktreeCleanliness` encodes the exact decision table above. */
7507
7877
  private async worktreeCleanliness(binding: ThreadBinding): Promise<WorktreeCleanliness> {
7878
+ if (!(await this.poolUserOwnerMatches(binding.user, `thread:${binding.threadKey}`)))
7879
+ return { clean: false, reason: "pool-owner-mismatch: cleanliness probe refused" };
7508
7880
  const injected = { GIT_TERMINAL_PROMPT: "0" }; // same injection as threadRun — fail fast, never prompt
7509
7881
  validateEnvNames(injected);
7510
7882
  const r = await this.run(["sh", "-c", worktreeCleanlinessScript(binding.worktreePath, binding.user)], {
@@ -7537,7 +7909,11 @@ export class ResidentDO extends Sandbox<Env> {
7537
7909
  * reads as busy. */
7538
7910
  private runsInFlightCount(): number {
7539
7911
  const threadOps = [...this.threadOpsInFlight.values()].reduce((a, n) => a + n, 0);
7540
- return threadOps + this.opUsersInUse.size + this.attachesInFlight;
7912
+ return (
7913
+ threadOps +
7914
+ Math.max(this.opUsersInUse.size, this.opAdmissionsInFlight) +
7915
+ Math.max(this.attachesInFlight, this.attachAdmissionsInFlight)
7916
+ );
7541
7917
  }
7542
7918
  /** The fleet drain in force (item 69), read from the registry at this clock;
7543
7919
  * a registry that cannot be read is NO drain: a run must never fail because
@@ -7582,6 +7958,11 @@ export class ResidentDO extends Sandbox<Env> {
7582
7958
  return n;
7583
7959
  }
7584
7960
  private attachesInFlight = 0;
7961
+ /** Entry-to-answer admissions include hydration before attachesInFlight starts. */
7962
+ private attachAdmissionsInFlight = 0;
7963
+ private opAdmissionsInFlight = 0;
7964
+ /** A refresh step in its entry gates is also unsafe to interrupt. */
7965
+ private refreshAdmissionsInFlight = 0;
7585
7966
  /** A refresh cycle past its idle/reconcile gates (fetching, rebuilding, snapshotting). */
7586
7967
  private refreshesInFlight = 0;
7587
7968
 
@@ -7630,18 +8011,18 @@ export class ResidentDO extends Sandbox<Env> {
7630
8011
  // `active` is re-read per binding: the container can wake mid-sweep (an
7631
8012
  // attach), and an eviction decided on a stale "inactive" would skip the
7632
8013
  // rm and orphan a real tree.
7633
- const activeNow = await this.isRuntimeActive().catch(() => false);
8014
+ const activeNow = await this.isRuntimeActive().catch(() => true);
7634
8015
  // What the tree holds goes on the eviction's record (item 17), measured
7635
8016
  // while the runtime is up; a slept container has no tree to measure.
7636
8017
  const tree = activeNow ? await this.measureTreeBeforeEviction(current) : undefined;
7637
8018
  if (
7638
- await this.evictBinding(
8019
+ (await this.evictBinding(
7639
8020
  current,
7640
8021
  activeNow,
7641
8022
  `worktree-sweep ${resource}`,
7642
8023
  last >= cutoff ? "clean-idle" : "ttl",
7643
8024
  tree,
7644
- )
8025
+ )) === "evicted"
7645
8026
  )
7646
8027
  evicted.push(binding.threadKey);
7647
8028
  else kept++;
@@ -7656,15 +8037,16 @@ export class ResidentDO extends Sandbox<Env> {
7656
8037
  * (its orphaned OPS_DIR entry dies with the container disk at the latest).
7657
8038
  * Both this allocator and allocateThreadUser exclude the set, so an op
7658
8039
  * never shares an OS user with a thread or another op. */
7659
- private opUsersInUse = new Set<string>();
8040
+ private opUsersInUse = new Map<string, string>();
7660
8041
 
7661
- /** Transient allocation: storage reads + a synchronous set-add in the same
7662
- * microtask (atomic under the DO input gate, like allocateThreadUser).
7663
- * Returns null when threads + ops have the whole pool busy. */
7664
- private async allocateOpUser(): Promise<string | null> {
7665
- const user = await this.findFreePoolUser();
7666
- if (user) this.opUsersInUse.add(user);
7667
- return user ?? null;
8042
+ /** The inspection reservation transfers to the op reservation synchronously. */
8043
+ private async allocateOpUser(): Promise<string | ThreadErr> {
8044
+ const owner = `op:${crypto.randomUUID()}`;
8045
+ const reserved = await this.reserveSafePoolUser(owner);
8046
+ if (typeof reserved !== "string") return reserved;
8047
+ this.opUsersInUse.set(reserved, owner);
8048
+ this.poolUsersInspecting.delete(reserved);
8049
+ return reserved;
7668
8050
  }
7669
8051
 
7670
8052
  /** POST /op work half: run ONE readonly command-table entry in a
@@ -7681,7 +8063,17 @@ export class ResidentDO extends Sandbox<Env> {
7681
8063
  // One step trace per op (docs/reference/specs/tracing.md item 19), like an attach.
7682
8064
  const t0 = systemClock();
7683
8065
  const trace = createStepTrace(t0);
7684
- const res = await this.stepTrace.run(trace, () => this.runOpTraced(op, refArg, t0));
8066
+ let res: OpRunOk | ThreadErr;
8067
+ if (this.recreateAdmission.pending) {
8068
+ res = this.recreateRefusal();
8069
+ } else {
8070
+ this.opAdmissionsInFlight++;
8071
+ try {
8072
+ res = await this.stepTrace.run(trace, () => this.runOpTraced(op, refArg, t0));
8073
+ } finally {
8074
+ this.opAdmissionsInFlight--;
8075
+ }
8076
+ }
7685
8077
  emitStepRoot("resident.op", t0, trace.steps(), traceparent, "error" in res ? refusalOutcome(res) : "ok", {
7686
8078
  command: op,
7687
8079
  });
@@ -7689,6 +8081,7 @@ export class ResidentDO extends Sandbox<Env> {
7689
8081
  }
7690
8082
 
7691
8083
  private async runOpTraced(op: "test" | "build", refArg: string | null, t0: number): Promise<OpRunOk | ThreadErr> {
8084
+ if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
7692
8085
  // The fleet drain (item 69; issue 2044): a typed op is a new piece of work
7693
8086
  // like a new run's attach, and during the incident one waited silently at
7694
8087
  // the drain — so it is answered with the drain's own record at once, a
@@ -7715,6 +8108,7 @@ export class ResidentDO extends Sandbox<Env> {
7715
8108
  cause: "system",
7716
8109
  };
7717
8110
  }
8111
+ if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
7718
8112
  // One storage round trip for the two facts; the registry lookup stays (an
7719
8113
  // op resolves ONLY through the onboard-time command table).
7720
8114
  const stored = await this.ctx.storage.get<string | RepoFacts>([RESOURCE_KEY, FACTS_KEY]);
@@ -7734,12 +8128,7 @@ export class ResidentDO extends Sandbox<Env> {
7734
8128
  if (!command) return { error: `op-unavailable: the command table has no "${op}" entry`, status: 400 };
7735
8129
 
7736
8130
  const user = await this.allocateOpUser();
7737
- if (!user) {
7738
- return {
7739
- error: `user-pool-exhausted: all ${THREAD_USERS.length} pool users are busy (threads or in-flight ops); try again shortly`,
7740
- status: 429,
7741
- };
7742
- }
8131
+ if (typeof user !== "string") return user;
7743
8132
  const opDir = `${OPS_DIR}/${crypto.randomUUID()}`;
7744
8133
  const checkout = `${opDir}/checkout`;
7745
8134
  try {
@@ -7883,8 +8272,13 @@ export class ResidentDO extends Sandbox<Env> {
7883
8272
  const all = await this.ctx.storage.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX });
7884
8273
  const decision = selectBindingsToPurge([...all.values()], prefix);
7885
8274
  if (!decision.ok) return { error: decision.error };
7886
- if (decision.purge.length > 0) await this.ctx.storage.delete(decision.purge.map(threadBindingKey));
7887
- return { purged: decision.purge, keptLive: decision.keptLive };
8275
+ const purged: string[] = [];
8276
+ const keptLive = [...decision.keptLive];
8277
+ for (const threadKey of decision.purge) {
8278
+ if (await this.deleteThreadBinding(threadKey, true)) purged.push(threadKey);
8279
+ else keptLive.push(threadKey);
8280
+ }
8281
+ return { purged, keptLive };
7888
8282
  }
7889
8283
 
7890
8284
  /** Debug fault injection: age a binding so the sweep's TTL path can be
@@ -7893,13 +8287,15 @@ export class ResidentDO extends Sandbox<Env> {
7893
8287
  const binding = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
7894
8288
  if (!binding) return { ok: false };
7895
8289
  const lastAttachAt = new Date(systemClock() - days * 86_400_000).toISOString();
7896
- await this.ctx.storage.put(threadBindingKey(threadKey), { ...binding, lastAttachAt } satisfies ThreadBinding);
8290
+ await this.putThreadBinding({ ...binding, lastAttachAt } satisfies ThreadBinding);
7897
8291
  return { ok: true, lastAttachAt };
7898
8292
  }
7899
8293
 
7900
8294
  /** Debug: run the sweep pass now (the exact function the `sweep` step runs). */
7901
- async debugSweepNow(): Promise<{ evicted: string[]; kept: number }> {
7902
- return this.sweepWorktrees((await this.ctx.storage.get<string>(RESOURCE_KEY)) ?? "");
8295
+ async debugSweepNow(): Promise<{ evicted: string[]; kept: number } | ThreadErr> {
8296
+ return this.withRecreateSafeAdmin(async () =>
8297
+ this.sweepWorktrees((await this.ctx.storage.get<string>(RESOURCE_KEY)) ?? ""),
8298
+ );
7903
8299
  }
7904
8300
 
7905
8301
  // -- event-triggered reclamation ---------------------------------------------
@@ -8035,25 +8431,35 @@ export class ResidentDO extends Sandbox<Env> {
8035
8431
  kept.push({ threadKey: binding.threadKey, ref: binding.ref, why: "re-attached" });
8036
8432
  continue;
8037
8433
  }
8038
- const activeNow = await this.isRuntimeActive().catch(() => false);
8434
+ const activeNow = await this.isRuntimeActive().catch(() => true);
8039
8435
  const why = `${decision.why}${detail}`;
8040
- if (await this.evictBinding(current, activeNow, `reclaim ${resource}`, why, tree)) {
8436
+ const eviction = await this.evictBinding(current, activeNow, `reclaim ${resource}`, why, tree);
8437
+ if (eviction === "evicted") {
8041
8438
  reclaimed.push({ threadKey: binding.threadKey, ref: binding.ref, why });
8042
8439
  console.log(`reclaim ${resource}: evicted ${binding.threadKey} on ${binding.ref} — ${why}`);
8043
- } else kept.push({ threadKey: binding.threadKey, ref: binding.ref, why: "re-attached" });
8440
+ } else
8441
+ kept.push({
8442
+ threadKey: binding.threadKey,
8443
+ ref: binding.ref,
8444
+ why: eviction === "changed" ? "re-attached" : "cleanup-failed",
8445
+ });
8044
8446
  }
8045
8447
  return { reclaimed, kept };
8046
8448
  }
8047
8449
 
8048
8450
  /** Debug: measure the disk now (admin) — the exact cycle/attach function. */
8049
- async debugMeasureDisk(): Promise<DiskSample | null> {
8050
- return this.measureDisk();
8451
+ async debugMeasureDisk(): Promise<DiskSample | null | ThreadErr> {
8452
+ return this.withRecreateSafeAdmin(() => this.measureDisk());
8051
8453
  }
8052
8454
 
8053
8455
  /** Debug: run the reclamation pass now (the exact refresh-cycle function,
8054
8456
  * with a fresh mint when the App is configured). Runs a `fetch --prune`
8055
8457
  * first so a branch deleted seconds ago already reads as gone. */
8056
- async debugReclaimNow(): Promise<{ reclaimed: unknown[]; kept: unknown[]; fetch: string }> {
8458
+ async debugReclaimNow(): Promise<{ reclaimed: unknown[]; kept: unknown[]; fetch: string } | ThreadErr> {
8459
+ return this.withRecreateSafeAdmin(() => this.debugReclaimNowBody());
8460
+ }
8461
+
8462
+ private async debugReclaimNowBody(): Promise<{ reclaimed: unknown[]; kept: unknown[]; fetch: string }> {
8057
8463
  const resource = (await this.ctx.storage.get<string>(RESOURCE_KEY)) ?? "";
8058
8464
  const facts = await this.ctx.storage.get<RepoFacts>(FACTS_KEY);
8059
8465
  if (!facts) return { reclaimed: [], kept: [], fetch: "skipped" };
@@ -8126,12 +8532,15 @@ export class ResidentDO extends Sandbox<Env> {
8126
8532
  }
8127
8533
 
8128
8534
  async getStatus(): Promise<ResidentStatus> {
8129
- const map = await this.ctx.storage.get<string>([STATE_KEY, REASON_KEY]);
8535
+ const map = await this.ctx.storage.get<string | boolean>([STATE_KEY, REASON_KEY, DESTROY_UNCONFIRMED_KEY]);
8536
+ if (this.destroying || map.get(DESTROY_UNCONFIRMED_KEY)) {
8537
+ return { state: "down", reason: "resident-destroy-unconfirmed: container reuse refused" };
8538
+ }
8130
8539
  const state = map.get(STATE_KEY) as ResidentState | undefined;
8131
8540
  if (!state) {
8132
8541
  return { state: "down", reason: "no resident state persisted (never onboarded, or already offboarded)" };
8133
8542
  }
8134
- return { state, reason: map.get(REASON_KEY) ?? "" };
8543
+ return { state, reason: (map.get(REASON_KEY) as string | undefined) ?? "" };
8135
8544
  }
8136
8545
 
8137
8546
  /** Full admin-facing view (surfaced via GET /residents and /debug info):
@@ -8176,6 +8585,9 @@ export class ResidentDO extends Sandbox<Env> {
8176
8585
  INFRA_STREAK_KEY,
8177
8586
  MEMORY_KEY,
8178
8587
  IMAGE_REPORT_PENDING_KEY,
8588
+ DESTROY_UNCONFIRMED_KEY,
8589
+ RECREATE_ADMISSION_KEY,
8590
+ SPENT_POOL_USERS_KEY,
8179
8591
  ]);
8180
8592
  const facts = map.get(FACTS_KEY) as RepoFacts | undefined;
8181
8593
  const snap = map.get(SNAPSHOT_KEY) as SnapshotRecord | undefined;
@@ -8233,8 +8645,11 @@ export class ResidentDO extends Sandbox<Env> {
8233
8645
  const registeredRuns = await this.registeredRunsBeyondOps();
8234
8646
  return {
8235
8647
  resource: map.get(RESOURCE_KEY) ?? null,
8236
- state: map.get(STATE_KEY) ?? "down",
8237
- reason: map.get(REASON_KEY) ?? "",
8648
+ state: this.destroying || map.get(DESTROY_UNCONFIRMED_KEY) ? "down" : (map.get(STATE_KEY) ?? "down"),
8649
+ reason:
8650
+ this.destroying || map.get(DESTROY_UNCONFIRMED_KEY)
8651
+ ? "resident-destroy-unconfirmed: container reuse refused"
8652
+ : (map.get(REASON_KEY) ?? ""),
8238
8653
  updatedAt: map.get(UPDATED_KEY) ?? null,
8239
8654
  defaultRef: facts?.defaultRef ?? null,
8240
8655
  sha: facts?.sha ?? null,
@@ -8258,6 +8673,8 @@ export class ResidentDO extends Sandbox<Env> {
8258
8673
  // the deploy's readers never print a bare "warm" for a repository whose
8259
8674
  // running container the last deploy could not verify.
8260
8675
  imageReport: map.get(IMAGE_REPORT_PENDING_KEY) !== undefined ? "pending" : "current",
8676
+ recreateAdmissionHeld: this.recreateAdmission.pending || map.get(RECREATE_ADMISSION_KEY) === true,
8677
+ poolUsersSpent: parseSpentPoolUsers(map.get(SPENT_POOL_USERS_KEY), THREAD_USERS)?.size ?? null,
8261
8678
  // Item 22: who holds what, as the rows say — the mirror mutex and the
8262
8679
  // cycle/hydration leases, each judged against this incarnation.
8263
8680
  incarnation: this.incarnation,
@@ -8348,6 +8765,15 @@ export class ResidentDO extends Sandbox<Env> {
8348
8765
  * recorded here, so the marker never strands `restoring`. */
8349
8766
  async debugRecreateContainer(): Promise<
8350
8767
  { recreated: true; restoreStartedAt: string } | { recreated: false; error: string; status: number }
8768
+ > {
8769
+ const admission = await this.recreateAdmission.run(() => this.debugRecreateContainerChecked());
8770
+ return admission.busy
8771
+ ? { recreated: false, status: 409, error: "recreate-refused: a VM recycle is already checking idleness" }
8772
+ : admission.value;
8773
+ }
8774
+
8775
+ private async debugRecreateContainerChecked(): Promise<
8776
+ { recreated: true; restoreStartedAt: string } | { recreated: false; error: string; status: number }
8351
8777
  > {
8352
8778
  const from = await this.getStatus();
8353
8779
  if (from.state === "onboarding" || from.state === "refreshing" || from.state === "restoring") {
@@ -8364,6 +8790,15 @@ export class ResidentDO extends Sandbox<Env> {
8364
8790
  error: `recreate-refused: the resident is down (${from.reason}) — POST /rebuild is its exit`,
8365
8791
  };
8366
8792
  }
8793
+ const active =
8794
+ this.inFlightCount() +
8795
+ this.refreshAdmissionsInFlight +
8796
+ this.adminWorkInFlight +
8797
+ (this.hydration ? 1 : 0) +
8798
+ (await this.registeredRunsBeyondOps());
8799
+ if (active > 0) {
8800
+ return { recreated: false, status: 409, error: `recreate-refused: ${active} run(s) still in flight` };
8801
+ }
8367
8802
  await this.recreateContainer(
8368
8803
  "runtime-unreachable: the container was recreated by an operator (recreate-container), snapshots kept — the restore from the snapshot is starting",
8369
8804
  );
@@ -8494,7 +8929,12 @@ export class ResidentDO extends Sandbox<Env> {
8494
8929
  };
8495
8930
  if (dryRun) return plan;
8496
8931
 
8497
- // Old snapshot objects go FIRST: initResident wipes the stored handles,
8932
+ // A failed destroy must leave the old snapshot and budget intact, and
8933
+ // must never arm provisioning against an unconfirmed old VM.
8934
+ this.swapIncarnation();
8935
+ await this.destroyConfirmed();
8936
+
8937
+ // Old snapshot objects go before initResident wipes the stored handles,
8498
8938
  // and backups/<id>/ lives outside the resident/<resource>/ prefix — this
8499
8939
  // is the only path that can still reach them (same ordering as teardown).
8500
8940
  let backupObjectsDeleted = 0;
@@ -8510,17 +8950,8 @@ export class ResidentDO extends Sandbox<Env> {
8510
8950
  // rebuild has seen it. The automatic one keeps the history — it IS the
8511
8951
  // budget's count.
8512
8952
  if (!opts.auto) await this.ctx.storage.delete(AUTO_REBUILDS_KEY);
8513
- // From scratch means a fresh container too: the SDK's runtime identity
8514
- // forgotten and the VM destroyed (SIGKILL, a fresh disk) before
8515
- // provisioning clones onto it. A rebuild that reprovisioned onto the
8516
- // running container inherited its wedged runtime once — every exec of the
8517
- // new provisioning met the same unanswered control port (item 64).
8518
- this.swapIncarnation(); // deliberate incarnation swap
8519
- await this.forgetRuntimeIdentity();
8520
- await this.destroy().catch((err) =>
8521
- console.log(`rebuild: destroy failed (provisioning starts anyway): ${errMsg(err)}`),
8522
- );
8523
- await this.initResident(resource, provisioningTimeoutMs);
8953
+ // Provisioning starts only after confirmed destruction.
8954
+ await this.initResident(resource, provisioningTimeoutMs, true);
8524
8955
  return { ...plan, backupObjectsDeleted, state: "onboarding" as const };
8525
8956
  }
8526
8957
 
@@ -8551,11 +8982,10 @@ export class ResidentDO extends Sandbox<Env> {
8551
8982
  };
8552
8983
  }
8553
8984
 
8554
- /** Offboard teardown: cancel timers, delete the R2 objects behind the SDK
8555
- * backup handles (they live under backups/<uuid>/, OUTSIDE the
8556
- * resident/<resource>/ prefix, and the handles die with deleteAll — so
8557
- * this must happen first), stop the container (best effort), wipe DO
8558
- * storage. resident/<resource>/ objects are deleted by the Worker. */
8985
+ /** Offboard teardown: confirm container destruction before deleting timers
8986
+ * or state. Then delete the R2 objects behind the SDK backup handles,
8987
+ * which live outside the resident prefix and must be removed before the
8988
+ * handles in DO storage. The Worker deletes resident-prefix objects. */
8559
8989
  async teardown(): Promise<{
8560
8990
  schedulesCancelled: boolean;
8561
8991
  containerStopped: boolean;
@@ -8564,8 +8994,9 @@ export class ResidentDO extends Sandbox<Env> {
8564
8994
  errors: string[];
8565
8995
  }> {
8566
8996
  const errors: string[] = [];
8567
- let containerStopped = false;
8568
8997
  let backupObjectsDeleted = 0;
8998
+ await this.destroyConfirmed();
8999
+ const containerStopped = true;
8569
9000
  this.deleteSchedules(PROVISIONING_CALLBACK);
8570
9001
  this.deleteSchedules(PROVISION_RUN_CALLBACK);
8571
9002
  const recorded = await this.recordedBackupIds(await this.ctx.storage.get<SnapshotRecord>(SNAPSHOT_KEY));
@@ -8583,12 +9014,6 @@ export class ResidentDO extends Sandbox<Env> {
8583
9014
  } catch (err) {
8584
9015
  errors.push(`deps backup deletion failed: ${errMsg(err)}`);
8585
9016
  }
8586
- try {
8587
- await this.destroy();
8588
- containerStopped = true;
8589
- } catch (err) {
8590
- errors.push(`destroy failed: ${errMsg(err)}`);
8591
- }
8592
9017
  // Retired DO: clear the alarm the Container base may have armed for its
8593
9018
  // schedules, then delete every stored key — ours and the SDK's (its runtime
8594
9019
  // identity among them) — so nothing ever wakes this object again. Keys, not
@@ -9107,9 +9532,15 @@ async function handleOnboard(env: Env, body: Record<string, unknown>): Promise<R
9107
9532
  // destroyed for nothing. Only once the newcomer holds the slot is the
9108
9533
  // victim's DO/R2 state torn down (its registry row is already gone, so no
9109
9534
  // new work routes to it meanwhile).
9535
+ const victim = await registry.getRecord(pick.candidate.resource);
9536
+ if (!victim) return json({ error: "eviction refused: candidate registry row vanished" }, 409);
9110
9537
  result = await registry.replace(pick.candidate.resource, record);
9111
9538
  if ("error" in result) return json({ error: result.error, wouldHaveEvicted: pick.candidate }, result.status);
9112
9539
  const teardown = await teardownResident(env, pick.candidate.resource);
9540
+ if (!teardown.containerStopped) {
9541
+ await registry.replace(record.resource, victim);
9542
+ return json({ error: "eviction refused: old container destruction unconfirmed", evicted: teardown }, 503);
9543
+ }
9113
9544
  evicted = { ...pick.candidate, registryRemoved: true, ...teardown };
9114
9545
  console.log(
9115
9546
  `lru-evict: offboarded ${pick.candidate.resource} (last activity ${pick.candidate.lastActivityAt}) to make room for ${resource.resource}`,
@@ -9172,7 +9603,12 @@ async function handleOffboard(env: Env, body: Record<string, unknown>): Promise<
9172
9603
 
9173
9604
  const registry = registryStub(env);
9174
9605
  const record = await registry.getRecord(resource.resource);
9175
- if (!record) return json({ error: `${resource.resource} is not onboarded` }, 404);
9606
+ if (!record) {
9607
+ // A prior destroy may have failed after registry removal. Retry teardown
9608
+ // on that same DO identity; a fully removed resident has no resource row.
9609
+ const old = await residentStub(env, resource.resource).getResidentInfo();
9610
+ if (old.resource !== resource.resource) return json({ error: `${resource.resource} is not onboarded` }, 404);
9611
+ }
9176
9612
 
9177
9613
  // --dry-run: the itemized plan of what the real teardown below would
9178
9614
  // remove, computed READ-ONLY — the resident stays fully intact (state,
@@ -9204,7 +9640,8 @@ async function handleOffboard(env: Env, body: Record<string, unknown>): Promise<
9204
9640
  });
9205
9641
  }
9206
9642
 
9207
- return json(await offboardResident(env, resource.resource));
9643
+ const result = await offboardResident(env, resource.resource);
9644
+ return json(result, result.containerStopped ? 200 : 503);
9208
9645
  }
9209
9646
 
9210
9647
  /** The full offboard teardown (item 11), shared by POST /offboard and the LRU
@@ -9220,7 +9657,7 @@ async function offboardResident(
9220
9657
  return { resource, registryRemoved, ...(await teardownResident(env, resource)) };
9221
9658
  }
9222
9659
 
9223
- /** Everything AFTER the registry removal: DO teardown + R2 prefix sweep.
9660
+ /** Everything AFTER the registry removal: DO teardown, then R2 prefix sweep.
9224
9661
  * Split out so the LRU path can reserve the slot atomically (registry
9225
9662
  * `replace`) before destroying anything. */
9226
9663
  async function teardownResident(
@@ -9234,26 +9671,25 @@ async function teardownResident(
9234
9671
  r2ObjectsDeleted: number;
9235
9672
  errors: string[];
9236
9673
  }> {
9237
- const [teardown, r2Sweep] = await Promise.all([
9238
- residentStub(env, resource)
9239
- .teardown()
9240
- .catch((err: unknown): Awaited<ReturnType<ResidentDO["teardown"]>> => ({
9241
- schedulesCancelled: false,
9242
- containerStopped: false,
9243
- storageCleared: false,
9244
- backupObjectsDeleted: 0,
9245
- errors: [`teardown failed: ${errMsg(err)}`],
9246
- })),
9247
- // The registry is already gone, so the offboard cannot be retried; a
9248
- // transient R2 failure must degrade to a reported partial success (naming
9249
- // the prefix left behind) rather than throw an unretryable 500.
9250
- deleteR2Prefix(env.BACKUP_BUCKET, r2Prefix(resource))
9251
- .then((deleted) => ({ deleted, error: undefined as string | undefined }))
9252
- .catch((err: unknown) => ({
9253
- deleted: 0,
9254
- error: `r2 prefix sweep failed for ${r2Prefix(resource)}: ${errMsg(err)}`,
9255
- })),
9256
- ]);
9674
+ const teardown = await residentStub(env, resource)
9675
+ .teardown()
9676
+ .catch((err: unknown): Awaited<ReturnType<ResidentDO["teardown"]>> => ({
9677
+ schedulesCancelled: false,
9678
+ containerStopped: false,
9679
+ storageCleared: false,
9680
+ backupObjectsDeleted: 0,
9681
+ errors: [`teardown failed: ${errMsg(err)}`],
9682
+ }));
9683
+ // If destroy was uncertain, preserve its durable refusal and all resident
9684
+ // objects. A later admin offboard can retry the same DO identity.
9685
+ const r2Sweep = teardown.containerStopped
9686
+ ? await deleteR2Prefix(env.BACKUP_BUCKET, r2Prefix(resource))
9687
+ .then((deleted) => ({ deleted, error: undefined as string | undefined }))
9688
+ .catch((err: unknown) => ({
9689
+ deleted: 0,
9690
+ error: `r2 prefix sweep failed for ${r2Prefix(resource)}: ${errMsg(err)}`,
9691
+ }))
9692
+ : { deleted: 0, error: undefined };
9257
9693
 
9258
9694
  const errors = [...teardown.errors];
9259
9695
  if (r2Sweep.error) errors.push(r2Sweep.error);
@@ -9948,10 +10384,14 @@ async function handleDebug(env: Env, body: Record<string, unknown>): Promise<Res
9948
10384
  return json(await stub.debugThreads());
9949
10385
  case "deps-backups":
9950
10386
  return json(await stub.debugDepsBackups());
9951
- case "sweep-now":
9952
- return json(await stub.debugSweepNow());
9953
- case "reclaim-now":
9954
- return json(await stub.debugReclaimNow());
10387
+ case "sweep-now": {
10388
+ const result = await stub.debugSweepNow();
10389
+ return json(result, "error" in result ? result.status : 200);
10390
+ }
10391
+ case "reclaim-now": {
10392
+ const result = await stub.debugReclaimNow();
10393
+ return json(result, "error" in result ? result.status : 200);
10394
+ }
9955
10395
  case "purge-bindings": {
9956
10396
  // Item 56: drop a load run's evicted synthetic bindings. `prefix` is
9957
10397
  // validated by the pure decision (a whole non-production namespace).
@@ -9959,10 +10399,12 @@ async function handleDebug(env: Env, body: Record<string, unknown>): Promise<Res
9959
10399
  const r = await stub.debugPurgeBindings(prefix);
9960
10400
  return "error" in r ? json({ error: r.error }, 400) : json(r);
9961
10401
  }
9962
- case "measure-disk":
10402
+ case "measure-disk": {
9963
10403
  // Item 55: take the sample now (df + one du) and answer it — the live
9964
10404
  // check's way to read the gauge without waiting for a cycle.
9965
- return json({ disk: await stub.debugMeasureDisk() });
10405
+ const disk = await stub.debugMeasureDisk();
10406
+ return disk && "error" in disk ? json(disk, disk.status) : json({ disk });
10407
+ }
9966
10408
  case "backdate-thread": {
9967
10409
  const threadKey = parseThreadKey(body.threadKey);
9968
10410
  if ("error" in threadKey) return json({ error: threadKey.error }, 400);