@coreplane/switchboard 1.267.0 → 1.267.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/deploy/cloudflare-resident/gc.ts +1 -0
- package/dist/assets/deploy/cloudflare-resident/keyedAsyncLock.ts +20 -0
- package/dist/assets/deploy/cloudflare-resident/legacyCredentials.ts +57 -0
- package/dist/assets/deploy/cloudflare-resident/orphanThreadUsers.ts +14 -0
- package/dist/assets/deploy/cloudflare-resident/worker.ts +646 -204
- package/dist/assets/package-lock.json +51 -4
- package/dist/assets/package.json +1 -1
- package/dist/assets/source.json +3 -3
- package/dist/assets/src/core/budgets.ts +10 -0
- package/dist/assets/src/core/coordinator/contract.ts +29 -0
- package/dist/assets/src/core/coordinator/driver.ts +15 -2
- package/dist/assets/src/core/publicationPush.ts +31 -9
- package/dist/assets/src/core/runRecord.ts +32 -0
- package/dist/assets/src/core/ship/coordinator.ts +53 -5
- package/dist/assets/src/execution/githubMintScope.ts +49 -0
- package/dist/assets/src/execution/residentCredentials.ts +9 -37
- package/dist/assets/src/execution/residentDestroyGate.ts +12 -0
- package/dist/assets/src/execution/residentPoolSpends.ts +96 -0
- package/dist/assets/src/execution/residentRecreateAdmission.ts +41 -0
- package/dist/assets/src/execution/residentSteps.ts +1 -0
- package/dist/assets/web/dist/.vite/manifest.json +505 -505
- package/dist/assets/web/dist/assets/AppShell-CCPwPqvc.js +1 -0
- package/dist/assets/web/dist/assets/{CostChart-4TeDdbvJ.js → CostChart-Y5BqJKLA.js} +2 -2
- package/dist/assets/web/dist/assets/CostsPage-RRggVvIx.js +1 -0
- package/dist/assets/web/dist/assets/DeliveryPage-CuIBLfup.js +1 -0
- package/dist/assets/web/dist/assets/HomePage-BQmun9kS.js +1 -0
- package/dist/assets/web/dist/assets/InputMenu-ymuqElZB.js +1 -0
- package/dist/assets/web/dist/assets/{MetricsPage-BQBuIuYU.js → MetricsPage-D2YGsI8C.js} +1 -1
- package/dist/assets/web/dist/assets/NotFoundPage-DZ6ysTXd.js +1 -0
- package/dist/assets/web/dist/assets/PendingTurnRow-BPyJ2f-i.js +1 -0
- package/dist/assets/web/dist/assets/PlanePage-C_gVx0Dg.js +1 -0
- package/dist/assets/web/dist/assets/ResidentDetailPage-CT6v9INM.js +1 -0
- package/dist/assets/web/dist/assets/ResidentsIndexPage-CWqPFf40.js +1 -0
- package/dist/assets/web/dist/assets/{RunFoldRow-DrfFauYJ.js → RunFoldRow-C1w-mwch.js} +1 -1
- package/dist/assets/web/dist/assets/RunRoutePage-zWMHTzla.js +9 -0
- package/dist/assets/web/dist/assets/RunsIndexPage-DF16D2Gg.js +1 -0
- package/dist/assets/web/dist/assets/RunsTabs-D7rsC4GR.js +1 -0
- package/dist/assets/web/dist/assets/ScheduledPage-CyRs0fXN.js +1 -0
- package/dist/assets/web/dist/assets/SettingSelect-CEkCLVWd.js +1 -0
- package/dist/assets/web/dist/assets/SettingsPage-BRuMCHSj.js +1 -0
- package/dist/assets/web/dist/assets/SilentTurn-D4MMDjWO.js +2 -0
- package/dist/assets/web/dist/assets/SlackMark-DjJTK2tD.js +1 -0
- package/dist/assets/web/dist/assets/StatusDot-BIAQN7iD.js +1 -0
- package/dist/assets/web/dist/assets/Tooltip-CxiQ7luL.js +1 -0
- package/dist/assets/web/dist/assets/UnitRoutePage-BQB5mAAl.js +1 -0
- package/dist/assets/web/dist/assets/{angular-html-BRRhsdQE.js → angular-html-DR_aiq_z.js} +1 -1
- package/dist/assets/web/dist/assets/{angular-ts-BFj-mqCm.js → angular-ts-DdAdPC99.js} +1 -1
- package/dist/assets/web/dist/assets/{apl-CsBuy1IL.js → apl-CFsVwJXV.js} +1 -1
- package/dist/assets/web/dist/assets/{astro-cVaqxBZq.js → astro-CV6XNX3T.js} +1 -1
- package/dist/assets/web/dist/assets/{blade-MEP5mWha.js → blade-BGvEqTv7.js} +1 -1
- package/dist/assets/web/dist/assets/browser-BWnzWwaT.js +6 -0
- package/dist/assets/web/dist/assets/budgets-CJHiTZmM.js +1 -0
- package/dist/assets/web/dist/assets/{c-D4NGQ_Pd.js → c-ViUYuFAx.js} +1 -1
- package/dist/assets/web/dist/assets/{chapel-BkWImsIy.js → chapel-CZ7_YbfP.js} +1 -1
- package/dist/assets/web/dist/assets/{cobol-RPDSPSET.js → cobol-Dc8kYvS6.js} +1 -1
- package/dist/assets/web/dist/assets/{coffee-DZ5z7oVX.js → coffee-C0musOvn.js} +1 -1
- package/dist/assets/web/dist/assets/{cpp-CrBtIaRZ.js → cpp-ToPQGcGd.js} +1 -1
- package/dist/assets/web/dist/assets/{crystal-DJesYi6f.js → crystal-DUJIaO8W.js} +1 -1
- package/dist/assets/web/dist/assets/{css-DO9mjau-.js → css-BTcEgWyQ.js} +1 -1
- package/dist/assets/web/dist/assets/{dist-v3Py3_QR.js → dist-I2OEmPqz.js} +2 -2
- package/dist/assets/web/dist/assets/{edge-C3VzbSW0.js → edge-D8sh1uAj.js} +1 -1
- package/dist/assets/web/dist/assets/{elixir-BkD_Vhfe.js → elixir-D5vOtHrB.js} +1 -1
- package/dist/assets/web/dist/assets/{elm-KIM0mFPk.js → elm-B_5xqTuz.js} +1 -1
- package/dist/assets/web/dist/assets/{erb-D2ruCqb2.js → erb-DBEct7_l.js} +1 -1
- package/dist/assets/web/dist/assets/{eventSource-Cv0AFlMi.js → eventSource-Ds6_gjPD.js} +1 -1
- package/dist/assets/web/dist/assets/{git-rebase-CnfbQTxA.js → git-rebase-lWoTL0Qh.js} +1 -1
- package/dist/assets/web/dist/assets/{glimmer-js-CbiwFXWH.js → glimmer-js-DNfSP4GT.js} +1 -1
- package/dist/assets/web/dist/assets/{glimmer-ts-DV8ii0Pv.js → glimmer-ts-DpPeWMLs.js} +1 -1
- package/dist/assets/web/dist/assets/{glsl-CgmznWkA.js → glsl-m6Ziz8q0.js} +1 -1
- package/dist/assets/web/dist/assets/{graphql-BK-bBYfH.js → graphql-shD9521e.js} +1 -1
- package/dist/assets/web/dist/assets/{hack-DnwTTP5n.js → hack-D_Mf_2gv.js} +1 -1
- package/dist/assets/web/dist/assets/{haml-DAVJsWwT.js → haml-BuEAF47S.js} +1 -1
- package/dist/assets/web/dist/assets/{handlebars-voOVfxFL.js → handlebars-D2aiSYtv.js} +1 -1
- package/dist/assets/web/dist/assets/{html-3FTqwaPk.js → html-CRjXJZot.js} +1 -1
- package/dist/assets/web/dist/assets/{html-derivative-CSlhLDQn.js → html-derivative-CYpQX_KK.js} +1 -1
- package/dist/assets/web/dist/assets/{http-Vx4ilPfS.js → http-D6vXMpc_.js} +1 -1
- package/dist/assets/web/dist/assets/{hurl-DxyZDE3R.js → hurl-B634cMJT.js} +1 -1
- package/dist/assets/web/dist/assets/{indexRow-BsWcHHVh.js → indexRow-BGjjLnN5.js} +1 -1
- package/dist/assets/web/dist/assets/{java-nWqdjNJG.js → java-BlINdAve.js} +1 -1
- package/dist/assets/web/dist/assets/{javascript-Dr82HOwp.js → javascript-BBEMmUOu.js} +1 -1
- package/dist/assets/web/dist/assets/{jinja-BhjLrPOt.js → jinja-BIA8TYHJ.js} +1 -1
- package/dist/assets/web/dist/assets/{jison-DilGIUa8.js → jison-aepfTbzH.js} +1 -1
- package/dist/assets/web/dist/assets/{json-CrOcvmhJ.js → json-DmYZ1boG.js} +1 -1
- package/dist/assets/web/dist/assets/{jsx-BW_RX-fo.js → jsx-GkaaF4_W.js} +1 -1
- package/dist/assets/web/dist/assets/{julia-CWH071DB.js → julia-DRtMztOz.js} +1 -1
- package/dist/assets/web/dist/assets/{just-DkhhoWBy.js → just-PvoeEWoy.js} +1 -1
- package/dist/assets/web/dist/assets/{latex-cuHJ4TUT.js → latex-Bujuo3AZ.js} +1 -1
- package/dist/assets/web/dist/assets/{liquid-DG6afYQN.js → liquid-B0h0HkS3.js} +1 -1
- package/dist/assets/web/dist/assets/{lua-C5CMnCPS.js → lua-6O-O1CSO.js} +1 -1
- package/dist/assets/web/dist/assets/main-DYwKyLY5.js +28 -0
- package/dist/assets/web/dist/assets/{marko-Dy2_DuC4.js → marko-XS_tpV0N.js} +1 -1
- package/dist/assets/web/dist/assets/{mdc-BMKb_EeA.js → mdc-DxULuTS1.js} +1 -1
- package/dist/assets/web/dist/assets/{nginx-og0RgMKf.js → nginx-n2RAfCKE.js} +1 -1
- package/dist/assets/web/dist/assets/{nim-CQxTQQP0.js → nim-DJxdqD8U.js} +1 -1
- package/dist/assets/web/dist/assets/{org-b5lkq_Pl.js → org-R9yxwZ2v.js} +1 -1
- package/dist/assets/web/dist/assets/{perl-D9X5_i_J.js → perl-zKAUtl73.js} +1 -1
- package/dist/assets/web/dist/assets/{php-CwUxvLX0.js → php-Dk50x1WE.js} +1 -1
- package/dist/assets/web/dist/assets/{pug-uOIOExr0.js → pug-C3TSg8JJ.js} +1 -1
- package/dist/assets/web/dist/assets/{qml-g5RoADf-.js → qml-CL1nqvCt.js} +1 -1
- package/dist/assets/web/dist/assets/{r-5LdAopD3.js → r-Bo44Yl7y.js} +1 -1
- package/dist/assets/web/dist/assets/{razor-DwrvFLYQ.js → razor-k2Dt2qXz.js} +1 -1
- package/dist/assets/web/dist/assets/{regexp-dGtAaxC-.js → regexp-Uy_HNe6G.js} +1 -1
- package/dist/assets/web/dist/assets/{rst-wFdW0brr.js → rst-U0zcRJ60.js} +1 -1
- package/dist/assets/web/dist/assets/{ruby-DPglN6I9.js → ruby-DUyqwP37.js} +1 -1
- package/dist/assets/web/dist/assets/{sas-CUbCs40d.js → sas-CxC8bRy1.js} +1 -1
- package/dist/assets/web/dist/assets/{scss-B-DUiI1c.js → scss-DaqXj6RQ.js} +1 -1
- package/dist/assets/web/dist/assets/{shellscript-lGOV7Ram.js → shellscript-CI49IoFh.js} +1 -1
- package/dist/assets/web/dist/assets/{shellsession-BDWtAAjr.js → shellsession-DJxotIOK.js} +1 -1
- package/dist/assets/web/dist/assets/{soy-zod_dcOy.js → soy-BjVAVuCU.js} +1 -1
- package/dist/assets/web/dist/assets/{sql-BG0HSSoM.js → sql-CzxBKQd1.js} +1 -1
- package/dist/assets/web/dist/assets/{sseReplay-DuALaBPq.js → sseReplay-BDVyc8HX.js} +4 -4
- package/dist/assets/web/dist/assets/{stata-BVUMLp1W.js → stata-DLMv_9_X.js} +1 -1
- package/dist/assets/web/dist/assets/{surrealql-DiSUbHKF.js → surrealql-BzbVEZbO.js} +1 -1
- package/dist/assets/web/dist/assets/{svelte-DZenT45Y.js → svelte-DTSfopUt.js} +1 -1
- package/dist/assets/web/dist/assets/{templ-BWKRNM0-.js → templ-BPxFlKQj.js} +1 -1
- package/dist/assets/web/dist/assets/{tex-CbMzBlkK.js → tex-B_KndQwK.js} +1 -1
- package/dist/assets/web/dist/assets/{ts-tags-CZsVorDN.js → ts-tags-CG7tgvBl.js} +1 -1
- package/dist/assets/web/dist/assets/{tsx-R1r3mH4W.js → tsx-CN9eqIdJ.js} +1 -1
- package/dist/assets/web/dist/assets/{twig-BjPMyjKH.js → twig-CKPbB5ca.js} +1 -1
- package/dist/assets/web/dist/assets/{typescript-Bx-66VN_.js → typescript-BQAXVjHJ.js} +1 -1
- package/dist/assets/web/dist/assets/{typst-fW037Eo6.js → typst-D3EaP9Ui.js} +1 -1
- package/dist/assets/web/dist/assets/{vue-D1V_GNxW.js → vue-DuQwQ__D.js} +1 -1
- package/dist/assets/web/dist/assets/{vue-html-C-4pTZ-s.js → vue-html-UEJn5cLD.js} +1 -1
- package/dist/assets/web/dist/assets/{vue-vine-C-QsX3hR.js → vue-vine-DDdijZY2.js} +1 -1
- package/dist/assets/web/dist/assets/{wallClock-x0hjr0J_.js → wallClock-DaewxNMY.js} +1 -1
- package/dist/assets/web/dist/assets/{xml-BJTJXPgC.js → xml-BezNDynk.js} +1 -1
- package/dist/assets/web/dist/assets/{xsl-Xc9lukqe.js → xsl-Bzb2zIr8.js} +1 -1
- package/dist/assets/web/dist/assets/{yaml-Dw-FMsQa.js → yaml-DOccVRRG.js} +1 -1
- package/dist/cli.js +1002 -366
- package/package.json +1 -1
- package/dist/assets/web/dist/assets/AppShell-DPJVhUGP.js +0 -1
- package/dist/assets/web/dist/assets/CostsPage-8HDgvuWB.js +0 -1
- package/dist/assets/web/dist/assets/DeliveryPage-BUJckSuW.js +0 -1
- package/dist/assets/web/dist/assets/HomePage-BfpIrKDX.js +0 -1
- package/dist/assets/web/dist/assets/InputMenu-BRV35UWP.js +0 -1
- package/dist/assets/web/dist/assets/NotFoundPage-CDjMfCcj.js +0 -1
- package/dist/assets/web/dist/assets/PendingTurnRow-C6hroa00.js +0 -1
- package/dist/assets/web/dist/assets/PlanePage-C2Cb1mkT.js +0 -1
- package/dist/assets/web/dist/assets/ResidentDetailPage-Bd9_Q87l.js +0 -1
- package/dist/assets/web/dist/assets/ResidentsIndexPage-C8eS7hxp.js +0 -1
- package/dist/assets/web/dist/assets/RunRoutePage-B_eXfNAF.js +0 -9
- package/dist/assets/web/dist/assets/RunsIndexPage-BpQu36Dm.js +0 -1
- package/dist/assets/web/dist/assets/RunsTabs-C7Ua3XX5.js +0 -1
- package/dist/assets/web/dist/assets/ScheduledPage-DGHcjcSt.js +0 -1
- package/dist/assets/web/dist/assets/SettingSelect-CEeA4iuC.js +0 -1
- package/dist/assets/web/dist/assets/SettingsPage-IfQ71QjN.js +0 -1
- package/dist/assets/web/dist/assets/SilentTurn-D9ON15a8.js +0 -2
- package/dist/assets/web/dist/assets/SlackMark-DB0MGiF5.js +0 -1
- package/dist/assets/web/dist/assets/StatusDot-DyFjGg4V.js +0 -1
- package/dist/assets/web/dist/assets/Tooltip-BvPkV4kZ.js +0 -1
- package/dist/assets/web/dist/assets/UnitRoutePage-BNe3nBIG.js +0 -1
- package/dist/assets/web/dist/assets/browser-CCImen5O.js +0 -6
- package/dist/assets/web/dist/assets/budgets-BQgcCyvb.js +0 -1
- package/dist/assets/web/dist/assets/main-ks3kBojV.js +0 -28
|
@@ -82,8 +82,15 @@ import { AsyncLocalStorage } from "node:async_hooks";
|
|
|
82
82
|
import type { DirectoryBackup, SandboxCommand } from "@cloudflare/sandbox";
|
|
83
83
|
import { createExtensionProcessSandbox } from "@cloudflare/sandbox/extensions";
|
|
84
84
|
import { DurableObject } from "cloudflare:workers";
|
|
85
|
-
import {
|
|
85
|
+
import {
|
|
86
|
+
legacyCredentialScrubCommand,
|
|
87
|
+
legacyStageContentScanCommand,
|
|
88
|
+
legacyStageReuseScrubCommand,
|
|
89
|
+
} from "./legacyCredentials.js";
|
|
90
|
+
import { hasUnexpectedOwnedThreadDir } from "./orphanThreadUsers.js";
|
|
91
|
+
import { KeyedAsyncLock } from "./keyedAsyncLock.js";
|
|
86
92
|
import { BASH_TIMEOUT_MAX_MS, clampBashTimeout } from "../../src/execution/bashTimeout.js";
|
|
93
|
+
import { RESIDENT_MIRROR_PERMISSIONS, verifyGithubMintScope } from "../../src/execution/githubMintScope.js";
|
|
87
94
|
import { selectBindingsToPurge } from "../../src/execution/bindingPurge.js";
|
|
88
95
|
import { busyAfterKillReason, planForceDetach } from "../../src/execution/residentDetach.js";
|
|
89
96
|
import {
|
|
@@ -164,6 +171,17 @@ import {
|
|
|
164
171
|
import { shellQuote } from "../../src/execution/shellQuote.js";
|
|
165
172
|
import { envFromRequest } from "../../src/execution/sandboxEnv.js";
|
|
166
173
|
import { CREDENTIAL_EXPIRY_MARGIN_MS } from "../../src/execution/residentCredentials.js";
|
|
174
|
+
import { destroyWithPersistentFence } from "../../src/execution/residentDestroyGate.js";
|
|
175
|
+
import { ResidentRecreateAdmission } from "../../src/execution/residentRecreateAdmission.js";
|
|
176
|
+
import {
|
|
177
|
+
claimPoolBinding,
|
|
178
|
+
mayRunAsPoolUser,
|
|
179
|
+
parsePoolBindings,
|
|
180
|
+
parseSpentPoolUsers,
|
|
181
|
+
rebuildPoolBindingIndex,
|
|
182
|
+
releasePoolBinding,
|
|
183
|
+
spendPoolUser,
|
|
184
|
+
} from "../../src/execution/residentPoolSpends.js";
|
|
167
185
|
import {
|
|
168
186
|
recordFiring,
|
|
169
187
|
scheduleForCron,
|
|
@@ -941,8 +959,9 @@ const tail = (s: string, n: number): string => s.trim().slice(-n);
|
|
|
941
959
|
// — Worker/DO scope only; the private key never
|
|
942
960
|
// enters the container. RS256 App JWT on WebCrypto (node:crypto is not
|
|
943
961
|
// available here), then POST /app/installations/:id/access_tokens with
|
|
944
|
-
// `repositories: [<own repo name>]` so a minted
|
|
945
|
-
// the resident's
|
|
962
|
+
// `repositories: [<own repo name>]` and mirror-read permissions, so a minted
|
|
963
|
+
// token grants only the resident's root-owned clone/fetch and PR lookup. Cache
|
|
964
|
+
// per slug, but only serve a cached token while
|
|
946
965
|
// it has more than `CREDENTIAL_EXPIRY_MARGIN_MS` of life left, so a new
|
|
947
966
|
// attach never inherits a near-expiry token minted for an earlier thread.
|
|
948
967
|
// ---------------------------------------------------------------------------
|
|
@@ -991,7 +1010,7 @@ export async function mintRepoScopedToken(env: Env, slug: string, opts?: { fresh
|
|
|
991
1010
|
"content-type": "application/json",
|
|
992
1011
|
"user-agent": "switchboard-resident",
|
|
993
1012
|
},
|
|
994
|
-
body: JSON.stringify({ repositories: [repoName] }),
|
|
1013
|
+
body: JSON.stringify({ repositories: [repoName], permissions: RESIDENT_MIRROR_PERMISSIONS }),
|
|
995
1014
|
// A slow GitHub must not hang attach/refresh. The 10s abort surfaces as a
|
|
996
1015
|
// command-level Error (below), never a lifecycle transition.
|
|
997
1016
|
signal: AbortSignal.timeout(10_000),
|
|
@@ -1010,6 +1029,7 @@ export async function mintRepoScopedToken(env: Env, slug: string, opts?: { fresh
|
|
|
1010
1029
|
throw new Error(`github-token-mint-failed: HTTP ${res.status} ${body.slice(0, 300)}`);
|
|
1011
1030
|
}
|
|
1012
1031
|
const data = (await res.json()) as { token: string; expires_at: string };
|
|
1032
|
+
verifyGithubMintScope(data, slug, RESIDENT_MIRROR_PERMISSIONS);
|
|
1013
1033
|
const minted: MintedToken = { token: data.token, expiresAtMs: Date.parse(data.expires_at) };
|
|
1014
1034
|
githubTokenCache.set(slug, minted);
|
|
1015
1035
|
return minted;
|
|
@@ -1204,15 +1224,10 @@ interface ThreadBinding {
|
|
|
1204
1224
|
* file, origin = the unreadable mirror. An attach in the other mode
|
|
1205
1225
|
* recreates the tree. Absent (pre-field bindings) = writable. */
|
|
1206
1226
|
readonly?: boolean;
|
|
1207
|
-
/**
|
|
1208
|
-
*
|
|
1209
|
-
* (`shouldRefreshThreadCredentials`); cleared by a read-only attach. */
|
|
1227
|
+
/** Legacy pre-door binding field. No new model credential file is written;
|
|
1228
|
+
* attach clears the old value after scrubbing that file. */
|
|
1210
1229
|
credentialsWrittenAt?: number;
|
|
1211
|
-
/**
|
|
1212
|
-
* on this — not `credentialsWrittenAt` — so a near-expiry token inherited
|
|
1213
|
-
* from an earlier thread's mint is re-minted before the first writable exec.
|
|
1214
|
-
* Absent (pre-field binding) → the next writable exec falls back to the
|
|
1215
|
-
* `credentialsWrittenAt` file-age rule; cleared alongside it on attach. */
|
|
1230
|
+
/** Legacy pre-door token expiry; cleared with `credentialsWrittenAt`. */
|
|
1216
1231
|
tokenExpiresAtMs?: number;
|
|
1217
1232
|
/** Non-secret host of the run-bearer Git door. The bearer lives only in
|
|
1218
1233
|
* the attach request and the run's per-exec environment. */
|
|
@@ -1720,6 +1735,11 @@ const AUTO_REBUILDS_KEY = "resident:autoRebuilds";
|
|
|
1720
1735
|
const RETIRED_REBUILD_STRIKES_KEY = "resident:rebuildStrikes";
|
|
1721
1736
|
/** Consecutive connects the container's control port did not answer (item 64): the ladder's count. */
|
|
1722
1737
|
const RUNTIME_UNREACHABLE_KEY = "resident:runtimeUnreachable";
|
|
1738
|
+
/** A failed or interrupted VM destroy forbids another process on an unconfirmed disk. */
|
|
1739
|
+
const DESTROY_UNCONFIRMED_KEY = "resident:destroyUnconfirmed";
|
|
1740
|
+
const RECREATE_ADMISSION_KEY = "resident:recreateAdmission";
|
|
1741
|
+
/** UIDs that have run untrusted code on this VM. Missing means an unknown old generation. */
|
|
1742
|
+
const SPENT_POOL_USERS_KEY = "resident:spentPoolUsers";
|
|
1723
1743
|
|
|
1724
1744
|
/** The row under RUNTIME_UNREACHABLE_KEY: the count and both instants, so the
|
|
1725
1745
|
* fleet watch sees how long the runtime has been silent. Cleared by any exec
|
|
@@ -1733,6 +1753,8 @@ interface RuntimeUnreachableRow {
|
|
|
1733
1753
|
/** Thread bindings live under their own prefix, keyed by threadKey. */
|
|
1734
1754
|
const THREAD_KEY_PREFIX = "thread:";
|
|
1735
1755
|
const threadBindingKey = (threadKey: string) => `${THREAD_KEY_PREFIX}${threadKey}`;
|
|
1756
|
+
const POOL_BINDING_KEY_PREFIX = "resident:poolBindings:";
|
|
1757
|
+
const poolBindingKey = (user: string) => `${POOL_BINDING_KEY_PREFIX}${user}`;
|
|
1736
1758
|
|
|
1737
1759
|
const parentDir = (p: string): string => p.slice(0, p.lastIndexOf("/"));
|
|
1738
1760
|
|
|
@@ -1839,16 +1861,53 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
1839
1861
|
|
|
1840
1862
|
constructor(...args: ConstructorParameters<typeof Sandbox<Env>>) {
|
|
1841
1863
|
super(...args);
|
|
1842
|
-
// The
|
|
1843
|
-
//
|
|
1844
|
-
// its first event (RETIRED_SCHEDULE_CALLBACKS).
|
|
1864
|
+
// The SDK deletes these rows synchronously. Do it before scheduling any
|
|
1865
|
+
// asynchronous initialization so retired callbacks cannot fire on boot.
|
|
1845
1866
|
for (const name of RETIRED_SCHEDULE_CALLBACKS) this.deleteSchedules(name);
|
|
1867
|
+
// Build the bounded UID view from primary rows before serving this DO
|
|
1868
|
+
// instance. A new instance repairs an old-image write or interrupted
|
|
1869
|
+
// migration; no durable "ready" flag can bless a stale index.
|
|
1870
|
+
void this.ctx.blockConcurrencyWhile(() => this.rebuildPoolBindingIndex());
|
|
1846
1871
|
}
|
|
1847
1872
|
|
|
1848
1873
|
/** Serializes concurrent hydration attempts within one DO lifetime. Never
|
|
1849
1874
|
* used as a "hydrated" flag — the container can sleep while the DO object
|
|
1850
1875
|
* survives, so hydration state is always probed from disk. */
|
|
1851
1876
|
private hydration: Promise<void> | null = null;
|
|
1877
|
+
/** Concurrent lifecycle paths share one destruction attempt. */
|
|
1878
|
+
private destroying: Promise<void> | null = null;
|
|
1879
|
+
/** An operator's idle check and destroy are one admission interval. */
|
|
1880
|
+
private readonly recreateAdmission = new ResidentRecreateAdmission({
|
|
1881
|
+
mark: () => this.ctx.storage.put(RECREATE_ADMISSION_KEY, true),
|
|
1882
|
+
clear: async () => {
|
|
1883
|
+
await this.ctx.storage.delete(RECREATE_ADMISSION_KEY);
|
|
1884
|
+
},
|
|
1885
|
+
held: async () => (await this.ctx.storage.get(RECREATE_ADMISSION_KEY)) === true,
|
|
1886
|
+
});
|
|
1887
|
+
|
|
1888
|
+
private recreateRefusal(): ThreadErr {
|
|
1889
|
+
return {
|
|
1890
|
+
error: "recreate-in-progress: new resident work is paused during the operator's checked VM recycle",
|
|
1891
|
+
status: 503,
|
|
1892
|
+
reason: "recreate-in-progress",
|
|
1893
|
+
cause: "system",
|
|
1894
|
+
};
|
|
1895
|
+
}
|
|
1896
|
+
|
|
1897
|
+
/** Direct admin disk work is admitted before its first await, so the
|
|
1898
|
+
* operator's idle check cannot pass while it is about to mutate the VM. */
|
|
1899
|
+
private adminWorkInFlight = 0;
|
|
1900
|
+
|
|
1901
|
+
private async withRecreateSafeAdmin<T>(action: () => Promise<T>): Promise<T | ThreadErr> {
|
|
1902
|
+
if (this.recreateAdmission.pending) return this.recreateRefusal();
|
|
1903
|
+
this.adminWorkInFlight++;
|
|
1904
|
+
try {
|
|
1905
|
+
if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
|
|
1906
|
+
return await action();
|
|
1907
|
+
} finally {
|
|
1908
|
+
this.adminWorkInFlight--;
|
|
1909
|
+
}
|
|
1910
|
+
}
|
|
1852
1911
|
|
|
1853
1912
|
/** Whether this resident knows the container it held is gone
|
|
1854
1913
|
* (docs/reference/specs/resident-repos.md item 43): a restore under way for
|
|
@@ -2245,6 +2304,9 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
2245
2304
|
argv: readonly string[],
|
|
2246
2305
|
opts: { cwd?: string; timeoutMs?: number; env?: Record<string, string> } = {},
|
|
2247
2306
|
): Promise<{ stdout: string; stderr: string; exitCode: number; timedOut: boolean; truncated?: boolean }> {
|
|
2307
|
+
if (this.destroying || (await this.ctx.storage.get(DESTROY_UNCONFIRMED_KEY))) {
|
|
2308
|
+
throw new Error("resident-destroy-unconfirmed: container reuse refused until destruction is confirmed");
|
|
2309
|
+
}
|
|
2248
2310
|
const timeout = opts.timeoutMs ?? DEFAULT_EXEC_TIMEOUT_MS;
|
|
2249
2311
|
const launch = {
|
|
2250
2312
|
...(opts.cwd ? { cwd: opts.cwd } : {}),
|
|
@@ -2316,6 +2378,11 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
2316
2378
|
console.log(
|
|
2317
2379
|
`exec: runtime replaced before the process started (SDK says retryable) — retrying once: ${errMsg(err)}`,
|
|
2318
2380
|
);
|
|
2381
|
+
if (this.destroying || (await this.ctx.storage.get(DESTROY_UNCONFIRMED_KEY))) {
|
|
2382
|
+
throw new Error("resident-destroy-unconfirmed: container reuse refused until destruction is confirmed", {
|
|
2383
|
+
cause: err,
|
|
2384
|
+
});
|
|
2385
|
+
}
|
|
2319
2386
|
proc = await createExtensionProcessSandbox(this).exec(argv as unknown as SandboxCommand, launch);
|
|
2320
2387
|
}
|
|
2321
2388
|
// The spawn is the proof the control port answers: a persisted count of
|
|
@@ -3156,7 +3223,16 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
3156
3223
|
* the provisioning work and its fail-closed deadline. Does NOT start the
|
|
3157
3224
|
* container — onboard must return immediately; the container first starts
|
|
3158
3225
|
* when runProvisioning's schedule fires (~1s later). */
|
|
3159
|
-
async initResident(resource: string, provisioningTimeoutMs: number): Promise<ResidentStatus> {
|
|
3226
|
+
async initResident(resource: string, provisioningTimeoutMs: number, rebuilding = false): Promise<ResidentStatus> {
|
|
3227
|
+
if (this.destroying || (await this.ctx.storage.get(DESTROY_UNCONFIRMED_KEY))) {
|
|
3228
|
+
throw new Error("resident-destroy-unconfirmed: onboarding refused until destruction is confirmed");
|
|
3229
|
+
}
|
|
3230
|
+
if (!rebuilding && (await this.ctx.storage.get(RESOURCE_KEY))) {
|
|
3231
|
+
throw new Error("resident-old-state-present: onboarding refused until prior teardown is confirmed");
|
|
3232
|
+
}
|
|
3233
|
+
// Offboard can clear storage without replacing the DO isolate. Restore
|
|
3234
|
+
// claimant rows before this same isolate records a new onboard.
|
|
3235
|
+
await this.rebuildPoolBindingIndex();
|
|
3160
3236
|
await this.ctx.storage.put({
|
|
3161
3237
|
[RESOURCE_KEY]: resource,
|
|
3162
3238
|
[STATE_KEY]: "onboarding" satisfies ResidentState,
|
|
@@ -3192,6 +3268,13 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
3192
3268
|
const resource = payload || ((await this.ctx.storage.get<string>(RESOURCE_KEY)) ?? "");
|
|
3193
3269
|
if ((await this.ctx.storage.get<ResidentState>(STATE_KEY)) !== "onboarding") return; // stale schedule
|
|
3194
3270
|
try {
|
|
3271
|
+
// An older image could have deleted every DO key after a failed
|
|
3272
|
+
// offboard destroy. No prior row is evidence of a fresh VM; confirm
|
|
3273
|
+
// destruction here, after onboard returned, before any provision step.
|
|
3274
|
+
if (!parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS)) {
|
|
3275
|
+
this.swapIncarnation();
|
|
3276
|
+
await this.destroyConfirmed();
|
|
3277
|
+
}
|
|
3195
3278
|
const record = await this.registry().getRecord(resource);
|
|
3196
3279
|
if (!record) throw new StepError("registry", "registry record missing (offboarded mid-onboard?)");
|
|
3197
3280
|
const slug = resource.slice("repo:".length);
|
|
@@ -3894,8 +3977,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
3894
3977
|
// on a fresh one.
|
|
3895
3978
|
const reason = infraStreakReason(row, "down");
|
|
3896
3979
|
this.swapIncarnation(); // deliberate incarnation swap
|
|
3897
|
-
await this.
|
|
3898
|
-
await this.destroy().catch((err) => console.log(`infra-streak: destroy failed: ${errMsg(err)}`));
|
|
3980
|
+
await this.destroyConfirmed();
|
|
3899
3981
|
await this.ctx.storage.delete(INFRA_STREAK_KEY);
|
|
3900
3982
|
await this.goDown(reason);
|
|
3901
3983
|
return;
|
|
@@ -4015,9 +4097,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4015
4097
|
// (REHYDRATION_FAILURE_RE) — goDown rebuilds on the transition, item 36 —
|
|
4016
4098
|
// and the VM destroyed, so the rebuild's provisioning starts on a new one.
|
|
4017
4099
|
this.swapIncarnation(); // deliberate incarnation swap
|
|
4018
|
-
await this.
|
|
4019
|
-
console.log(`runtime-unreachable: destroy failed: ${errMsg(destroyErr)}`),
|
|
4020
|
-
);
|
|
4100
|
+
await this.destroyConfirmed();
|
|
4021
4101
|
await this.clearInstanceLease(instance);
|
|
4022
4102
|
return { status: "failed", reason: (await this.goDown(reason)).reason };
|
|
4023
4103
|
}
|
|
@@ -4039,8 +4119,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4039
4119
|
`recreate: destroying the container — snapshots kept; the next exec restores from R2 (${reason.slice(0, 200)})`,
|
|
4040
4120
|
);
|
|
4041
4121
|
this.swapIncarnation(); // deliberate incarnation swap
|
|
4042
|
-
await this.
|
|
4043
|
-
await this.destroy().catch((err) => console.log(`runtime-unreachable: destroy failed: ${errMsg(err)}`));
|
|
4122
|
+
await this.destroyConfirmed();
|
|
4044
4123
|
await this.setResidentState("degraded", reason);
|
|
4045
4124
|
}
|
|
4046
4125
|
|
|
@@ -4055,6 +4134,34 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4055
4134
|
await this.ctx.storage.delete(SDK_RUNTIME_RECORD_KEY);
|
|
4056
4135
|
}
|
|
4057
4136
|
|
|
4137
|
+
/** Mark uncertainty before touching SDK runtime identity. A failed destroy
|
|
4138
|
+
* remains visible after an isolate restart and refuses every new exec. */
|
|
4139
|
+
private async destroyConfirmed(): Promise<void> {
|
|
4140
|
+
const pending =
|
|
4141
|
+
this.destroying ??
|
|
4142
|
+
destroyWithPersistentFence(
|
|
4143
|
+
{
|
|
4144
|
+
mark: () => this.ctx.storage.put(DESTROY_UNCONFIRMED_KEY, true),
|
|
4145
|
+
clear: async () => {
|
|
4146
|
+
await this.ctx.storage.delete(DESTROY_UNCONFIRMED_KEY);
|
|
4147
|
+
},
|
|
4148
|
+
},
|
|
4149
|
+
async () => {
|
|
4150
|
+
await this.forgetRuntimeIdentity();
|
|
4151
|
+
await this.destroy();
|
|
4152
|
+
// Only confirmed destruction creates an empty generation. If this
|
|
4153
|
+
// write fails, the uncertainty fence remains and no UID is reused.
|
|
4154
|
+
await this.ctx.storage.put(SPENT_POOL_USERS_KEY, []);
|
|
4155
|
+
},
|
|
4156
|
+
);
|
|
4157
|
+
this.destroying = pending;
|
|
4158
|
+
try {
|
|
4159
|
+
await pending;
|
|
4160
|
+
} finally {
|
|
4161
|
+
if (this.destroying === pending) this.destroying = null;
|
|
4162
|
+
}
|
|
4163
|
+
}
|
|
4164
|
+
|
|
4058
4165
|
// -- the refresh cycle as a Workflow instance (item 7) --------------------------
|
|
4059
4166
|
//
|
|
4060
4167
|
// `ResidentRefresh` (the Workflow entrypoint, refresh.ts) calls these
|
|
@@ -4227,6 +4334,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4227
4334
|
): Promise<InstanceStepAnswer<T>> {
|
|
4228
4335
|
const startedAt = systemClock();
|
|
4229
4336
|
const trace = createStepTrace(startedAt);
|
|
4337
|
+
this.refreshAdmissionsInFlight++;
|
|
4230
4338
|
let counted = false;
|
|
4231
4339
|
const count = () => {
|
|
4232
4340
|
if (counted) return;
|
|
@@ -4235,6 +4343,11 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4235
4343
|
};
|
|
4236
4344
|
let outcome = "done";
|
|
4237
4345
|
try {
|
|
4346
|
+
if (await this.recreateAdmission.blocked()) {
|
|
4347
|
+
outcome = "stopped (recreate-in-progress)";
|
|
4348
|
+
await this.clearInstanceLease(instance);
|
|
4349
|
+
return { status: "stopped", why: "recreate-in-progress", startedAt, trace: trace.steps() };
|
|
4350
|
+
}
|
|
4238
4351
|
const result = await this.stepTrace.run(trace, () => fn({ count }));
|
|
4239
4352
|
if (result.status !== "done") {
|
|
4240
4353
|
outcome = result.status === "stopped" ? `stopped (${result.why})` : `failed (${result.reason})`;
|
|
@@ -4297,6 +4410,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4297
4410
|
outcome = `failed (${failure.reason})`;
|
|
4298
4411
|
return { status: "failed", reason: failure.reason, startedAt, trace: trace.steps() };
|
|
4299
4412
|
} finally {
|
|
4413
|
+
this.refreshAdmissionsInFlight--;
|
|
4300
4414
|
if (counted) this.refreshesInFlight--;
|
|
4301
4415
|
await this.recordInstanceStep(instance, step, outcome).catch((err) =>
|
|
4302
4416
|
console.log(`refresh instance ${instance}: recording ${step} failed: ${errMsg(err)}`),
|
|
@@ -4648,7 +4762,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4648
4762
|
private async recordRecycledTree(binding: ThreadBinding, tree: EvictedTree | undefined): Promise<void> {
|
|
4649
4763
|
const now = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(binding.threadKey));
|
|
4650
4764
|
if (!now || now.evicted || now.lastAttachAt !== binding.lastAttachAt) return;
|
|
4651
|
-
await this.
|
|
4765
|
+
await this.putThreadBinding({
|
|
4652
4766
|
...now,
|
|
4653
4767
|
recycledLeftBehind: tree && "leftBehind" in tree ? tree.leftBehind : undefined,
|
|
4654
4768
|
recycledUnmeasured: tree && "unmeasured" in tree ? tree.unmeasured : undefined,
|
|
@@ -4773,7 +4887,10 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4773
4887
|
const at = new Date(systemClock()).toISOString();
|
|
4774
4888
|
const all = await this.ctx.storage.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX });
|
|
4775
4889
|
const used = new Set([...all.values()].filter((b) => !b.evicted && b.user).map((b) => b.user));
|
|
4776
|
-
|
|
4890
|
+
const spent = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
|
|
4891
|
+
for (const user of spent?.keys() ?? THREAD_USERS) used.add(user);
|
|
4892
|
+
for (const u of this.opUsersInUse.keys()) used.add(u);
|
|
4893
|
+
for (const u of this.poolUsersInspecting) used.add(u);
|
|
4777
4894
|
const reading = await this.memoryGauge();
|
|
4778
4895
|
const generation = this.containerIdMemo ?? this.incarnation;
|
|
4779
4896
|
const seat = { side: seatSide(used.size, THREAD_USERS.length), used: used.size, total: THREAD_USERS.length };
|
|
@@ -4949,7 +5066,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4949
5066
|
kept.push({ threadKey: c.threadKey, why: "busy" });
|
|
4950
5067
|
continue;
|
|
4951
5068
|
}
|
|
4952
|
-
if (
|
|
5069
|
+
if ((await this.evictBinding(current, true, "disk-pressure", DISK_PRESSURE_REASON, tree)) !== "evicted") {
|
|
4953
5070
|
kept.push({ threadKey: c.threadKey, why: "other" });
|
|
4954
5071
|
continue;
|
|
4955
5072
|
}
|
|
@@ -5340,6 +5457,8 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5340
5457
|
capFiles?: { out: string; err: string },
|
|
5341
5458
|
env?: Record<string, string>,
|
|
5342
5459
|
): Promise<{ stdout: string; stderr: string; exitCode: number; timedOut: boolean; truncated?: boolean }> {
|
|
5460
|
+
if (THREAD_USERS.includes(user) && !(await this.poolUserOwnerMatches(user)))
|
|
5461
|
+
throw new Error("pool-owner-mismatch: model command refused before UID ownership");
|
|
5343
5462
|
// The caller's variables (an /exec body's `env`, docs/reference/specs/
|
|
5344
5463
|
// harness-pi.md item 4) under the Worker's own: a caller never overrides
|
|
5345
5464
|
// what the Worker injects. `su` without `-` keeps this environment for the
|
|
@@ -5396,23 +5515,192 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5396
5515
|
return this.assertOk(await this.threadRun(user, worktreePath, command, timeoutMs), step);
|
|
5397
5516
|
}
|
|
5398
5517
|
|
|
5399
|
-
/**
|
|
5400
|
-
|
|
5401
|
-
|
|
5402
|
-
|
|
5403
|
-
*
|
|
5404
|
-
|
|
5518
|
+
/** A candidate remains reserved across the disk inspection's awaits. */
|
|
5519
|
+
private poolUsersInspecting = new Set<string>();
|
|
5520
|
+
|
|
5521
|
+
/** A UID is spent before any untrusted command receives it. The storage
|
|
5522
|
+
* transaction prevents concurrent admissions from losing each other's
|
|
5523
|
+
* marks. An old resident without this row is an unknown generation. */
|
|
5524
|
+
private async markPoolUserSpent(user: string, owner: string): Promise<boolean> {
|
|
5525
|
+
return this.ctx.storage.transaction(async (txn) => {
|
|
5526
|
+
const current = await txn.get<unknown>(SPENT_POOL_USERS_KEY);
|
|
5527
|
+
const next = spendPoolUser(current, THREAD_USERS, user, owner);
|
|
5528
|
+
if (!next) return false;
|
|
5529
|
+
await txn.put(SPENT_POOL_USERS_KEY, next);
|
|
5530
|
+
return true;
|
|
5531
|
+
});
|
|
5532
|
+
}
|
|
5533
|
+
|
|
5534
|
+
private async rebuildPoolBindingIndex(): Promise<void> {
|
|
5535
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
5536
|
+
const bindings = await txn.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX });
|
|
5537
|
+
const byUser = rebuildPoolBindingIndex(bindings.values(), THREAD_USERS);
|
|
5538
|
+
if (!byUser) throw new Error("pool-binding-index: malformed live binding; refusing resident work");
|
|
5539
|
+
for (const [user, keys] of byUser) await txn.put(poolBindingKey(user), keys);
|
|
5540
|
+
});
|
|
5541
|
+
}
|
|
5542
|
+
|
|
5543
|
+
/** Primary and per-UID rows change atomically. The index keeps every live
|
|
5544
|
+
* claimant, including a legacy duplicate that must block shell access. */
|
|
5545
|
+
private async putThreadBinding(next: ThreadBinding): Promise<void> {
|
|
5546
|
+
const key = threadBindingKey(next.threadKey);
|
|
5547
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
5548
|
+
const prior = await txn.get<ThreadBinding>(key);
|
|
5549
|
+
const oldUser = prior && !prior.evicted ? prior.user : "";
|
|
5550
|
+
const newUser = !next.evicted ? next.user : "";
|
|
5551
|
+
if (oldUser && !THREAD_USERS.includes(oldUser)) throw new Error("pool-binding-index: unknown old UID");
|
|
5552
|
+
if (newUser && !THREAD_USERS.includes(newUser)) throw new Error("pool-binding-index: unknown new UID");
|
|
5553
|
+
if (oldUser !== newUser) {
|
|
5554
|
+
if (oldUser) {
|
|
5555
|
+
const keys = releasePoolBinding(await txn.get<unknown>(poolBindingKey(oldUser)), next.threadKey);
|
|
5556
|
+
if (!keys) throw new Error("pool-binding-index: missing old claimant");
|
|
5557
|
+
await txn.put(poolBindingKey(oldUser), keys);
|
|
5558
|
+
}
|
|
5559
|
+
if (newUser) {
|
|
5560
|
+
const keys = claimPoolBinding(await txn.get<unknown>(poolBindingKey(newUser)), next.threadKey);
|
|
5561
|
+
if (!keys) throw new Error("pool-binding-index: conflicting new claimant");
|
|
5562
|
+
await txn.put(poolBindingKey(newUser), keys);
|
|
5563
|
+
}
|
|
5564
|
+
}
|
|
5565
|
+
await txn.put(key, next);
|
|
5566
|
+
});
|
|
5567
|
+
}
|
|
5568
|
+
|
|
5569
|
+
private async deleteThreadBinding(threadKey: string, onlyEvicted = false): Promise<boolean> {
|
|
5570
|
+
return this.ctx.storage.transaction(async (txn) => {
|
|
5571
|
+
const key = threadBindingKey(threadKey);
|
|
5572
|
+
const prior = await txn.get<ThreadBinding>(key);
|
|
5573
|
+
if (!prior || (onlyEvicted && !prior.evicted)) return false;
|
|
5574
|
+
if (!prior.evicted && prior.user) {
|
|
5575
|
+
const keys = releasePoolBinding(await txn.get<unknown>(poolBindingKey(prior.user)), threadKey);
|
|
5576
|
+
if (!keys) throw new Error("pool-binding-index: missing deleted claimant");
|
|
5577
|
+
await txn.put(poolBindingKey(prior.user), keys);
|
|
5578
|
+
}
|
|
5579
|
+
await txn.delete(key);
|
|
5580
|
+
return true;
|
|
5581
|
+
});
|
|
5582
|
+
}
|
|
5583
|
+
|
|
5584
|
+
/** Every UID-scoped action checks both the durable claimant and the live
|
|
5585
|
+
* bindings. Two retained bindings naming one UID fail closed, even when one
|
|
5586
|
+
* is the ledger owner. Ops have a separate exact in-flight owner. */
|
|
5587
|
+
private async poolUserOwnerMatches(user: string, expectedOwner?: string): Promise<boolean> {
|
|
5588
|
+
if (!THREAD_USERS.includes(user)) return false;
|
|
5589
|
+
const [spent, bindings] = await Promise.all([
|
|
5590
|
+
this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY),
|
|
5591
|
+
this.ctx.storage.get<unknown>(poolBindingKey(user)),
|
|
5592
|
+
]);
|
|
5593
|
+
const liveThreadKeys = parsePoolBindings(bindings);
|
|
5594
|
+
if (!liveThreadKeys) return false;
|
|
5595
|
+
return mayRunAsPoolUser(spent, THREAD_USERS, user, liveThreadKeys, this.opUsersInUse.get(user), expectedOwner);
|
|
5596
|
+
}
|
|
5597
|
+
|
|
5598
|
+
/** Storage bindings, active ops and inspections all reserve a pool user.
|
|
5599
|
+
* Claim inside this method, before its promise resolves: two concurrent
|
|
5600
|
+
* callers must not each read the same free UID before either can claim it. */
|
|
5601
|
+
private async findFreePoolUser(excludeThreadKey?: string, excluded = new Set<string>()): Promise<string | undefined> {
|
|
5405
5602
|
const all = await this.ctx.storage.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX });
|
|
5406
5603
|
const used = new Set(
|
|
5407
5604
|
[...all.values()].filter((b) => !b.evicted && b.user && b.threadKey !== excludeThreadKey).map((b) => b.user),
|
|
5408
5605
|
);
|
|
5409
|
-
for (const u of this.opUsersInUse) used.add(u);
|
|
5410
|
-
|
|
5606
|
+
for (const u of this.opUsersInUse.keys()) used.add(u);
|
|
5607
|
+
for (const u of this.poolUsersInspecting) used.add(u);
|
|
5608
|
+
for (const u of excluded) used.add(u);
|
|
5609
|
+
const user = THREAD_USERS.find((u) => !used.has(u));
|
|
5610
|
+
if (user) this.poolUsersInspecting.add(user);
|
|
5611
|
+
return user;
|
|
5612
|
+
}
|
|
5613
|
+
|
|
5614
|
+
/** Root-side inspection fails closed on a missing/partial answer. A former
|
|
5615
|
+
* binding may have been freed by an old image while its private tree stayed
|
|
5616
|
+
* on disk; assigning that UID would let a new run read the old tree. */
|
|
5617
|
+
private async poolUserHasOldThreadDir(user: string, allowed: readonly string[]): Promise<boolean> {
|
|
5618
|
+
for (const dir of [THREADS_DIR, OPS_DIR]) {
|
|
5619
|
+
const result = await this.run([
|
|
5620
|
+
"sh",
|
|
5621
|
+
"-c",
|
|
5622
|
+
'if test -L "$1"; then exit 42; fi; if test -d "$1"; then find "$1" -mindepth 1 -maxdepth 1 -type d -user "$2" -print; fi',
|
|
5623
|
+
"_",
|
|
5624
|
+
dir,
|
|
5625
|
+
user,
|
|
5626
|
+
]);
|
|
5627
|
+
if (result.exitCode !== 0 || result.timedOut || result.truncated)
|
|
5628
|
+
throw new Error("pool-user-disk-scan-incomplete");
|
|
5629
|
+
if (hasUnexpectedOwnedThreadDir(result.stdout, dir, dir === THREADS_DIR ? allowed : [])) return true;
|
|
5630
|
+
}
|
|
5631
|
+
return false;
|
|
5632
|
+
}
|
|
5633
|
+
|
|
5634
|
+
/** A freed UID may still have a pre-fix credential or staged write in its
|
|
5635
|
+
* private staging directory. Inspect as root before assigning it to an op
|
|
5636
|
+
* (which does not run attach's credential scrub) or a fresh thread. */
|
|
5637
|
+
private async poolUserHasOldStageContent(user: string): Promise<boolean> {
|
|
5638
|
+
const result = await this.run(legacyStageContentScanCommand(`/workspace/.stage-${user}`));
|
|
5639
|
+
if (result.exitCode !== 0 || result.timedOut || result.truncated || result.stderr !== "")
|
|
5640
|
+
throw new Error("pool-user-stage-scan-incomplete");
|
|
5641
|
+
if (result.stdout === "stale\n") return true;
|
|
5642
|
+
if (result.stdout === "clean\n") return false;
|
|
5643
|
+
throw new Error("pool-user-stage-scan-invalid");
|
|
5644
|
+
}
|
|
5645
|
+
|
|
5646
|
+
private async reserveSafePoolUser(owner: string, excludeThreadKey?: string): Promise<string | ThreadErr> {
|
|
5647
|
+
const spent = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
|
|
5648
|
+
if (!spent) return { error: "pool-generation-unknown: fresh VM required before UID assignment", status: 503 };
|
|
5649
|
+
const excluded = new Set(spent.keys());
|
|
5650
|
+
let contaminated = false;
|
|
5651
|
+
for (;;) {
|
|
5652
|
+
const user = await this.findFreePoolUser(excludeThreadKey, excluded);
|
|
5653
|
+
if (!user)
|
|
5654
|
+
return contaminated
|
|
5655
|
+
? { error: "pool-user-contaminated: no clean pool user is available", status: 503 }
|
|
5656
|
+
: excluded.size === THREAD_USERS.length
|
|
5657
|
+
? { error: "pool-recycle-required: all UIDs spent on this VM; confirm idle and recreate it", status: 503 }
|
|
5658
|
+
: { error: "user-pool-exhausted: no safe pool user is available", status: 429 };
|
|
5659
|
+
let safe = false;
|
|
5660
|
+
try {
|
|
5661
|
+
const clean = !(await this.poolUserHasOldThreadDir(user, [])) && !(await this.poolUserHasOldStageContent(user));
|
|
5662
|
+
if (clean) {
|
|
5663
|
+
safe = await this.markPoolUserSpent(user, owner);
|
|
5664
|
+
if (safe) return user;
|
|
5665
|
+
} else contaminated = true;
|
|
5666
|
+
excluded.add(user);
|
|
5667
|
+
} catch {
|
|
5668
|
+
return { error: "pool-user-inspection-failed: allocation refused", status: 503 };
|
|
5669
|
+
} finally {
|
|
5670
|
+
if (!safe) this.poolUsersInspecting.delete(user);
|
|
5671
|
+
}
|
|
5672
|
+
// A rejected UID may still own a private tree. It stays unavailable to
|
|
5673
|
+
// this request, and every later allocation checks disk again.
|
|
5674
|
+
}
|
|
5675
|
+
}
|
|
5676
|
+
|
|
5677
|
+
/** The prior binding read, allocation and possible rollback are one
|
|
5678
|
+
* same-thread operation, even while an attach awaits disk or Git. */
|
|
5679
|
+
private readonly threadAttaches = new KeyedAsyncLock();
|
|
5680
|
+
|
|
5681
|
+
/** Claim an old retained binding before any user-scoped probe. A duplicate
|
|
5682
|
+
* legacy binding may name the same UID, but only its ledger owner proceeds. */
|
|
5683
|
+
private async claimRetainedThreadUser(binding: ThreadBinding): Promise<ThreadErr | null> {
|
|
5684
|
+
const user = binding.user;
|
|
5685
|
+
if (!THREAD_USERS.includes(user)) return { error: "pool-user-invalid: binding names a non-pool user", status: 409 };
|
|
5686
|
+
try {
|
|
5687
|
+
if (await this.poolUserHasOldThreadDir(user, [parentDir(binding.worktreePath)]))
|
|
5688
|
+
return { error: "pool-user-contaminated: another thread directory has this user", status: 409 };
|
|
5689
|
+
} catch {
|
|
5690
|
+
return { error: "pool-user-inspection-failed: allocation refused", status: 503 };
|
|
5691
|
+
}
|
|
5692
|
+
try {
|
|
5693
|
+
if (!(await this.markPoolUserSpent(user, `thread:${binding.threadKey}`)))
|
|
5694
|
+
return { error: "pool-owner-mismatch: existing binding cannot share a UID", status: 503 };
|
|
5695
|
+
} catch {
|
|
5696
|
+
return { error: "pool-generation-write-failed: existing binding cannot run", status: 503 };
|
|
5697
|
+
}
|
|
5698
|
+
if (!(await this.poolUserOwnerMatches(user, `thread:${binding.threadKey}`)))
|
|
5699
|
+
return { error: "pool-owner-mismatch: conflicting retained binding names this UID", status: 503 };
|
|
5700
|
+
return null;
|
|
5411
5701
|
}
|
|
5412
5702
|
|
|
5413
|
-
/**
|
|
5414
|
-
* put touches nothing but this object's storage, so two concurrent
|
|
5415
|
-
* attaches cannot both claim the same user). A hint that names no branch
|
|
5703
|
+
/** Allocation stays provisional until attach finishes. A hint that names no branch
|
|
5416
5704
|
* keeps the sticky ref. An explicit named ref records named authority even
|
|
5417
5705
|
* when it matches the sticky ref; only a changed ref replaces the path. */
|
|
5418
5706
|
private async allocateThreadUser(
|
|
@@ -5429,49 +5717,54 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5429
5717
|
const replacingRef = existing !== undefined && replaceRef && existing.ref !== ref;
|
|
5430
5718
|
const recordingNamedAuthority = existing !== undefined && replaceRef && existing.boundBy !== boundBy;
|
|
5431
5719
|
const deferRefWrite = replacingRef && !existing.evicted && existing.user !== "";
|
|
5432
|
-
|
|
5433
|
-
|
|
5434
|
-
|
|
5435
|
-
|
|
5436
|
-
|
|
5437
|
-
return
|
|
5438
|
-
|
|
5439
|
-
|
|
5720
|
+
const existingUser = existing && !existing.evicted && existing.user ? existing.user : undefined;
|
|
5721
|
+
if (existing && existingUser) {
|
|
5722
|
+
if (existing.threadKey !== threadKey)
|
|
5723
|
+
return { error: "pool-owner-mismatch: retained binding belongs to another thread", status: 503 };
|
|
5724
|
+
const refusal = await this.claimRetainedThreadUser(existing);
|
|
5725
|
+
if (refusal) return refusal;
|
|
5726
|
+
}
|
|
5727
|
+
if (existingUser && !replacingRef && !recordingNamedAuthority) return { binding: existing!, wrote: false };
|
|
5728
|
+
const reserved = existingUser ?? (await this.reserveSafePoolUser(`thread:${threadKey}`, threadKey));
|
|
5729
|
+
if (typeof reserved !== "string") return reserved;
|
|
5730
|
+
const user = reserved;
|
|
5731
|
+
try {
|
|
5732
|
+
const now = new Date(systemClock()).toISOString();
|
|
5733
|
+
const binding: ThreadBinding = {
|
|
5734
|
+
threadKey,
|
|
5735
|
+
ref: replacingRef ? ref : (existing?.ref ?? ref),
|
|
5736
|
+
user,
|
|
5737
|
+
worktreePath: replacingRef ? worktreePath : (existing?.worktreePath ?? worktreePath),
|
|
5738
|
+
boundAt: existing?.boundAt ?? now,
|
|
5739
|
+
lastAttachAt: now,
|
|
5740
|
+
evicted: false,
|
|
5741
|
+
// A named ref starts a new binding choice, so stale move/return records
|
|
5742
|
+
// do not make it look default-bound. The thread's pushed-branch memory
|
|
5743
|
+
// remains its history across every ref choice.
|
|
5744
|
+
...(existing
|
|
5745
|
+
? {
|
|
5746
|
+
...(replacingRef || recordingNamedAuthority
|
|
5747
|
+
? { boundBy }
|
|
5748
|
+
: {
|
|
5749
|
+
...(existing.boundBy !== undefined ? { boundBy: existing.boundBy } : {}),
|
|
5750
|
+
...(existing.rebound !== undefined ? { rebound: existing.rebound } : {}),
|
|
5751
|
+
...(existing.returned !== undefined ? { returned: existing.returned } : {}),
|
|
5752
|
+
}),
|
|
5753
|
+
...(existing.ownBranches !== undefined ? { ownBranches: existing.ownBranches } : {}),
|
|
5754
|
+
}
|
|
5755
|
+
: { boundBy }),
|
|
5440
5756
|
};
|
|
5757
|
+
// A live existing binding already reserves this user. Its named-ref
|
|
5758
|
+
// replacement stays provisional until deps and credentials succeed;
|
|
5759
|
+
// keeping the old row here lets a failed attach discard the new checkout
|
|
5760
|
+
// without ever publishing a binding that did not complete. An evicted row
|
|
5761
|
+
// has no user reservation, so its fresh allocation is written and rolled
|
|
5762
|
+
// back through the ordinary path.
|
|
5763
|
+
if (!deferRefWrite) await this.putThreadBinding(binding);
|
|
5764
|
+
return { binding, wrote: !deferRefWrite };
|
|
5765
|
+
} finally {
|
|
5766
|
+
if (!existingUser) this.poolUsersInspecting.delete(user);
|
|
5441
5767
|
}
|
|
5442
|
-
const now = new Date(systemClock()).toISOString();
|
|
5443
|
-
const binding: ThreadBinding = {
|
|
5444
|
-
threadKey,
|
|
5445
|
-
ref: replacingRef ? ref : (existing?.ref ?? ref),
|
|
5446
|
-
user,
|
|
5447
|
-
worktreePath: replacingRef ? worktreePath : (existing?.worktreePath ?? worktreePath),
|
|
5448
|
-
boundAt: existing?.boundAt ?? now,
|
|
5449
|
-
lastAttachAt: now,
|
|
5450
|
-
evicted: false,
|
|
5451
|
-
// A named ref starts a new binding choice, so stale move/return records
|
|
5452
|
-
// do not make it look default-bound. The thread's pushed-branch memory
|
|
5453
|
-
// remains its history across every ref choice.
|
|
5454
|
-
...(existing
|
|
5455
|
-
? {
|
|
5456
|
-
...(replacingRef || recordingNamedAuthority
|
|
5457
|
-
? { boundBy }
|
|
5458
|
-
: {
|
|
5459
|
-
...(existing.boundBy !== undefined ? { boundBy: existing.boundBy } : {}),
|
|
5460
|
-
...(existing.rebound !== undefined ? { rebound: existing.rebound } : {}),
|
|
5461
|
-
...(existing.returned !== undefined ? { returned: existing.returned } : {}),
|
|
5462
|
-
}),
|
|
5463
|
-
...(existing.ownBranches !== undefined ? { ownBranches: existing.ownBranches } : {}),
|
|
5464
|
-
}
|
|
5465
|
-
: { boundBy }),
|
|
5466
|
-
};
|
|
5467
|
-
// A live existing binding already reserves this user. Its named-ref
|
|
5468
|
-
// replacement stays provisional until deps and credentials succeed;
|
|
5469
|
-
// keeping the old row here lets a failed attach discard the new checkout
|
|
5470
|
-
// without ever publishing a binding that did not complete. An evicted row
|
|
5471
|
-
// has no user reservation, so its fresh allocation is written and rolled
|
|
5472
|
-
// back through the ordinary path.
|
|
5473
|
-
if (!deferRefWrite) await this.ctx.storage.put(key, binding);
|
|
5474
|
-
return { binding, wrote: !deferRefWrite };
|
|
5475
5768
|
}
|
|
5476
5769
|
|
|
5477
5770
|
/** Best-effort cleanup after a provisional named-ref replacement settles.
|
|
@@ -5510,9 +5803,19 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5510
5803
|
// the attach runs lands on it, and the answer carries it.
|
|
5511
5804
|
const t0 = systemClock();
|
|
5512
5805
|
const trace = createStepTrace(t0);
|
|
5513
|
-
|
|
5514
|
-
|
|
5515
|
-
|
|
5806
|
+
let res: AttachOk | ThreadErr;
|
|
5807
|
+
if (this.recreateAdmission.pending) {
|
|
5808
|
+
res = this.recreateRefusal();
|
|
5809
|
+
} else {
|
|
5810
|
+
this.attachAdmissionsInFlight++;
|
|
5811
|
+
try {
|
|
5812
|
+
res = await this.stepTrace.run(trace, () =>
|
|
5813
|
+
this.attachThreadTraced(threadKey, refHint, readonly, wantSha, reuse, record, t0, reason, githubDoor),
|
|
5814
|
+
);
|
|
5815
|
+
} finally {
|
|
5816
|
+
this.attachAdmissionsInFlight--;
|
|
5817
|
+
}
|
|
5818
|
+
}
|
|
5516
5819
|
// The same steps as the resident's own `resident.attach` root (item 22).
|
|
5517
5820
|
emitStepRoot("resident.attach", t0, trace.steps(), traceparent, "error" in res ? refusalOutcome(res) : "ok");
|
|
5518
5821
|
// A refusal carries the steps that led to it; a success already does.
|
|
@@ -5531,7 +5834,9 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5531
5834
|
githubDoor?: { baseUrl: string; bearer: string },
|
|
5532
5835
|
): Promise<AttachOk | ThreadErr> {
|
|
5533
5836
|
try {
|
|
5837
|
+
if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
|
|
5534
5838
|
await this.ensureHydrated();
|
|
5839
|
+
if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
|
|
5535
5840
|
// The fleet drain (item 69): a deploy is waiting for the runs in flight
|
|
5536
5841
|
// to end, and a NEW run's attach is refused with the record the bot
|
|
5537
5842
|
// waits on — a real 503 in the streamed document, read by the client as
|
|
@@ -5576,22 +5881,24 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5576
5881
|
// container under it (and isIdle never parks the cycle mid-attach).
|
|
5577
5882
|
this.attachesInFlight++;
|
|
5578
5883
|
try {
|
|
5579
|
-
|
|
5580
|
-
|
|
5581
|
-
|
|
5582
|
-
|
|
5583
|
-
|
|
5584
|
-
|
|
5585
|
-
|
|
5586
|
-
|
|
5587
|
-
|
|
5588
|
-
|
|
5589
|
-
|
|
5590
|
-
|
|
5591
|
-
|
|
5592
|
-
|
|
5593
|
-
|
|
5594
|
-
|
|
5884
|
+
return await this.threadAttaches.run(threadKey, async () => {
|
|
5885
|
+
const res = await this.attachThreadBody(
|
|
5886
|
+
threadKey,
|
|
5887
|
+
refHint,
|
|
5888
|
+
readonly,
|
|
5889
|
+
wantSha,
|
|
5890
|
+
reuse,
|
|
5891
|
+
resourceId,
|
|
5892
|
+
t0,
|
|
5893
|
+
record,
|
|
5894
|
+
reason,
|
|
5895
|
+
githubDoor,
|
|
5896
|
+
);
|
|
5897
|
+
// The run this attach opens is now in flight until its release —
|
|
5898
|
+
// whatever its op counters read between the bot's calls (item 44).
|
|
5899
|
+
if (!("error" in res)) await this.registerRun(threadKey);
|
|
5900
|
+
return res;
|
|
5901
|
+
});
|
|
5595
5902
|
} finally {
|
|
5596
5903
|
this.attachesInFlight--;
|
|
5597
5904
|
}
|
|
@@ -5674,6 +5981,23 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5674
5981
|
if (!readonly && storedPrior?.githubDoorHost && !githubDoor)
|
|
5675
5982
|
return { error: "Git door credential is required to reattach this writable tree", status: 403, cause: "request" };
|
|
5676
5983
|
const namedRef = refHint !== null && !reason.refByDefault && reason.ownPr === null;
|
|
5984
|
+
// The own-PR rebind may run a branch probe as this UID before ordinary
|
|
5985
|
+
// allocation. Establish exact ownership first on a recycled VM.
|
|
5986
|
+
if (!namedRef && storedPrior && !storedPrior.evicted && storedPrior.user) {
|
|
5987
|
+
if (storedPrior.threadKey !== threadKey)
|
|
5988
|
+
return { error: "pool-owner-mismatch: retained binding belongs to another thread", status: 503 };
|
|
5989
|
+
const refusal = await this.claimRetainedThreadUser(storedPrior);
|
|
5990
|
+
if (refusal) return refusal;
|
|
5991
|
+
// The own-PR fallback may inspect a local branch as this pool user.
|
|
5992
|
+
// Remove any legacy App token before that first user-scoped Git probe.
|
|
5993
|
+
if (reason.ownPr !== null) {
|
|
5994
|
+
try {
|
|
5995
|
+
await this.scrubThreadCredentialFiles(storedPrior);
|
|
5996
|
+
} catch (err) {
|
|
5997
|
+
return this.attachFailed(err);
|
|
5998
|
+
}
|
|
5999
|
+
}
|
|
6000
|
+
}
|
|
5677
6001
|
const rebind = namedRef
|
|
5678
6002
|
? { binding: storedPrior }
|
|
5679
6003
|
: await this.rebindToOwnPr(storedPrior, reason.ownPr, reuse, facts.defaultRef, slug);
|
|
@@ -5715,10 +6039,23 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5715
6039
|
// bind sticky garbage nor leak a pool user.
|
|
5716
6040
|
const rollback = async (): Promise<void> => {
|
|
5717
6041
|
if (!alloc.wrote) return;
|
|
5718
|
-
if (prior) await this.
|
|
5719
|
-
else await this.
|
|
6042
|
+
if (prior) await this.putThreadBinding(prior);
|
|
6043
|
+
else await this.deleteThreadBinding(threadKey);
|
|
5720
6044
|
};
|
|
5721
6045
|
|
|
6046
|
+
// A named ref builds beside the prior checkout and removes that sibling
|
|
6047
|
+
// only after the new binding commits. If cleanup later fails, the same
|
|
6048
|
+
// pool user can still traverse the old tree. Remove its legacy App token
|
|
6049
|
+
// before admitting the replacement or any of its model commands.
|
|
6050
|
+
if (replacingNamedRef && prior && !prior.evicted) {
|
|
6051
|
+
try {
|
|
6052
|
+
await this.scrubThreadCredentialFiles(prior);
|
|
6053
|
+
} catch (err) {
|
|
6054
|
+
await rollback();
|
|
6055
|
+
return this.attachFailed(err);
|
|
6056
|
+
}
|
|
6057
|
+
}
|
|
6058
|
+
|
|
5722
6059
|
// Disk admission (item 55): before the lock, since making room takes it.
|
|
5723
6060
|
// An evicted prior binding has already published its replacement to
|
|
5724
6061
|
// reserve the user; keep that row provisional if admission itself throws.
|
|
@@ -5910,7 +6247,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5910
6247
|
at: new Date(systemClock()).toISOString(),
|
|
5911
6248
|
};
|
|
5912
6249
|
const moved: ThreadBinding = { ...current, ref: plan.to, rebound };
|
|
5913
|
-
await this.
|
|
6250
|
+
await this.putThreadBinding(moved);
|
|
5914
6251
|
return { kind: "rebound", moved, rebound };
|
|
5915
6252
|
}
|
|
5916
6253
|
|
|
@@ -5937,7 +6274,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5937
6274
|
const current = (await this.ctx.storage.get<ThreadBinding>(key)) ?? binding;
|
|
5938
6275
|
const back = returnToDefault(current, defaultRef, new Date(systemClock()).toISOString());
|
|
5939
6276
|
if (back === undefined) return { binding: current };
|
|
5940
|
-
await this.
|
|
6277
|
+
await this.putThreadBinding(back.binding);
|
|
5941
6278
|
console.log(
|
|
5942
6279
|
`attach ${binding.threadKey}: ${back.returned.from} is gone from the mirror (the thread's own pull request #${back.returned.pr}) — returned to ${back.returned.to}; the tree is provisioned there`,
|
|
5943
6280
|
);
|
|
@@ -5954,7 +6291,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5954
6291
|
private async rememberOwnBranches(threadKey: string, pushed: readonly PushedBranch[]): Promise<void> {
|
|
5955
6292
|
const binding = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
|
|
5956
6293
|
if (!binding) return;
|
|
5957
|
-
await this.
|
|
6294
|
+
await this.putThreadBinding({
|
|
5958
6295
|
...binding,
|
|
5959
6296
|
ownBranches: rememberOwnBranches(binding.ownBranches, pushed, new Date(systemClock()).toISOString()),
|
|
5960
6297
|
} satisfies ThreadBinding);
|
|
@@ -6219,7 +6556,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
6219
6556
|
githubDoorHost: _priorDoor,
|
|
6220
6557
|
...bindingSansCred
|
|
6221
6558
|
} = binding;
|
|
6222
|
-
await this.
|
|
6559
|
+
await this.putThreadBinding({
|
|
6223
6560
|
...bindingSansCred,
|
|
6224
6561
|
lastAttachAt: new Date(systemClock()).toISOString(),
|
|
6225
6562
|
deps: deps.deps,
|
|
@@ -7028,6 +7365,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7028
7365
|
* binding, worktree actually on disk (the container may have slept since
|
|
7029
7366
|
* the last attach — disk is cache, re-attach recreates). */
|
|
7030
7367
|
private async threadPreflight(threadKey: string): Promise<{ binding: ThreadBinding } | ThreadErr> {
|
|
7368
|
+
if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
|
|
7031
7369
|
try {
|
|
7032
7370
|
await this.ensureHydrated();
|
|
7033
7371
|
} catch (err) {
|
|
@@ -7056,6 +7394,9 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7056
7394
|
needs: "attach",
|
|
7057
7395
|
};
|
|
7058
7396
|
}
|
|
7397
|
+
if (!(await this.poolUserOwnerMatches(binding.user, `thread:${threadKey}`))) {
|
|
7398
|
+
return { error: "pool-owner-mismatch: model command refused before UID ownership", status: 503 };
|
|
7399
|
+
}
|
|
7059
7400
|
if ((await this.run(["test", "-d", `${binding.worktreePath}/.git`])).exitCode !== 0) {
|
|
7060
7401
|
return {
|
|
7061
7402
|
error: "worktree-missing: the container disk was recycled since the last attach — POST /attach to recreate",
|
|
@@ -7155,7 +7496,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7155
7496
|
!/^sbr_[A-Za-z0-9_-]{1,64}\.[A-Za-z0-9_-]{20,128}$/.test(env?.GH_ENTERPRISE_TOKEN ?? ""))
|
|
7156
7497
|
)
|
|
7157
7498
|
return { error: "Git door run bearer is missing from this command", status: 403, cause: "request" };
|
|
7158
|
-
await this.
|
|
7499
|
+
await this.putThreadBinding({
|
|
7159
7500
|
...binding,
|
|
7160
7501
|
lastAttachAt: new Date(systemClock()).toISOString(), // exec counts as activity for the sweep
|
|
7161
7502
|
} satisfies ThreadBinding);
|
|
@@ -7281,9 +7622,15 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7281
7622
|
path: string,
|
|
7282
7623
|
content: string,
|
|
7283
7624
|
): Promise<{ ok: true; bytes: number } | ThreadErr> {
|
|
7284
|
-
return this.withThreadBusy(threadKey, () =>
|
|
7625
|
+
return this.withThreadBusy(threadKey, () =>
|
|
7626
|
+
this.threadWrites.run(threadKey, () => this.writeThreadFileImpl(threadKey, path, content)),
|
|
7627
|
+
);
|
|
7285
7628
|
}
|
|
7286
7629
|
|
|
7630
|
+
/** All writes for a thread share its stage file; keep staging and cleanup
|
|
7631
|
+
* in one operation so a failed write cannot remove another write's data. */
|
|
7632
|
+
private readonly threadWrites = new KeyedAsyncLock();
|
|
7633
|
+
|
|
7287
7634
|
private async writeThreadFileImpl(
|
|
7288
7635
|
threadKey: string,
|
|
7289
7636
|
path: string,
|
|
@@ -7312,6 +7659,12 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7312
7659
|
DEFAULT_EXEC_TIMEOUT_MS,
|
|
7313
7660
|
);
|
|
7314
7661
|
} catch (err) {
|
|
7662
|
+
try {
|
|
7663
|
+
await this.runOk(legacyStageReuseScrubCommand(stageDir), "stage-write-cleanup");
|
|
7664
|
+
} catch (cleanupErr) {
|
|
7665
|
+
// The next UID reservation still refuses nonempty staging content.
|
|
7666
|
+
console.log(`write: stage cleanup failed for ${threadKey}: ${errMsg(cleanupErr)}`);
|
|
7667
|
+
}
|
|
7315
7668
|
if (err instanceof ControlResetError) return controlResetErr(err);
|
|
7316
7669
|
if (err instanceof RuntimeReplacedError) return runtimeReplacedErr(err);
|
|
7317
7670
|
if (err instanceof SandboxRuntimeBusyError) return runtimeBusyErr(err);
|
|
@@ -7338,20 +7691,31 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7338
7691
|
* as the thread user with the runtime up); absent when the tree was clean,
|
|
7339
7692
|
* already gone with the disk, or not measured (a force detach). */
|
|
7340
7693
|
tree?: EvictedTree,
|
|
7341
|
-
): Promise<
|
|
7694
|
+
): Promise<"evicted" | "changed" | "cleanup-failed"> {
|
|
7695
|
+
if (!(await this.poolUserOwnerMatches(binding.user, `thread:${binding.threadKey}`))) return "cleanup-failed";
|
|
7342
7696
|
const threadDir = parentDir(binding.worktreePath);
|
|
7343
|
-
if (runtimeActive && threadDir.startsWith(`${THREADS_DIR}/`))
|
|
7697
|
+
if (runtimeActive && !threadDir.startsWith(`${THREADS_DIR}/`)) return "cleanup-failed";
|
|
7698
|
+
if (runtimeActive) {
|
|
7344
7699
|
try {
|
|
7345
7700
|
// Worktree removal counts as a mirror-adjacent mutation — same mutex.
|
|
7346
|
-
await this.withMirrorLock(() =>
|
|
7701
|
+
await this.withMirrorLock(async () => {
|
|
7702
|
+
if (!(await this.poolUserOwnerMatches(binding.user, `thread:${binding.threadKey}`)))
|
|
7703
|
+
throw new Error("pool-owner-mismatch: eviction refused before directory removal");
|
|
7704
|
+
await this.runOk(["rm", "-rf", threadDir], "evict");
|
|
7705
|
+
});
|
|
7347
7706
|
} catch (err) {
|
|
7348
7707
|
console.log(`${logCtx}: rm failed for ${binding.threadKey}: ${errMsg(err)}`);
|
|
7708
|
+
// The directory may still contain a credential file readable by this
|
|
7709
|
+
// UID. Keep the binding live so the allocator cannot give it to a new
|
|
7710
|
+
// thread; a later sweep or detach may retry the removal.
|
|
7711
|
+
return "cleanup-failed";
|
|
7349
7712
|
}
|
|
7350
7713
|
// Item 55: what an `install` thread's package manager left OUTSIDE the
|
|
7351
7714
|
// tree — its pnpm store (the tree's hardlink source: 0 unique bytes while
|
|
7352
7715
|
// the tree lived, all of them now), npm/yarn/bun caches — goes with it.
|
|
7353
7716
|
// Pool users only, never the build user (its store backs the warm checkout).
|
|
7354
7717
|
if ((THREAD_USERS as readonly string[]).includes(binding.user)) {
|
|
7718
|
+
if (!(await this.poolUserOwnerMatches(binding.user, `thread:${binding.threadKey}`))) return "cleanup-failed";
|
|
7355
7719
|
await this.run(threadUserCacheCleanArgv(`/home/${binding.user}`)).catch((err) =>
|
|
7356
7720
|
console.log(`${logCtx}: home cache rm failed for ${binding.user}: ${errMsg(err)}`),
|
|
7357
7721
|
);
|
|
@@ -7365,9 +7729,9 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7365
7729
|
if (!now || now.evicted || now.lastAttachAt !== binding.lastAttachAt) {
|
|
7366
7730
|
const why = !now ? "binding deleted" : now.evicted ? "already evicted (concurrent eviction)" : "re-attached";
|
|
7367
7731
|
console.log(`${logCtx}: ${binding.threadKey} ${why} during eviction — binding left as is`);
|
|
7368
|
-
return
|
|
7732
|
+
return "changed";
|
|
7369
7733
|
}
|
|
7370
|
-
await this.
|
|
7734
|
+
await this.putThreadBinding({
|
|
7371
7735
|
...now,
|
|
7372
7736
|
user: "",
|
|
7373
7737
|
evicted: true,
|
|
@@ -7380,7 +7744,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7380
7744
|
// never on a give-way above — a re-attach has re-registered it anyway.
|
|
7381
7745
|
await this.ctx.storage.delete(runRegKey(binding.threadKey));
|
|
7382
7746
|
if (tree) console.log(`${logCtx}: ${binding.threadKey} evicted (${why}) — ${evictedTreeSentence(tree)}`);
|
|
7383
|
-
return
|
|
7747
|
+
return "evicted";
|
|
7384
7748
|
}
|
|
7385
7749
|
|
|
7386
7750
|
/** POST /detach: a run has ended — give the thread's pool user back now
|
|
@@ -7404,6 +7768,8 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7404
7768
|
const binding = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
|
|
7405
7769
|
if (!binding) return { error: `no-binding: ${threadKey} has never attached to this resident`, status: 404 };
|
|
7406
7770
|
if (binding.evicted || !binding.user) return { released: false, reason: "already-evicted" };
|
|
7771
|
+
if (!(await this.poolUserOwnerMatches(binding.user, `thread:${threadKey}`)))
|
|
7772
|
+
return { error: "pool-owner-mismatch: detach refused for conflicting UID owner", status: 503 };
|
|
7407
7773
|
const plan = planForceDetach({
|
|
7408
7774
|
force,
|
|
7409
7775
|
inFlight: this.threadOpsInFlight.get(threadKey) ?? 0,
|
|
@@ -7412,7 +7778,8 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7412
7778
|
});
|
|
7413
7779
|
if (plan.action === "refuse") return { released: false, reason: plan.reason, user: binding.user };
|
|
7414
7780
|
if (plan.action === "kill") {
|
|
7415
|
-
await this.killThreadUserProcesses(plan.user)
|
|
7781
|
+
if (!(await this.killThreadUserProcesses(plan.user, threadKey)))
|
|
7782
|
+
return { error: "pool-owner-mismatch: force detach refused before kill", status: 503 };
|
|
7416
7783
|
console.log(
|
|
7417
7784
|
`detach: force — killed ${plan.user}'s processes for ${threadKey} (${plan.inFlight} op(s) were in flight)`,
|
|
7418
7785
|
);
|
|
@@ -7446,10 +7813,11 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7446
7813
|
const user = current.user;
|
|
7447
7814
|
// Same as the sweep: `active` was read before the measurement's awaits; a
|
|
7448
7815
|
// container that woke meanwhile must get the rm, not an orphaned tree.
|
|
7449
|
-
const activeNow = await this.isRuntimeActive().catch(() =>
|
|
7450
|
-
|
|
7451
|
-
|
|
7452
|
-
|
|
7816
|
+
const activeNow = await this.isRuntimeActive().catch(() => true);
|
|
7817
|
+
const eviction = await this.evictBinding(current, activeNow, `detach`, "detach", tree);
|
|
7818
|
+
if (eviction === "cleanup-failed")
|
|
7819
|
+
return { released: false, reason: "thread-cleanup-failed: pool user kept", user };
|
|
7820
|
+
if (eviction === "changed") return { released: false, reason: "re-attached during eviction — kept", user };
|
|
7453
7821
|
// Item 55: the tree is gone; the gauge catches up at the next refresh
|
|
7454
7822
|
// instance's `measure` step, and the admission's `df` sees the space now.
|
|
7455
7823
|
const leftBehind = tree && "leftBehind" in tree ? tree.leftBehind : undefined;
|
|
@@ -7464,12 +7832,14 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7464
7832
|
* (runtime replaced mid-kill) is fine as well: the drain wait after it is
|
|
7465
7833
|
* what decides, and it is bounded. Only ever called with a plan from
|
|
7466
7834
|
* `planForceDetach`, which refuses anything but a `THREAD_USERS` member. */
|
|
7467
|
-
private async killThreadUserProcesses(user: string): Promise<
|
|
7835
|
+
private async killThreadUserProcesses(user: string, threadKey: string): Promise<boolean> {
|
|
7836
|
+
if (!(await this.poolUserOwnerMatches(user, `thread:${threadKey}`))) return false;
|
|
7468
7837
|
try {
|
|
7469
7838
|
await this.run(["su", "-s", "/bin/bash", user, "-c", "kill -9 -1"], { timeoutMs: FORCE_DETACH_KILL_TIMEOUT_MS });
|
|
7470
7839
|
} catch (err) {
|
|
7471
7840
|
console.log(`detach: force — kill as ${user} threw (continuing to the drain wait): ${errMsg(err)}`);
|
|
7472
7841
|
}
|
|
7842
|
+
return true;
|
|
7473
7843
|
}
|
|
7474
7844
|
|
|
7475
7845
|
/** Wait (bounded, see FORCE_DETACH_DRAIN_MS) for this thread's in-flight op
|
|
@@ -7505,6 +7875,8 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7505
7875
|
* commands inside a single privilege-dropped `su`, tagged lines out);
|
|
7506
7876
|
* `parseWorktreeCleanliness` encodes the exact decision table above. */
|
|
7507
7877
|
private async worktreeCleanliness(binding: ThreadBinding): Promise<WorktreeCleanliness> {
|
|
7878
|
+
if (!(await this.poolUserOwnerMatches(binding.user, `thread:${binding.threadKey}`)))
|
|
7879
|
+
return { clean: false, reason: "pool-owner-mismatch: cleanliness probe refused" };
|
|
7508
7880
|
const injected = { GIT_TERMINAL_PROMPT: "0" }; // same injection as threadRun — fail fast, never prompt
|
|
7509
7881
|
validateEnvNames(injected);
|
|
7510
7882
|
const r = await this.run(["sh", "-c", worktreeCleanlinessScript(binding.worktreePath, binding.user)], {
|
|
@@ -7537,7 +7909,11 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7537
7909
|
* reads as busy. */
|
|
7538
7910
|
private runsInFlightCount(): number {
|
|
7539
7911
|
const threadOps = [...this.threadOpsInFlight.values()].reduce((a, n) => a + n, 0);
|
|
7540
|
-
return
|
|
7912
|
+
return (
|
|
7913
|
+
threadOps +
|
|
7914
|
+
Math.max(this.opUsersInUse.size, this.opAdmissionsInFlight) +
|
|
7915
|
+
Math.max(this.attachesInFlight, this.attachAdmissionsInFlight)
|
|
7916
|
+
);
|
|
7541
7917
|
}
|
|
7542
7918
|
/** The fleet drain in force (item 69), read from the registry at this clock;
|
|
7543
7919
|
* a registry that cannot be read is NO drain: a run must never fail because
|
|
@@ -7582,6 +7958,11 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7582
7958
|
return n;
|
|
7583
7959
|
}
|
|
7584
7960
|
private attachesInFlight = 0;
|
|
7961
|
+
/** Entry-to-answer admissions include hydration before attachesInFlight starts. */
|
|
7962
|
+
private attachAdmissionsInFlight = 0;
|
|
7963
|
+
private opAdmissionsInFlight = 0;
|
|
7964
|
+
/** A refresh step in its entry gates is also unsafe to interrupt. */
|
|
7965
|
+
private refreshAdmissionsInFlight = 0;
|
|
7585
7966
|
/** A refresh cycle past its idle/reconcile gates (fetching, rebuilding, snapshotting). */
|
|
7586
7967
|
private refreshesInFlight = 0;
|
|
7587
7968
|
|
|
@@ -7630,18 +8011,18 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7630
8011
|
// `active` is re-read per binding: the container can wake mid-sweep (an
|
|
7631
8012
|
// attach), and an eviction decided on a stale "inactive" would skip the
|
|
7632
8013
|
// rm and orphan a real tree.
|
|
7633
|
-
const activeNow = await this.isRuntimeActive().catch(() =>
|
|
8014
|
+
const activeNow = await this.isRuntimeActive().catch(() => true);
|
|
7634
8015
|
// What the tree holds goes on the eviction's record (item 17), measured
|
|
7635
8016
|
// while the runtime is up; a slept container has no tree to measure.
|
|
7636
8017
|
const tree = activeNow ? await this.measureTreeBeforeEviction(current) : undefined;
|
|
7637
8018
|
if (
|
|
7638
|
-
await this.evictBinding(
|
|
8019
|
+
(await this.evictBinding(
|
|
7639
8020
|
current,
|
|
7640
8021
|
activeNow,
|
|
7641
8022
|
`worktree-sweep ${resource}`,
|
|
7642
8023
|
last >= cutoff ? "clean-idle" : "ttl",
|
|
7643
8024
|
tree,
|
|
7644
|
-
)
|
|
8025
|
+
)) === "evicted"
|
|
7645
8026
|
)
|
|
7646
8027
|
evicted.push(binding.threadKey);
|
|
7647
8028
|
else kept++;
|
|
@@ -7656,15 +8037,16 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7656
8037
|
* (its orphaned OPS_DIR entry dies with the container disk at the latest).
|
|
7657
8038
|
* Both this allocator and allocateThreadUser exclude the set, so an op
|
|
7658
8039
|
* never shares an OS user with a thread or another op. */
|
|
7659
|
-
private opUsersInUse = new
|
|
8040
|
+
private opUsersInUse = new Map<string, string>();
|
|
7660
8041
|
|
|
7661
|
-
/**
|
|
7662
|
-
|
|
7663
|
-
|
|
7664
|
-
|
|
7665
|
-
|
|
7666
|
-
|
|
7667
|
-
|
|
8042
|
+
/** The inspection reservation transfers to the op reservation synchronously. */
|
|
8043
|
+
private async allocateOpUser(): Promise<string | ThreadErr> {
|
|
8044
|
+
const owner = `op:${crypto.randomUUID()}`;
|
|
8045
|
+
const reserved = await this.reserveSafePoolUser(owner);
|
|
8046
|
+
if (typeof reserved !== "string") return reserved;
|
|
8047
|
+
this.opUsersInUse.set(reserved, owner);
|
|
8048
|
+
this.poolUsersInspecting.delete(reserved);
|
|
8049
|
+
return reserved;
|
|
7668
8050
|
}
|
|
7669
8051
|
|
|
7670
8052
|
/** POST /op work half: run ONE readonly command-table entry in a
|
|
@@ -7681,7 +8063,17 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7681
8063
|
// One step trace per op (docs/reference/specs/tracing.md item 19), like an attach.
|
|
7682
8064
|
const t0 = systemClock();
|
|
7683
8065
|
const trace = createStepTrace(t0);
|
|
7684
|
-
|
|
8066
|
+
let res: OpRunOk | ThreadErr;
|
|
8067
|
+
if (this.recreateAdmission.pending) {
|
|
8068
|
+
res = this.recreateRefusal();
|
|
8069
|
+
} else {
|
|
8070
|
+
this.opAdmissionsInFlight++;
|
|
8071
|
+
try {
|
|
8072
|
+
res = await this.stepTrace.run(trace, () => this.runOpTraced(op, refArg, t0));
|
|
8073
|
+
} finally {
|
|
8074
|
+
this.opAdmissionsInFlight--;
|
|
8075
|
+
}
|
|
8076
|
+
}
|
|
7685
8077
|
emitStepRoot("resident.op", t0, trace.steps(), traceparent, "error" in res ? refusalOutcome(res) : "ok", {
|
|
7686
8078
|
command: op,
|
|
7687
8079
|
});
|
|
@@ -7689,6 +8081,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7689
8081
|
}
|
|
7690
8082
|
|
|
7691
8083
|
private async runOpTraced(op: "test" | "build", refArg: string | null, t0: number): Promise<OpRunOk | ThreadErr> {
|
|
8084
|
+
if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
|
|
7692
8085
|
// The fleet drain (item 69; issue 2044): a typed op is a new piece of work
|
|
7693
8086
|
// like a new run's attach, and during the incident one waited silently at
|
|
7694
8087
|
// the drain — so it is answered with the drain's own record at once, a
|
|
@@ -7715,6 +8108,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7715
8108
|
cause: "system",
|
|
7716
8109
|
};
|
|
7717
8110
|
}
|
|
8111
|
+
if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
|
|
7718
8112
|
// One storage round trip for the two facts; the registry lookup stays (an
|
|
7719
8113
|
// op resolves ONLY through the onboard-time command table).
|
|
7720
8114
|
const stored = await this.ctx.storage.get<string | RepoFacts>([RESOURCE_KEY, FACTS_KEY]);
|
|
@@ -7734,12 +8128,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7734
8128
|
if (!command) return { error: `op-unavailable: the command table has no "${op}" entry`, status: 400 };
|
|
7735
8129
|
|
|
7736
8130
|
const user = await this.allocateOpUser();
|
|
7737
|
-
if (
|
|
7738
|
-
return {
|
|
7739
|
-
error: `user-pool-exhausted: all ${THREAD_USERS.length} pool users are busy (threads or in-flight ops); try again shortly`,
|
|
7740
|
-
status: 429,
|
|
7741
|
-
};
|
|
7742
|
-
}
|
|
8131
|
+
if (typeof user !== "string") return user;
|
|
7743
8132
|
const opDir = `${OPS_DIR}/${crypto.randomUUID()}`;
|
|
7744
8133
|
const checkout = `${opDir}/checkout`;
|
|
7745
8134
|
try {
|
|
@@ -7883,8 +8272,13 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7883
8272
|
const all = await this.ctx.storage.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX });
|
|
7884
8273
|
const decision = selectBindingsToPurge([...all.values()], prefix);
|
|
7885
8274
|
if (!decision.ok) return { error: decision.error };
|
|
7886
|
-
|
|
7887
|
-
|
|
8275
|
+
const purged: string[] = [];
|
|
8276
|
+
const keptLive = [...decision.keptLive];
|
|
8277
|
+
for (const threadKey of decision.purge) {
|
|
8278
|
+
if (await this.deleteThreadBinding(threadKey, true)) purged.push(threadKey);
|
|
8279
|
+
else keptLive.push(threadKey);
|
|
8280
|
+
}
|
|
8281
|
+
return { purged, keptLive };
|
|
7888
8282
|
}
|
|
7889
8283
|
|
|
7890
8284
|
/** Debug fault injection: age a binding so the sweep's TTL path can be
|
|
@@ -7893,13 +8287,15 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7893
8287
|
const binding = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
|
|
7894
8288
|
if (!binding) return { ok: false };
|
|
7895
8289
|
const lastAttachAt = new Date(systemClock() - days * 86_400_000).toISOString();
|
|
7896
|
-
await this.
|
|
8290
|
+
await this.putThreadBinding({ ...binding, lastAttachAt } satisfies ThreadBinding);
|
|
7897
8291
|
return { ok: true, lastAttachAt };
|
|
7898
8292
|
}
|
|
7899
8293
|
|
|
7900
8294
|
/** Debug: run the sweep pass now (the exact function the `sweep` step runs). */
|
|
7901
|
-
async debugSweepNow(): Promise<{ evicted: string[]; kept: number }> {
|
|
7902
|
-
return this.
|
|
8295
|
+
async debugSweepNow(): Promise<{ evicted: string[]; kept: number } | ThreadErr> {
|
|
8296
|
+
return this.withRecreateSafeAdmin(async () =>
|
|
8297
|
+
this.sweepWorktrees((await this.ctx.storage.get<string>(RESOURCE_KEY)) ?? ""),
|
|
8298
|
+
);
|
|
7903
8299
|
}
|
|
7904
8300
|
|
|
7905
8301
|
// -- event-triggered reclamation ---------------------------------------------
|
|
@@ -8035,25 +8431,35 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
8035
8431
|
kept.push({ threadKey: binding.threadKey, ref: binding.ref, why: "re-attached" });
|
|
8036
8432
|
continue;
|
|
8037
8433
|
}
|
|
8038
|
-
const activeNow = await this.isRuntimeActive().catch(() =>
|
|
8434
|
+
const activeNow = await this.isRuntimeActive().catch(() => true);
|
|
8039
8435
|
const why = `${decision.why}${detail}`;
|
|
8040
|
-
|
|
8436
|
+
const eviction = await this.evictBinding(current, activeNow, `reclaim ${resource}`, why, tree);
|
|
8437
|
+
if (eviction === "evicted") {
|
|
8041
8438
|
reclaimed.push({ threadKey: binding.threadKey, ref: binding.ref, why });
|
|
8042
8439
|
console.log(`reclaim ${resource}: evicted ${binding.threadKey} on ${binding.ref} — ${why}`);
|
|
8043
|
-
} else
|
|
8440
|
+
} else
|
|
8441
|
+
kept.push({
|
|
8442
|
+
threadKey: binding.threadKey,
|
|
8443
|
+
ref: binding.ref,
|
|
8444
|
+
why: eviction === "changed" ? "re-attached" : "cleanup-failed",
|
|
8445
|
+
});
|
|
8044
8446
|
}
|
|
8045
8447
|
return { reclaimed, kept };
|
|
8046
8448
|
}
|
|
8047
8449
|
|
|
8048
8450
|
/** Debug: measure the disk now (admin) — the exact cycle/attach function. */
|
|
8049
|
-
async debugMeasureDisk(): Promise<DiskSample | null> {
|
|
8050
|
-
return this.measureDisk();
|
|
8451
|
+
async debugMeasureDisk(): Promise<DiskSample | null | ThreadErr> {
|
|
8452
|
+
return this.withRecreateSafeAdmin(() => this.measureDisk());
|
|
8051
8453
|
}
|
|
8052
8454
|
|
|
8053
8455
|
/** Debug: run the reclamation pass now (the exact refresh-cycle function,
|
|
8054
8456
|
* with a fresh mint when the App is configured). Runs a `fetch --prune`
|
|
8055
8457
|
* first so a branch deleted seconds ago already reads as gone. */
|
|
8056
|
-
async debugReclaimNow(): Promise<{ reclaimed: unknown[]; kept: unknown[]; fetch: string }> {
|
|
8458
|
+
async debugReclaimNow(): Promise<{ reclaimed: unknown[]; kept: unknown[]; fetch: string } | ThreadErr> {
|
|
8459
|
+
return this.withRecreateSafeAdmin(() => this.debugReclaimNowBody());
|
|
8460
|
+
}
|
|
8461
|
+
|
|
8462
|
+
private async debugReclaimNowBody(): Promise<{ reclaimed: unknown[]; kept: unknown[]; fetch: string }> {
|
|
8057
8463
|
const resource = (await this.ctx.storage.get<string>(RESOURCE_KEY)) ?? "";
|
|
8058
8464
|
const facts = await this.ctx.storage.get<RepoFacts>(FACTS_KEY);
|
|
8059
8465
|
if (!facts) return { reclaimed: [], kept: [], fetch: "skipped" };
|
|
@@ -8126,12 +8532,15 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
8126
8532
|
}
|
|
8127
8533
|
|
|
8128
8534
|
async getStatus(): Promise<ResidentStatus> {
|
|
8129
|
-
const map = await this.ctx.storage.get<string>([STATE_KEY, REASON_KEY]);
|
|
8535
|
+
const map = await this.ctx.storage.get<string | boolean>([STATE_KEY, REASON_KEY, DESTROY_UNCONFIRMED_KEY]);
|
|
8536
|
+
if (this.destroying || map.get(DESTROY_UNCONFIRMED_KEY)) {
|
|
8537
|
+
return { state: "down", reason: "resident-destroy-unconfirmed: container reuse refused" };
|
|
8538
|
+
}
|
|
8130
8539
|
const state = map.get(STATE_KEY) as ResidentState | undefined;
|
|
8131
8540
|
if (!state) {
|
|
8132
8541
|
return { state: "down", reason: "no resident state persisted (never onboarded, or already offboarded)" };
|
|
8133
8542
|
}
|
|
8134
|
-
return { state, reason: map.get(REASON_KEY) ?? "" };
|
|
8543
|
+
return { state, reason: (map.get(REASON_KEY) as string | undefined) ?? "" };
|
|
8135
8544
|
}
|
|
8136
8545
|
|
|
8137
8546
|
/** Full admin-facing view (surfaced via GET /residents and /debug info):
|
|
@@ -8176,6 +8585,9 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
8176
8585
|
INFRA_STREAK_KEY,
|
|
8177
8586
|
MEMORY_KEY,
|
|
8178
8587
|
IMAGE_REPORT_PENDING_KEY,
|
|
8588
|
+
DESTROY_UNCONFIRMED_KEY,
|
|
8589
|
+
RECREATE_ADMISSION_KEY,
|
|
8590
|
+
SPENT_POOL_USERS_KEY,
|
|
8179
8591
|
]);
|
|
8180
8592
|
const facts = map.get(FACTS_KEY) as RepoFacts | undefined;
|
|
8181
8593
|
const snap = map.get(SNAPSHOT_KEY) as SnapshotRecord | undefined;
|
|
@@ -8233,8 +8645,11 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
8233
8645
|
const registeredRuns = await this.registeredRunsBeyondOps();
|
|
8234
8646
|
return {
|
|
8235
8647
|
resource: map.get(RESOURCE_KEY) ?? null,
|
|
8236
|
-
state: map.get(STATE_KEY) ?? "down",
|
|
8237
|
-
reason:
|
|
8648
|
+
state: this.destroying || map.get(DESTROY_UNCONFIRMED_KEY) ? "down" : (map.get(STATE_KEY) ?? "down"),
|
|
8649
|
+
reason:
|
|
8650
|
+
this.destroying || map.get(DESTROY_UNCONFIRMED_KEY)
|
|
8651
|
+
? "resident-destroy-unconfirmed: container reuse refused"
|
|
8652
|
+
: (map.get(REASON_KEY) ?? ""),
|
|
8238
8653
|
updatedAt: map.get(UPDATED_KEY) ?? null,
|
|
8239
8654
|
defaultRef: facts?.defaultRef ?? null,
|
|
8240
8655
|
sha: facts?.sha ?? null,
|
|
@@ -8258,6 +8673,8 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
8258
8673
|
// the deploy's readers never print a bare "warm" for a repository whose
|
|
8259
8674
|
// running container the last deploy could not verify.
|
|
8260
8675
|
imageReport: map.get(IMAGE_REPORT_PENDING_KEY) !== undefined ? "pending" : "current",
|
|
8676
|
+
recreateAdmissionHeld: this.recreateAdmission.pending || map.get(RECREATE_ADMISSION_KEY) === true,
|
|
8677
|
+
poolUsersSpent: parseSpentPoolUsers(map.get(SPENT_POOL_USERS_KEY), THREAD_USERS)?.size ?? null,
|
|
8261
8678
|
// Item 22: who holds what, as the rows say — the mirror mutex and the
|
|
8262
8679
|
// cycle/hydration leases, each judged against this incarnation.
|
|
8263
8680
|
incarnation: this.incarnation,
|
|
@@ -8348,6 +8765,15 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
8348
8765
|
* recorded here, so the marker never strands `restoring`. */
|
|
8349
8766
|
async debugRecreateContainer(): Promise<
|
|
8350
8767
|
{ recreated: true; restoreStartedAt: string } | { recreated: false; error: string; status: number }
|
|
8768
|
+
> {
|
|
8769
|
+
const admission = await this.recreateAdmission.run(() => this.debugRecreateContainerChecked());
|
|
8770
|
+
return admission.busy
|
|
8771
|
+
? { recreated: false, status: 409, error: "recreate-refused: a VM recycle is already checking idleness" }
|
|
8772
|
+
: admission.value;
|
|
8773
|
+
}
|
|
8774
|
+
|
|
8775
|
+
private async debugRecreateContainerChecked(): Promise<
|
|
8776
|
+
{ recreated: true; restoreStartedAt: string } | { recreated: false; error: string; status: number }
|
|
8351
8777
|
> {
|
|
8352
8778
|
const from = await this.getStatus();
|
|
8353
8779
|
if (from.state === "onboarding" || from.state === "refreshing" || from.state === "restoring") {
|
|
@@ -8364,6 +8790,15 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
8364
8790
|
error: `recreate-refused: the resident is down (${from.reason}) — POST /rebuild is its exit`,
|
|
8365
8791
|
};
|
|
8366
8792
|
}
|
|
8793
|
+
const active =
|
|
8794
|
+
this.inFlightCount() +
|
|
8795
|
+
this.refreshAdmissionsInFlight +
|
|
8796
|
+
this.adminWorkInFlight +
|
|
8797
|
+
(this.hydration ? 1 : 0) +
|
|
8798
|
+
(await this.registeredRunsBeyondOps());
|
|
8799
|
+
if (active > 0) {
|
|
8800
|
+
return { recreated: false, status: 409, error: `recreate-refused: ${active} run(s) still in flight` };
|
|
8801
|
+
}
|
|
8367
8802
|
await this.recreateContainer(
|
|
8368
8803
|
"runtime-unreachable: the container was recreated by an operator (recreate-container), snapshots kept — the restore from the snapshot is starting",
|
|
8369
8804
|
);
|
|
@@ -8494,7 +8929,12 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
8494
8929
|
};
|
|
8495
8930
|
if (dryRun) return plan;
|
|
8496
8931
|
|
|
8497
|
-
//
|
|
8932
|
+
// A failed destroy must leave the old snapshot and budget intact, and
|
|
8933
|
+
// must never arm provisioning against an unconfirmed old VM.
|
|
8934
|
+
this.swapIncarnation();
|
|
8935
|
+
await this.destroyConfirmed();
|
|
8936
|
+
|
|
8937
|
+
// Old snapshot objects go before initResident wipes the stored handles,
|
|
8498
8938
|
// and backups/<id>/ lives outside the resident/<resource>/ prefix — this
|
|
8499
8939
|
// is the only path that can still reach them (same ordering as teardown).
|
|
8500
8940
|
let backupObjectsDeleted = 0;
|
|
@@ -8510,17 +8950,8 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
8510
8950
|
// rebuild has seen it. The automatic one keeps the history — it IS the
|
|
8511
8951
|
// budget's count.
|
|
8512
8952
|
if (!opts.auto) await this.ctx.storage.delete(AUTO_REBUILDS_KEY);
|
|
8513
|
-
//
|
|
8514
|
-
|
|
8515
|
-
// provisioning clones onto it. A rebuild that reprovisioned onto the
|
|
8516
|
-
// running container inherited its wedged runtime once — every exec of the
|
|
8517
|
-
// new provisioning met the same unanswered control port (item 64).
|
|
8518
|
-
this.swapIncarnation(); // deliberate incarnation swap
|
|
8519
|
-
await this.forgetRuntimeIdentity();
|
|
8520
|
-
await this.destroy().catch((err) =>
|
|
8521
|
-
console.log(`rebuild: destroy failed (provisioning starts anyway): ${errMsg(err)}`),
|
|
8522
|
-
);
|
|
8523
|
-
await this.initResident(resource, provisioningTimeoutMs);
|
|
8953
|
+
// Provisioning starts only after confirmed destruction.
|
|
8954
|
+
await this.initResident(resource, provisioningTimeoutMs, true);
|
|
8524
8955
|
return { ...plan, backupObjectsDeleted, state: "onboarding" as const };
|
|
8525
8956
|
}
|
|
8526
8957
|
|
|
@@ -8551,11 +8982,10 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
8551
8982
|
};
|
|
8552
8983
|
}
|
|
8553
8984
|
|
|
8554
|
-
/** Offboard teardown:
|
|
8555
|
-
*
|
|
8556
|
-
* resident
|
|
8557
|
-
*
|
|
8558
|
-
* storage. resident/<resource>/ objects are deleted by the Worker. */
|
|
8985
|
+
/** Offboard teardown: confirm container destruction before deleting timers
|
|
8986
|
+
* or state. Then delete the R2 objects behind the SDK backup handles,
|
|
8987
|
+
* which live outside the resident prefix and must be removed before the
|
|
8988
|
+
* handles in DO storage. The Worker deletes resident-prefix objects. */
|
|
8559
8989
|
async teardown(): Promise<{
|
|
8560
8990
|
schedulesCancelled: boolean;
|
|
8561
8991
|
containerStopped: boolean;
|
|
@@ -8564,8 +8994,9 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
8564
8994
|
errors: string[];
|
|
8565
8995
|
}> {
|
|
8566
8996
|
const errors: string[] = [];
|
|
8567
|
-
let containerStopped = false;
|
|
8568
8997
|
let backupObjectsDeleted = 0;
|
|
8998
|
+
await this.destroyConfirmed();
|
|
8999
|
+
const containerStopped = true;
|
|
8569
9000
|
this.deleteSchedules(PROVISIONING_CALLBACK);
|
|
8570
9001
|
this.deleteSchedules(PROVISION_RUN_CALLBACK);
|
|
8571
9002
|
const recorded = await this.recordedBackupIds(await this.ctx.storage.get<SnapshotRecord>(SNAPSHOT_KEY));
|
|
@@ -8583,12 +9014,6 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
8583
9014
|
} catch (err) {
|
|
8584
9015
|
errors.push(`deps backup deletion failed: ${errMsg(err)}`);
|
|
8585
9016
|
}
|
|
8586
|
-
try {
|
|
8587
|
-
await this.destroy();
|
|
8588
|
-
containerStopped = true;
|
|
8589
|
-
} catch (err) {
|
|
8590
|
-
errors.push(`destroy failed: ${errMsg(err)}`);
|
|
8591
|
-
}
|
|
8592
9017
|
// Retired DO: clear the alarm the Container base may have armed for its
|
|
8593
9018
|
// schedules, then delete every stored key — ours and the SDK's (its runtime
|
|
8594
9019
|
// identity among them) — so nothing ever wakes this object again. Keys, not
|
|
@@ -9107,9 +9532,15 @@ async function handleOnboard(env: Env, body: Record<string, unknown>): Promise<R
|
|
|
9107
9532
|
// destroyed for nothing. Only once the newcomer holds the slot is the
|
|
9108
9533
|
// victim's DO/R2 state torn down (its registry row is already gone, so no
|
|
9109
9534
|
// new work routes to it meanwhile).
|
|
9535
|
+
const victim = await registry.getRecord(pick.candidate.resource);
|
|
9536
|
+
if (!victim) return json({ error: "eviction refused: candidate registry row vanished" }, 409);
|
|
9110
9537
|
result = await registry.replace(pick.candidate.resource, record);
|
|
9111
9538
|
if ("error" in result) return json({ error: result.error, wouldHaveEvicted: pick.candidate }, result.status);
|
|
9112
9539
|
const teardown = await teardownResident(env, pick.candidate.resource);
|
|
9540
|
+
if (!teardown.containerStopped) {
|
|
9541
|
+
await registry.replace(record.resource, victim);
|
|
9542
|
+
return json({ error: "eviction refused: old container destruction unconfirmed", evicted: teardown }, 503);
|
|
9543
|
+
}
|
|
9113
9544
|
evicted = { ...pick.candidate, registryRemoved: true, ...teardown };
|
|
9114
9545
|
console.log(
|
|
9115
9546
|
`lru-evict: offboarded ${pick.candidate.resource} (last activity ${pick.candidate.lastActivityAt}) to make room for ${resource.resource}`,
|
|
@@ -9172,7 +9603,12 @@ async function handleOffboard(env: Env, body: Record<string, unknown>): Promise<
|
|
|
9172
9603
|
|
|
9173
9604
|
const registry = registryStub(env);
|
|
9174
9605
|
const record = await registry.getRecord(resource.resource);
|
|
9175
|
-
if (!record)
|
|
9606
|
+
if (!record) {
|
|
9607
|
+
// A prior destroy may have failed after registry removal. Retry teardown
|
|
9608
|
+
// on that same DO identity; a fully removed resident has no resource row.
|
|
9609
|
+
const old = await residentStub(env, resource.resource).getResidentInfo();
|
|
9610
|
+
if (old.resource !== resource.resource) return json({ error: `${resource.resource} is not onboarded` }, 404);
|
|
9611
|
+
}
|
|
9176
9612
|
|
|
9177
9613
|
// --dry-run: the itemized plan of what the real teardown below would
|
|
9178
9614
|
// remove, computed READ-ONLY — the resident stays fully intact (state,
|
|
@@ -9204,7 +9640,8 @@ async function handleOffboard(env: Env, body: Record<string, unknown>): Promise<
|
|
|
9204
9640
|
});
|
|
9205
9641
|
}
|
|
9206
9642
|
|
|
9207
|
-
|
|
9643
|
+
const result = await offboardResident(env, resource.resource);
|
|
9644
|
+
return json(result, result.containerStopped ? 200 : 503);
|
|
9208
9645
|
}
|
|
9209
9646
|
|
|
9210
9647
|
/** The full offboard teardown (item 11), shared by POST /offboard and the LRU
|
|
@@ -9220,7 +9657,7 @@ async function offboardResident(
|
|
|
9220
9657
|
return { resource, registryRemoved, ...(await teardownResident(env, resource)) };
|
|
9221
9658
|
}
|
|
9222
9659
|
|
|
9223
|
-
/** Everything AFTER the registry removal: DO teardown
|
|
9660
|
+
/** Everything AFTER the registry removal: DO teardown, then R2 prefix sweep.
|
|
9224
9661
|
* Split out so the LRU path can reserve the slot atomically (registry
|
|
9225
9662
|
* `replace`) before destroying anything. */
|
|
9226
9663
|
async function teardownResident(
|
|
@@ -9234,26 +9671,25 @@ async function teardownResident(
|
|
|
9234
9671
|
r2ObjectsDeleted: number;
|
|
9235
9672
|
errors: string[];
|
|
9236
9673
|
}> {
|
|
9237
|
-
const
|
|
9238
|
-
|
|
9239
|
-
|
|
9240
|
-
|
|
9241
|
-
|
|
9242
|
-
|
|
9243
|
-
|
|
9244
|
-
|
|
9245
|
-
|
|
9246
|
-
|
|
9247
|
-
|
|
9248
|
-
|
|
9249
|
-
|
|
9250
|
-
|
|
9251
|
-
|
|
9252
|
-
|
|
9253
|
-
|
|
9254
|
-
|
|
9255
|
-
|
|
9256
|
-
]);
|
|
9674
|
+
const teardown = await residentStub(env, resource)
|
|
9675
|
+
.teardown()
|
|
9676
|
+
.catch((err: unknown): Awaited<ReturnType<ResidentDO["teardown"]>> => ({
|
|
9677
|
+
schedulesCancelled: false,
|
|
9678
|
+
containerStopped: false,
|
|
9679
|
+
storageCleared: false,
|
|
9680
|
+
backupObjectsDeleted: 0,
|
|
9681
|
+
errors: [`teardown failed: ${errMsg(err)}`],
|
|
9682
|
+
}));
|
|
9683
|
+
// If destroy was uncertain, preserve its durable refusal and all resident
|
|
9684
|
+
// objects. A later admin offboard can retry the same DO identity.
|
|
9685
|
+
const r2Sweep = teardown.containerStopped
|
|
9686
|
+
? await deleteR2Prefix(env.BACKUP_BUCKET, r2Prefix(resource))
|
|
9687
|
+
.then((deleted) => ({ deleted, error: undefined as string | undefined }))
|
|
9688
|
+
.catch((err: unknown) => ({
|
|
9689
|
+
deleted: 0,
|
|
9690
|
+
error: `r2 prefix sweep failed for ${r2Prefix(resource)}: ${errMsg(err)}`,
|
|
9691
|
+
}))
|
|
9692
|
+
: { deleted: 0, error: undefined };
|
|
9257
9693
|
|
|
9258
9694
|
const errors = [...teardown.errors];
|
|
9259
9695
|
if (r2Sweep.error) errors.push(r2Sweep.error);
|
|
@@ -9948,10 +10384,14 @@ async function handleDebug(env: Env, body: Record<string, unknown>): Promise<Res
|
|
|
9948
10384
|
return json(await stub.debugThreads());
|
|
9949
10385
|
case "deps-backups":
|
|
9950
10386
|
return json(await stub.debugDepsBackups());
|
|
9951
|
-
case "sweep-now":
|
|
9952
|
-
|
|
9953
|
-
|
|
9954
|
-
|
|
10387
|
+
case "sweep-now": {
|
|
10388
|
+
const result = await stub.debugSweepNow();
|
|
10389
|
+
return json(result, "error" in result ? result.status : 200);
|
|
10390
|
+
}
|
|
10391
|
+
case "reclaim-now": {
|
|
10392
|
+
const result = await stub.debugReclaimNow();
|
|
10393
|
+
return json(result, "error" in result ? result.status : 200);
|
|
10394
|
+
}
|
|
9955
10395
|
case "purge-bindings": {
|
|
9956
10396
|
// Item 56: drop a load run's evicted synthetic bindings. `prefix` is
|
|
9957
10397
|
// validated by the pure decision (a whole non-production namespace).
|
|
@@ -9959,10 +10399,12 @@ async function handleDebug(env: Env, body: Record<string, unknown>): Promise<Res
|
|
|
9959
10399
|
const r = await stub.debugPurgeBindings(prefix);
|
|
9960
10400
|
return "error" in r ? json({ error: r.error }, 400) : json(r);
|
|
9961
10401
|
}
|
|
9962
|
-
case "measure-disk":
|
|
10402
|
+
case "measure-disk": {
|
|
9963
10403
|
// Item 55: take the sample now (df + one du) and answer it — the live
|
|
9964
10404
|
// check's way to read the gauge without waiting for a cycle.
|
|
9965
|
-
|
|
10405
|
+
const disk = await stub.debugMeasureDisk();
|
|
10406
|
+
return disk && "error" in disk ? json(disk, disk.status) : json({ disk });
|
|
10407
|
+
}
|
|
9966
10408
|
case "backdate-thread": {
|
|
9967
10409
|
const threadKey = parseThreadKey(body.threadKey);
|
|
9968
10410
|
if ("error" in threadKey) return json({ error: threadKey.error }, 400);
|