@coreplane/switchboard 1.267.2 → 1.268.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/deploy/cloudflare-memory/testDiagnostics.ts +44 -3
- package/dist/assets/deploy/cloudflare-memory/testDiagnosticsProtocol.ts +7 -0
- package/dist/assets/deploy/cloudflare-memory/testDiagnosticsReporter.ts +22 -0
- package/dist/assets/deploy/cloudflare-memory/testFetch.ts +21 -0
- package/dist/assets/deploy/cloudflare-memory/worker.ts +66 -1
- package/dist/assets/deploy/cloudflare-resident/imageReconcileState.ts +28 -0
- package/dist/assets/deploy/cloudflare-resident/runRegistration.ts +68 -0
- package/dist/assets/deploy/cloudflare-resident/shared.ts +2 -3
- package/dist/assets/deploy/cloudflare-resident/worker.ts +541 -192
- package/dist/assets/deploy/cloudflare-sandbox/worker.ts +17 -2
- package/dist/assets/package-lock.json +3 -3
- package/dist/assets/package.json +1 -1
- package/dist/assets/source.json +3 -3
- package/dist/assets/src/agents/registry.ts +2 -2
- package/dist/assets/src/core/budgets.ts +6 -0
- package/dist/assets/src/core/coordinator/contract.ts +14 -8
- package/dist/assets/src/core/costs.ts +2 -0
- package/dist/assets/src/core/runEvents.ts +18 -1
- package/dist/assets/src/core/runLedger/types.ts +2 -0
- package/dist/assets/src/core/runRecord.ts +27 -1
- package/dist/assets/src/core/runUsage.ts +53 -0
- package/dist/assets/src/core/ship/coordinator.ts +4 -1
- package/dist/assets/src/core/shipSpend.ts +52 -0
- package/dist/assets/src/core/trace/attrs.ts +22 -0
- package/dist/assets/src/core/trace/tracer.ts +10 -2
- package/dist/assets/src/core/trace/types.ts +12 -1
- package/dist/assets/src/core/trace/workerTrace.ts +2 -2
- package/dist/assets/src/execution/residentPoolSpends.ts +9 -0
- package/dist/assets/src/execution/residentRecreateAdmission.ts +28 -0
- package/dist/assets/src/execution/seedPlan.ts +13 -4
- package/dist/assets/web/dist/.vite/manifest.json +31 -31
- package/dist/assets/web/dist/assets/CostsPage-6P9rJ7k_.js +1 -0
- package/dist/assets/web/dist/assets/{HomePage-BQmun9kS.js → HomePage-B4LGwXHO.js} +1 -1
- package/dist/assets/web/dist/assets/ResidentDetailPage-jpGfsR5n.js +1 -0
- package/dist/assets/web/dist/assets/ResidentsIndexPage-C4-5nG9M.js +1 -0
- package/dist/assets/web/dist/assets/{RunFoldRow-C1w-mwch.js → RunFoldRow-u4FTh57c.js} +1 -1
- package/dist/assets/web/dist/assets/{RunRoutePage-zWMHTzla.js → RunRoutePage-CxOKb7oC.js} +4 -4
- package/dist/assets/web/dist/assets/{RunsIndexPage-DF16D2Gg.js → RunsIndexPage-CULSlv_d.js} +1 -1
- package/dist/assets/web/dist/assets/{ScheduledPage-CyRs0fXN.js → ScheduledPage-CDjkZuqM.js} +1 -1
- package/dist/assets/web/dist/assets/{SettingsPage-BRuMCHSj.js → SettingsPage-BMSomotV.js} +1 -1
- package/dist/assets/web/dist/assets/{StatusDot-BIAQN7iD.js → StatusDot-XxLkmRaK.js} +1 -1
- package/dist/assets/web/dist/assets/{Tooltip-CxiQ7luL.js → Tooltip-qB73RL4b.js} +1 -1
- package/dist/assets/web/dist/assets/{UnitRoutePage-BQB5mAAl.js → UnitRoutePage-DFRWAdOT.js} +1 -1
- package/dist/assets/web/dist/assets/{dist-I2OEmPqz.js → dist-BsLBAkQJ.js} +1 -1
- package/dist/assets/web/dist/assets/{main-DYwKyLY5.js → main-9ynpo780.js} +10 -10
- package/dist/assets/web/dist/assets/main-DSiJ9sHU.css +1 -0
- package/dist/cli.js +1178 -431
- package/package.json +1 -1
- package/dist/assets/web/dist/assets/CostsPage-RRggVvIx.js +0 -1
- package/dist/assets/web/dist/assets/ResidentDetailPage-CT6v9INM.js +0 -1
- package/dist/assets/web/dist/assets/ResidentsIndexPage-CWqPFf40.js +0 -1
- package/dist/assets/web/dist/assets/main-DPB0UO1-.css +0 -1
|
@@ -89,6 +89,18 @@ import {
|
|
|
89
89
|
} from "./legacyCredentials.js";
|
|
90
90
|
import { hasUnexpectedOwnedThreadDir } from "./orphanThreadUsers.js";
|
|
91
91
|
import { KeyedAsyncLock } from "./keyedAsyncLock.js";
|
|
92
|
+
import {
|
|
93
|
+
nextDeployImageReconcile,
|
|
94
|
+
replacementContainerStarted,
|
|
95
|
+
type DeployImageReconcileState,
|
|
96
|
+
} from "./imageReconcileState.js";
|
|
97
|
+
import {
|
|
98
|
+
registeredRunAllowsClaim,
|
|
99
|
+
registeredRunAllowsReattach,
|
|
100
|
+
registeredRunNeedsProtection,
|
|
101
|
+
registeredRunOwnsRelease,
|
|
102
|
+
} from "./runRegistration.js";
|
|
103
|
+
import { DAY_MS, RUN_REGISTRATION_GRACE_MS } from "../../src/core/budgets.js";
|
|
92
104
|
import { BASH_TIMEOUT_MAX_MS, clampBashTimeout } from "../../src/execution/bashTimeout.js";
|
|
93
105
|
import { RESIDENT_MIRROR_PERMISSIONS, verifyGithubMintScope } from "../../src/execution/githubMintScope.js";
|
|
94
106
|
import { selectBindingsToPurge } from "../../src/execution/bindingPurge.js";
|
|
@@ -172,15 +184,17 @@ import { shellQuote } from "../../src/execution/shellQuote.js";
|
|
|
172
184
|
import { envFromRequest } from "../../src/execution/sandboxEnv.js";
|
|
173
185
|
import { CREDENTIAL_EXPIRY_MARGIN_MS } from "../../src/execution/residentCredentials.js";
|
|
174
186
|
import { destroyWithPersistentFence } from "../../src/execution/residentDestroyGate.js";
|
|
175
|
-
import { ResidentRecreateAdmission } from "../../src/execution/residentRecreateAdmission.js";
|
|
187
|
+
import { ResidentRecreateAdmission, idleForPoolRecycle } from "../../src/execution/residentRecreateAdmission.js";
|
|
176
188
|
import {
|
|
177
189
|
claimPoolBinding,
|
|
178
190
|
mayRunAsPoolUser,
|
|
191
|
+
ownedPoolUsers,
|
|
179
192
|
parsePoolBindings,
|
|
180
193
|
parseSpentPoolUsers,
|
|
181
194
|
rebuildPoolBindingIndex,
|
|
182
195
|
releasePoolBinding,
|
|
183
196
|
spendPoolUser,
|
|
197
|
+
unavailablePoolUsers,
|
|
184
198
|
} from "../../src/execution/residentPoolSpends.js";
|
|
185
199
|
import {
|
|
186
200
|
recordFiring,
|
|
@@ -584,16 +598,10 @@ const THREADS_DIR = "/workspace/threads";
|
|
|
584
598
|
* at the latest (disk is cache). */
|
|
585
599
|
const OPS_DIR = "/workspace/ops";
|
|
586
600
|
|
|
587
|
-
/**
|
|
588
|
-
*
|
|
589
|
-
*
|
|
590
|
-
*
|
|
591
|
-
/** Pool of OS users for thread worktrees (worker1 is the build user). Sized
|
|
592
|
-
* for SIMULTANEOUS runs, not for every thread ever seen: a run returns its
|
|
593
|
-
* user via /detach when it ends, so the pool only fills when 16 runs on one
|
|
594
|
-
* repo are genuinely concurrent. Memory, not this list, is the real ceiling
|
|
595
|
-
* — see the instance_type note in wrangler.jsonc. Must match the useradd loop
|
|
596
|
-
* in the Dockerfile. */
|
|
601
|
+
/** Pool of OS users for thread worktrees and disposable ops. A UID stays with
|
|
602
|
+
* its first owner for this VM generation; detach releases the live binding,
|
|
603
|
+
* not the UID spend. Exhaustion may recycle a proven-idle VM. Must match the
|
|
604
|
+
* useradd loop in the Dockerfile. */
|
|
597
605
|
const THREAD_USERS = Array.from({ length: THREAD_POOL_SIZE }, (_, i) => `worker${i + 2}`);
|
|
598
606
|
|
|
599
607
|
/** Force-detach: after killing the thread user's processes, how long
|
|
@@ -1194,6 +1202,8 @@ interface ThreadBinding {
|
|
|
1194
1202
|
lastAttachAt: string;
|
|
1195
1203
|
evicted?: boolean;
|
|
1196
1204
|
evictedAt?: string;
|
|
1205
|
+
/** Owner of the registration removed with this tree, for a delayed push receipt. */
|
|
1206
|
+
lastRunOwner?: { runId?: string; ownerGen?: string; ownerFence?: number };
|
|
1197
1207
|
/** Why the last eviction happened (the audit trail): `ttl`, `clean-idle`,
|
|
1198
1208
|
* `detach`, `disk-pressure`, or a reclamation fate — `merged #N` /
|
|
1199
1209
|
* `closed #N` / `gone`. */
|
|
@@ -1287,7 +1297,7 @@ interface AttachOk {
|
|
|
1287
1297
|
returned?: Returned;
|
|
1288
1298
|
}
|
|
1289
1299
|
|
|
1290
|
-
/** What `POST /detach` answers: whether the
|
|
1300
|
+
/** What `POST /detach` answers: whether the live binding was released, why not, and
|
|
1291
1301
|
* — on a release — what the tree still held (item 16a), now gone with it. */
|
|
1292
1302
|
interface DetachAnswer {
|
|
1293
1303
|
released: boolean;
|
|
@@ -1469,6 +1479,7 @@ const DRAIN_KEY = "drain";
|
|
|
1469
1479
|
* verify this resident's container on the new image; the next reconcile that
|
|
1470
1480
|
* finds it current reports to the registry and clears it. */
|
|
1471
1481
|
const IMAGE_REPORT_PENDING_KEY = "imageReportPending";
|
|
1482
|
+
const IMAGE_RECONCILE_KEY = "imageReconcile";
|
|
1472
1483
|
|
|
1473
1484
|
type OnboardResult = { ok: true; record: ResidentRecord } | { ok: false; status: number; error: string };
|
|
1474
1485
|
|
|
@@ -1764,9 +1775,17 @@ const parentDir = (p: string): string => p.slice(0, p.lastIndexOf("/"));
|
|
|
1764
1775
|
* a fresh isolate still counts the process that survived in the container. */
|
|
1765
1776
|
const RUN_REG_KEY_PREFIX = "runReg:";
|
|
1766
1777
|
const runRegKey = (threadKey: string) => `${RUN_REG_KEY_PREFIX}${threadKey}`;
|
|
1778
|
+
// A thread's highest accepted ledger claim survives registration release and
|
|
1779
|
+
// binding purge: a delayed older /attach must never reopen an evicted tree.
|
|
1780
|
+
const RUN_FENCE_KEY_PREFIX = "runFence:";
|
|
1781
|
+
const runFenceKey = (threadKey: string) => `${RUN_FENCE_KEY_PREFIX}${threadKey}`;
|
|
1767
1782
|
interface RunRegistration {
|
|
1768
1783
|
threadKey: string;
|
|
1769
1784
|
registeredAt: string;
|
|
1785
|
+
deadlineAt?: number;
|
|
1786
|
+
runId?: string;
|
|
1787
|
+
ownerGen?: string;
|
|
1788
|
+
ownerFence?: number;
|
|
1770
1789
|
}
|
|
1771
1790
|
|
|
1772
1791
|
/** Deterministic per-thread+ref worktree path. Slugs replace anything outside
|
|
@@ -3893,8 +3912,8 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
3893
3912
|
await this.ctx.storage.delete(INFRA_STREAK_KEY);
|
|
3894
3913
|
await this.setResidentState("warm");
|
|
3895
3914
|
// Event-triggered reclamation: the prune above already told the
|
|
3896
|
-
// mirror which branches died; finished refs
|
|
3897
|
-
//
|
|
3915
|
+
// mirror which branches died; finished refs release their worktree and
|
|
3916
|
+
// live binding now, not at the idle TTL. Housekeeping, never a
|
|
3898
3917
|
// lifecycle flip — a failure here is a log line.
|
|
3899
3918
|
try {
|
|
3900
3919
|
const gc = await this.reclaimFinishedRefs(resource, facts.defaultRef, cycle.token);
|
|
@@ -4644,23 +4663,21 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4644
4663
|
|
|
4645
4664
|
/** Whether a run may be using this disk: a live binding attached to or
|
|
4646
4665
|
* used (an exec bumps `lastAttachAt` too, item 21) within IDLE_AFTER_S.
|
|
4647
|
-
* The
|
|
4648
|
-
*
|
|
4649
|
-
*
|
|
4650
|
-
*
|
|
4651
|
-
* what a tree holds (item 17): a dirty tree no run is using protects
|
|
4652
|
-
* nothing — the next attach wipes it. Bindings are storage, so this needs
|
|
4653
|
-
* no container. */
|
|
4666
|
+
* The run registration protects the gap between tool calls past this floor.
|
|
4667
|
+
* The idle-sleep gate (item 16b) and disk-full recycle (item 54) both read
|
|
4668
|
+
* the floor and the bounded registrations, never what a tree holds (item
|
|
4669
|
+
* 17): a dirty tree no run is using protects nothing. */
|
|
4654
4670
|
private recentlyUsed(live: ThreadBinding[]): boolean {
|
|
4655
4671
|
const recent = systemClock() - IDLE_AFTER_S * 1000;
|
|
4656
4672
|
return live.some((b) => Date.parse(b.lastAttachAt) >= recent);
|
|
4657
4673
|
}
|
|
4658
4674
|
|
|
4659
|
-
/**
|
|
4660
|
-
*
|
|
4675
|
+
/** A live run registration prevents sleep even after an hour without an op;
|
|
4676
|
+
* an expired one cannot keep the container awake. */
|
|
4661
4677
|
private async isIdle(): Promise<boolean> {
|
|
4662
4678
|
if (this.recentlyUsed(await this.liveBindings())) return false;
|
|
4663
|
-
|
|
4679
|
+
if (this.inFlightCount() > 0) return false;
|
|
4680
|
+
return (await this.registeredRunsBeyondOps()) === 0;
|
|
4664
4681
|
}
|
|
4665
4682
|
|
|
4666
4683
|
/** Bindings that hold a pool user and a tree on disk (not evicted). */
|
|
@@ -4698,10 +4715,10 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4698
4715
|
* path as a platform sleep. Only when the pure plan allows it: no recycle
|
|
4699
4716
|
* within the cooldown, nothing in flight (`selfInFlight` excludes the
|
|
4700
4717
|
* calling refresh cycle from the count) and no live binding used within
|
|
4701
|
-
* the idle floor — the idle-sleep gate's own
|
|
4702
|
-
* because a sleep and a recycle destroy the same disk: it may
|
|
4703
|
-
* no run is using it, never for what the trees hold (item 17).
|
|
4704
|
-
* discards every live tree the way an eviction does, so each is measured
|
|
4718
|
+
* the idle floor and no live run registration — the idle-sleep gate's own
|
|
4719
|
+
* predicates, because a sleep and a recycle destroy the same disk: it may
|
|
4720
|
+
* go away when no run is using it, never for what the trees hold (item 17).
|
|
4721
|
+
* A recycle discards every live tree the way an eviction does, so each is measured
|
|
4705
4722
|
* first, as its thread user, and its binding records what went
|
|
4706
4723
|
* (`recordRecycledTree`); the plan is decided again after those awaits,
|
|
4707
4724
|
* right before the stop. A refused recycle is written to
|
|
@@ -4710,21 +4727,23 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4710
4727
|
* whether the container was recycled. */
|
|
4711
4728
|
private async recoverFromDiskFull(reason: string, selfInFlight: number): Promise<boolean> {
|
|
4712
4729
|
const lastRecycleAt = await this.ctx.storage.get<number>(DISK_FULL_RECYCLE_KEY);
|
|
4713
|
-
const plan = (live: ThreadBinding[]) =>
|
|
4714
|
-
|
|
4730
|
+
const plan = async (live: ThreadBinding[]) => {
|
|
4731
|
+
const liveRegistrations = await this.registeredRunsBeyondOps();
|
|
4732
|
+
return planDiskFullRecovery({
|
|
4715
4733
|
now: systemClock(),
|
|
4716
4734
|
lastRecycleAt,
|
|
4717
4735
|
inFlight: this.inFlightCount() - selfInFlight,
|
|
4718
|
-
recentlyUsed: this.recentlyUsed(live),
|
|
4736
|
+
recentlyUsed: this.recentlyUsed(live) || liveRegistrations > 0,
|
|
4719
4737
|
idleFloorS: IDLE_AFTER_S,
|
|
4720
4738
|
});
|
|
4739
|
+
};
|
|
4721
4740
|
const kept = async (why: string) => {
|
|
4722
4741
|
console.log(`disk-full: container kept — ${why}`);
|
|
4723
4742
|
await this.recordRefreshError(`${reason} — container kept: ${why}`);
|
|
4724
4743
|
return false;
|
|
4725
4744
|
};
|
|
4726
4745
|
const live = await this.liveBindings();
|
|
4727
|
-
const first = plan(live);
|
|
4746
|
+
const first = await plan(live);
|
|
4728
4747
|
if (first.action === "wait") return kept(first.why);
|
|
4729
4748
|
// What each live tree holds, for its record — never a reason to keep the
|
|
4730
4749
|
// container. Concurrent: each probe touches only its own tree, one spawn
|
|
@@ -4737,7 +4756,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4737
4756
|
: live.map((binding) => [binding, undefined] as const);
|
|
4738
4757
|
// The measurement awaited (the DO yields at each await): decide again over
|
|
4739
4758
|
// fresh facts — an attach or an op that landed meanwhile keeps the container.
|
|
4740
|
-
const verdict = plan(await this.liveBindings());
|
|
4759
|
+
const verdict = await plan(await this.liveBindings());
|
|
4741
4760
|
if (verdict.action === "wait") return kept(verdict.why);
|
|
4742
4761
|
console.log(
|
|
4743
4762
|
`disk-full: recycling the container — the next cycle restores mirror + checkout from R2 onto an empty disk (${reason})`,
|
|
@@ -5153,8 +5172,8 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5153
5172
|
* run — stop the container so it restarts on the current image (state is DO
|
|
5154
5173
|
* storage + R2 — the disk is a cache). A deferred restart re-checks on
|
|
5155
5174
|
* every later refresh cycle until the resident is quiet; a registration
|
|
5156
|
-
* whose release never came defers it only
|
|
5157
|
-
*
|
|
5175
|
+
* whose release never came defers it only through the clean-idle window;
|
|
5176
|
+
* a later sweep removes its binding. The attach path never stops the container (issue 2101,
|
|
5158
5177
|
* the `stale` answer below); the stop is the refresh cycle's or the
|
|
5159
5178
|
* deploy's alone. Answers `restarted` when a stop was issued, else why
|
|
5160
5179
|
* not. */
|
|
@@ -5233,7 +5252,12 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5233
5252
|
}
|
|
5234
5253
|
console.log(`image-stale (${where}): ${stale} — stopping so it restarts on the current image`);
|
|
5235
5254
|
this.swapIncarnation(); // deliberate incarnation swap
|
|
5236
|
-
|
|
5255
|
+
try {
|
|
5256
|
+
await this.stop();
|
|
5257
|
+
} catch (err) {
|
|
5258
|
+
console.log(`image-stale: stop failed: ${errMsg(err)}`);
|
|
5259
|
+
if (where === "deploy") return "deferred";
|
|
5260
|
+
}
|
|
5237
5261
|
// A successful stop proves only that the old process is gone. The pending
|
|
5238
5262
|
// marker deliberately survives it: the replacement's own hydration reports
|
|
5239
5263
|
// after reaching `warm`, so `imageReport: current` can never precede a
|
|
@@ -5289,9 +5313,66 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5289
5313
|
* added afterwards would wait for a report nothing sends (until the drain's
|
|
5290
5314
|
* `until`). Held first, every report finds its hold. */
|
|
5291
5315
|
async reconcileForDeploy(resource: string): Promise<{ result: ImageReconcileResult; verified: boolean }> {
|
|
5292
|
-
|
|
5293
|
-
|
|
5316
|
+
// The deploy's image report and an idle pool recycle both reason about the
|
|
5317
|
+
// same VM. Admit reconciliation as disk work before its first await, so a
|
|
5318
|
+
// recycle sees it as busy; a recycle already holding admission defers this
|
|
5319
|
+
// pass for the deploy's bounded readiness retry.
|
|
5320
|
+
const admitted = await this.withRecreateSafeAdmin(() => this.reconcileForDeployAdmitted(resource));
|
|
5321
|
+
return "error" in admitted ? { result: "deferred", verified: false } : admitted;
|
|
5322
|
+
}
|
|
5323
|
+
|
|
5324
|
+
private async reconcileForDeployAdmitted(
|
|
5325
|
+
resource: string,
|
|
5326
|
+
): Promise<{ result: ImageReconcileResult; verified: boolean }> {
|
|
5327
|
+
const progress = await this.ctx.storage.get<DeployImageReconcileState>(IMAGE_RECONCILE_KEY);
|
|
5328
|
+
const pending = await this.ctx.storage.get<{ resource: string }>(IMAGE_REPORT_PENDING_KEY);
|
|
5329
|
+
const action = nextDeployImageReconcile(BUILD.commit, progress, pending !== undefined);
|
|
5330
|
+
if (action === "verified") return { result: "current", verified: true };
|
|
5331
|
+
if (action === "start") {
|
|
5332
|
+
await registryStub(this.env).holdDrainFor([resource]);
|
|
5333
|
+
await this.ctx.storage.put(IMAGE_REPORT_PENDING_KEY, { resource });
|
|
5334
|
+
await this.ctx.storage.put(IMAGE_RECONCILE_KEY, { build: BUILD.commit, cycleIssued: false });
|
|
5335
|
+
}
|
|
5336
|
+
if (action === "await-report") {
|
|
5337
|
+
const active = await this.isRuntimeActive().catch(() => null);
|
|
5338
|
+
if (active === false) await this.reportPendingImageCurrent("deploy");
|
|
5339
|
+
else if (active === true) {
|
|
5340
|
+
const [state, facts] = await Promise.all([
|
|
5341
|
+
this.ctx.storage.get<ResidentState>(STATE_KEY),
|
|
5342
|
+
this.ctx.storage.get<RepoFacts>(FACTS_KEY),
|
|
5343
|
+
]);
|
|
5344
|
+
// A replacement can reuse the disk and leave `warm` unchanged; that
|
|
5345
|
+
// hydrate shortcut does not send a report. A different boot after the
|
|
5346
|
+
// successful stop is proof that a replacement started. Finish its
|
|
5347
|
+
// hydration before reporting; an old warm state alone proves nothing.
|
|
5348
|
+
if (replacementContainerStarted(progress?.containerBeforeCycle, await this.containerIdentity())) {
|
|
5349
|
+
await this.ensureHydrated();
|
|
5350
|
+
await this.reportPendingImageCurrent("deploy");
|
|
5351
|
+
} else if (
|
|
5352
|
+
state === "warm" &&
|
|
5353
|
+
facts?.lastRestore?.at &&
|
|
5354
|
+
Date.parse(facts.lastRestore.at) >= Date.parse(progress?.cycleStoppedAt ?? "")
|
|
5355
|
+
) {
|
|
5356
|
+
await this.reportPendingImageCurrent("deploy");
|
|
5357
|
+
}
|
|
5358
|
+
}
|
|
5359
|
+
return {
|
|
5360
|
+
result: active === false ? "inactive" : "deferred",
|
|
5361
|
+
verified: (await this.ctx.storage.get(IMAGE_REPORT_PENDING_KEY)) === undefined,
|
|
5362
|
+
};
|
|
5363
|
+
}
|
|
5364
|
+
// Reading the boot ID runs a command and would wake a sleeping resident.
|
|
5365
|
+
// Preserve the inactive path's report-without-start behavior.
|
|
5366
|
+
const activeBeforeCycle = await this.isRuntimeActive().catch(() => null);
|
|
5367
|
+
const containerBeforeCycle = activeBeforeCycle === true ? await this.containerIdentity() : undefined;
|
|
5294
5368
|
const result = await this.reconcileImage("deploy", true);
|
|
5369
|
+
if (result === "restarted")
|
|
5370
|
+
await this.ctx.storage.put(IMAGE_RECONCILE_KEY, {
|
|
5371
|
+
build: BUILD.commit,
|
|
5372
|
+
cycleIssued: true,
|
|
5373
|
+
cycleStoppedAt: new Date(systemClock()).toISOString(),
|
|
5374
|
+
containerBeforeCycle,
|
|
5375
|
+
} satisfies DeployImageReconcileState);
|
|
5295
5376
|
if (result === "deferred") return { result, verified: false };
|
|
5296
5377
|
// Inactivity reports synchronously; a successful stop does not. A
|
|
5297
5378
|
// concurrent fresh start may also have hydrated while this reconcile
|
|
@@ -5598,7 +5679,11 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5598
5679
|
/** Storage bindings, active ops and inspections all reserve a pool user.
|
|
5599
5680
|
* Claim inside this method, before its promise resolves: two concurrent
|
|
5600
5681
|
* callers must not each read the same free UID before either can claim it. */
|
|
5601
|
-
private async findFreePoolUser(
|
|
5682
|
+
private async findFreePoolUser(
|
|
5683
|
+
excludeThreadKey?: string,
|
|
5684
|
+
excluded = new Set<string>(),
|
|
5685
|
+
preferred: readonly string[] = [],
|
|
5686
|
+
): Promise<string | undefined> {
|
|
5602
5687
|
const all = await this.ctx.storage.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX });
|
|
5603
5688
|
const used = new Set(
|
|
5604
5689
|
[...all.values()].filter((b) => !b.evicted && b.user && b.threadKey !== excludeThreadKey).map((b) => b.user),
|
|
@@ -5606,7 +5691,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5606
5691
|
for (const u of this.opUsersInUse.keys()) used.add(u);
|
|
5607
5692
|
for (const u of this.poolUsersInspecting) used.add(u);
|
|
5608
5693
|
for (const u of excluded) used.add(u);
|
|
5609
|
-
const user = THREAD_USERS.find((u) => !used.has(u));
|
|
5694
|
+
const user = [...preferred, ...THREAD_USERS].find((u) => !used.has(u));
|
|
5610
5695
|
if (user) this.poolUsersInspecting.add(user);
|
|
5611
5696
|
return user;
|
|
5612
5697
|
}
|
|
@@ -5643,19 +5728,82 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5643
5728
|
throw new Error("pool-user-stage-scan-invalid");
|
|
5644
5729
|
}
|
|
5645
5730
|
|
|
5731
|
+
/** A full one-use pool can serve the next request by destroying its VM,
|
|
5732
|
+
* never by assigning another owner's UID. The admission mark closes every
|
|
5733
|
+
* other entry before the idle read and stays held through the restore. */
|
|
5734
|
+
private async recycleSpentPoolForAdmission(): Promise<true | ThreadErr> {
|
|
5735
|
+
try {
|
|
5736
|
+
const admission = await this.recreateAdmission.run(async (): Promise<true | ThreadErr> => {
|
|
5737
|
+
const spent = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
|
|
5738
|
+
if (!spent) return { error: "pool-generation-unknown: fresh VM required before UID assignment", status: 503 };
|
|
5739
|
+
if (spent.size < THREAD_USERS.length) return true; // another checked recreate already cleared the generation
|
|
5740
|
+
const [status, drain, imagePending, snapshot, registeredRuns, bindings] = await Promise.all([
|
|
5741
|
+
this.getStatus(),
|
|
5742
|
+
this.registry().getDrain(),
|
|
5743
|
+
this.ctx.storage.get(IMAGE_REPORT_PENDING_KEY),
|
|
5744
|
+
this.ctx.storage.get(SNAPSHOT_KEY),
|
|
5745
|
+
this.registeredRunsBeyondOps(),
|
|
5746
|
+
this.ctx.storage.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX }),
|
|
5747
|
+
]);
|
|
5748
|
+
if (
|
|
5749
|
+
!snapshot ||
|
|
5750
|
+
!idleForPoolRecycle({
|
|
5751
|
+
state: status.state,
|
|
5752
|
+
draining: liveDrain(drain, systemClock()) !== null,
|
|
5753
|
+
imagePending: imagePending !== undefined,
|
|
5754
|
+
inFlight: this.inFlightCount(),
|
|
5755
|
+
refreshAdmissions: this.refreshAdmissionsInFlight,
|
|
5756
|
+
adminWork: this.adminWorkInFlight,
|
|
5757
|
+
hydrating: this.hydration !== null,
|
|
5758
|
+
registeredRuns,
|
|
5759
|
+
liveBindings: [...bindings.values()].filter((binding) => !binding.evicted).length,
|
|
5760
|
+
inspecting: this.poolUsersInspecting.size,
|
|
5761
|
+
})
|
|
5762
|
+
)
|
|
5763
|
+
return {
|
|
5764
|
+
error: "pool-recycle-required: all UIDs spent and the resident is not idle for checked VM recycle",
|
|
5765
|
+
status: 503,
|
|
5766
|
+
};
|
|
5767
|
+
await this.recreateContainer("pool-recycle: all UIDs spent; no other resident work owns this VM");
|
|
5768
|
+
await this.ensureHydrated();
|
|
5769
|
+
const fresh = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
|
|
5770
|
+
const ready = await this.getStatus();
|
|
5771
|
+
const imageStillPending = (await this.ctx.storage.get(IMAGE_REPORT_PENDING_KEY)) !== undefined;
|
|
5772
|
+
if (!fresh || fresh.size !== 0 || ready.state !== "warm" || imageStillPending)
|
|
5773
|
+
return { error: "pool-recycle-failed: the restored VM is not warm, current and empty", status: 503 };
|
|
5774
|
+
return true;
|
|
5775
|
+
});
|
|
5776
|
+
return admission.busy ? this.recreateRefusal() : admission.value;
|
|
5777
|
+
} catch (err) {
|
|
5778
|
+
console.log(`pool-recycle: checked VM recycle failed (${errMsg(err)})`);
|
|
5779
|
+
return { error: "pool-recycle-failed: the checked VM restore did not complete", status: 503 };
|
|
5780
|
+
}
|
|
5781
|
+
}
|
|
5782
|
+
|
|
5646
5783
|
private async reserveSafePoolUser(owner: string, excludeThreadKey?: string): Promise<string | ThreadErr> {
|
|
5647
|
-
|
|
5784
|
+
let spent = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
|
|
5648
5785
|
if (!spent) return { error: "pool-generation-unknown: fresh VM required before UID assignment", status: 503 };
|
|
5649
|
-
|
|
5786
|
+
let excluded = unavailablePoolUsers(spent, owner);
|
|
5650
5787
|
let contaminated = false;
|
|
5788
|
+
let recycled = false;
|
|
5651
5789
|
for (;;) {
|
|
5652
|
-
const user = await this.findFreePoolUser(excludeThreadKey, excluded);
|
|
5653
|
-
if (!user)
|
|
5790
|
+
const user = await this.findFreePoolUser(excludeThreadKey, excluded, ownedPoolUsers(spent, owner));
|
|
5791
|
+
if (!user) {
|
|
5792
|
+
if (!contaminated && excluded.size === THREAD_USERS.length && !recycled) {
|
|
5793
|
+
recycled = true;
|
|
5794
|
+
const result = await this.recycleSpentPoolForAdmission();
|
|
5795
|
+
if (result !== true) return result;
|
|
5796
|
+
spent = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
|
|
5797
|
+
if (!spent) return { error: "pool-generation-unknown: fresh VM required before UID assignment", status: 503 };
|
|
5798
|
+
excluded = unavailablePoolUsers(spent, owner);
|
|
5799
|
+
continue;
|
|
5800
|
+
}
|
|
5654
5801
|
return contaminated
|
|
5655
5802
|
? { error: "pool-user-contaminated: no clean pool user is available", status: 503 }
|
|
5656
5803
|
: excluded.size === THREAD_USERS.length
|
|
5657
5804
|
? { error: "pool-recycle-required: all UIDs spent on this VM; confirm idle and recreate it", status: 503 }
|
|
5658
5805
|
: { error: "user-pool-exhausted: no safe pool user is available", status: 429 };
|
|
5806
|
+
}
|
|
5659
5807
|
let safe = false;
|
|
5660
5808
|
try {
|
|
5661
5809
|
const clean = !(await this.poolUserHasOldThreadDir(user, [])) && !(await this.poolUserHasOldStageContent(user));
|
|
@@ -5798,23 +5946,37 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5798
5946
|
traceparent?: string,
|
|
5799
5947
|
reason: RefHintReason = NO_REF_HINT_REASON,
|
|
5800
5948
|
githubDoor?: { baseUrl: string; bearer: string },
|
|
5949
|
+
runBudgetMs?: number,
|
|
5950
|
+
runId?: string,
|
|
5951
|
+
ownerGen?: string,
|
|
5952
|
+
ownerFence?: number,
|
|
5801
5953
|
): Promise<AttachOk | ThreadErr> {
|
|
5802
5954
|
// One step trace per attach (docs/reference/specs/tracing.md item 19): every command
|
|
5803
5955
|
// the attach runs lands on it, and the answer carries it.
|
|
5804
5956
|
const t0 = systemClock();
|
|
5805
5957
|
const trace = createStepTrace(t0);
|
|
5806
5958
|
let res: AttachOk | ThreadErr;
|
|
5807
|
-
|
|
5808
|
-
|
|
5809
|
-
|
|
5810
|
-
|
|
5811
|
-
|
|
5812
|
-
|
|
5813
|
-
|
|
5814
|
-
|
|
5815
|
-
|
|
5816
|
-
|
|
5817
|
-
|
|
5959
|
+
this.attachAdmissionsInFlight++;
|
|
5960
|
+
try {
|
|
5961
|
+
res = await this.stepTrace.run(trace, () =>
|
|
5962
|
+
this.attachThreadTraced(
|
|
5963
|
+
threadKey,
|
|
5964
|
+
refHint,
|
|
5965
|
+
readonly,
|
|
5966
|
+
wantSha,
|
|
5967
|
+
reuse,
|
|
5968
|
+
record,
|
|
5969
|
+
t0,
|
|
5970
|
+
reason,
|
|
5971
|
+
githubDoor,
|
|
5972
|
+
runBudgetMs,
|
|
5973
|
+
runId,
|
|
5974
|
+
ownerGen,
|
|
5975
|
+
ownerFence,
|
|
5976
|
+
),
|
|
5977
|
+
);
|
|
5978
|
+
} finally {
|
|
5979
|
+
this.attachAdmissionsInFlight--;
|
|
5818
5980
|
}
|
|
5819
5981
|
// The same steps as the resident's own `resident.attach` root (item 22).
|
|
5820
5982
|
emitStepRoot("resident.attach", t0, trace.steps(), traceparent, "error" in res ? refusalOutcome(res) : "ok");
|
|
@@ -5832,56 +5994,80 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5832
5994
|
t0: number,
|
|
5833
5995
|
reason: RefHintReason,
|
|
5834
5996
|
githubDoor?: { baseUrl: string; bearer: string },
|
|
5997
|
+
runBudgetMs?: number,
|
|
5998
|
+
runId?: string,
|
|
5999
|
+
ownerGen?: string,
|
|
6000
|
+
ownerFence?: number,
|
|
5835
6001
|
): Promise<AttachOk | ThreadErr> {
|
|
5836
6002
|
try {
|
|
5837
|
-
|
|
5838
|
-
|
|
5839
|
-
|
|
5840
|
-
|
|
5841
|
-
|
|
5842
|
-
|
|
5843
|
-
|
|
5844
|
-
|
|
5845
|
-
|
|
5846
|
-
|
|
5847
|
-
|
|
5848
|
-
|
|
5849
|
-
|
|
5850
|
-
|
|
5851
|
-
|
|
5852
|
-
|
|
5853
|
-
|
|
5854
|
-
|
|
5855
|
-
|
|
5856
|
-
|
|
5857
|
-
|
|
5858
|
-
|
|
5859
|
-
|
|
5860
|
-
|
|
5861
|
-
|
|
5862
|
-
|
|
5863
|
-
|
|
5864
|
-
|
|
5865
|
-
|
|
5866
|
-
|
|
5867
|
-
|
|
5868
|
-
|
|
5869
|
-
|
|
5870
|
-
|
|
5871
|
-
|
|
5872
|
-
|
|
5873
|
-
|
|
5874
|
-
|
|
5875
|
-
|
|
5876
|
-
|
|
5877
|
-
|
|
5878
|
-
|
|
5879
|
-
|
|
5880
|
-
|
|
5881
|
-
|
|
5882
|
-
|
|
5883
|
-
|
|
5884
|
-
|
|
6003
|
+
return await this.threadAttaches.run(threadKey, async () => {
|
|
6004
|
+
// The attachment fence takes precedence over every 503 gate. Keep the
|
|
6005
|
+
// same thread lock through the gates and registration write so a newer
|
|
6006
|
+
// attach cannot slip between this check and a fallback-eligible refusal.
|
|
6007
|
+
const current = await this.ctx.storage.get<RunRegistration>(runRegKey(threadKey));
|
|
6008
|
+
const accepted = await this.ctx.storage.get<Pick<RunRegistration, "runId" | "ownerGen" | "ownerFence">>(
|
|
6009
|
+
runFenceKey(threadKey),
|
|
6010
|
+
);
|
|
6011
|
+
if (
|
|
6012
|
+
!registeredRunAllowsClaim(current, runId, ownerGen, ownerFence) ||
|
|
6013
|
+
!registeredRunAllowsClaim(accepted, runId, ownerGen, ownerFence)
|
|
6014
|
+
)
|
|
6015
|
+
return { error: "run-registration-mismatch: a newer generation owns the thread", status: 409 };
|
|
6016
|
+
if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
|
|
6017
|
+
await this.ensureHydrated();
|
|
6018
|
+
if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
|
|
6019
|
+
// The fleet drain (item 69): a deploy is waiting for the runs in flight
|
|
6020
|
+
// to end, and a NEW run's attach is refused with the record the bot
|
|
6021
|
+
// waits on — a real 503 in the streamed document, read by the client as
|
|
6022
|
+
// `draining`, never as the platform's transient. A run already in flight
|
|
6023
|
+
// — with an owned, live registration (item 44) — re-attaches
|
|
6024
|
+
// through: a rolled container, an evicted worktree, a resumed run are
|
|
6025
|
+
// the runs the drain waits FOR, and refusing them would hold the fleet
|
|
6026
|
+
// closed on the run it is closed for. Read before the image reconcile so
|
|
6027
|
+
// a refused attach never restarts a container.
|
|
6028
|
+
const drain = await this.fleetDrain();
|
|
6029
|
+
const registration = await this.ctx.storage.get<RunRegistration>(runRegKey(threadKey));
|
|
6030
|
+
const registered = registeredRunAllowsReattach(
|
|
6031
|
+
registration,
|
|
6032
|
+
runId,
|
|
6033
|
+
systemClock(),
|
|
6034
|
+
RUN_REGISTRATION_GRACE_MS,
|
|
6035
|
+
ownerGen,
|
|
6036
|
+
ownerFence,
|
|
6037
|
+
);
|
|
6038
|
+
if (drain && !registered) {
|
|
6039
|
+
const refusal: ThreadErr & { draining: DrainRecord } = drainRefusal(drain);
|
|
6040
|
+
return refusal;
|
|
6041
|
+
}
|
|
6042
|
+
// Item 70: above the soft memory threshold a NEW attach is refused like
|
|
6043
|
+
// `mirror-busy` (the bot falls back or waits, the card says why) — after
|
|
6044
|
+
// the drain (storage only, cheaper) and before the image reconcile, so a
|
|
6045
|
+
// refused attach never restarts a container. An owned, live run's
|
|
6046
|
+
// re-attach passes for the same reason it passes the drain above.
|
|
6047
|
+
const memory = await this.memoryGate("attach", registered);
|
|
6048
|
+
if (memory) return memory;
|
|
6049
|
+
const resourceId = (await this.ctx.storage.get<string>(RESOURCE_KEY)) ?? "";
|
|
6050
|
+
// An attach never restarts the container (issue 2101): a `stale` verdict
|
|
6051
|
+
// refuses the NEW run — it falls back to the seeded sandbox — while a
|
|
6052
|
+
// owned, live run's re-attach passes exactly as it passes the drain and
|
|
6053
|
+
// the memory gate; the restart itself is the refresh cycle's or the
|
|
6054
|
+
// deploy's.
|
|
6055
|
+
if ((await this.reconcileImage("attach")) === "stale" && !registered) {
|
|
6056
|
+
const s = await this.getStatus();
|
|
6057
|
+
return {
|
|
6058
|
+
error:
|
|
6059
|
+
"image-stale: the container predates the deploy and restarts on the next quiet refresh; new runs use the fallback sandbox until then",
|
|
6060
|
+
status: 503,
|
|
6061
|
+
state: s.state,
|
|
6062
|
+
stateReason: s.reason,
|
|
6063
|
+
reason: "image-stale",
|
|
6064
|
+
};
|
|
6065
|
+
}
|
|
6066
|
+
// From here the attach may hold the mirror lock through clone/install:
|
|
6067
|
+
// count it so a concurrent refresh-cycle reconcileImage never stops the
|
|
6068
|
+
// container under it (and isIdle never parks the cycle mid-attach).
|
|
6069
|
+
this.attachesInFlight++;
|
|
6070
|
+
try {
|
|
5885
6071
|
const res = await this.attachThreadBody(
|
|
5886
6072
|
threadKey,
|
|
5887
6073
|
refHint,
|
|
@@ -5896,12 +6082,12 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5896
6082
|
);
|
|
5897
6083
|
// The run this attach opens is now in flight until its release —
|
|
5898
6084
|
// whatever its op counters read between the bot's calls (item 44).
|
|
5899
|
-
if (!("error" in res)) await this.registerRun(threadKey);
|
|
6085
|
+
if (!("error" in res)) await this.registerRun(threadKey, runBudgetMs, runId, ownerGen, ownerFence);
|
|
5900
6086
|
return res;
|
|
5901
|
-
}
|
|
5902
|
-
|
|
5903
|
-
|
|
5904
|
-
}
|
|
6087
|
+
} finally {
|
|
6088
|
+
this.attachesInFlight--;
|
|
6089
|
+
}
|
|
6090
|
+
});
|
|
5905
6091
|
} catch (err) {
|
|
5906
6092
|
return catchAllErr(err, "attach-failed");
|
|
5907
6093
|
}
|
|
@@ -7736,6 +7922,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7736
7922
|
user: "",
|
|
7737
7923
|
evicted: true,
|
|
7738
7924
|
evictedAt: new Date(systemClock()).toISOString(),
|
|
7925
|
+
lastRunOwner: await this.ctx.storage.get<RunRegistration>(runRegKey(binding.threadKey)),
|
|
7739
7926
|
evictedWhy: why,
|
|
7740
7927
|
evictedLeftBehind: tree && "leftBehind" in tree ? tree.leftBehind : undefined,
|
|
7741
7928
|
evictedUnmeasured: tree && "unmeasured" in tree ? tree.unmeasured : undefined,
|
|
@@ -7747,9 +7934,9 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7747
7934
|
return "evicted";
|
|
7748
7935
|
}
|
|
7749
7936
|
|
|
7750
|
-
/** POST /detach: a run has ended —
|
|
7751
|
-
*
|
|
7752
|
-
*
|
|
7937
|
+
/** POST /detach: a run has ended — release its live binding now instead of
|
|
7938
|
+
* holding it until the TTL sweep. The UID remains spent for this VM
|
|
7939
|
+
* generation. A run starts from a clean tree (item 17), so the tree
|
|
7753
7940
|
* goes whatever it holds: what it held — uncommitted changes, unpushed
|
|
7754
7941
|
* commits — is measured once, as the thread user, and named in the answer
|
|
7755
7942
|
* (`leftBehind`) so the loss is never silent. The one thing that keeps a
|
|
@@ -7763,65 +7950,77 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7763
7950
|
force: boolean,
|
|
7764
7951
|
/** What the ending run pushed (item 16a): remembered on the binding first, whatever the detach then decides. */
|
|
7765
7952
|
pushed: readonly PushedBranch[] = [],
|
|
7953
|
+
runId?: string,
|
|
7954
|
+
ownerGen?: string,
|
|
7955
|
+
ownerFence?: number,
|
|
7766
7956
|
): Promise<DetachAnswer | ThreadErr> {
|
|
7767
|
-
|
|
7768
|
-
|
|
7769
|
-
|
|
7770
|
-
|
|
7771
|
-
|
|
7772
|
-
|
|
7773
|
-
|
|
7774
|
-
|
|
7775
|
-
|
|
7776
|
-
|
|
7777
|
-
|
|
7778
|
-
|
|
7779
|
-
|
|
7780
|
-
|
|
7781
|
-
|
|
7782
|
-
|
|
7783
|
-
|
|
7784
|
-
`detach: force — killed ${plan.user}'s processes for ${threadKey} (${plan.inFlight} op(s) were in flight)`,
|
|
7785
|
-
);
|
|
7786
|
-
const left = await this.waitForThreadDrain(threadKey);
|
|
7787
|
-
if (left > 0) return { released: false, reason: busyAfterKillReason(left), user: binding.user };
|
|
7788
|
-
}
|
|
7789
|
-
const active = await this.isRuntimeActive().catch(() => false);
|
|
7790
|
-
// What the tree still holds, for the answer and the eviction's record —
|
|
7791
|
-
// never a reason to keep it. Not measured on a force release: a read-only
|
|
7792
|
-
// tree holds nothing, and a hard stop's tree is whatever the killed
|
|
7793
|
-
// command left. A probe that fails names nothing in the answer (never a
|
|
7794
|
-
// guess); the record and the log say it could not be measured.
|
|
7795
|
-
let tree: EvictedTree | undefined;
|
|
7796
|
-
if (!force && active) tree = await this.measureTreeBeforeEviction(binding);
|
|
7797
|
-
// Re-check right before removal: the measurement above awaited (the DO
|
|
7798
|
-
// yields at each await), so an exec that arrived mid-detach would otherwise
|
|
7799
|
-
// have its tree removed under it.
|
|
7800
|
-
const busyNow = this.threadOpsInFlight.get(threadKey) ?? 0;
|
|
7801
|
-
if (busyNow > 0)
|
|
7802
|
-
return {
|
|
7803
|
-
released: false,
|
|
7804
|
-
reason: `busy: ${busyNow} operation(s) started during detach — kept`,
|
|
7957
|
+
return this.threadAttaches.run(threadKey, async () => {
|
|
7958
|
+
const binding = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
|
|
7959
|
+
if (!binding) return { error: `no-binding: ${threadKey} has never attached to this resident`, status: 404 };
|
|
7960
|
+
if (binding.evicted || !binding.user) {
|
|
7961
|
+
if (pushed.length > 0 && registeredRunOwnsRelease(binding.lastRunOwner, runId, ownerGen, ownerFence))
|
|
7962
|
+
await this.rememberOwnBranches(threadKey, pushed);
|
|
7963
|
+
return { released: false, reason: "already-evicted" };
|
|
7964
|
+
}
|
|
7965
|
+
const registration = await this.ctx.storage.get<RunRegistration>(runRegKey(threadKey));
|
|
7966
|
+
if (!registeredRunOwnsRelease(registration, runId, ownerGen, ownerFence))
|
|
7967
|
+
return { released: false, reason: "run-registration-mismatch: the workspace belongs to another run" };
|
|
7968
|
+
if (pushed.length > 0) await this.rememberOwnBranches(threadKey, pushed);
|
|
7969
|
+
if (!(await this.poolUserOwnerMatches(binding.user, `thread:${threadKey}`)))
|
|
7970
|
+
return { error: "pool-owner-mismatch: detach refused for conflicting UID owner", status: 503 };
|
|
7971
|
+
const plan = planForceDetach({
|
|
7972
|
+
force,
|
|
7973
|
+
inFlight: this.threadOpsInFlight.get(threadKey) ?? 0,
|
|
7805
7974
|
user: binding.user,
|
|
7806
|
-
|
|
7807
|
-
|
|
7808
|
-
|
|
7809
|
-
|
|
7810
|
-
|
|
7811
|
-
|
|
7812
|
-
|
|
7813
|
-
|
|
7814
|
-
|
|
7815
|
-
|
|
7816
|
-
|
|
7817
|
-
|
|
7818
|
-
|
|
7819
|
-
|
|
7820
|
-
|
|
7821
|
-
|
|
7822
|
-
|
|
7823
|
-
|
|
7824
|
-
|
|
7975
|
+
poolUsers: THREAD_USERS,
|
|
7976
|
+
});
|
|
7977
|
+
if (plan.action === "refuse") return { released: false, reason: plan.reason, user: binding.user };
|
|
7978
|
+
if (plan.action === "kill") {
|
|
7979
|
+
if (!(await this.killThreadUserProcesses(plan.user, threadKey)))
|
|
7980
|
+
return { error: "pool-owner-mismatch: force detach refused before kill", status: 503 };
|
|
7981
|
+
console.log(
|
|
7982
|
+
`detach: force — killed ${plan.user}'s processes for ${threadKey} (${plan.inFlight} op(s) were in flight)`,
|
|
7983
|
+
);
|
|
7984
|
+
const left = await this.waitForThreadDrain(threadKey);
|
|
7985
|
+
if (left > 0) return { released: false, reason: busyAfterKillReason(left), user: binding.user };
|
|
7986
|
+
}
|
|
7987
|
+
const active = await this.isRuntimeActive().catch(() => false);
|
|
7988
|
+
// What the tree still holds, for the answer and the eviction's record —
|
|
7989
|
+
// never a reason to keep it. Not measured on a force release: a read-only
|
|
7990
|
+
// tree holds nothing, and a hard stop's tree is whatever the killed
|
|
7991
|
+
// command left. A probe that fails names nothing in the answer (never a
|
|
7992
|
+
// guess); the record and the log say it could not be measured.
|
|
7993
|
+
let tree: EvictedTree | undefined;
|
|
7994
|
+
if (!force && active) tree = await this.measureTreeBeforeEviction(binding);
|
|
7995
|
+
// Re-check right before removal: the measurement above awaited (the DO
|
|
7996
|
+
// yields at each await), so an exec that arrived mid-detach would otherwise
|
|
7997
|
+
// have its tree removed under it.
|
|
7998
|
+
const busyNow = this.threadOpsInFlight.get(threadKey) ?? 0;
|
|
7999
|
+
if (busyNow > 0)
|
|
8000
|
+
return {
|
|
8001
|
+
released: false,
|
|
8002
|
+
reason: `busy: ${busyNow} operation(s) started during detach — kept`,
|
|
8003
|
+
user: binding.user,
|
|
8004
|
+
};
|
|
8005
|
+
// Same re-read as the sweep: a re-attach during the measurement means a
|
|
8006
|
+
// fresh tree we must not remove from a stale snapshot.
|
|
8007
|
+
const current = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
|
|
8008
|
+
if (!current || current.evicted) return { released: false, reason: "already-evicted" };
|
|
8009
|
+
if (current.lastAttachAt !== binding.lastAttachAt)
|
|
8010
|
+
return { released: false, reason: "re-attached during the detach — kept", user: current.user };
|
|
8011
|
+
const user = current.user;
|
|
8012
|
+
// Same as the sweep: `active` was read before the measurement's awaits; a
|
|
8013
|
+
// container that woke meanwhile must get the rm, not an orphaned tree.
|
|
8014
|
+
const activeNow = await this.isRuntimeActive().catch(() => true);
|
|
8015
|
+
const eviction = await this.evictBinding(current, activeNow, `detach`, "detach", tree);
|
|
8016
|
+
if (eviction === "cleanup-failed")
|
|
8017
|
+
return { released: false, reason: "thread-cleanup-failed: pool user kept", user };
|
|
8018
|
+
if (eviction === "changed") return { released: false, reason: "re-attached during eviction — kept", user };
|
|
8019
|
+
// Item 55: the tree is gone; the gauge catches up at the next refresh
|
|
8020
|
+
// instance's `measure` step, and the admission's `df` sees the space now.
|
|
8021
|
+
const leftBehind = tree && "leftBehind" in tree ? tree.leftBehind : undefined;
|
|
8022
|
+
return { released: true, user, ...(leftBehind !== undefined ? { leftBehind } : {}) };
|
|
8023
|
+
});
|
|
7825
8024
|
}
|
|
7826
8025
|
|
|
7827
8026
|
/** Force-detach's kill: end every process owned by the pool user —
|
|
@@ -7936,25 +8135,74 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7936
8135
|
async getInFlightCount(): Promise<number> {
|
|
7937
8136
|
return this.inFlightCount() + (await this.registeredRunsBeyondOps());
|
|
7938
8137
|
}
|
|
7939
|
-
/** A run
|
|
7940
|
-
* sweep, disk pressure
|
|
7941
|
-
*
|
|
7942
|
-
private async registerRun(
|
|
7943
|
-
|
|
7944
|
-
|
|
7945
|
-
|
|
7946
|
-
|
|
8138
|
+
/** A run's registration protects its worktree until release or the bounded
|
|
8139
|
+
* deadline. `/detach`, the sweep, and disk pressure end in `evictBinding`,
|
|
8140
|
+
* which clears the row; a re-attach refreshes it. */
|
|
8141
|
+
private async registerRun(
|
|
8142
|
+
threadKey: string,
|
|
8143
|
+
runBudgetMs?: number,
|
|
8144
|
+
runId?: string,
|
|
8145
|
+
ownerGen?: string,
|
|
8146
|
+
ownerFence?: number,
|
|
8147
|
+
): Promise<void> {
|
|
8148
|
+
const now = systemClock();
|
|
8149
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
8150
|
+
if (ownerFence !== undefined) await txn.put(runFenceKey(threadKey), { runId, ownerGen, ownerFence });
|
|
8151
|
+
await txn.put(runRegKey(threadKey), {
|
|
8152
|
+
threadKey,
|
|
8153
|
+
registeredAt: new Date(now).toISOString(),
|
|
8154
|
+
...(runBudgetMs !== undefined ? { deadlineAt: now + runBudgetMs } : {}),
|
|
8155
|
+
...(runId !== undefined ? { runId } : {}),
|
|
8156
|
+
...(ownerGen !== undefined ? { ownerGen } : {}),
|
|
8157
|
+
...(ownerFence !== undefined ? { ownerFence } : {}),
|
|
8158
|
+
} satisfies RunRegistration);
|
|
8159
|
+
});
|
|
8160
|
+
}
|
|
8161
|
+
/** The harness's lease begins after attach. Tighten the provisional setup
|
|
8162
|
+
* bound to its actual remaining clock, only for the run that attached. */
|
|
8163
|
+
async updateRunDeadline(
|
|
8164
|
+
threadKey: string,
|
|
8165
|
+
runId: string,
|
|
8166
|
+
remainingMs: number,
|
|
8167
|
+
ownerGen?: string,
|
|
8168
|
+
ownerFence?: number,
|
|
8169
|
+
): Promise<{ deadlineAt: number } | ThreadErr> {
|
|
8170
|
+
return this.ctx.storage.transaction(async (txn) => {
|
|
8171
|
+
const registration = await txn.get<RunRegistration>(runRegKey(threadKey));
|
|
8172
|
+
if (
|
|
8173
|
+
!registration ||
|
|
8174
|
+
registration.runId !== runId ||
|
|
8175
|
+
registration.ownerGen !== ownerGen ||
|
|
8176
|
+
registration.ownerFence !== ownerFence
|
|
8177
|
+
)
|
|
8178
|
+
return { error: "run-registration-mismatch: this run does not own the thread", status: 409 };
|
|
8179
|
+
const deadlineAt = systemClock() + remainingMs;
|
|
8180
|
+
await txn.put(runRegKey(threadKey), { ...registration, deadlineAt });
|
|
8181
|
+
return { deadlineAt };
|
|
8182
|
+
});
|
|
7947
8183
|
}
|
|
7948
8184
|
/** The registrations the op counters do not already see: a thread with an
|
|
7949
8185
|
* op in flight is counted by `runsInFlightCount`, so its registration is
|
|
7950
|
-
* not counted again.
|
|
7951
|
-
*
|
|
7952
|
-
*
|
|
7953
|
-
* drains a registration whose release never came. */
|
|
8186
|
+
* not counted again. The idle, image-reconcile and disk-recycle gates read
|
|
8187
|
+
* this bounded count alongside their op counters, so a live run protects
|
|
8188
|
+
* its container while a stale registration cannot pin it awake. */
|
|
7954
8189
|
private async registeredRunsBeyondOps(): Promise<number> {
|
|
7955
8190
|
const regs = await this.ctx.storage.list<RunRegistration>({ prefix: RUN_REG_KEY_PREFIX });
|
|
8191
|
+
const cutoff = systemClock() - CLEAN_IDLE_RELEASE_S * 1000;
|
|
7956
8192
|
let n = 0;
|
|
7957
|
-
for (const r of regs.values())
|
|
8193
|
+
for (const r of regs.values()) {
|
|
8194
|
+
const binding = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(r.threadKey));
|
|
8195
|
+
if (
|
|
8196
|
+
registeredRunNeedsProtection(
|
|
8197
|
+
binding?.lastAttachAt,
|
|
8198
|
+
this.threadOpsInFlight.get(r.threadKey) ?? 0,
|
|
8199
|
+
cutoff,
|
|
8200
|
+
r.deadlineAt === undefined ? undefined : r.deadlineAt + RUN_REGISTRATION_GRACE_MS,
|
|
8201
|
+
systemClock(),
|
|
8202
|
+
)
|
|
8203
|
+
)
|
|
8204
|
+
n++;
|
|
8205
|
+
}
|
|
7958
8206
|
return n;
|
|
7959
8207
|
}
|
|
7960
8208
|
private attachesInFlight = 0;
|
|
@@ -7983,6 +8231,15 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7983
8231
|
const idleCutoff = systemClock() - CLEAN_IDLE_RELEASE_S * 1000;
|
|
7984
8232
|
for (const binding of all.values()) {
|
|
7985
8233
|
if (binding.evicted || !binding.user) continue;
|
|
8234
|
+
const registration = await this.ctx.storage.get<RunRegistration>(runRegKey(binding.threadKey));
|
|
8235
|
+
if (
|
|
8236
|
+
registration?.deadlineAt !== undefined &&
|
|
8237
|
+
Number.isSafeInteger(registration.deadlineAt) &&
|
|
8238
|
+
systemClock() <= registration.deadlineAt + RUN_REGISTRATION_GRACE_MS
|
|
8239
|
+
) {
|
|
8240
|
+
kept++;
|
|
8241
|
+
continue;
|
|
8242
|
+
}
|
|
7986
8243
|
const last = Date.parse(binding.lastAttachAt);
|
|
7987
8244
|
if (last >= cutoff) {
|
|
7988
8245
|
// Not past the TTL. Still release it if it has been idle for an hour
|
|
@@ -7991,9 +8248,9 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7991
8248
|
// a clean tree (item 17) — so there is nothing to keep it for. This is
|
|
7992
8249
|
// what drains the bindings of runs whose release never came (a
|
|
7993
8250
|
// resident that was sick at the run's end, a run older than
|
|
7994
|
-
// `/detach`). A live run is protected by its
|
|
7995
|
-
// dirt. A slept container has no tree any more
|
|
7996
|
-
// the disk).
|
|
8251
|
+
// `/detach`). A live run is protected by its budget above or an op
|
|
8252
|
+
// in flight, not by its dirt. A slept container has no tree any more
|
|
8253
|
+
// anyway (sleep destroys the disk).
|
|
7997
8254
|
const busy = this.threadOpsInFlight.get(binding.threadKey) ?? 0;
|
|
7998
8255
|
if (last >= idleCutoff || busy > 0) {
|
|
7999
8256
|
kept++;
|
|
@@ -8390,13 +8647,30 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
8390
8647
|
fates.set(ref, { fate: looked.fate, detail });
|
|
8391
8648
|
}),
|
|
8392
8649
|
);
|
|
8393
|
-
// The runs registered on this resident (item 44): a
|
|
8394
|
-
//
|
|
8650
|
+
// The runs registered on this resident (item 44): a live registration
|
|
8651
|
+
// holds its tree through the deadline, and its process lives in the container
|
|
8395
8652
|
// between the bot's calls — the op counters read 0 while its model thinks.
|
|
8653
|
+
const registrations = new Map(
|
|
8654
|
+
[...(await this.ctx.storage.list<RunRegistration>({ prefix: RUN_REG_KEY_PREFIX })).values()].map((r) => [
|
|
8655
|
+
r.threadKey,
|
|
8656
|
+
r,
|
|
8657
|
+
]),
|
|
8658
|
+
);
|
|
8659
|
+
const protectsTree = (binding: ThreadBinding, registration: RunRegistration | undefined): boolean => {
|
|
8660
|
+
if (!registration) return false;
|
|
8661
|
+
const now = systemClock();
|
|
8662
|
+
return registeredRunNeedsProtection(
|
|
8663
|
+
binding.lastAttachAt,
|
|
8664
|
+
0,
|
|
8665
|
+
now - CLEAN_IDLE_RELEASE_S * 1000,
|
|
8666
|
+
registration.deadlineAt === undefined ? undefined : registration.deadlineAt + RUN_REGISTRATION_GRACE_MS,
|
|
8667
|
+
now,
|
|
8668
|
+
);
|
|
8669
|
+
};
|
|
8396
8670
|
const registered = new Set(
|
|
8397
|
-
|
|
8398
|
-
(
|
|
8399
|
-
|
|
8671
|
+
live
|
|
8672
|
+
.filter((binding) => protectsTree(binding, registrations.get(binding.threadKey)))
|
|
8673
|
+
.map((binding) => binding.threadKey),
|
|
8400
8674
|
);
|
|
8401
8675
|
for (const binding of live) {
|
|
8402
8676
|
const isDefaultRef = binding.ref === defaultRef;
|
|
@@ -8423,7 +8697,8 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
8423
8697
|
continue;
|
|
8424
8698
|
}
|
|
8425
8699
|
// A run attached while the tree was measured holds it now (item 44).
|
|
8426
|
-
|
|
8700
|
+
const currentRegistration = await this.ctx.storage.get<RunRegistration>(runRegKey(binding.threadKey));
|
|
8701
|
+
if (protectsTree(current ?? binding, currentRegistration)) {
|
|
8427
8702
|
kept.push({ threadKey: binding.threadKey, ref: binding.ref, why: "run-held" });
|
|
8428
8703
|
continue;
|
|
8429
8704
|
}
|
|
@@ -8643,6 +8918,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
8643
8918
|
// The runs whose process lives in the container between operator calls
|
|
8644
8919
|
// (item 44): what the deploy preflight must see, added to both counts.
|
|
8645
8920
|
const registeredRuns = await this.registeredRunsBeyondOps();
|
|
8921
|
+
const poolSpends = parseSpentPoolUsers(map.get(SPENT_POOL_USERS_KEY), THREAD_USERS);
|
|
8646
8922
|
return {
|
|
8647
8923
|
resource: map.get(RESOURCE_KEY) ?? null,
|
|
8648
8924
|
state: this.destroying || map.get(DESTROY_UNCONFIRMED_KEY) ? "down" : (map.get(STATE_KEY) ?? "down"),
|
|
@@ -8674,7 +8950,10 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
8674
8950
|
// running container the last deploy could not verify.
|
|
8675
8951
|
imageReport: map.get(IMAGE_REPORT_PENDING_KEY) !== undefined ? "pending" : "current",
|
|
8676
8952
|
recreateAdmissionHeld: this.recreateAdmission.pending || map.get(RECREATE_ADMISSION_KEY) === true,
|
|
8677
|
-
poolUsersSpent:
|
|
8953
|
+
poolUsersSpent: poolSpends?.size ?? null,
|
|
8954
|
+
// Read-scoped owner receipts explain historical occupancy even after
|
|
8955
|
+
// every live binding and disposable op has ended.
|
|
8956
|
+
poolUserSpends: poolSpends ? [...poolSpends].map(([user, owner]) => ({ user, owner })) : null,
|
|
8678
8957
|
// Item 22: who holds what, as the rows say — the mirror mutex and the
|
|
8679
8958
|
// cycle/hydration leases, each judged against this incarnation.
|
|
8680
8959
|
incarnation: this.incarnation,
|
|
@@ -9249,6 +9528,7 @@ const ROUTES: Record<string, { scope: Scope; method: string }> = {
|
|
|
9249
9528
|
"/debug": { scope: "read", method: "POST" }, // per-op: READ_DEBUG_OPS for read scope, everything for admin
|
|
9250
9529
|
"/status": { scope: "operator", method: "GET" },
|
|
9251
9530
|
"/attach": { scope: "operator", method: "POST" },
|
|
9531
|
+
"/run-deadline": { scope: "operator", method: "POST" },
|
|
9252
9532
|
"/detach": { scope: "operator", method: "POST" },
|
|
9253
9533
|
"/exec": { scope: "operator", method: "POST" },
|
|
9254
9534
|
"/read": { scope: "operator", method: "POST" },
|
|
@@ -9366,6 +9646,8 @@ export default {
|
|
|
9366
9646
|
return await handleStatus(env, url);
|
|
9367
9647
|
case "/attach":
|
|
9368
9648
|
return await handleAttach(env, body, traceparent);
|
|
9649
|
+
case "/run-deadline":
|
|
9650
|
+
return await handleRunDeadline(env, body);
|
|
9369
9651
|
case "/detach":
|
|
9370
9652
|
return await handleDetach(env, body);
|
|
9371
9653
|
case "/exec":
|
|
@@ -9996,6 +10278,27 @@ async function handleAttach(env: Env, body: Record<string, unknown>, traceparent
|
|
|
9996
10278
|
if ("error" in want) return json({ error: want.error }, 400);
|
|
9997
10279
|
const reuse = parseReuse(body.reuse);
|
|
9998
10280
|
if ("error" in reuse) return json({ error: reuse.error }, 400);
|
|
10281
|
+
let runBudgetMs: number | undefined;
|
|
10282
|
+
if (body.runBudgetMs !== undefined) {
|
|
10283
|
+
const value = body.runBudgetMs;
|
|
10284
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 0 || value > DAY_MS)
|
|
10285
|
+
return json({ error: "invalid run budget" }, 400);
|
|
10286
|
+
runBudgetMs = value;
|
|
10287
|
+
}
|
|
10288
|
+
const runId = body.runId;
|
|
10289
|
+
if (runId !== undefined && (typeof runId !== "string" || runId.length === 0 || runId.length > 128))
|
|
10290
|
+
return json({ error: "invalid run id" }, 400);
|
|
10291
|
+
const ownerGen = body.ownerGen;
|
|
10292
|
+
if (ownerGen !== undefined && (typeof ownerGen !== "string" || ownerGen.length === 0 || ownerGen.length > 128))
|
|
10293
|
+
return json({ error: "invalid run generation" }, 400);
|
|
10294
|
+
const ownerFence = body.ownerFence;
|
|
10295
|
+
if (
|
|
10296
|
+
ownerFence !== undefined &&
|
|
10297
|
+
(typeof ownerFence !== "number" || !Number.isSafeInteger(ownerFence) || ownerFence <= 0)
|
|
10298
|
+
)
|
|
10299
|
+
return json({ error: "invalid run fence" }, 400);
|
|
10300
|
+
if (ownerFence !== undefined && (runId === undefined || ownerGen === undefined))
|
|
10301
|
+
return json({ error: "run fence requires a run id and generation" }, 400);
|
|
9999
10302
|
// Why the hint is what it is (item 16): the thread's own pull request and
|
|
10000
10303
|
// its head branch — the branch checked against the one ref pattern like
|
|
10001
10304
|
// every ref, before it can become a git argument — and the bound-by-default flag.
|
|
@@ -10050,6 +10353,10 @@ async function handleAttach(env: Env, body: Record<string, unknown>, traceparent
|
|
|
10050
10353
|
traceparent,
|
|
10051
10354
|
reason,
|
|
10052
10355
|
githubDoor,
|
|
10356
|
+
runBudgetMs,
|
|
10357
|
+
runId,
|
|
10358
|
+
ownerGen,
|
|
10359
|
+
ownerFence,
|
|
10053
10360
|
),
|
|
10054
10361
|
),
|
|
10055
10362
|
({ result, levels }) => ({ ...(result as object), ...(levels ? { levels } : {}) }),
|
|
@@ -10057,6 +10364,29 @@ async function handleAttach(env: Env, body: Record<string, unknown>, traceparent
|
|
|
10057
10364
|
);
|
|
10058
10365
|
}
|
|
10059
10366
|
|
|
10367
|
+
async function handleRunDeadline(env: Env, body: Record<string, unknown>): Promise<Response> {
|
|
10368
|
+
const ctx = await resolveThreadRoute(env, body);
|
|
10369
|
+
if (ctx instanceof Response) return ctx;
|
|
10370
|
+
const runId = body.runId;
|
|
10371
|
+
const ownerGen = body.ownerGen;
|
|
10372
|
+
const ownerFence = body.ownerFence;
|
|
10373
|
+
const remainingMs = body.remainingMs;
|
|
10374
|
+
if (typeof runId !== "string" || runId.length === 0 || runId.length > 128)
|
|
10375
|
+
return json({ error: "invalid run id" }, 400);
|
|
10376
|
+
if (ownerGen !== undefined && (typeof ownerGen !== "string" || ownerGen.length === 0 || ownerGen.length > 128))
|
|
10377
|
+
return json({ error: "invalid run generation" }, 400);
|
|
10378
|
+
if (
|
|
10379
|
+
ownerFence !== undefined &&
|
|
10380
|
+
(typeof ownerFence !== "number" || !Number.isSafeInteger(ownerFence) || ownerFence <= 0)
|
|
10381
|
+
)
|
|
10382
|
+
return json({ error: "invalid run fence" }, 400);
|
|
10383
|
+
if (typeof remainingMs !== "number" || !Number.isSafeInteger(remainingMs) || remainingMs <= 0 || remainingMs > DAY_MS)
|
|
10384
|
+
return json({ error: "invalid remaining lease" }, 400);
|
|
10385
|
+
const result = await ctx.stub.updateRunDeadline(ctx.threadKey, runId, remainingMs, ownerGen, ownerFence);
|
|
10386
|
+
if ("error" in result) return threadErrResponse(result);
|
|
10387
|
+
return json(result);
|
|
10388
|
+
}
|
|
10389
|
+
|
|
10060
10390
|
/** POST /await-restore (docs/reference/specs/execution.md item 27): the bot's
|
|
10061
10391
|
* one held request while the resident restores. Held server-side by the DO's
|
|
10062
10392
|
* waiter ledger; the answer streams like /attach (heartbeat whitespace then
|
|
@@ -10078,6 +10408,18 @@ async function handleAwaitRestore(env: Env, body: Record<string, unknown>): Prom
|
|
|
10078
10408
|
async function handleDetach(env: Env, body: Record<string, unknown>): Promise<Response> {
|
|
10079
10409
|
const ctx = await resolveThreadRoute(env, body);
|
|
10080
10410
|
if (ctx instanceof Response) return ctx;
|
|
10411
|
+
const runId = body.runId;
|
|
10412
|
+
if (runId !== undefined && (typeof runId !== "string" || runId.length === 0 || runId.length > 128))
|
|
10413
|
+
return json({ error: "invalid run id" }, 400);
|
|
10414
|
+
const ownerGen = body.ownerGen;
|
|
10415
|
+
if (ownerGen !== undefined && (typeof ownerGen !== "string" || ownerGen.length === 0 || ownerGen.length > 128))
|
|
10416
|
+
return json({ error: "invalid run generation" }, 400);
|
|
10417
|
+
const ownerFence = body.ownerFence;
|
|
10418
|
+
if (
|
|
10419
|
+
ownerFence !== undefined &&
|
|
10420
|
+
(typeof ownerFence !== "number" || !Number.isSafeInteger(ownerFence) || ownerFence <= 0)
|
|
10421
|
+
)
|
|
10422
|
+
return json({ error: "invalid run fence" }, 400);
|
|
10081
10423
|
// What the run pushed (item 16a), each ref through the one ref pattern
|
|
10082
10424
|
// before it can be stored or become a git argument.
|
|
10083
10425
|
const pushed = parsePushed(body.pushed);
|
|
@@ -10086,7 +10428,14 @@ async function handleDetach(env: Env, body: Record<string, unknown>): Promise<Re
|
|
|
10086
10428
|
const ref = parseRef(entry.ref, "pushed[].ref");
|
|
10087
10429
|
if ("error" in ref) return json({ error: ref.error }, 400);
|
|
10088
10430
|
}
|
|
10089
|
-
const result = await ctx.stub.detachThread(
|
|
10431
|
+
const result = await ctx.stub.detachThread(
|
|
10432
|
+
ctx.threadKey,
|
|
10433
|
+
body.force === true,
|
|
10434
|
+
pushed.pushed,
|
|
10435
|
+
runId,
|
|
10436
|
+
ownerGen,
|
|
10437
|
+
ownerFence,
|
|
10438
|
+
);
|
|
10090
10439
|
if ("error" in result) return threadErrResponse(result);
|
|
10091
10440
|
return json(result);
|
|
10092
10441
|
}
|