@coreplane/switchboard 1.233.0 → 1.235.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/deploy/cloudflare-resident/gc.ts +6 -9
- package/dist/assets/deploy/cloudflare-resident/worker.ts +310 -277
- package/dist/assets/deploy/cloudflare-sandbox/worker.ts +82 -13
- package/dist/assets/package-lock.json +3 -3
- package/dist/assets/package.json +1 -1
- package/dist/assets/source.json +3 -3
- package/dist/assets/src/agents/registry.ts +13 -7
- package/dist/assets/src/core/provider.ts +9 -6
- package/dist/assets/src/core/runEvents.ts +25 -0
- package/dist/assets/src/execution/residentCleanliness.ts +69 -5
- package/dist/assets/src/execution/residentDiskBudget.ts +7 -6
- package/dist/assets/src/execution/residentRebind.ts +92 -77
- package/dist/assets/src/execution/residentReuse.ts +7 -18
- package/dist/assets/src/execution/residentSteps.ts +0 -1
- package/dist/assets/src/execution/sandboxErrors.ts +5 -0
- package/dist/assets/src/execution/sandboxIdle.ts +220 -0
- package/dist/assets/web/dist/.vite/manifest.json +68 -28
- package/dist/assets/web/dist/assets/{ResidentDetailPage-V4K35SYX.js → ResidentDetailPage-DSpsye5e.js} +1 -1
- package/dist/assets/web/dist/assets/{ResidentsIndexPage-3HuYaDwD.js → ResidentsIndexPage-CT8fI1M3.js} +1 -1
- package/dist/assets/web/dist/assets/RunFoldRow-Cqh8pTXC.js +9 -0
- package/dist/assets/web/dist/assets/RunRoutePage-oN9GkVEA.js +6 -0
- package/dist/assets/web/dist/assets/RunsIndexPage-sjXr8iCV.js +1 -0
- package/dist/assets/web/dist/assets/{ScheduledPage-dfrjQ5E9.js → ScheduledPage-Tz__zLzi.js} +1 -1
- package/dist/assets/web/dist/assets/{StatusDot-DexKtvUC.js → StatusDot-D14iJMy7.js} +1 -1
- package/dist/assets/web/dist/assets/{Tooltip-DGQ2gu9M.js → Tooltip-B0Ob5MQ4.js} +1 -1
- package/dist/assets/web/dist/assets/UnitRoutePage-CbKCL58v.js +1 -0
- package/dist/assets/web/dist/assets/{dist-boTdDLF4.js → dist-pOdUzj6D.js} +1 -1
- package/dist/assets/web/dist/assets/favicon-BQsePYv5.js +1 -0
- package/dist/assets/web/dist/assets/indexRow-Bnj883ii.js +1 -0
- package/dist/assets/web/dist/assets/{main-BHhLVMQh.js → main-D3lG-yQ9.js} +2 -2
- package/dist/assets/web/dist/assets/main-Dm11o0hc.css +1 -0
- package/dist/cli.js +1231 -586
- package/package.json +1 -1
- package/dist/assets/web/dist/assets/RunRoutePage-Dr867-Ha.js +0 -13
- package/dist/assets/web/dist/assets/RunsIndexPage-D_lpXsr8.js +0 -1
- package/dist/assets/web/dist/assets/favicon-CWPcvWvp.js +0 -1
- package/dist/assets/web/dist/assets/main-CAVqMbiX.css +0 -1
|
@@ -16,21 +16,34 @@
|
|
|
16
16
|
*
|
|
17
17
|
* Shape: the caller names the reason for its hint (`ownPr`: the pull request
|
|
18
18
|
* the thread's own run opened and its head branch — never a PR a person
|
|
19
|
-
* named). The
|
|
19
|
+
* named). The move is a decision about the BINDING alone; the tree is the
|
|
20
|
+
* attach's business afterwards (item 17: a run starts from a clean tree at
|
|
21
|
+
* the bound ref, so the attach provisions the tree at the moved ref as it
|
|
22
|
+
* provisions any other). The resident moves the binding only when all of
|
|
23
|
+
* these hold:
|
|
20
24
|
* - the binding was made by default (the first message named no branch), read
|
|
21
25
|
* off `boundBy`, or, for a binding made before that field, off whether the
|
|
22
26
|
* ref is the default branch — a ref a person named is never moved;
|
|
23
|
-
* - the thread was not rebound before
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
27
|
+
* - the thread was not rebound before, or its earlier move was returned (the
|
|
28
|
+
* second movement below) — a thread moves once per pull request;
|
|
29
|
+
* - the branch is the thread's own: remembered from a release (`ownBranches`,
|
|
30
|
+
* the branches its runs pushed), or, when nothing was remembered, a local
|
|
31
|
+
* branch of the thread's surviving tree — the physical fact that this
|
|
32
|
+
* thread's run created it. A branch neither remembered nor local is
|
|
33
|
+
* refused whatever the caller says; so is one the mirror does not hold
|
|
34
|
+
* even after a fetch (the Worker's check: the tree is cloned from it).
|
|
35
|
+
* Every refusal is named in the attach answer so the bot can say why the
|
|
36
|
+
* follow-up runs where it does.
|
|
37
|
+
*
|
|
38
|
+
* The second movement: a rebound binding names a branch that can die — the
|
|
39
|
+
* pull request merges and the branch is deleted. A binding left on it would
|
|
40
|
+
* fail every later attach (`unknown-ref`) for the thread's whole life. So a
|
|
41
|
+
* binding a rebind moved, whose branch the mirror no longer holds after a
|
|
42
|
+
* fetch, goes back to the default it was bound to (`canReturnToDefault`,
|
|
43
|
+
* `returnToDefault`); the attach provisions the tree there, clean, and the
|
|
44
|
+
* thread is default-bound again, so a later own pull request may move it once
|
|
45
|
+
* more. A ref a person named that vanished keeps the `unknown-ref` refusal:
|
|
46
|
+
* that branch is the person's to sort out. */
|
|
34
47
|
|
|
35
48
|
/** The pull request the thread's own run opened, and its head branch — the
|
|
36
49
|
* reason a caller's refHint is that branch. */
|
|
@@ -145,15 +158,29 @@ export function boundByOf(binding: { ref: string; boundBy?: BoundBy }, defaultRe
|
|
|
145
158
|
return binding.boundBy ?? (binding.ref === defaultRef ? "default" : "name");
|
|
146
159
|
}
|
|
147
160
|
|
|
148
|
-
/** The record a rebind leaves on the binding and in the attach answer
|
|
161
|
+
/** The record a rebind leaves on the binding and in the attach answer: from
|
|
162
|
+
* which ref, onto which branch, for which pull request, when. `returnedAt`:
|
|
163
|
+
* that branch was gone from the mirror at a later attach and the binding
|
|
164
|
+
* went back to the default (the second movement) — a returned move no
|
|
165
|
+
* longer counts as the thread's one move. */
|
|
149
166
|
export interface Rebound {
|
|
150
167
|
from: string;
|
|
151
168
|
to: string;
|
|
152
169
|
pr: number;
|
|
153
170
|
at: string;
|
|
171
|
+
returnedAt?: string;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** The move back, in the attach answer: from the branch that is gone, to the
|
|
175
|
+
* default, for the pull request whose branch it was, when. */
|
|
176
|
+
export interface Returned {
|
|
177
|
+
from: string;
|
|
178
|
+
to: string;
|
|
179
|
+
pr: number;
|
|
180
|
+
at: string;
|
|
154
181
|
}
|
|
155
182
|
|
|
156
|
-
export type RebindRefusal = "named-ref" | "already-rebound" | "branch-absent"
|
|
183
|
+
export type RebindRefusal = "named-ref" | "already-rebound" | "branch-absent";
|
|
157
184
|
|
|
158
185
|
/** Why the binding stood, in the attach answer: the branch it was asked to
|
|
159
186
|
* move to, the pull request, the reason and its sentence. */
|
|
@@ -182,9 +209,10 @@ export type RebindPlan =
|
|
|
182
209
|
| { kind: "none" }
|
|
183
210
|
/** The binding alone rules it out; nothing on disk is consulted. */
|
|
184
211
|
| { kind: "refuse"; refused: RebindRefused }
|
|
185
|
-
/** The binding allows it and has a live tree
|
|
186
|
-
*
|
|
187
|
-
*
|
|
212
|
+
/** The binding allows it and has a live tree. `own`: the thread's own runs
|
|
213
|
+
* pushed the branch, as the binding remembers it — the fact that decides;
|
|
214
|
+
* when nothing was remembered, the tree's local branch is the fallback
|
|
215
|
+
* evidence (`rebindVerdict`). */
|
|
188
216
|
| { kind: "measure"; from: string; to: string; pr: number; own: boolean }
|
|
189
217
|
/** The binding allows it, its tree was evicted, and the thread's own runs
|
|
190
218
|
* pushed the branch: the binding moves and the attach recreates the tree
|
|
@@ -217,7 +245,10 @@ export function rebindPlan(input: {
|
|
|
217
245
|
),
|
|
218
246
|
};
|
|
219
247
|
}
|
|
220
|
-
|
|
248
|
+
// A move that was returned (its branch gone, the binding back on the
|
|
249
|
+
// default) no longer stands in the way: the thread may follow its next
|
|
250
|
+
// pull request as it followed the first.
|
|
251
|
+
if (binding.rebound && binding.rebound.returnedAt === undefined) {
|
|
221
252
|
const r = binding.rebound;
|
|
222
253
|
return {
|
|
223
254
|
kind: "refuse",
|
|
@@ -243,42 +274,30 @@ export function rebindPlan(input: {
|
|
|
243
274
|
return { kind: "measure", from: binding.ref, ...plan, own };
|
|
244
275
|
}
|
|
245
276
|
|
|
246
|
-
/** What the attach measured about the thread's tree, as the thread user
|
|
247
|
-
*
|
|
277
|
+
/** What the attach measured about the thread's tree, as the thread user —
|
|
278
|
+
* only when the binding remembers no push of the branch: the tree is then
|
|
279
|
+
* the only place the branch's origin can be read. */
|
|
248
280
|
export interface RebindTreeFacts {
|
|
249
281
|
/** `<worktree>/.git` is a directory. */
|
|
250
282
|
exists: boolean;
|
|
251
|
-
/** `git rev-parse --verify --quiet refs/heads/<to>` succeeded in the tree
|
|
283
|
+
/** `git rev-parse --verify --quiet refs/heads/<to>` succeeded in the tree;
|
|
284
|
+
* false for a branch the tree never made and for a tree git cannot read
|
|
285
|
+
* (neither verifies anything). */
|
|
252
286
|
branchExists?: boolean;
|
|
253
|
-
/** `git status --porcelain -uno` ran: false is a tree git cannot read
|
|
254
|
-
* (corrupt, or owned by an earlier pool user), where nothing is verifiable. */
|
|
255
|
-
readable?: boolean;
|
|
256
|
-
/** `git status --porcelain -uno` listed a tracked change (untracked scratch
|
|
257
|
-
* files are the thread's own state and survive a checkout). */
|
|
258
|
-
dirty?: boolean;
|
|
259
|
-
/** `git rev-parse --abbrev-ref HEAD` in the tree — the branch checked out
|
|
260
|
-
* (`HEAD` when detached) — measured once the tree is dirty: the one fact
|
|
261
|
-
* that tells a dirty tree already on the branch from one elsewhere. */
|
|
262
|
-
head?: string;
|
|
263
287
|
}
|
|
264
288
|
|
|
265
289
|
export type RebindVerdict =
|
|
266
|
-
/**
|
|
267
|
-
*
|
|
268
|
-
*
|
|
269
|
-
|
|
270
|
-
* by the rebind (`note` says so for the log). */
|
|
271
|
-
| { kind: "rebind"; checkout: true }
|
|
272
|
-
| { kind: "rebind"; checkout: false; note: string }
|
|
273
|
-
/** The tree is gone (a slept container) and the thread's own runs pushed the
|
|
274
|
-
* branch: move the binding and let the attach recreate the tree at it. */
|
|
275
|
-
| { kind: "recreate" }
|
|
276
|
-
| { kind: "refuse"; refused: RebindRefused };
|
|
290
|
+
/** The branch is the thread's own: move the binding. The tree is not this
|
|
291
|
+
* verdict's concern — the attach provisions it at the moved ref (item 17),
|
|
292
|
+
* once the Worker has seen the mirror hold the branch. */
|
|
293
|
+
{ kind: "rebind" } | { kind: "refuse"; refused: RebindRefused };
|
|
277
294
|
|
|
278
|
-
/**
|
|
295
|
+
/** Whether the branch is the thread's own, for a measured plan: the memory of
|
|
296
|
+
* a release decides by itself; without it, the surviving tree must hold the
|
|
297
|
+
* branch as a local branch. */
|
|
279
298
|
export function rebindVerdict(plan: { to: string; pr: number; own?: boolean }, tree: RebindTreeFacts): RebindVerdict {
|
|
299
|
+
if (plan.own === true) return { kind: "rebind" };
|
|
280
300
|
if (!tree.exists) {
|
|
281
|
-
if (plan.own === true) return { kind: "recreate" };
|
|
282
301
|
return {
|
|
283
302
|
kind: "refuse",
|
|
284
303
|
refused: rebindRefused(
|
|
@@ -288,46 +307,42 @@ export function rebindVerdict(plan: { to: string; pr: number; own?: boolean }, t
|
|
|
288
307
|
),
|
|
289
308
|
};
|
|
290
309
|
}
|
|
291
|
-
if (tree.readable === false) {
|
|
292
|
-
return {
|
|
293
|
-
kind: "refuse",
|
|
294
|
-
refused: rebindRefused(
|
|
295
|
-
plan,
|
|
296
|
-
"branch-absent",
|
|
297
|
-
"the thread's worktree cannot be read; the branch cannot be verified there",
|
|
298
|
-
),
|
|
299
|
-
};
|
|
300
|
-
}
|
|
301
310
|
if (tree.branchExists !== true) {
|
|
302
311
|
return {
|
|
303
312
|
kind: "refuse",
|
|
304
313
|
refused: rebindRefused(
|
|
305
314
|
plan,
|
|
306
315
|
"branch-absent",
|
|
307
|
-
`${JSON.stringify(plan.to)} is not a local branch of the thread's worktree; only a branch this thread's own run made moves it`,
|
|
316
|
+
`${JSON.stringify(plan.to)} is not a local branch of the thread's worktree and none of its runs pushed it; only a branch this thread's own run made moves it`,
|
|
308
317
|
),
|
|
309
318
|
};
|
|
310
319
|
}
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
320
|
+
return { kind: "rebind" };
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
/** Whether a binding whose ref the mirror no longer holds goes back to the
|
|
324
|
+
* default branch (the second movement): only a binding a rebind moved onto
|
|
325
|
+
* its own pull request's branch — bound by default in the first place, still
|
|
326
|
+
* on that branch, the move not yet returned. Anything else keeps the
|
|
327
|
+
* attach's `unknown-ref` refusal: a ref a person named is that person's to
|
|
328
|
+
* sort out, and a binding on the default cannot lose its ref. */
|
|
329
|
+
export function canReturnToDefault(
|
|
330
|
+
binding: { ref: string; boundBy?: BoundBy; rebound?: Rebound },
|
|
331
|
+
defaultRef: string,
|
|
332
|
+
): boolean {
|
|
333
|
+
const r = binding.rebound;
|
|
334
|
+
if (r === undefined || r.returnedAt !== undefined || binding.ref !== r.to) return false;
|
|
335
|
+
return binding.ref !== defaultRef && boundByOf(binding, defaultRef) === "default";
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/** The move back: the binding's ref becomes the default and the move that
|
|
339
|
+
* brought it here is stamped returned, so the thread may move again. Only
|
|
340
|
+
* ever applied to a binding `canReturnToDefault` admitted. */
|
|
341
|
+
export function returnToDefault<B extends { ref: string; rebound?: Rebound }>(
|
|
342
|
+
binding: B & { rebound: Rebound },
|
|
343
|
+
defaultRef: string,
|
|
344
|
+
at: string,
|
|
345
|
+
): { binding: B; returned: Returned } {
|
|
346
|
+
const returned: Returned = { from: binding.ref, to: defaultRef, pr: binding.rebound.pr, at };
|
|
347
|
+
return { binding: { ...binding, ref: defaultRef, rebound: { ...binding.rebound, returnedAt: at } }, returned };
|
|
333
348
|
}
|
|
@@ -16,11 +16,11 @@
|
|
|
16
16
|
* PROVISIONS (a fresh one). A reusing attach keeps a readable tree exactly
|
|
17
17
|
* as it stands, dirt and stale HEAD included, and refuses by name a tree it
|
|
18
18
|
* cannot keep (gone, unreadable, built for the other mode) without touching
|
|
19
|
-
* it. A provisioning attach keeps the dirty/stale discipline byte for byte
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
19
|
+
* it. A provisioning attach keeps the dirty/stale discipline byte for byte:
|
|
20
|
+
* a run starts from a clean tree at the bound ref's tip, and what a run
|
|
21
|
+
* wants kept it commits and pushes (docs/reference/specs/resident-repos.md
|
|
22
|
+
* item 17) — so a tree left dirty, or on a branch the binding has since
|
|
23
|
+
* moved away from, is recreated, never repaired. */
|
|
24
24
|
|
|
25
25
|
export type ParsedReuse = { reuse: boolean } | { error: string };
|
|
26
26
|
|
|
@@ -62,15 +62,6 @@ export type WorktreeDecision =
|
|
|
62
62
|
export function decideWorktree(input: {
|
|
63
63
|
/** True for a resumed run's attach: keep the tree, never wipe it. */
|
|
64
64
|
reuse: boolean;
|
|
65
|
-
/** True when this attach's rebind moved the binding onto the branch the
|
|
66
|
-
* tree already had checked out, dirty, and promised not to touch it
|
|
67
|
-
* (residentRebind.ts, the `checkout: false` verdict): the dirt is the
|
|
68
|
-
* thread's own uncommitted work on its own pull request branch, so a
|
|
69
|
-
* readable tree is kept as it stands — dirt and HEAD included, like a
|
|
70
|
-
* resumed run's — while a tree that turns out missing, unreadable or built
|
|
71
|
-
* for the other mode has nothing to keep and is provisioned like any fresh
|
|
72
|
-
* attach's. Absent on every attach that did not make that move. */
|
|
73
|
-
keepTree?: boolean;
|
|
74
65
|
/** The tree was built for the other mode (read-only against writable). */
|
|
75
66
|
modeSwitch: boolean;
|
|
76
67
|
/** The commit a provisioning attach checks out: the ref's tip, or the expected head. */
|
|
@@ -78,7 +69,7 @@ export function decideWorktree(input: {
|
|
|
78
69
|
worktreePath: string;
|
|
79
70
|
facts: WorktreeFacts;
|
|
80
71
|
}): WorktreeDecision {
|
|
81
|
-
const { reuse,
|
|
72
|
+
const { reuse, modeSwitch, sha, worktreePath, facts } = input;
|
|
82
73
|
if (modeSwitch) {
|
|
83
74
|
return reuse
|
|
84
75
|
? {
|
|
@@ -105,9 +96,7 @@ export function decideWorktree(input: {
|
|
|
105
96
|
}
|
|
106
97
|
// A reusing attach judges nothing past readability: the dirt and the HEAD are the run's own state.
|
|
107
98
|
if (reuse) return { kind: "reuse" };
|
|
108
|
-
|
|
109
|
-
// the dirt is the reason it made that promise, so it is not a reason to wipe.
|
|
110
|
-
if (facts.dirty) return keepTree ? { kind: "reuse" } : { kind: "recreate", why: "dirty" };
|
|
99
|
+
if (facts.dirty) return { kind: "recreate", why: "dirty" };
|
|
111
100
|
if (facts.head !== sha && facts.descendsFromTip !== true) return { kind: "recreate", why: "stale" };
|
|
112
101
|
return { kind: "reuse" };
|
|
113
102
|
}
|
|
@@ -18,7 +18,6 @@ export const RESIDENT_STEP_LABELS = {
|
|
|
18
18
|
"for-each-ref": "listing the branches",
|
|
19
19
|
checkout: "checking out the branch",
|
|
20
20
|
"checkout-update": "updating the checkout",
|
|
21
|
-
"rebind-checkout": "checking out the thread's own branch",
|
|
22
21
|
"rev-parse": "reading the commit",
|
|
23
22
|
"cat-file": "checking the mirror for the commit",
|
|
24
23
|
"show-ref": "reading the branch tip",
|
|
@@ -48,6 +48,11 @@ const FLEET_BUSY_PATTERNS: readonly RegExp[] = [
|
|
|
48
48
|
/^Failed to create session: 503\b/i,
|
|
49
49
|
/no container instance (?:that can be provided|available)/i,
|
|
50
50
|
/\bCONTAINER_UNAVAILABLE\b/,
|
|
51
|
+
// The platform's wording since the 0.13 line ("… Try again later, or try
|
|
52
|
+
// configuring a higher value for max_instances"): the 0.13 SDK's own warm
|
|
53
|
+
// pool matches on this exact phrase. Seen live passing through as a plain
|
|
54
|
+
// in-body error and ending two reviews in under a minute each.
|
|
55
|
+
/Maximum number of running container instances exceeded/i,
|
|
51
56
|
];
|
|
52
57
|
|
|
53
58
|
export function isFleetBusy(message: string): boolean {
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
// The per-thread sandbox's idle deadline, decided by the Durable Object from
|
|
2
|
+
// the one fact it owns (docs/reference/specs/execution.md item 22): when it
|
|
3
|
+
// last served a request. Deliberately free of node: imports so wrangler can
|
|
4
|
+
// bundle it into the sandbox Worker, like sandboxErrors.ts and
|
|
5
|
+
// sandboxLifecycle.ts.
|
|
6
|
+
//
|
|
7
|
+
// Why this exists: on the 0.13 SDK line the Container's `sleepAfter` is no
|
|
8
|
+
// longer a deadline. At expiry the SDK asks its runtime whether any tracked
|
|
9
|
+
// process or terminal is still active and, if so — or if the probe fails at
|
|
10
|
+
// all — renews the timeout instead of stopping. A detached pi holds its
|
|
11
|
+
// command's stdio open, a bot deploy mid-run orphans that pi until the same
|
|
12
|
+
// thread's next run, and the container is awake for good: twenty-five
|
|
13
|
+
// sandboxes were found running 10–16 h after their last request against a
|
|
14
|
+
// 5-minute sleepAfter with three runs in flight, and the fleet answered every
|
|
15
|
+
// new thread "Maximum number of running container instances exceeded". The
|
|
16
|
+
// guard below makes the deadline ours again: served-time is recorded on every
|
|
17
|
+
// request, a sweep the Durable Object schedules for itself checks it once a
|
|
18
|
+
// minute (and the SDK's own expiry hook is answered by the same verdict), and
|
|
19
|
+
// a container past the window is destroyed — through the SDK's clean teardown
|
|
20
|
+
// when that finishes in time, by the platform's own kill when it does not.
|
|
21
|
+
// Nothing inside the container can extend its life; only a request can.
|
|
22
|
+
|
|
23
|
+
import { BASH_TIMEOUT_MAX_MS } from "./bashTimeout.js";
|
|
24
|
+
|
|
25
|
+
/** The idle window in milliseconds — the twin of `SANDBOX_SLEEP_AFTER` ("5m",
|
|
26
|
+
* sandboxLifecycle.ts), which stays the SDK's own setting so its alarm loop
|
|
27
|
+
* still calls `onActivityExpired` on this cadence. A test holds the two
|
|
28
|
+
* together. */
|
|
29
|
+
export const SANDBOX_SLEEP_AFTER_MS = 5 * 60_000;
|
|
30
|
+
|
|
31
|
+
/** The guard's own cadence: a scheduled callback the Durable Object re-arms
|
|
32
|
+
* after each run while there is a container to guard. Independent of the
|
|
33
|
+
* SDK's activity renewals, so a deadline is met within a minute of passing
|
|
34
|
+
* even when the SDK's busy poll renews its timeout every second. */
|
|
35
|
+
export const IDLE_SWEEP_INTERVAL_MS = 60_000;
|
|
36
|
+
|
|
37
|
+
/** A request in flight this long is stuck, not service: the longest command
|
|
38
|
+
* `/exec` admits (`BASH_TIMEOUT_MAX_MS`, 20 min) plus the SDK backstop and
|
|
39
|
+
* output-wait margins with room to spare. Requests are measured one by one
|
|
40
|
+
* (each carries its own start), so an overlapping chain of short polls beside
|
|
41
|
+
* a long command is never read as one long request. */
|
|
42
|
+
export const INFLIGHT_STUCK_MS = BASH_TIMEOUT_MAX_MS + 10 * 60_000;
|
|
43
|
+
|
|
44
|
+
/** How often the served-time reaches Durable Object storage: the in-memory
|
|
45
|
+
* fact is exact while the object lives, storage is the baseline for the next
|
|
46
|
+
* wake, and the pi harness polls every 750 ms — one write per poll would be
|
|
47
|
+
* the loudest thing the Worker does. */
|
|
48
|
+
export const PERSIST_EVERY_MS = 10_000;
|
|
49
|
+
|
|
50
|
+
/** How long the SDK's clean `destroy()` may take before the platform's own
|
|
51
|
+
* kill ends the container regardless. The SDK bounds its runtime cleanup at
|
|
52
|
+
* 30 s; a teardown that has not finished twice that is not going to. */
|
|
53
|
+
export const DESTROY_GRACE_MS = 60_000;
|
|
54
|
+
|
|
55
|
+
/** The facts the verdict is drawn from. `inflight` holds the start time of
|
|
56
|
+
* every request being served right now — a list, not a count, so the oldest
|
|
57
|
+
* LIVE request decides "stuck" and a finished one stops counting. */
|
|
58
|
+
export interface IdleLedger {
|
|
59
|
+
lastServedAt: number;
|
|
60
|
+
inflight: number[];
|
|
61
|
+
lastPersistedAt: number;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export function newIdleLedger(now: number): IdleLedger {
|
|
65
|
+
return { lastServedAt: now, inflight: [], lastPersistedAt: now };
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export type IdleVerdict =
|
|
69
|
+
| { action: "destroy"; why: "idle" | "stuck"; idleMs: number }
|
|
70
|
+
| { action: "keep"; why: "warm" | "serving"; recheckInMs: number };
|
|
71
|
+
|
|
72
|
+
/** The decision, pure: destroy when nothing is in flight and the last request
|
|
73
|
+
* finished a full window ago, or when the oldest request in flight has been
|
|
74
|
+
* there longer than any command may run; keep otherwise. A clock that went
|
|
75
|
+
* backwards reads as a fresh request, never as an idle container. */
|
|
76
|
+
export function idleVerdict(ledger: IdleLedger, now: number, sleepAfterMs = SANDBOX_SLEEP_AFTER_MS): IdleVerdict {
|
|
77
|
+
if (ledger.inflight.length > 0) {
|
|
78
|
+
const oldest = Math.min(...ledger.inflight);
|
|
79
|
+
const inflightMs = now - oldest;
|
|
80
|
+
if (inflightMs >= INFLIGHT_STUCK_MS) return { action: "destroy", why: "stuck", idleMs: inflightMs };
|
|
81
|
+
return { action: "keep", why: "serving", recheckInMs: IDLE_SWEEP_INTERVAL_MS };
|
|
82
|
+
}
|
|
83
|
+
const idleMs = now - ledger.lastServedAt;
|
|
84
|
+
if (idleMs >= sleepAfterMs) return { action: "destroy", why: "idle", idleMs };
|
|
85
|
+
return { action: "keep", why: "warm", recheckInMs: sleepAfterMs - idleMs };
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** What the guard needs from the Durable Object, as plain functions so the
|
|
89
|
+
* whole decision — arming, counting, destroying, forcing — is exercised
|
|
90
|
+
* against a fake in tests and the Worker's class only forwards. */
|
|
91
|
+
export interface IdleGuardHost {
|
|
92
|
+
now(): number;
|
|
93
|
+
/** `ctx.container?.running`: false when the platform says stopped, true when
|
|
94
|
+
* running, undefined when the object cannot tell — guarded like running. */
|
|
95
|
+
containerRunning(): boolean | undefined;
|
|
96
|
+
/** Whether a sweep callback is already scheduled (the SDK's schedule table). */
|
|
97
|
+
sweepScheduled(): Promise<boolean>;
|
|
98
|
+
scheduleSweep(delayMs: number): Promise<void>;
|
|
99
|
+
/** The SDK's clean teardown (`Sandbox.destroy()`): sessions closed, the
|
|
100
|
+
* container SIGKILLed at the end. May hang or throw; the guard bounds it. */
|
|
101
|
+
destroySandbox(): Promise<void>;
|
|
102
|
+
/** The platform primitive (`ctx.container.destroy()`): the microVM is gone. */
|
|
103
|
+
killContainer(): Promise<void>;
|
|
104
|
+
loadLastServedAt(): Promise<number | undefined>;
|
|
105
|
+
saveLastServedAt(at: number): Promise<void>;
|
|
106
|
+
log(event: Record<string, unknown>): void;
|
|
107
|
+
wait(ms: number): Promise<void>;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export type IdleStopSource = "sweep" | "sdk-expiry";
|
|
111
|
+
|
|
112
|
+
export class IdleGuard {
|
|
113
|
+
readonly ledger: IdleLedger;
|
|
114
|
+
private armed = false;
|
|
115
|
+
private destroying: Promise<void> | null = null;
|
|
116
|
+
|
|
117
|
+
constructor(private readonly host: IdleGuardHost) {
|
|
118
|
+
this.ledger = newIdleLedger(host.now());
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** On every Durable Object wake (its constructor): the stored served-time is
|
|
122
|
+
* the baseline when there is one — a container found awake with no record
|
|
123
|
+
* is idle from now, so a fleet leaked before this code arrived is gone one
|
|
124
|
+
* window after the deploy — and one sweep is armed when a container may be
|
|
125
|
+
* running and none is scheduled. */
|
|
126
|
+
async wake(): Promise<void> {
|
|
127
|
+
const stored = await this.host.loadLastServedAt();
|
|
128
|
+
if (stored !== undefined) {
|
|
129
|
+
this.ledger.lastServedAt = stored;
|
|
130
|
+
this.ledger.lastPersistedAt = stored;
|
|
131
|
+
}
|
|
132
|
+
if (this.guarding()) await this.arm();
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Every request the object serves runs inside this: counted in flight from
|
|
136
|
+
* its start (so a live command is never idle), recorded at its finish
|
|
137
|
+
* (success or failure alike), persisted on the persist cadence. */
|
|
138
|
+
async served<T>(op: () => Promise<T>): Promise<T> {
|
|
139
|
+
const startedAt = this.host.now();
|
|
140
|
+
this.ledger.inflight.push(startedAt);
|
|
141
|
+
try {
|
|
142
|
+
await this.arm();
|
|
143
|
+
return await op();
|
|
144
|
+
} finally {
|
|
145
|
+
const i = this.ledger.inflight.indexOf(startedAt);
|
|
146
|
+
if (i >= 0) this.ledger.inflight.splice(i, 1);
|
|
147
|
+
const now = this.host.now();
|
|
148
|
+
this.ledger.lastServedAt = now;
|
|
149
|
+
if (now - this.ledger.lastPersistedAt >= PERSIST_EVERY_MS) {
|
|
150
|
+
this.ledger.lastPersistedAt = now;
|
|
151
|
+
try {
|
|
152
|
+
await this.host.saveLastServedAt(now);
|
|
153
|
+
} catch (err) {
|
|
154
|
+
this.host.log({ event: "sandbox.idle-ledger.persist-failed", error: String(err) });
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** The scheduled callback. The SDK deletes a schedule row once it has run,
|
|
161
|
+
* so the guard re-arms itself here while there is a container to guard;
|
|
162
|
+
* with none (destroyed, or never started), the next request arms it. */
|
|
163
|
+
async sweep(): Promise<void> {
|
|
164
|
+
this.armed = false;
|
|
165
|
+
await this.enforce("sweep");
|
|
166
|
+
if (this.guarding()) await this.arm();
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** The SDK's `onActivityExpired`, answered by the same verdict — its
|
|
170
|
+
* process and terminal probes never decide. */
|
|
171
|
+
async expired(): Promise<void> {
|
|
172
|
+
await this.enforce("sdk-expiry");
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
private guarding(): boolean {
|
|
176
|
+
return this.host.containerRunning() !== false || this.ledger.inflight.length > 0;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
private async arm(): Promise<void> {
|
|
180
|
+
if (this.armed) return;
|
|
181
|
+
if (!(await this.host.sweepScheduled())) await this.host.scheduleSweep(IDLE_SWEEP_INTERVAL_MS);
|
|
182
|
+
this.armed = true;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
private async enforce(source: IdleStopSource): Promise<void> {
|
|
186
|
+
if (this.destroying) return this.destroying;
|
|
187
|
+
if (!this.guarding()) return;
|
|
188
|
+
const verdict = idleVerdict(this.ledger, this.host.now());
|
|
189
|
+
if (verdict.action === "keep") return;
|
|
190
|
+
this.destroying = this.destroy(verdict, source).finally(() => {
|
|
191
|
+
this.destroying = null;
|
|
192
|
+
});
|
|
193
|
+
return this.destroying;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** The SDK's clean destroy, bounded; then the platform's kill unless the
|
|
197
|
+
* container is known stopped. The guarantee lives in the second step. */
|
|
198
|
+
private async destroy(verdict: Extract<IdleVerdict, { action: "destroy" }>, source: IdleStopSource): Promise<void> {
|
|
199
|
+
const base = { why: verdict.why, idleMs: verdict.idleMs, source, sleepAfterMs: SANDBOX_SLEEP_AFTER_MS };
|
|
200
|
+
this.host.log({ event: "sandbox.idle-stop", ...base });
|
|
201
|
+
let outcome: "done" | "timeout" | "failed";
|
|
202
|
+
let error: string | undefined;
|
|
203
|
+
try {
|
|
204
|
+
outcome = await Promise.race([
|
|
205
|
+
this.host.destroySandbox().then(() => "done" as const),
|
|
206
|
+
this.host.wait(DESTROY_GRACE_MS).then(() => "timeout" as const),
|
|
207
|
+
]);
|
|
208
|
+
} catch (err) {
|
|
209
|
+
outcome = "failed";
|
|
210
|
+
error = String(err);
|
|
211
|
+
}
|
|
212
|
+
if (outcome === "done" && this.host.containerRunning() === false) return;
|
|
213
|
+
this.host.log({ event: "sandbox.idle-stop.forced", ...base, outcome, ...(error ? { error } : {}) });
|
|
214
|
+
try {
|
|
215
|
+
await this.host.killContainer();
|
|
216
|
+
} catch (err) {
|
|
217
|
+
this.host.log({ event: "sandbox.idle-stop.kill-failed", ...base, error: String(err) });
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|