@coreplane/switchboard 1.235.0 → 1.236.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/Dockerfile +11 -0
- package/dist/assets/config/config.example.yaml +9 -4
- package/dist/assets/deploy/cloudflare-memory/worker.ts +12 -0
- package/dist/assets/deploy/cloudflare-resident/Dockerfile +18 -0
- package/dist/assets/deploy/cloudflare-resident/worker.ts +115 -56
- package/dist/assets/deploy/cloudflare-sandbox/Dockerfile +11 -0
- package/dist/assets/deploy/cloudflare-sandbox/worker.ts +68 -7
- package/dist/assets/package-lock.json +212 -3
- package/dist/assets/package.json +4 -1
- package/dist/assets/source.json +3 -3
- package/dist/assets/src/core/coordinator/contract.ts +24 -4
- package/dist/assets/src/core/coordinator/driver.ts +31 -3
- package/dist/assets/src/core/runEvents.ts +36 -1
- package/dist/assets/src/core/runFriction.ts +1 -0
- package/dist/assets/src/core/ship/contract.ts +3 -1
- package/dist/assets/src/core/ship/coordinator.ts +213 -29
- package/dist/assets/src/core/trace/types.ts +5 -0
- package/dist/assets/src/execution/residentDisk.ts +21 -10
- package/dist/assets/src/execution/sandboxErrors.ts +72 -0
- package/dist/assets/src/execution/sandboxStart.ts +90 -0
- package/dist/assets/web/dist/.vite/manifest.json +437 -423
- package/dist/assets/web/dist/assets/{AppShell-lYVcj-k7.js → AppShell-DZdz_sl7.js} +1 -1
- package/dist/assets/web/dist/assets/{CostsPage-D-v8am88.js → CostsPage-BfJsol9y.js} +1 -1
- package/dist/assets/web/dist/assets/{DeliveryPage-CvlWP7Eq.js → DeliveryPage-CvInfu7y.js} +1 -1
- package/dist/assets/web/dist/assets/{NotFoundPage-BzWS4Ca1.js → NotFoundPage-d37EP8hS.js} +1 -1
- package/dist/assets/web/dist/assets/ResidentDetailPage-DQSD0SZD.js +1 -0
- package/dist/assets/web/dist/assets/ResidentsIndexPage-CZUGLen6.js +1 -0
- package/dist/assets/web/dist/assets/RunFoldRow-Bngn4GjF.js +9 -0
- package/dist/assets/web/dist/assets/{RunRoutePage-oN9GkVEA.js → RunRoutePage-a-tMrirB.js} +3 -3
- package/dist/assets/web/dist/assets/RunsIndexPage-s0VKmRda.js +1 -0
- package/dist/assets/web/dist/assets/{RunsTabs-DEICQ4FY.js → RunsTabs-CyCdOhJ2.js} +1 -1
- package/dist/assets/web/dist/assets/ScheduledPage-ByCmyrDs.js +1 -0
- package/dist/assets/web/dist/assets/SettingsPage-DhKEokGK.js +1 -0
- package/dist/assets/web/dist/assets/{StatusDot-D14iJMy7.js → StatusDot-D-BPSPpw.js} +1 -1
- package/dist/assets/web/dist/assets/{Tooltip-B0Ob5MQ4.js → Tooltip-DPDx5tXT.js} +1 -1
- package/dist/assets/web/dist/assets/UnitRoutePage-0eiekTHQ.js +1 -0
- package/dist/assets/web/dist/assets/{angular-html-Cw130Zyi.js → angular-html-Ni1Xil2G.js} +1 -1
- package/dist/assets/web/dist/assets/{angular-ts-DY1m4C2s.js → angular-ts-oCgo6p05.js} +1 -1
- package/dist/assets/web/dist/assets/{apl-W6Ty7v05.js → apl-CiWYl0gh.js} +1 -1
- package/dist/assets/web/dist/assets/{astro-DlLVHwVk.js → astro-B-VeKZ_J.js} +1 -1
- package/dist/assets/web/dist/assets/{blade-Bco3tDh4.js → blade-BIWEISKd.js} +1 -1
- package/dist/assets/web/dist/assets/{c-1LaFY_cj.js → c-DHxA5Byp.js} +1 -1
- package/dist/assets/web/dist/assets/{chapel-B-OvrOL5.js → chapel-D4qGLw-Q.js} +1 -1
- package/dist/assets/web/dist/assets/{cobol-Ch3EPlHu.js → cobol-BznX5Q-u.js} +1 -1
- package/dist/assets/web/dist/assets/{coffee-hZuITnto.js → coffee-CVr8Idlj.js} +1 -1
- package/dist/assets/web/dist/assets/{cpp-B8iRVo8m.js → cpp-wrk6NfW4.js} +1 -1
- package/dist/assets/web/dist/assets/{crystal-DKveI7lr.js → crystal-CGYWMeH1.js} +1 -1
- package/dist/assets/web/dist/assets/{css-_2_CD-rr.js → css-BgGytMsy.js} +1 -1
- package/dist/assets/web/dist/assets/{dist-pOdUzj6D.js → dist-DSkVB16z.js} +2 -2
- package/dist/assets/web/dist/assets/durationTone-DYogEPEk.js +1 -0
- package/dist/assets/web/dist/assets/{edge-BenMDPct.js → edge-CdR6AU9v.js} +1 -1
- package/dist/assets/web/dist/assets/{elixir-CBbdEqG6.js → elixir-CQRb3PZx.js} +1 -1
- package/dist/assets/web/dist/assets/{elm-DMub_2lY.js → elm-CpuA6W8B.js} +1 -1
- package/dist/assets/web/dist/assets/{erb-CI9He_Up.js → erb-CvlrblqH.js} +1 -1
- package/dist/assets/web/dist/assets/{favicon-BQsePYv5.js → favicon-CSt-qDvc.js} +1 -1
- package/dist/assets/web/dist/assets/{git-rebase-BBfhnyF8.js → git-rebase-COnovCVc.js} +1 -1
- package/dist/assets/web/dist/assets/{glimmer-js-CR9Fxrau.js → glimmer-js-C-DS4g_J.js} +1 -1
- package/dist/assets/web/dist/assets/{glimmer-ts-CwFwX_XS.js → glimmer-ts-DTkorWRI.js} +1 -1
- package/dist/assets/web/dist/assets/{glsl-Daiodp1r.js → glsl-CH51-JW7.js} +1 -1
- package/dist/assets/web/dist/assets/{graphql-DZvrsEkB.js → graphql-BO-XymNv.js} +1 -1
- package/dist/assets/web/dist/assets/{hack-BaXOzdg7.js → hack-C-N0oicf.js} +1 -1
- package/dist/assets/web/dist/assets/{haml-CMskq_VA.js → haml-BfmdtWoP.js} +1 -1
- package/dist/assets/web/dist/assets/{handlebars-Dhw_f7jD.js → handlebars-BqhO_te-.js} +1 -1
- package/dist/assets/web/dist/assets/{html-C9k99z0z.js → html-DUEZlTKV.js} +1 -1
- package/dist/assets/web/dist/assets/{html-derivative-VckPItXN.js → html-derivative-DOUJTS8A.js} +1 -1
- package/dist/assets/web/dist/assets/{http-DXU_3h9v.js → http-j7UH0Cqv.js} +1 -1
- package/dist/assets/web/dist/assets/{hurl-KU24JNdp.js → hurl-D6ZaBLJk.js} +1 -1
- package/dist/assets/web/dist/assets/indexRow-BjXBzI5Z.js +1 -0
- package/dist/assets/web/dist/assets/{java-BbZjTNgf.js → java-dugWx3wi.js} +1 -1
- package/dist/assets/web/dist/assets/{javascript-DXCdqcZZ.js → javascript-CK4GC4JO.js} +1 -1
- package/dist/assets/web/dist/assets/{jinja-pDlsjTVp.js → jinja-DrLKIyKG.js} +1 -1
- package/dist/assets/web/dist/assets/{jison-DsXDk66e.js → jison-CS1Th-TO.js} +1 -1
- package/dist/assets/web/dist/assets/{json-tCxXfRgG.js → json-CkBqdVys.js} +1 -1
- package/dist/assets/web/dist/assets/{jsx-B0ZYkCmi.js → jsx-CCxdg7n3.js} +1 -1
- package/dist/assets/web/dist/assets/{julia-BLzzeI6x.js → julia-BalKW2hA.js} +1 -1
- package/dist/assets/web/dist/assets/{just-DvD2_60c.js → just-BKw1G40D.js} +1 -1
- package/dist/assets/web/dist/assets/{latex-511zn3h7.js → latex-Dfad6ZN-.js} +1 -1
- package/dist/assets/web/dist/assets/{liquid-BD0BkMox.js → liquid-f_CMGBvd.js} +1 -1
- package/dist/assets/web/dist/assets/{indexFormat-B-pd4r7Y.js → localIso-CNA-bhS-.js} +1 -1
- package/dist/assets/web/dist/assets/{lua-NFicKm6N.js → lua-BcMK_uKM.js} +1 -1
- package/dist/assets/web/dist/assets/main-B77vAZVg.css +1 -0
- package/dist/assets/web/dist/assets/main-DR8iKpNt.js +28 -0
- package/dist/assets/web/dist/assets/{marko-1-G_nEXK.js → marko-B0oNLx2i.js} +1 -1
- package/dist/assets/web/dist/assets/{mdc-DUe3AQLp.js → mdc-Bc_MmIyq.js} +1 -1
- package/dist/assets/web/dist/assets/{nginx-Dg379_bA.js → nginx-Cmxv8RIt.js} +1 -1
- package/dist/assets/web/dist/assets/{nim-BnFq97ZO.js → nim-BtgRfaY0.js} +1 -1
- package/dist/assets/web/dist/assets/{org-F0uiwGvq.js → org-Dl9EzlHh.js} +1 -1
- package/dist/assets/web/dist/assets/{perl-B0euczkl.js → perl-BmF7x7BB.js} +1 -1
- package/dist/assets/web/dist/assets/{php-DkL3k_n7.js → php-rH6CVptQ.js} +1 -1
- package/dist/assets/web/dist/assets/{pug-L_OjjZP4.js → pug-CIN5Ccck.js} +1 -1
- package/dist/assets/web/dist/assets/{qml-BMjB00Zz.js → qml-BpLS3RFr.js} +1 -1
- package/dist/assets/web/dist/assets/{r-DoeLdnqR.js → r-BUMF3B-H.js} +1 -1
- package/dist/assets/web/dist/assets/{razor-BzYNkAWy.js → razor-B7sTBPdT.js} +1 -1
- package/dist/assets/web/dist/assets/{regexp-CPmElMk3.js → regexp-h2SvwrsJ.js} +1 -1
- package/dist/assets/web/dist/assets/residentsModel-VSJEepL1.js +1 -0
- package/dist/assets/web/dist/assets/{rst-DrsjgfI7.js → rst-DLOWEsD2.js} +1 -1
- package/dist/assets/web/dist/assets/{ruby-cB42ppvx.js → ruby-Cl_-I4k3.js} +1 -1
- package/dist/assets/web/dist/assets/{sas-R5Y3NM_I.js → sas-Cm4M-BvV.js} +1 -1
- package/dist/assets/web/dist/assets/{scss-C-zWxV9s.js → scss-CEFCEQUF.js} +1 -1
- package/dist/assets/web/dist/assets/{shellscript-oJF96aAU.js → shellscript-CE6zb5eS.js} +1 -1
- package/dist/assets/web/dist/assets/{shellsession-CznECKyi.js → shellsession-B4fDVzrM.js} +1 -1
- package/dist/assets/web/dist/assets/{soy-wjHLSRag.js → soy-DwXCMpjE.js} +1 -1
- package/dist/assets/web/dist/assets/{sql-C_BM8IOW.js → sql-DrnAKnyD.js} +1 -1
- package/dist/assets/web/dist/assets/{stata-D31HO8aQ.js → stata-BzlwPWD6.js} +1 -1
- package/dist/assets/web/dist/assets/{surrealql-CKCLyplA.js → surrealql-CeUieZXd.js} +1 -1
- package/dist/assets/web/dist/assets/{svelte-3geWk2iu.js → svelte-BERIpIKR.js} +1 -1
- package/dist/assets/web/dist/assets/{templ-DiJr_hTD.js → templ-6FSuUDwg.js} +1 -1
- package/dist/assets/web/dist/assets/{tex-BEJGWyeF.js → tex-SWCyMwNX.js} +1 -1
- package/dist/assets/web/dist/assets/{ts-tags-C8Mzdpxx.js → ts-tags-VWj7gWgY.js} +1 -1
- package/dist/assets/web/dist/assets/{tsx-UQiSm_p3.js → tsx-B6dzfL8L.js} +1 -1
- package/dist/assets/web/dist/assets/{twig-_Pvzkeo9.js → twig-BDXsr482.js} +1 -1
- package/dist/assets/web/dist/assets/{typescript-BWMkINKK.js → typescript-s100Dt8y.js} +1 -1
- package/dist/assets/web/dist/assets/{typst-Bdg9m-e7.js → typst-WfyEAUdt.js} +1 -1
- package/dist/assets/web/dist/assets/{vue-CDZrrAZ8.js → vue-CkXOOJ-6.js} +1 -1
- package/dist/assets/web/dist/assets/{vue-html-tAlpNhfN.js → vue-html-CEpzdcau.js} +1 -1
- package/dist/assets/web/dist/assets/{vue-vine-C27UF_rh.js → vue-vine-D9dsfiFA.js} +1 -1
- package/dist/assets/web/dist/assets/{xml-CxeDr9Zh.js → xml-iBDofbJq.js} +1 -1
- package/dist/assets/web/dist/assets/{xsl-Bpi0uUnM.js → xsl-CqbJF4hD.js} +1 -1
- package/dist/assets/web/dist/assets/{yaml-BtUgZ1GN.js → yaml-D9i6OxYe.js} +1 -1
- package/dist/cli.js +2255 -1170
- package/package.json +1 -1
- package/dist/assets/web/dist/assets/ResidentDetailPage-DSpsye5e.js +0 -1
- package/dist/assets/web/dist/assets/ResidentsIndexPage-CT8fI1M3.js +0 -1
- package/dist/assets/web/dist/assets/RunFoldRow-Cqh8pTXC.js +0 -9
- package/dist/assets/web/dist/assets/RunsIndexPage-sjXr8iCV.js +0 -1
- package/dist/assets/web/dist/assets/ScheduledPage-Tz__zLzi.js +0 -1
- package/dist/assets/web/dist/assets/UnitRoutePage-CbKCL58v.js +0 -1
- package/dist/assets/web/dist/assets/indexRow-Bnj883ii.js +0 -1
- package/dist/assets/web/dist/assets/localIso-L06jV29p.js +0 -1
- package/dist/assets/web/dist/assets/main-D3lG-yQ9.js +0 -28
- package/dist/assets/web/dist/assets/main-Dm11o0hc.css +0 -1
- package/dist/assets/web/dist/assets/residentDiskBudget-BMBKlYRH.js +0 -1
|
@@ -51,6 +51,29 @@ export interface ShipCaps {
|
|
|
51
51
|
* burning an attach and a model turn on a doomed round. */
|
|
52
52
|
export const SHIP_ROUND_RESERVE_MS = 3 * MIN;
|
|
53
53
|
|
|
54
|
+
/** What a round-0 coding child must leave on the pipeline's clock: two review
|
|
55
|
+
* rounds (a review and, after a fix, its re-review) and one merge poll — the
|
|
56
|
+
* loop the pipeline exists to run. The coding child's budget is clipped to
|
|
57
|
+
* the remaining wall clock MINUS this reserve (never under the two minutes a
|
|
58
|
+
* spawn accepts), so a child that uses its whole directive still hands the
|
|
59
|
+
* pipeline a clock that holds the review; without the clip a 40-minute
|
|
60
|
+
* pipeline hands 39 minutes to the child and caps out with the pull request
|
|
61
|
+
* shipped and unreviewed. */
|
|
62
|
+
export const SHIP_LOOP_RESERVE_MS = 2 * SHIP_ROUND_RESERVE_MS + 5 * MIN;
|
|
63
|
+
|
|
64
|
+
/** What a findings child (the fix after a review asked for changes) must leave
|
|
65
|
+
* on the clock: the re-review and one merge poll — one round's reserve and
|
|
66
|
+
* five minutes. Never the whole loop's: by the time a fix round runs, the
|
|
67
|
+
* first review has already happened, and holding two rounds back from a late
|
|
68
|
+
* fix would cap a pipeline that still has the time. */
|
|
69
|
+
export const SHIP_FIX_RESERVE_MS = SHIP_ROUND_RESERVE_MS + 5 * MIN;
|
|
70
|
+
|
|
71
|
+
/** The floor `validateShip` holds `ship.maxMinutes` to: the loop's reserve
|
|
72
|
+
* plus one round's — under it no coding child can both work and leave the
|
|
73
|
+
* review its time, so the config is refused at load rather than left to cap
|
|
74
|
+
* out on every unit. */
|
|
75
|
+
export const SHIP_MIN_MAX_MINUTES = (SHIP_LOOP_RESERVE_MS + SHIP_ROUND_RESERVE_MS) / MIN;
|
|
76
|
+
|
|
54
77
|
/** What a ship pipeline's thread and card say when the bot died under it (run-
|
|
55
78
|
* history item 36): the work it did stands on GitHub with nobody driving it,
|
|
56
79
|
* so the note names the PR when one was opened and the exact re-issue that
|
|
@@ -407,7 +430,15 @@ export type CoordinatorAction =
|
|
|
407
430
|
| { type: "spawn"; step: string; preset: ChildPreset; round: RoundRef; budgetMinutes: number; brief: Brief }
|
|
408
431
|
| { type: "wait"; step: string; runId: string; timeoutMs: number }
|
|
409
432
|
| { type: "read-record"; step: string; runId: string }
|
|
410
|
-
| {
|
|
433
|
+
| {
|
|
434
|
+
type: "pr-check";
|
|
435
|
+
step: string;
|
|
436
|
+
/** Set after a coding child died: the bot opens the pull request from the
|
|
437
|
+
* pushed branch itself (title from the unit, body from this run's
|
|
438
|
+
* submitted description when the record holds one) instead of answering
|
|
439
|
+
* `none` over stranded work. */
|
|
440
|
+
recover?: { runId: string };
|
|
441
|
+
}
|
|
411
442
|
| { type: "merge"; step: string; prNumber: number; headSha: string }
|
|
412
443
|
| { type: "sleep"; step: string; ms: number }
|
|
413
444
|
| { type: "end"; step: string; ending: UnitEnding };
|
|
@@ -442,7 +473,14 @@ export type ChildFacts =
|
|
|
442
473
|
/** What heads the unit's branch on GitHub: nothing, an open pull request, or —
|
|
443
474
|
* with no open one — a merged one, `sha` the merge commit on the base. */
|
|
444
475
|
export type PrCheck =
|
|
445
|
-
| {
|
|
476
|
+
| {
|
|
477
|
+
state: "none";
|
|
478
|
+
/** After a recover pr-check (a dead coding child): why nothing was
|
|
479
|
+
* recovered — `no_commits` (GitHub refused the create: nothing between
|
|
480
|
+
* the base and the head) or `no_base` (the instance names no base to
|
|
481
|
+
* open against, so no create was tried). Absent on a plain check. */
|
|
482
|
+
unrecovered?: "no_commits" | "no_base";
|
|
483
|
+
}
|
|
446
484
|
| { state: "open"; prNumber: number; url: string; headSha?: string; autoMergeEnabled?: boolean }
|
|
447
485
|
| { state: "merged"; prNumber: number; url: string; sha: string; mergedAt: string };
|
|
448
486
|
|
|
@@ -473,7 +511,12 @@ export type UnitEnding =
|
|
|
473
511
|
| { kind: "merge_ready"; pr: PrRef; reviewRounds: number }
|
|
474
512
|
| { kind: "merge_refused"; pr: PrRef; reason: string; reviewRounds: number }
|
|
475
513
|
| { kind: "round_cap"; maxRounds: number; reviewRounds: number }
|
|
476
|
-
| { kind: "wall_clock_cap"; remainingMs: number; reviewRounds: number }
|
|
514
|
+
| { kind: "wall_clock_cap"; remainingMs: number; reviewRounds: number; spent: ShipBudgetSpent }
|
|
515
|
+
/** The wall clock capped AFTER the coding child opened or updated the pull
|
|
516
|
+
* request: the work stands and only the review is missing, so the ending
|
|
517
|
+
* names the pull request and "review pending" instead of calling the unit a
|
|
518
|
+
* failure — the re-issued attempt starts at the review round (`lastPush`). */
|
|
519
|
+
| { kind: "review_pending"; pr: PrRef; headSha?: string; reviewRounds: number; spent: ShipBudgetSpent }
|
|
477
520
|
| {
|
|
478
521
|
kind: "stopped";
|
|
479
522
|
mode: "soft" | "hard";
|
|
@@ -494,6 +537,12 @@ export type CoordinatorNote =
|
|
|
494
537
|
| { type: "round"; index: number; agent: ChildPreset; outcome: ShipRoundOutcome }
|
|
495
538
|
| { type: "ended"; ending: UnitEnding };
|
|
496
539
|
|
|
540
|
+
/** How the pipeline's budget went, in ms: the coding rounds' (round 0 and the
|
|
541
|
+
* findings steps), the review rounds', and everything else (branching,
|
|
542
|
+
* pr-checks, busy waits, merge polls) — reported on the cap endings so a
|
|
543
|
+
* person can see whether the cap or the child is the problem. */
|
|
544
|
+
export type ShipBudgetSpent = Readonly<Record<"coding" | "review" | "waiting", number>>;
|
|
545
|
+
|
|
497
546
|
export interface UnitPipelineInput {
|
|
498
547
|
unit: { id: string; branch: string };
|
|
499
548
|
repo: string;
|
|
@@ -513,6 +562,11 @@ export interface UnitPipelineInput {
|
|
|
513
562
|
generated: boolean;
|
|
514
563
|
/** Resume at review: an open pull request of ship's own the requester named. */
|
|
515
564
|
resume?: { pr: number; headSha?: string; url?: string };
|
|
565
|
+
/** The head the previous attempt's coding child last pushed (a
|
|
566
|
+
* `review_pending` ending's `headSha`, carried on the unit's row): when the
|
|
567
|
+
* pre-check finds the open pull request still at exactly this head, there is
|
|
568
|
+
* nothing to code and the attempt starts at the review round. */
|
|
569
|
+
lastPush?: string;
|
|
516
570
|
}
|
|
517
571
|
|
|
518
572
|
type Phase =
|
|
@@ -524,7 +578,17 @@ type Phase =
|
|
|
524
578
|
/** `until`: when the child's budget plus the margin runs out, counted from the spawn's answer — the wait's last slice ends there. */
|
|
525
579
|
| { at: "wait"; round: RoundRef; runId: string; n: number; until: number }
|
|
526
580
|
| { at: "read"; round: RoundRef; runId: string; n: number; until: number }
|
|
527
|
-
| {
|
|
581
|
+
| {
|
|
582
|
+
at: "pr-check";
|
|
583
|
+
round: RoundRef;
|
|
584
|
+
runId: string;
|
|
585
|
+
childHead?: string;
|
|
586
|
+
finalReply?: string;
|
|
587
|
+
/** The coding child died (`failed` or `interrupted`) after it may have
|
|
588
|
+
* pushed: the pr-check recovers a pushed branch by opening its pull
|
|
589
|
+
* request; with nothing pushed the unit ends with the child's own reason. */
|
|
590
|
+
dead?: "failed" | "interrupted";
|
|
591
|
+
}
|
|
528
592
|
| { at: "merge"; pr: PrRef; headSha: string; n: number; since: number }
|
|
529
593
|
| { at: "merge-sleep"; pr: PrRef; headSha: string; n: number; since: number }
|
|
530
594
|
| { at: "ended" };
|
|
@@ -551,6 +615,8 @@ export interface UnitPipelineState {
|
|
|
551
615
|
readonly findingsRunByRound: Readonly<Record<number, string>>;
|
|
552
616
|
/** The last coding run (round 0's child or a findings step's): its record carries the unit's handoff. */
|
|
553
617
|
readonly lastCodingRunId?: string;
|
|
618
|
+
/** How the budget went so far, accrued as each answer moves the clock. */
|
|
619
|
+
readonly spentMs: ShipBudgetSpent;
|
|
554
620
|
readonly ending?: UnitEnding;
|
|
555
621
|
}
|
|
556
622
|
|
|
@@ -579,6 +645,7 @@ export function openUnitPipeline(input: UnitPipelineInput, at: number): UnitPipe
|
|
|
579
645
|
clock: at,
|
|
580
646
|
phase: { at: "pre-check" },
|
|
581
647
|
reviewRounds: 0,
|
|
648
|
+
spentMs: { coding: 0, review: 0, waiting: 0 },
|
|
582
649
|
findingsByRound: {},
|
|
583
650
|
dispositionsByRound: {},
|
|
584
651
|
reviewRunByRound: {},
|
|
@@ -609,8 +676,25 @@ function waitSliceMs(clock: number, until: number): number {
|
|
|
609
676
|
/** The child's budget: its preset's own, clipped to the pipeline's remaining
|
|
610
677
|
* wall clock (agent-ship item 8's clip), never under the two minutes a
|
|
611
678
|
* spawn accepts. */
|
|
612
|
-
function budgetMinutesFor(s: UnitPipelineState,
|
|
613
|
-
|
|
679
|
+
function budgetMinutesFor(s: UnitPipelineState, round: RoundRef): number {
|
|
680
|
+
const headroom = remainingMs(s) - reserveFor(round.kind);
|
|
681
|
+
return Math.max(2, Math.min(s.input.childMinutes[presetOf(round.kind)], Math.floor(headroom / MIN)));
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
/** What a child of each round kind leaves on the pipeline's clock: round 0's
|
|
685
|
+
* coding child the whole loop (two reviews and the merge poll,
|
|
686
|
+
* SHIP_LOOP_RESERVE_MS), a findings child the re-review and the merge poll
|
|
687
|
+
* (SHIP_FIX_RESERVE_MS), a review child nothing — the review is what the
|
|
688
|
+
* reserve was held for. */
|
|
689
|
+
function reserveFor(kind: RoundKind): number {
|
|
690
|
+
switch (kind) {
|
|
691
|
+
case "coding":
|
|
692
|
+
return SHIP_LOOP_RESERVE_MS;
|
|
693
|
+
case "findings":
|
|
694
|
+
return SHIP_FIX_RESERVE_MS;
|
|
695
|
+
case "review":
|
|
696
|
+
return 0;
|
|
697
|
+
}
|
|
614
698
|
}
|
|
615
699
|
|
|
616
700
|
const roundStep = (s: UnitPipelineState, round: RoundRef) => `${s.input.unit.id}/${round.index}/${round.kind}`;
|
|
@@ -668,7 +752,7 @@ export function nextAction(s: UnitPipelineState): CoordinatorAction {
|
|
|
668
752
|
step: roundStep(s, p.round),
|
|
669
753
|
preset,
|
|
670
754
|
round: p.round,
|
|
671
|
-
budgetMinutes: budgetMinutesFor(s,
|
|
755
|
+
budgetMinutes: budgetMinutesFor(s, p.round),
|
|
672
756
|
brief: briefFor(s, p.round),
|
|
673
757
|
};
|
|
674
758
|
}
|
|
@@ -687,7 +771,11 @@ export function nextAction(s: UnitPipelineState): CoordinatorAction {
|
|
|
687
771
|
case "read":
|
|
688
772
|
return { type: "read-record", step: `${roundStep(s, p.round)}/read/${p.n}`, runId: p.runId };
|
|
689
773
|
case "pr-check":
|
|
690
|
-
return {
|
|
774
|
+
return {
|
|
775
|
+
type: "pr-check",
|
|
776
|
+
step: `${roundStep(s, p.round)}/pr-check`,
|
|
777
|
+
...(p.dead !== undefined ? { recover: { runId: p.runId } } : {}),
|
|
778
|
+
};
|
|
691
779
|
case "merge":
|
|
692
780
|
return { type: "merge", step: `${unit}/merge/${p.n}`, prNumber: p.pr.number, headSha: p.headSha };
|
|
693
781
|
case "merge-sleep":
|
|
@@ -717,12 +805,26 @@ const roundNote = (round: RoundRef, outcome: ShipRoundOutcome): CoordinatorNote
|
|
|
717
805
|
outcome,
|
|
718
806
|
});
|
|
719
807
|
|
|
808
|
+
/** The cap's ending: `review_pending` when the clock ran out entering a
|
|
809
|
+
* review round with the child's pull request standing — the work shipped and
|
|
810
|
+
* only the review is missing — else the wall-clock cap. Both carry the split. */
|
|
811
|
+
function capEnding(s: UnitPipelineState, round?: RoundRef): UnitEnding {
|
|
812
|
+
if (round?.kind === "review" && s.pr !== undefined)
|
|
813
|
+
return {
|
|
814
|
+
kind: "review_pending",
|
|
815
|
+
pr: s.pr,
|
|
816
|
+
...(s.lastReviewHead !== undefined ? { headSha: s.lastReviewHead } : {}),
|
|
817
|
+
reviewRounds: s.reviewRounds,
|
|
818
|
+
spent: s.spentMs,
|
|
819
|
+
};
|
|
820
|
+
return { kind: "wall_clock_cap", remainingMs: remainingMs(s), reviewRounds: s.reviewRounds, spent: s.spentMs };
|
|
821
|
+
}
|
|
822
|
+
|
|
720
823
|
/** Start a round if the reservation holds (agent-ship item 8's check: a
|
|
721
824
|
* child clipped under the reserve cannot do useful work). */
|
|
722
825
|
function enterRound(s: UnitPipelineState, round: RoundRef, notes: CoordinatorNote[] = []): Transition {
|
|
723
826
|
const remaining = remainingMs(s);
|
|
724
|
-
if (remaining < SHIP_ROUND_RESERVE_MS)
|
|
725
|
-
return end(s, { kind: "wall_clock_cap", remainingMs: remaining, reviewRounds: s.reviewRounds }, notes);
|
|
827
|
+
if (remaining < SHIP_ROUND_RESERVE_MS) return end(s, capEnding(s, round), notes);
|
|
726
828
|
const reviewRounds = round.kind === "review" ? round.index : s.reviewRounds;
|
|
727
829
|
return { state: { ...s, reviewRounds, phase: { at: "spawn", round, busy: 0 } }, notes };
|
|
728
830
|
}
|
|
@@ -773,17 +875,12 @@ function settleCoding(
|
|
|
773
875
|
},
|
|
774
876
|
[roundNote(round, "stopped")],
|
|
775
877
|
);
|
|
878
|
+
// A failed coding child no longer aborts outright: the pr-check looks at the
|
|
879
|
+
// branch first — a push before the death is recovered as the round's pull
|
|
880
|
+
// request (agent-ship items 10 and 15), and only a branch with
|
|
881
|
+
// nothing on it ends the unit with the child's own reason.
|
|
776
882
|
if (facts.status === "failed")
|
|
777
|
-
return
|
|
778
|
-
next,
|
|
779
|
-
{
|
|
780
|
-
kind: "aborted",
|
|
781
|
-
reason: `⚠️ The coding child of round ${round.index} (run ${runId}) ended \`failed\` — its run page has the error; nothing was opened or edited from it.`,
|
|
782
|
-
round,
|
|
783
|
-
reviewRounds: next.reviewRounds,
|
|
784
|
-
},
|
|
785
|
-
[roundNote(round, "aborted")],
|
|
786
|
-
);
|
|
883
|
+
return { state: { ...next, phase: { at: "pr-check", round, runId, dead: "failed" } }, notes: [] };
|
|
787
884
|
next = {
|
|
788
885
|
...next,
|
|
789
886
|
phase: {
|
|
@@ -919,6 +1016,32 @@ function settlePrCheck(s: UnitPipelineState, phase: Extract<Phase, { at: "pr-che
|
|
|
919
1016
|
// completed without a pull request of its own, and the unit is done.
|
|
920
1017
|
if (pr.state === "merged") return foundMerged(s, pr, [roundNote(round, "completed")]);
|
|
921
1018
|
if (pr.state === "none") {
|
|
1019
|
+
// A dead child left nothing on the branch to recover: the unit ends with
|
|
1020
|
+
// the child's own reason — never the budget clip.
|
|
1021
|
+
if (phase.dead === "interrupted")
|
|
1022
|
+
return end(s, { kind: "interrupted", round, runId: phase.runId, reviewRounds: s.reviewRounds }, [
|
|
1023
|
+
roundNote(round, "aborted"),
|
|
1024
|
+
]);
|
|
1025
|
+
if (phase.dead === "failed") {
|
|
1026
|
+
// The abort repeats the bot's reason for recovering nothing, and claims
|
|
1027
|
+
// no more than the answer carried.
|
|
1028
|
+
const why =
|
|
1029
|
+
pr.unrecovered === "no_commits"
|
|
1030
|
+
? `nothing heads \`${s.input.unit.branch}\`: no commits were pushed, so there was no work to recover`
|
|
1031
|
+
: pr.unrecovered === "no_base"
|
|
1032
|
+
? `\`${s.input.unit.branch}\` could not be given a pull request: the instance names no base branch to open it against, so whatever was pushed stays on the branch`
|
|
1033
|
+
: `the pr-check found no pull request heading \`${s.input.unit.branch}\`, so nothing was recovered`;
|
|
1034
|
+
return end(
|
|
1035
|
+
s,
|
|
1036
|
+
{
|
|
1037
|
+
kind: "aborted",
|
|
1038
|
+
reason: `⚠️ The coding child of round ${round.index} (run ${phase.runId}) ended \`failed\` — its run page has the error — and ${why}.`,
|
|
1039
|
+
round,
|
|
1040
|
+
reviewRounds: s.reviewRounds,
|
|
1041
|
+
},
|
|
1042
|
+
[roundNote(round, "aborted")],
|
|
1043
|
+
);
|
|
1044
|
+
}
|
|
922
1045
|
const reason =
|
|
923
1046
|
round.index === 0
|
|
924
1047
|
? `⚠️ Ship ended at round 0: the coding round ended without opening a pull request (a clarifying question, a budget write-up, an unproven push or a description-less push ends the pipeline here). No review round ran.`
|
|
@@ -944,6 +1067,25 @@ function settlePrCheck(s: UnitPipelineState, phase: Extract<Phase, { at: "pr-che
|
|
|
944
1067
|
const codingHead = normalizeHead(head);
|
|
945
1068
|
const reviewedAt = normalizeHead(s.lastReviewHead);
|
|
946
1069
|
if (codingHead !== undefined && reviewedAt !== undefined && sameCommit(codingHead, reviewedAt)) {
|
|
1070
|
+
// A findings child that died before it pushed: the recover pr-check found
|
|
1071
|
+
// the round's own pull request, still at the reviewed head. The unit
|
|
1072
|
+
// ends with the child's own reason — the ship-restart note for a bot
|
|
1073
|
+
// roll, the failure for a failed run — never as the round's inaction.
|
|
1074
|
+
if (phase.dead === "interrupted")
|
|
1075
|
+
return end(next, { kind: "interrupted", round, runId: phase.runId, reviewRounds: next.reviewRounds }, [
|
|
1076
|
+
roundNote(round, "aborted"),
|
|
1077
|
+
]);
|
|
1078
|
+
if (phase.dead === "failed")
|
|
1079
|
+
return end(
|
|
1080
|
+
next,
|
|
1081
|
+
{
|
|
1082
|
+
kind: "aborted",
|
|
1083
|
+
reason: `⚠️ The findings child of round ${round.index} (run ${phase.runId}) ended \`failed\` — its run page has the error — and the branch still sits at \`${codingHead.slice(0, 7)}\`, the commit the review already read, so nothing new was pushed to re-review.`,
|
|
1084
|
+
round,
|
|
1085
|
+
reviewRounds: next.reviewRounds,
|
|
1086
|
+
},
|
|
1087
|
+
[roundNote(round, "aborted")],
|
|
1088
|
+
);
|
|
947
1089
|
const findings = s.findingsByRound[round.index] ?? [];
|
|
948
1090
|
const dispositions = s.dispositionsByRound[round.index] ?? [];
|
|
949
1091
|
const allDeclined =
|
|
@@ -967,6 +1109,17 @@ function settlePrCheck(s: UnitPipelineState, phase: Extract<Phase, { at: "pr-che
|
|
|
967
1109
|
]);
|
|
968
1110
|
}
|
|
969
1111
|
|
|
1112
|
+
/** Move the clock and charge the elapsed time to the budget's bucket: a phase
|
|
1113
|
+
* inside a coding or findings round is the coding child's time, a review
|
|
1114
|
+
* round's is the review's, everything else — the pre-check, the branch, busy
|
|
1115
|
+
* waits, the merge polls — is waiting. The split rides the cap endings. */
|
|
1116
|
+
function withClock(s: UnitPipelineState, at: number): UnitPipelineState {
|
|
1117
|
+
const delta = Math.max(0, at - s.clock);
|
|
1118
|
+
const round = "round" in s.phase ? s.phase.round : undefined;
|
|
1119
|
+
const bucket = round === undefined ? "waiting" : round.kind === "review" ? "review" : "coding";
|
|
1120
|
+
return { ...s, clock: at, spentMs: { ...s.spentMs, [bucket]: s.spentMs[bucket] + delta } };
|
|
1121
|
+
}
|
|
1122
|
+
|
|
970
1123
|
/**
|
|
971
1124
|
* Feed a step's answer to the machine. An answer for any step but the one the
|
|
972
1125
|
* machine is at — a duplicate `run finished`, a replayed spawn — changes
|
|
@@ -976,7 +1129,7 @@ export function applyReturn(s: UnitPipelineState, ret: StepReturn): Transition {
|
|
|
976
1129
|
const expected = nextAction(s);
|
|
977
1130
|
if (expected.type === "end" || ret.step !== expected.step || ret.type !== expected.type)
|
|
978
1131
|
return { state: s, notes: [] };
|
|
979
|
-
const clocked: UnitPipelineState = "at" in ret ?
|
|
1132
|
+
const clocked: UnitPipelineState = "at" in ret ? withClock(s, ret.at) : s;
|
|
980
1133
|
const p = s.phase;
|
|
981
1134
|
switch (p.at) {
|
|
982
1135
|
case "pre-check": {
|
|
@@ -987,6 +1140,20 @@ export function applyReturn(s: UnitPipelineState, ret: StepReturn): Transition {
|
|
|
987
1140
|
// by the coding child and adopted at the round's own pr-check.
|
|
988
1141
|
const r = ret as Extract<StepReturn, { type: "pr-check" }>;
|
|
989
1142
|
if (r.pr.state === "merged") return foundMerged(clocked, r.pr);
|
|
1143
|
+
// The open pull request still heads at the child's own last push (the
|
|
1144
|
+
// previous attempt ended `review_pending`): nothing to code, so the
|
|
1145
|
+
// attempt adopts it and starts at the review round — never a fresh
|
|
1146
|
+
// coding round on an already-shipped pull request.
|
|
1147
|
+
if (r.pr.state === "open") {
|
|
1148
|
+
const head = normalizeHead(r.pr.headSha);
|
|
1149
|
+
const lastPush = normalizeHead(s.input.lastPush);
|
|
1150
|
+
if (head !== undefined && lastPush !== undefined && sameCommit(head, lastPush))
|
|
1151
|
+
return nextReview({
|
|
1152
|
+
...clocked,
|
|
1153
|
+
pr: { number: r.pr.prNumber, url: r.pr.url },
|
|
1154
|
+
lastReviewHead: head,
|
|
1155
|
+
});
|
|
1156
|
+
}
|
|
990
1157
|
return { state: { ...clocked, phase: { at: "branch" } }, notes: [] };
|
|
991
1158
|
}
|
|
992
1159
|
case "branch": {
|
|
@@ -1004,7 +1171,7 @@ export function applyReturn(s: UnitPipelineState, ret: StepReturn): Transition {
|
|
|
1004
1171
|
case "spawned":
|
|
1005
1172
|
case "alreadySpawned": {
|
|
1006
1173
|
// The child's budget runs from the spawn's answer; the wait walks it, plus the margin, in chunks.
|
|
1007
|
-
const until = clocked.clock + budgetMinutesFor(s,
|
|
1174
|
+
const until = clocked.clock + budgetMinutesFor(s, p.round) * MIN + WAIT_MARGIN_MS;
|
|
1008
1175
|
const runs =
|
|
1009
1176
|
p.round.kind === "review"
|
|
1010
1177
|
? { reviewRunByRound: { ...s.reviewRunByRound, [p.round.index]: r.runId } }
|
|
@@ -1022,12 +1189,7 @@ export function applyReturn(s: UnitPipelineState, ret: StepReturn): Transition {
|
|
|
1022
1189
|
case "busy": {
|
|
1023
1190
|
// Another run holds the unit's thread: wait for its end, then ask
|
|
1024
1191
|
// again — unless the pipeline's wall clock ran out meanwhile.
|
|
1025
|
-
if (remainingMs(clocked) < SHIP_ROUND_RESERVE_MS)
|
|
1026
|
-
return end(clocked, {
|
|
1027
|
-
kind: "wall_clock_cap",
|
|
1028
|
-
remainingMs: remainingMs(clocked),
|
|
1029
|
-
reviewRounds: s.reviewRounds,
|
|
1030
|
-
});
|
|
1192
|
+
if (remainingMs(clocked) < SHIP_ROUND_RESERVE_MS) return end(clocked, capEnding(clocked, p.round));
|
|
1031
1193
|
return {
|
|
1032
1194
|
state: {
|
|
1033
1195
|
...clocked,
|
|
@@ -1076,10 +1238,19 @@ export function applyReturn(s: UnitPipelineState, ret: StepReturn): Transition {
|
|
|
1076
1238
|
},
|
|
1077
1239
|
notes: [],
|
|
1078
1240
|
};
|
|
1079
|
-
if (r.run.status === "interrupted")
|
|
1241
|
+
if (r.run.status === "interrupted") {
|
|
1242
|
+
// A dead CODING child may have pushed before the ledger closed it: the
|
|
1243
|
+
// pr-check recovers the branch. A review child has nothing on the
|
|
1244
|
+
// branch to recover, so its interruption still ends the unit at once.
|
|
1245
|
+
if (p.round.kind !== "review")
|
|
1246
|
+
return {
|
|
1247
|
+
state: { ...clocked, phase: { at: "pr-check", round: p.round, runId: p.runId, dead: "interrupted" } },
|
|
1248
|
+
notes: [],
|
|
1249
|
+
};
|
|
1080
1250
|
return end(clocked, { kind: "interrupted", round: p.round, runId: p.runId, reviewRounds: s.reviewRounds }, [
|
|
1081
1251
|
roundNote(p.round, "aborted"),
|
|
1082
1252
|
]);
|
|
1253
|
+
}
|
|
1083
1254
|
return p.round.kind === "review"
|
|
1084
1255
|
? settleReview(clocked, p.round, r.run)
|
|
1085
1256
|
: settleCoding(clocked, p.round, p.runId, r.run);
|
|
@@ -1135,6 +1306,12 @@ function dispositionFor(s: UnitPipelineState, finding: Finding, round: number):
|
|
|
1135
1306
|
return undefined;
|
|
1136
1307
|
}
|
|
1137
1308
|
|
|
1309
|
+
/** How the budget went, in the card's words: coding, review, waiting minutes. */
|
|
1310
|
+
function budgetSplitLine(spent: ShipBudgetSpent, maxMinutes: number): string {
|
|
1311
|
+
const min = (ms: number) => Math.round(ms / MIN);
|
|
1312
|
+
return `Budget split (${maxMinutes} min): coding ${min(spent.coding)} min, review ${min(spent.review)} min, waiting ${min(spent.waiting)} min.`;
|
|
1313
|
+
}
|
|
1314
|
+
|
|
1138
1315
|
/** The cap report's declined-vs-unaddressed split over the last review round's findings. */
|
|
1139
1316
|
function splitReport(s: UnitPipelineState): string {
|
|
1140
1317
|
if (s.reviewRounds === 0) return "No review round ran before the cap — there are no findings to report.";
|
|
@@ -1222,9 +1399,16 @@ export function renderUnitReport(s: UnitPipelineState, facts?: MergeReadyFacts):
|
|
|
1222
1399
|
case "wall_clock_cap":
|
|
1223
1400
|
return join([
|
|
1224
1401
|
`🧢 Ship stopped at a cap: the remaining pipeline time (~${Math.max(0, Math.round(e.remainingMs / MIN))} min of the ${s.input.caps.maxMinutes}-minute budget) cannot hold another round — no approval after ${rounds}.${prLine}`,
|
|
1402
|
+
budgetSplitLine(e.spent, s.input.caps.maxMinutes),
|
|
1225
1403
|
splitReport(s),
|
|
1226
1404
|
reissue,
|
|
1227
1405
|
]);
|
|
1406
|
+
case "review_pending":
|
|
1407
|
+
return join([
|
|
1408
|
+
`⏳ Review pending: the coding child shipped ${e.pr.url}${e.headSha !== undefined ? ` (head \`${e.headSha.slice(0, 7)}\`)` : ""} but the remaining pipeline time cannot hold the review round — the work stands, only the review is missing. The next attempt starts at the review round while the pull request still heads at the child's own last push.`,
|
|
1409
|
+
budgetSplitLine(e.spent, s.input.caps.maxMinutes),
|
|
1410
|
+
reissue,
|
|
1411
|
+
]);
|
|
1228
1412
|
case "stopped":
|
|
1229
1413
|
return join([
|
|
1230
1414
|
`${e.mode === "hard" ? "⛔" : "⏹"} Ship stopped by operator (${e.mode} stop) after ${rounds}.${prLine}`,
|
|
@@ -68,6 +68,11 @@ export interface SpanOptions {
|
|
|
68
68
|
* plain fetch it was. */
|
|
69
69
|
export interface TraceOptions {
|
|
70
70
|
span?: Span;
|
|
71
|
+
/** Which door the call came through when it was not the surface's own
|
|
72
|
+
* grammar: `route` for a command the request router bound from prose
|
|
73
|
+
* (record 0036). Copied onto the registry's audit line, never read by a
|
|
74
|
+
* handler. Absent for a typed command. */
|
|
75
|
+
source?: "route";
|
|
71
76
|
}
|
|
72
77
|
|
|
73
78
|
export interface Span {
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
// the disk. Two facts fix that: the failure is classified `disk-full` (never
|
|
12
12
|
// serviceable, so the bot skips the attach and the card names the disk), and
|
|
13
13
|
// the resident recycles its container — the disk is a cache; the next refresh
|
|
14
|
-
// cycle restores mirror + checkout from R2 — once
|
|
14
|
+
// cycle restores mirror + checkout from R2 — once no run is using it.
|
|
15
15
|
|
|
16
16
|
/** The errno wording tools print for ENOSPC: Node's `ENOSPC` code and libc's
|
|
17
17
|
* strerror text (git, cp, tar, pnpm all pass it through). A message carrying
|
|
@@ -76,17 +76,27 @@ export const DISK_FULL_RECYCLE_COOLDOWN_MS = 60 * 60_000;
|
|
|
76
76
|
export type DiskFullRecovery = { action: "recycle" } | { action: "wait"; why: string };
|
|
77
77
|
|
|
78
78
|
/** Whether a disk-full resident may stop its container now. The disk is a
|
|
79
|
-
* cache
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
79
|
+
* cache — the next cycle restores mirror + checkout from R2, and a live
|
|
80
|
+
* binding's tree is recreated on its next attach — so the one question is
|
|
81
|
+
* whether a RUN is using it, never what the trees hold (item 17: a run
|
|
82
|
+
* starts from a clean tree; what it wants kept, it commits and pushes). Two
|
|
83
|
+
* facts say a run may be: an operation in flight (a recycle kills it), and a
|
|
84
|
+
* live binding attached to or used within the idle floor — the op counter is
|
|
85
|
+
* 0 between a run's tool calls, so the recent-use floor is what stands for a
|
|
86
|
+
* run mid-flight. That is the idle-sleep gate's own predicate (item 16b): a
|
|
87
|
+
* platform sleep destroys the disk exactly as a recycle does, so one rule
|
|
88
|
+
* says when the disk may go away. Both keep the container; so does the
|
|
89
|
+
* cooldown. */
|
|
85
90
|
export function planDiskFullRecovery(input: {
|
|
86
91
|
now: number;
|
|
87
92
|
lastRecycleAt?: number;
|
|
88
93
|
inFlight: number;
|
|
89
|
-
|
|
94
|
+
/** A live binding was attached to or used (an exec bumps `lastAttachAt`
|
|
95
|
+
* too) within `idleFloorS` — computed once by the Worker, for the idle
|
|
96
|
+
* gate and this plan alike. */
|
|
97
|
+
recentlyUsed: boolean;
|
|
98
|
+
/** The floor `recentlyUsed` was measured against, named in the refusal. */
|
|
99
|
+
idleFloorS: number;
|
|
90
100
|
}): DiskFullRecovery {
|
|
91
101
|
if (input.lastRecycleAt !== undefined && input.now - input.lastRecycleAt < DISK_FULL_RECYCLE_COOLDOWN_MS) {
|
|
92
102
|
const min = Math.round((input.now - input.lastRecycleAt) / 60_000);
|
|
@@ -97,10 +107,11 @@ export function planDiskFullRecovery(input: {
|
|
|
97
107
|
}
|
|
98
108
|
if (input.inFlight > 0)
|
|
99
109
|
return { action: "wait", why: `${input.inFlight} operation(s) in flight — a recycle would kill them` };
|
|
100
|
-
if (
|
|
110
|
+
if (input.recentlyUsed) {
|
|
111
|
+
const min = Math.round(input.idleFloorS / 60);
|
|
101
112
|
return {
|
|
102
113
|
action: "wait",
|
|
103
|
-
why:
|
|
114
|
+
why: `a live worktree was attached to or used within the last ${min} min — a run may be mid-flight between two tool calls, and a recycle would destroy its tree`,
|
|
104
115
|
};
|
|
105
116
|
}
|
|
106
117
|
return { action: "recycle" };
|
|
@@ -125,6 +125,78 @@ export function fleetBusyExhaustedMessage(waitedMs: number): string {
|
|
|
125
125
|
);
|
|
126
126
|
}
|
|
127
127
|
|
|
128
|
+
// ---------------------------------------------------------------------------
|
|
129
|
+
// A container that is still starting (docs/reference/specs/execution.md item 23).
|
|
130
|
+
//
|
|
131
|
+
// Why this exists: a thread's first request finds no running container, and
|
|
132
|
+
// the SDK's first exec then carries the whole start — the platform's instance
|
|
133
|
+
// grant (its default wait 30 s), the image pull on a machine that has not seen
|
|
134
|
+
// this image, the microVM boot and the runtime's port (90 s) — before the
|
|
135
|
+
// command runs. The executor's per-send deadline for a 60 s command is 90 s,
|
|
136
|
+
// so every fresh-sandbox run died with "gave no answer within 90s" while its
|
|
137
|
+
// container came up a minute later and sat idle. The Worker now names the
|
|
138
|
+
// condition instead: it starts the container in the background and answers
|
|
139
|
+
// `sandbox-starting` at once, the executor waits on the token exactly as it
|
|
140
|
+
// waits on `fleet-busy` — the identical request re-sent, nothing ran — under a
|
|
141
|
+
// start budget of its own, and the command runs once the container is up.
|
|
142
|
+
|
|
143
|
+
/** The machine token the executor waits on, beside `fleet-busy`. */
|
|
144
|
+
export const SANDBOX_STARTING_REASON = "sandbox-starting" as const;
|
|
145
|
+
|
|
146
|
+
/** What the token means, in the words the model and the operator see. */
|
|
147
|
+
export const SANDBOX_STARTING_EXPLANATION =
|
|
148
|
+
"the thread's sandbox container is starting (image pull, boot, runtime) — nothing ran yet; the request is re-sent once it is up";
|
|
149
|
+
|
|
150
|
+
/** The executor waits at most this long for a container to start, whatever
|
|
151
|
+
* the command's own budget: the platform's own start allowances (30 s for an
|
|
152
|
+
* instance, 90 s for the port) plus a slow image pull fit inside it, and a
|
|
153
|
+
* 60 s command is never killed by a two-minute start it did not cause. */
|
|
154
|
+
export const SANDBOX_START_WAIT_MAX_MS = 5 * 60_000;
|
|
155
|
+
|
|
156
|
+
/** Backoff between re-sends while a container starts: 5 s, 10 s, then 15 s.
|
|
157
|
+
* A start takes tens of seconds, not minutes, so the poll is denser than the
|
|
158
|
+
* fleet wait's and a ready container is used within 15 s of coming up. */
|
|
159
|
+
export const SANDBOX_START_BACKOFF_MS: readonly number[] = [5_000, 10_000, 15_000];
|
|
160
|
+
|
|
161
|
+
/** The reasons whose answers the executor re-sends after a wait. Every other
|
|
162
|
+
* `reason` — or none — is an ordinary failure after one send. */
|
|
163
|
+
export type WaitReason = typeof FLEET_BUSY_REASON | typeof SANDBOX_STARTING_REASON;
|
|
164
|
+
|
|
165
|
+
export function isWaitReason(reason: unknown): reason is WaitReason {
|
|
166
|
+
return reason === FLEET_BUSY_REASON || reason === SANDBOX_STARTING_REASON;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** The Worker's answer on /read and /write (sent as HTTP 503) while the
|
|
170
|
+
* container starts: the token, and the start's own phase as the cause. */
|
|
171
|
+
export function sandboxStartingAnswer(cause: string): { error: string; reason: typeof SANDBOX_STARTING_REASON } {
|
|
172
|
+
return {
|
|
173
|
+
error: `${SANDBOX_STARTING_REASON}: ${SANDBOX_STARTING_EXPLANATION} (${cause})`,
|
|
174
|
+
reason: SANDBOX_STARTING_REASON,
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** The Worker's answer on /exec, in-body under the streamed HTTP 200 in the
|
|
179
|
+
* item-3 dual shape, like `fleetBusyExecAnswer`. */
|
|
180
|
+
export function sandboxStartingExecAnswer(cause: string): {
|
|
181
|
+
error: string;
|
|
182
|
+
reason: typeof SANDBOX_STARTING_REASON;
|
|
183
|
+
stdout: "";
|
|
184
|
+
stderr: string;
|
|
185
|
+
exitCode: 127;
|
|
186
|
+
} {
|
|
187
|
+
const { error, reason } = sandboxStartingAnswer(cause);
|
|
188
|
+
return { error, reason, stdout: "", stderr: error, exitCode: 127 };
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** The message `ExecCapacityError` carries when a container did not start
|
|
192
|
+
* inside the start budget: the wait, and what to do. */
|
|
193
|
+
export function startWaitExhaustedMessage(waitedMs: number): string {
|
|
194
|
+
return (
|
|
195
|
+
`sandbox not ready — the thread's container did not finish starting within ${Math.round(waitedMs / 1000)}s; ` +
|
|
196
|
+
"try again in a few minutes"
|
|
197
|
+
);
|
|
198
|
+
}
|
|
199
|
+
|
|
128
200
|
/** The text the Worker carries in-body for a thrown value: the SDK's own
|
|
129
201
|
* message when it has one, else a sentence that says the SDK gave none —
|
|
130
202
|
* naming the error's name and code, and the one condition known to produce
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
// The start gate of a thread's sandbox (docs/reference/specs/execution.md
|
|
2
|
+
// item 23): the Durable Object's decision, on every request, between running
|
|
3
|
+
// the operation and answering `sandbox-starting`. Deliberately free of node:
|
|
4
|
+
// imports so wrangler can bundle it into the sandbox Worker, like
|
|
5
|
+
// sandboxErrors.ts and sandboxIdle.ts.
|
|
6
|
+
//
|
|
7
|
+
// Why: a thread's first request finds no running container, and the SDK's
|
|
8
|
+
// first exec then carries the whole start — the platform's instance grant,
|
|
9
|
+
// the image pull, the microVM boot, the runtime's port — before the command
|
|
10
|
+
// runs; the executor's per-send deadline for a 60 s command is 90 s, so every
|
|
11
|
+
// fresh-sandbox run died with "gave no answer within 90s" while its container
|
|
12
|
+
// came up a minute later. The gate starts the container in the background
|
|
13
|
+
// through a warm-up the host provides, answers the named token at once, and
|
|
14
|
+
// lets the operation through only once the container is up. A warm-up that
|
|
15
|
+
// fails hands its error to the next request, so a full fleet or a silent
|
|
16
|
+
// runtime keeps its own name (item 14, item 9) — the gate never swallows it.
|
|
17
|
+
|
|
18
|
+
/** What the gate needs from the Durable Object. */
|
|
19
|
+
export interface StartGateHost {
|
|
20
|
+
/** `ctx.container?.running`: true once the platform runs the container,
|
|
21
|
+
* false when it is stopped, undefined when there is no container binding
|
|
22
|
+
* (treated as not running: the warm-up will say what is wrong). */
|
|
23
|
+
containerRunning(): boolean | undefined;
|
|
24
|
+
/** Start the container and wait for its runtime: one trivial command
|
|
25
|
+
* through the SDK, which does the start itself. Resolves when the runtime
|
|
26
|
+
* answered; rejects with the SDK's own error otherwise. */
|
|
27
|
+
warmUp(): Promise<void>;
|
|
28
|
+
now(): number;
|
|
29
|
+
log(event: Record<string, unknown>): void;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** The gate's answer when the operation cannot run yet: the phase the start
|
|
33
|
+
* is in, in words, for the answer's cause. */
|
|
34
|
+
export type StartingCause = "container not running; starting it" | "container starting";
|
|
35
|
+
|
|
36
|
+
export class StartGate {
|
|
37
|
+
private starting: Promise<void> | null = null;
|
|
38
|
+
private startedAt = 0;
|
|
39
|
+
private failure: { error: unknown } | null = null;
|
|
40
|
+
|
|
41
|
+
constructor(private readonly host: StartGateHost) {}
|
|
42
|
+
|
|
43
|
+
/** Run `op` when the container is up; otherwise answer `starting(cause)`
|
|
44
|
+
* at once — beginning the warm-up on the first such request — and let the
|
|
45
|
+
* executor's wait re-send. A warm-up that failed is thrown here once, on
|
|
46
|
+
* the next request, so the caller classifies it as it would any SDK error
|
|
47
|
+
* (a full fleet, a silent control port); the request after that starts a
|
|
48
|
+
* fresh warm-up if the container is still not running. */
|
|
49
|
+
async through<T, S>(op: () => Promise<T>, starting: (cause: StartingCause) => S): Promise<T | S> {
|
|
50
|
+
if (this.failure) {
|
|
51
|
+
const { error } = this.failure;
|
|
52
|
+
this.failure = null;
|
|
53
|
+
throw error;
|
|
54
|
+
}
|
|
55
|
+
if (this.starting) return starting("container starting");
|
|
56
|
+
if (this.host.containerRunning() !== true) {
|
|
57
|
+
this.begin();
|
|
58
|
+
return starting("container not running; starting it");
|
|
59
|
+
}
|
|
60
|
+
return op();
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Whether a warm-up is in flight — for the wiring test and the card. */
|
|
64
|
+
get isStarting(): boolean {
|
|
65
|
+
return this.starting !== null;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
private begin(): void {
|
|
69
|
+
this.startedAt = this.host.now();
|
|
70
|
+
this.host.log({ event: "sandbox.starting" });
|
|
71
|
+
this.starting = this.host
|
|
72
|
+
.warmUp()
|
|
73
|
+
.then(
|
|
74
|
+
() => {
|
|
75
|
+
this.host.log({ event: "sandbox.started", durationMs: this.host.now() - this.startedAt });
|
|
76
|
+
},
|
|
77
|
+
(error: unknown) => {
|
|
78
|
+
this.host.log({
|
|
79
|
+
event: "sandbox.start-failed",
|
|
80
|
+
durationMs: this.host.now() - this.startedAt,
|
|
81
|
+
error: String(error),
|
|
82
|
+
});
|
|
83
|
+
this.failure = { error };
|
|
84
|
+
},
|
|
85
|
+
)
|
|
86
|
+
.finally(() => {
|
|
87
|
+
this.starting = null;
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
}
|