@coreplane/switchboard 1.233.0 → 1.235.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/deploy/cloudflare-resident/gc.ts +6 -9
- package/dist/assets/deploy/cloudflare-resident/worker.ts +310 -277
- package/dist/assets/deploy/cloudflare-sandbox/worker.ts +82 -13
- package/dist/assets/package-lock.json +3 -3
- package/dist/assets/package.json +1 -1
- package/dist/assets/source.json +3 -3
- package/dist/assets/src/agents/registry.ts +13 -7
- package/dist/assets/src/core/provider.ts +9 -6
- package/dist/assets/src/core/runEvents.ts +25 -0
- package/dist/assets/src/execution/residentCleanliness.ts +69 -5
- package/dist/assets/src/execution/residentDiskBudget.ts +7 -6
- package/dist/assets/src/execution/residentRebind.ts +92 -77
- package/dist/assets/src/execution/residentReuse.ts +7 -18
- package/dist/assets/src/execution/residentSteps.ts +0 -1
- package/dist/assets/src/execution/sandboxErrors.ts +5 -0
- package/dist/assets/src/execution/sandboxIdle.ts +220 -0
- package/dist/assets/web/dist/.vite/manifest.json +68 -28
- package/dist/assets/web/dist/assets/{ResidentDetailPage-V4K35SYX.js → ResidentDetailPage-DSpsye5e.js} +1 -1
- package/dist/assets/web/dist/assets/{ResidentsIndexPage-3HuYaDwD.js → ResidentsIndexPage-CT8fI1M3.js} +1 -1
- package/dist/assets/web/dist/assets/RunFoldRow-Cqh8pTXC.js +9 -0
- package/dist/assets/web/dist/assets/RunRoutePage-oN9GkVEA.js +6 -0
- package/dist/assets/web/dist/assets/RunsIndexPage-sjXr8iCV.js +1 -0
- package/dist/assets/web/dist/assets/{ScheduledPage-dfrjQ5E9.js → ScheduledPage-Tz__zLzi.js} +1 -1
- package/dist/assets/web/dist/assets/{StatusDot-DexKtvUC.js → StatusDot-D14iJMy7.js} +1 -1
- package/dist/assets/web/dist/assets/{Tooltip-DGQ2gu9M.js → Tooltip-B0Ob5MQ4.js} +1 -1
- package/dist/assets/web/dist/assets/UnitRoutePage-CbKCL58v.js +1 -0
- package/dist/assets/web/dist/assets/{dist-boTdDLF4.js → dist-pOdUzj6D.js} +1 -1
- package/dist/assets/web/dist/assets/favicon-BQsePYv5.js +1 -0
- package/dist/assets/web/dist/assets/indexRow-Bnj883ii.js +1 -0
- package/dist/assets/web/dist/assets/{main-BHhLVMQh.js → main-D3lG-yQ9.js} +2 -2
- package/dist/assets/web/dist/assets/main-Dm11o0hc.css +1 -0
- package/dist/cli.js +1231 -586
- package/package.json +1 -1
- package/dist/assets/web/dist/assets/RunRoutePage-Dr867-Ha.js +0 -13
- package/dist/assets/web/dist/assets/RunsIndexPage-D_lpXsr8.js +0 -1
- package/dist/assets/web/dist/assets/favicon-CWPcvWvp.js +0 -1
- package/dist/assets/web/dist/assets/main-CAVqMbiX.css +0 -1
|
@@ -87,6 +87,7 @@ import { parseReadonly, planReadonlyAttach } from "../../src/execution/residentR
|
|
|
87
87
|
import { decideWorktree, parseReuse, type WorktreeFacts } from "../../src/execution/residentReuse.js";
|
|
88
88
|
import {
|
|
89
89
|
boundByFor,
|
|
90
|
+
canReturnToDefault,
|
|
90
91
|
parseOwnPr,
|
|
91
92
|
parsePushed,
|
|
92
93
|
parseRefByDefault,
|
|
@@ -94,10 +95,12 @@ import {
|
|
|
94
95
|
rebindRefused,
|
|
95
96
|
rebindVerdict,
|
|
96
97
|
rememberOwnBranches,
|
|
98
|
+
returnToDefault,
|
|
97
99
|
type BoundBy,
|
|
98
100
|
type OwnBranch,
|
|
99
101
|
type OwnPr,
|
|
100
102
|
type PushedBranch,
|
|
103
|
+
type Returned,
|
|
101
104
|
type Rebound,
|
|
102
105
|
type RebindRefused,
|
|
103
106
|
type RebindTreeFacts,
|
|
@@ -112,7 +115,15 @@ import {
|
|
|
112
115
|
type DepCacheMaterialization,
|
|
113
116
|
type ThreadDepsMechanism,
|
|
114
117
|
} from "../../src/execution/residentDepCache.js";
|
|
115
|
-
import {
|
|
118
|
+
import {
|
|
119
|
+
evictedTreeOf,
|
|
120
|
+
evictedTreeSentence,
|
|
121
|
+
parseWorktreeCleanliness,
|
|
122
|
+
worktreeCleanlinessScript,
|
|
123
|
+
type EvictedTree,
|
|
124
|
+
type LeftBehind,
|
|
125
|
+
type WorktreeCleanliness,
|
|
126
|
+
} from "../../src/execution/residentCleanliness.js";
|
|
116
127
|
import {
|
|
117
128
|
base64LengthOf,
|
|
118
129
|
chunkPlan,
|
|
@@ -994,15 +1005,17 @@ interface ThreadBinding {
|
|
|
994
1005
|
* binding made before the field: read as `default` iff `ref` is the default
|
|
995
1006
|
* branch. Only a `default` binding may ever move. */
|
|
996
1007
|
boundBy?: BoundBy;
|
|
997
|
-
/** The
|
|
998
|
-
*
|
|
999
|
-
*
|
|
1008
|
+
/** The move a binding may make (item 16): from the default it was bound to,
|
|
1009
|
+
* onto the branch the thread's own run opened a pull request on. Set, the
|
|
1010
|
+
* binding moves again only after the move is returned (`returnedAt`: the
|
|
1011
|
+
* branch was gone from the mirror and the binding went back to the
|
|
1012
|
+
* default), and then for the next own pull request. */
|
|
1000
1013
|
rebound?: Rebound;
|
|
1001
1014
|
/** The branches the thread's own runs pushed and the pull requests they
|
|
1002
1015
|
* head, as each run's release handed them over (`/detach` `pushed`, item
|
|
1003
1016
|
* 16a) — written before any eviction decision, so the fact outlives the
|
|
1004
|
-
* tree:
|
|
1005
|
-
* lets the follow-up's attach move onto the branch and
|
|
1017
|
+
* tree: the tree is released the moment its run ends, and this is what
|
|
1018
|
+
* lets the follow-up's attach move onto the branch and provision the tree
|
|
1006
1019
|
* there. Newest last; the oldest fall off past `OWN_BRANCHES_MAX`. */
|
|
1007
1020
|
ownBranches?: OwnBranch[];
|
|
1008
1021
|
/** Allocated OS user (worker2..worker17); "" once evicted (pool released). */
|
|
@@ -1013,8 +1026,17 @@ interface ThreadBinding {
|
|
|
1013
1026
|
evicted?: boolean;
|
|
1014
1027
|
evictedAt?: string;
|
|
1015
1028
|
/** Why the last eviction happened (the audit trail): `ttl`, `clean-idle`,
|
|
1016
|
-
* `detach`, or a reclamation fate — `merged #N` /
|
|
1029
|
+
* `detach`, `disk-pressure`, or a reclamation fate — `merged #N` /
|
|
1030
|
+
* `closed #N` / `gone`. */
|
|
1017
1031
|
evictedWhy?: string;
|
|
1032
|
+
/** What the tree held when that eviction removed it (item 17: never a
|
|
1033
|
+
* reason to keep the tree, never discarded silently): the tracked changes
|
|
1034
|
+
* and unpushed commits measured as the thread user. Absent when the tree
|
|
1035
|
+
* was clean, already gone with the disk, or not measured (a force detach). */
|
|
1036
|
+
evictedLeftBehind?: LeftBehind;
|
|
1037
|
+
/** Why the tree could not be measured before that eviction (git could not
|
|
1038
|
+
* read it) — so an unreadable tree is never recorded as clean. */
|
|
1039
|
+
evictedUnmeasured?: string;
|
|
1018
1040
|
/** How deps were last materialized (evidence that the per-branch reconciliation ran). */
|
|
1019
1041
|
deps?: ThreadDepsMechanism;
|
|
1020
1042
|
/** Commit the worktree was last attached at (the ref's tip in the mirror
|
|
@@ -1088,6 +1110,20 @@ interface AttachOk {
|
|
|
1088
1110
|
/** The caller asked for that move and the binding stood: the branch, the PR
|
|
1089
1111
|
* and why — so the bot can say where the follow-up runs and why. */
|
|
1090
1112
|
rebindRefused?: RebindRefused;
|
|
1113
|
+
/** This attach moved the binding BACK to the default branch (item 16's
|
|
1114
|
+
* second movement): the branch a rebind had moved it onto is gone from the
|
|
1115
|
+
* mirror, so the tree was provisioned on the default instead of the attach
|
|
1116
|
+
* failing `unknown-ref`. From where, to where, which PR's branch it was. */
|
|
1117
|
+
returned?: Returned;
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
/** What `POST /detach` answers: whether the pool user went back, why not, and
|
|
1121
|
+
* — on a release — what the tree still held (item 16a), now gone with it. */
|
|
1122
|
+
interface DetachAnswer {
|
|
1123
|
+
released: boolean;
|
|
1124
|
+
reason?: string;
|
|
1125
|
+
user?: string;
|
|
1126
|
+
leftBehind?: LeftBehind;
|
|
1091
1127
|
}
|
|
1092
1128
|
|
|
1093
1129
|
/** Why the caller's `refHint` is what it is (item 16), beside the hint itself:
|
|
@@ -1105,17 +1141,11 @@ const NO_REF_HINT_REASON: RefHintReason = { ownPr: null, refByDefault: false };
|
|
|
1105
1141
|
|
|
1106
1142
|
/** What the rebind decided under the mutex (item 16): nothing to move (the
|
|
1107
1143
|
* row is already on the branch), a named refusal, or the move — written on
|
|
1108
|
-
* the row
|
|
1109
|
-
* recreate the tree at. */
|
|
1144
|
+
* the row; the attach that follows provisions the tree at the moved ref. */
|
|
1110
1145
|
type RebindOutcome =
|
|
1111
1146
|
| { kind: "none"; binding: ThreadBinding }
|
|
1112
1147
|
| { kind: "refuse"; refused: RebindRefused }
|
|
1113
|
-
| { kind: "rebound"; moved: ThreadBinding; rebound: Rebound
|
|
1114
|
-
/** The move was the record alone — a dirty tree already on the branch, no
|
|
1115
|
-
* checkout run — with the verdict's `note` for the log line. The attach
|
|
1116
|
-
* that follows must keep that tree (`keepTree`, the worktree step's word
|
|
1117
|
-
* for it): the promise the note makes is the attach's to keep. */
|
|
1118
|
-
| { kind: "rebound"; moved: ThreadBinding; rebound: Rebound; keepTree: true; note: string };
|
|
1148
|
+
| { kind: "rebound"; moved: ThreadBinding; rebound: Rebound };
|
|
1119
1149
|
|
|
1120
1150
|
/** Result of one /op test/build execution. `ok` is the command's
|
|
1121
1151
|
* verdict — a failing test run is a RESULT with ok:false, never an error. */
|
|
@@ -3768,15 +3798,16 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
3768
3798
|
});
|
|
3769
3799
|
}
|
|
3770
3800
|
|
|
3771
|
-
/** Idle = no live binding attached within IDLE_AFTER_S
|
|
3772
|
-
*
|
|
3773
|
-
*
|
|
3801
|
+
/** Idle = no live binding attached within IDLE_AFTER_S and nothing in
|
|
3802
|
+
* flight. What a live tree holds is not asked (item 17): a dirty tree no
|
|
3803
|
+
* run is using protects nothing — the next attach wipes it — so it is
|
|
3804
|
+
* never a reason to keep the container awake; the sweep's release records
|
|
3805
|
+
* what it held. Bindings are storage, so this needs no container. */
|
|
3774
3806
|
private async isIdle(): Promise<boolean> {
|
|
3775
3807
|
const live = await this.liveBindings();
|
|
3776
3808
|
const recent = systemClock() - IDLE_AFTER_S * 1000;
|
|
3777
3809
|
if (live.some((b) => Date.parse(b.lastAttachAt) >= recent)) return false;
|
|
3778
|
-
|
|
3779
|
-
return this.liveTreesClean(live);
|
|
3810
|
+
return this.inFlightCount() === 0;
|
|
3780
3811
|
}
|
|
3781
3812
|
|
|
3782
3813
|
/** Bindings that hold a pool user and a tree on disk (not evicted). */
|
|
@@ -3785,14 +3816,13 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
3785
3816
|
return [...all.values()].filter((b) => !b.evicted && b.user);
|
|
3786
3817
|
}
|
|
3787
3818
|
|
|
3788
|
-
/**
|
|
3789
|
-
* is clean
|
|
3790
|
-
* has destroyed the disk, so there is nothing left to lose. */
|
|
3819
|
+
/** The disk-full recycle's tree guard (item 54), its one reader: true when
|
|
3820
|
+
* every live tree is clean as its thread user, or the runtime is already
|
|
3821
|
+
* down — a sleep has destroyed the disk, so there is nothing left to lose. */
|
|
3791
3822
|
private async liveTreesClean(live: ThreadBinding[]): Promise<boolean> {
|
|
3792
3823
|
if (!(await this.isRuntimeActive().catch(() => false))) return true;
|
|
3793
3824
|
// Concurrent: each check touches only its own (disjoint) tree, and one
|
|
3794
|
-
// spawn each —
|
|
3795
|
-
// 3-probe round-trip per live binding.
|
|
3825
|
+
// spawn each — no serial 3-probe round-trip per live binding.
|
|
3796
3826
|
const checks = await Promise.all(live.map((b) => this.worktreeCleanliness(b)));
|
|
3797
3827
|
return checks.every((c) => c.clean); // any dirty or unknown → not clean
|
|
3798
3828
|
}
|
|
@@ -3930,10 +3960,11 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
3930
3960
|
* already on disk is `reuse` (0 — a dirty/stale recreate frees the old one
|
|
3931
3961
|
* first). A ref not yet in the mirror (fetched under the lock, moments later)
|
|
3932
3962
|
* is projected as `hardlink`, the common case. When it does not fit, the
|
|
3933
|
-
* coldest
|
|
3963
|
+
* coldest idle trees go first (`orderEvictionCandidates`: never the
|
|
3934
3964
|
* requesting thread, a busy tree, the default branch, or one attached within
|
|
3935
|
-
* DISK_EVICT_MIN_IDLE_MS
|
|
3936
|
-
*
|
|
3965
|
+
* DISK_EVICT_MIN_IDLE_MS), whatever they hold — item 17: dirt never keeps a
|
|
3966
|
+
* tree; each is measured as its thread user for the eviction's record —
|
|
3967
|
+
* `df` re-probed after each; still
|
|
3937
3968
|
* short → `503 {reason:"disk-pressure"}` with the whole math in `error`, the
|
|
3938
3969
|
* same shape as `mirror-busy`, so the bot falls back cold legibly. Never a
|
|
3939
3970
|
* lifecycle flip: the checkout is intact and every existing tree keeps
|
|
@@ -4003,12 +4034,10 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4003
4034
|
if (verdict.fits) break;
|
|
4004
4035
|
const binding = live.find((b) => b.threadKey === c.threadKey);
|
|
4005
4036
|
if (!binding) continue;
|
|
4006
|
-
|
|
4007
|
-
|
|
4008
|
-
|
|
4009
|
-
|
|
4010
|
-
}
|
|
4011
|
-
// Same re-read guards as the sweep: the clean check awaited.
|
|
4037
|
+
// What the tree holds goes on the eviction's record (item 17), never
|
|
4038
|
+
// into a keep; the runtime is up — this attach is running on it.
|
|
4039
|
+
const tree = await this.measureTreeBeforeEviction(binding);
|
|
4040
|
+
// Same re-read guards as the sweep: the measurement awaited.
|
|
4012
4041
|
const current = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(c.threadKey));
|
|
4013
4042
|
if (
|
|
4014
4043
|
!current ||
|
|
@@ -4019,7 +4048,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4019
4048
|
kept.push({ threadKey: c.threadKey, why: "busy" });
|
|
4020
4049
|
continue;
|
|
4021
4050
|
}
|
|
4022
|
-
if (!(await this.evictBinding(current, true, "disk-pressure", DISK_PRESSURE_REASON))) {
|
|
4051
|
+
if (!(await this.evictBinding(current, true, "disk-pressure", DISK_PRESSURE_REASON, tree))) {
|
|
4023
4052
|
kept.push({ threadKey: c.threadKey, why: "other" });
|
|
4024
4053
|
continue;
|
|
4025
4054
|
}
|
|
@@ -4514,10 +4543,12 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4514
4543
|
|
|
4515
4544
|
// The binding's ref wins for the thread's whole life, with one exception
|
|
4516
4545
|
// (item 16): a thread bound to the repo default for want of a named branch
|
|
4517
|
-
// moves, once
|
|
4518
|
-
// request on — when that branch is
|
|
4519
|
-
// the
|
|
4520
|
-
// the attach (fetch,
|
|
4546
|
+
// moves, once per pull request, onto the branch its own run opened a pull
|
|
4547
|
+
// request on — when that branch is the thread's own and the mirror holds
|
|
4548
|
+
// it. A decision about the binding alone, made here, before the ref is
|
|
4549
|
+
// chosen, so the rest of the attach (fetch, the worktree step that
|
|
4550
|
+
// provisions the tree clean at the moved ref, deps, credentials) runs on
|
|
4551
|
+
// the moved ref.
|
|
4521
4552
|
const rebind = await this.rebindToOwnPr(
|
|
4522
4553
|
stored.get(threadBindingKey(threadKey)) as ThreadBinding | undefined,
|
|
4523
4554
|
reason.ownPr,
|
|
@@ -4573,7 +4604,6 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4573
4604
|
refHint,
|
|
4574
4605
|
wantSha,
|
|
4575
4606
|
reuse,
|
|
4576
|
-
keepTree: rebind.keepTree === true,
|
|
4577
4607
|
slug,
|
|
4578
4608
|
t0,
|
|
4579
4609
|
facts,
|
|
@@ -4591,54 +4621,36 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4591
4621
|
|
|
4592
4622
|
/** Item 16's one exception to the sticky binding: a thread bound to the repo
|
|
4593
4623
|
* default for want of a named branch moves onto the branch its own run
|
|
4594
|
-
* opened a pull request on (`ownPr`), once
|
|
4595
|
-
* pure `rebindPlan` / `rebindVerdict` of
|
|
4596
|
-
*
|
|
4597
|
-
*
|
|
4598
|
-
*
|
|
4599
|
-
*
|
|
4600
|
-
*
|
|
4601
|
-
*
|
|
4602
|
-
*
|
|
4603
|
-
*
|
|
4604
|
-
*
|
|
4605
|
-
*
|
|
4606
|
-
* the whole attach's. The binding records the move (`rebound`); the answer
|
|
4607
|
-
* carries it, or the named refusal, so the bot can say where the follow-up
|
|
4608
|
-
* runs and why. A refusal never fails the attach. */
|
|
4624
|
+
* opened a pull request on (`ownPr`), once per pull request. A decision
|
|
4625
|
+
* about the BINDING alone — the pure `rebindPlan` / `rebindVerdict` of
|
|
4626
|
+
* src/execution/residentRebind.ts: the branch must be the thread's own,
|
|
4627
|
+
* remembered from a release (`ownBranches`) or, when nothing was
|
|
4628
|
+
* remembered, a local branch of the thread's surviving tree, measured as
|
|
4629
|
+
* the thread user — and the mirror must hold it (`moveOntoOwnBranch`), since
|
|
4630
|
+
* the attach that follows provisions the tree at the moved ref from the
|
|
4631
|
+
* mirror as it provisions any tree (item 17: clean at the ref's tip, a tree
|
|
4632
|
+
* left dirty or on the old branch recreated, the binding's path kept). Under
|
|
4633
|
+
* the mirror mutex, on the row as it stands there. The binding records the
|
|
4634
|
+
* move (`rebound`); the answer carries it, or the named refusal, so the bot
|
|
4635
|
+
* can say where the follow-up runs and why. A refusal never fails the attach. */
|
|
4609
4636
|
private async rebindToOwnPr(
|
|
4610
4637
|
prior: ThreadBinding | undefined,
|
|
4611
4638
|
ownPr: OwnPr | null,
|
|
4612
4639
|
reuse: boolean,
|
|
4613
4640
|
defaultRef: string,
|
|
4614
4641
|
slug: string,
|
|
4615
|
-
): Promise<
|
|
4616
|
-
| {
|
|
4617
|
-
binding: ThreadBinding | undefined;
|
|
4618
|
-
rebound?: Rebound;
|
|
4619
|
-
rebindRefused?: RebindRefused;
|
|
4620
|
-
/** The move was the record alone — the tree is dirty with the branch
|
|
4621
|
-
* already checked out — and the attach must keep that tree as it
|
|
4622
|
-
* stands: the worktree step's item 17 wipe of a dirty tree would
|
|
4623
|
-
* destroy the very work the rebind declined to touch. */
|
|
4624
|
-
keepTree?: true;
|
|
4625
|
-
}
|
|
4626
|
-
| ThreadErr
|
|
4627
|
-
> {
|
|
4642
|
+
): Promise<{ binding: ThreadBinding | undefined; rebound?: Rebound; rebindRefused?: RebindRefused } | ThreadErr> {
|
|
4628
4643
|
const plan = rebindPlan({ ownPr, reuse, binding: prior, defaultRef });
|
|
4629
4644
|
if (plan.kind === "none" || prior === undefined) return { binding: prior };
|
|
4630
4645
|
if (plan.kind === "refuse") return { binding: prior, rebindRefused: plan.refused };
|
|
4631
4646
|
const key = threadBindingKey(prior.threadKey);
|
|
4632
|
-
//
|
|
4633
|
-
//
|
|
4634
|
-
//
|
|
4635
|
-
//
|
|
4636
|
-
|
|
4637
|
-
|
|
4638
|
-
|
|
4639
|
-
(plan.kind === "recreate" || plan.own) && githubAppConfigured(this.env)
|
|
4640
|
-
? ((await mintRepoScopedToken(this.env, slug).catch(() => null))?.token ?? null)
|
|
4641
|
-
: null;
|
|
4647
|
+
// The tree is provisioned at the branch from the mirror, so the mirror
|
|
4648
|
+
// must hold it first: a branch pushed since the last refresh cycle is
|
|
4649
|
+
// fetched for — with a token minted here, before the mutex, like the
|
|
4650
|
+
// attach's own recovery fetch.
|
|
4651
|
+
const fetchToken = githubAppConfigured(this.env)
|
|
4652
|
+
? ((await mintRepoScopedToken(this.env, slug).catch(() => null))?.token ?? null)
|
|
4653
|
+
: null;
|
|
4642
4654
|
let outcome: RebindOutcome;
|
|
4643
4655
|
try {
|
|
4644
4656
|
outcome = (
|
|
@@ -4653,79 +4665,25 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4653
4665
|
const again = rebindPlan({ ownPr, reuse, binding: current, defaultRef });
|
|
4654
4666
|
if (again.kind === "none") return { kind: "none", binding: current };
|
|
4655
4667
|
if (again.kind === "refuse") return again;
|
|
4656
|
-
if (again.kind === "
|
|
4657
|
-
|
|
4658
|
-
|
|
4659
|
-
|
|
4660
|
-
|
|
4661
|
-
|
|
4662
|
-
|
|
4663
|
-
|
|
4664
|
-
const branch = await this.threadRun(
|
|
4665
|
-
current.user,
|
|
4666
|
-
wt,
|
|
4667
|
-
`git rev-parse --verify --quiet ${shellQuote(`refs/heads/${again.to}`)}`,
|
|
4668
|
-
DEFAULT_EXEC_TIMEOUT_MS,
|
|
4669
|
-
);
|
|
4670
|
-
tree.branchExists = branch.exitCode === 0;
|
|
4671
|
-
const status = await this.threadRun(
|
|
4672
|
-
current.user,
|
|
4673
|
-
wt,
|
|
4674
|
-
"git status --porcelain -uno",
|
|
4675
|
-
DEFAULT_EXEC_TIMEOUT_MS,
|
|
4676
|
-
);
|
|
4677
|
-
tree.readable = status.exitCode === 0;
|
|
4678
|
-
if (tree.readable) tree.dirty = status.stdout.trim() !== "";
|
|
4679
|
-
if (tree.dirty) {
|
|
4680
|
-
// A dirty tree is refused unless its HEAD is already the branch
|
|
4681
|
-
// (the run made it here and left an edit after pushing): the
|
|
4682
|
-
// one fact that tells the two apart, measured only when it
|
|
4683
|
-
// decides anything.
|
|
4684
|
-
const head = await this.threadRun(
|
|
4668
|
+
if (again.kind === "measure" && !again.own) {
|
|
4669
|
+
// Nothing remembered from a release: the branch must be a local
|
|
4670
|
+
// branch of the thread's surviving tree — the physical trace of
|
|
4671
|
+
// this thread's run having made it — measured as the thread user.
|
|
4672
|
+
const wt = current.worktreePath;
|
|
4673
|
+
const tree: RebindTreeFacts = { exists: (await this.run(["test", "-d", `${wt}/.git`])).exitCode === 0 };
|
|
4674
|
+
if (tree.exists) {
|
|
4675
|
+
const branch = await this.threadRun(
|
|
4685
4676
|
current.user,
|
|
4686
4677
|
wt,
|
|
4687
|
-
|
|
4678
|
+
`git rev-parse --verify --quiet ${shellQuote(`refs/heads/${again.to}`)}`,
|
|
4688
4679
|
DEFAULT_EXEC_TIMEOUT_MS,
|
|
4689
4680
|
);
|
|
4690
|
-
|
|
4691
|
-
}
|
|
4692
|
-
}
|
|
4693
|
-
// Judged on the re-read plan: it carries whether the branch is the
|
|
4694
|
-
// thread's own, which the pre-lock plan lacks when it was a recreate.
|
|
4695
|
-
const verdict = rebindVerdict(again, tree);
|
|
4696
|
-
if (verdict.kind === "recreate") return this.recreateAtOwnBranch(current, again, fetchToken);
|
|
4697
|
-
if (verdict.kind !== "rebind") return verdict;
|
|
4698
|
-
if (verdict.checkout) {
|
|
4699
|
-
const startedAt = systemClock();
|
|
4700
|
-
const checkout = await this.threadRun(
|
|
4701
|
-
current.user,
|
|
4702
|
-
wt,
|
|
4703
|
-
`git checkout --quiet ${shellQuote(again.to)}`,
|
|
4704
|
-
DEFAULT_EXEC_TIMEOUT_MS,
|
|
4705
|
-
);
|
|
4706
|
-
this.stepTrace.getStore()?.record("rebind-checkout", {
|
|
4707
|
-
startedAt,
|
|
4708
|
-
endedAt: systemClock(),
|
|
4709
|
-
exitCode: checkout.exitCode,
|
|
4710
|
-
timedOut: checkout.timedOut,
|
|
4711
|
-
});
|
|
4712
|
-
if (checkout.exitCode !== 0) {
|
|
4713
|
-
const detail =
|
|
4714
|
-
checkout.stderr.trim().split("\n")[0]?.slice(0, 200) || `git checkout exited ${checkout.exitCode}`;
|
|
4715
|
-
return { kind: "refuse", refused: rebindRefused(again, "checkout-failed", detail) };
|
|
4681
|
+
tree.branchExists = branch.exitCode === 0;
|
|
4716
4682
|
}
|
|
4683
|
+
const verdict = rebindVerdict(again, tree);
|
|
4684
|
+
if (verdict.kind === "refuse") return verdict;
|
|
4717
4685
|
}
|
|
4718
|
-
|
|
4719
|
-
from: current.ref,
|
|
4720
|
-
to: again.to,
|
|
4721
|
-
pr: again.pr,
|
|
4722
|
-
at: new Date(systemClock()).toISOString(),
|
|
4723
|
-
};
|
|
4724
|
-
const moved: ThreadBinding = { ...current, ref: again.to, rebound };
|
|
4725
|
-
await this.ctx.storage.put(key, moved);
|
|
4726
|
-
return verdict.checkout
|
|
4727
|
-
? { kind: "rebound", moved, rebound, keepTree: false }
|
|
4728
|
-
: { kind: "rebound", moved, rebound, keepTree: true, note: verdict.note };
|
|
4686
|
+
return this.moveOntoOwnBranch(current, again, fetchToken);
|
|
4729
4687
|
}, ATTACH_MUTEX_WAIT_MS)
|
|
4730
4688
|
).value;
|
|
4731
4689
|
} catch (err) {
|
|
@@ -4743,28 +4701,22 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4743
4701
|
return { binding: prior, rebindRefused: outcome.refused };
|
|
4744
4702
|
}
|
|
4745
4703
|
const { moved, rebound } = outcome;
|
|
4746
|
-
if (!outcome.keepTree) {
|
|
4747
|
-
console.log(
|
|
4748
|
-
`attach ${prior.threadKey}: rebound ${rebound.from} → ${rebound.to} (the thread's own pull request #${rebound.pr})`,
|
|
4749
|
-
);
|
|
4750
|
-
return { binding: moved, rebound };
|
|
4751
|
-
}
|
|
4752
4704
|
console.log(
|
|
4753
|
-
`attach ${prior.threadKey}: rebound ${rebound.from} → ${rebound.to} (the thread's own pull request #${rebound.pr})
|
|
4705
|
+
`attach ${prior.threadKey}: rebound ${rebound.from} → ${rebound.to} (the thread's own pull request #${rebound.pr}); the tree is provisioned at it`,
|
|
4754
4706
|
);
|
|
4755
|
-
return { binding: moved, rebound
|
|
4756
|
-
}
|
|
4757
|
-
|
|
4758
|
-
/** The move
|
|
4759
|
-
*
|
|
4760
|
-
*
|
|
4761
|
-
*
|
|
4762
|
-
*
|
|
4763
|
-
*
|
|
4764
|
-
*
|
|
4765
|
-
*
|
|
4766
|
-
*
|
|
4767
|
-
private async
|
|
4707
|
+
return { binding: moved, rebound };
|
|
4708
|
+
}
|
|
4709
|
+
|
|
4710
|
+
/** The move itself (item 16): the binding's ref becomes the branch, and the
|
|
4711
|
+
* attach that follows provisions the tree at it — `ensureThreadWorktree`
|
|
4712
|
+
* clones the branch from the mirror when the tree is gone, dirty or on the
|
|
4713
|
+
* old ref. So the mirror must hold the branch: one pushed since the last
|
|
4714
|
+
* refresh cycle is fetched for here, under the mutex; a branch still
|
|
4715
|
+
* missing after the fetch (deleted after a merge, or never pushed) is a
|
|
4716
|
+
* `branch-absent` refusal and the binding stands — the attach then goes on
|
|
4717
|
+
* at the ref it had, never `unknown-ref`. Nothing on disk is touched here:
|
|
4718
|
+
* the row is written, the tree is the attach's. */
|
|
4719
|
+
private async moveOntoOwnBranch(
|
|
4768
4720
|
current: ThreadBinding,
|
|
4769
4721
|
plan: { to: string; pr: number },
|
|
4770
4722
|
fetchToken: string | null,
|
|
@@ -4795,7 +4747,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4795
4747
|
refused: rebindRefused(
|
|
4796
4748
|
plan,
|
|
4797
4749
|
"branch-absent",
|
|
4798
|
-
`the
|
|
4750
|
+
`the mirror does not hold ${JSON.stringify(plan.to)} even after a fetch (deleted after a merge, or never pushed); the tree cannot be provisioned at it`,
|
|
4799
4751
|
),
|
|
4800
4752
|
};
|
|
4801
4753
|
}
|
|
@@ -4807,13 +4759,43 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4807
4759
|
};
|
|
4808
4760
|
const moved: ThreadBinding = { ...current, ref: plan.to, rebound };
|
|
4809
4761
|
await this.ctx.storage.put(threadBindingKey(current.threadKey), moved);
|
|
4810
|
-
return { kind: "rebound", moved, rebound
|
|
4762
|
+
return { kind: "rebound", moved, rebound };
|
|
4763
|
+
}
|
|
4764
|
+
|
|
4765
|
+
/** Item 16's second movement, decided where the fact is established — under
|
|
4766
|
+
* the mirror mutex, after the attach's fetch found the bound ref gone: a
|
|
4767
|
+
* binding a rebind moved onto its own pull request's branch, that branch
|
|
4768
|
+
* now deleted (its pull request merged), goes back to the default it was
|
|
4769
|
+
* bound to, so the run starts clean there (item 17) instead of the thread
|
|
4770
|
+
* failing `unknown-ref` for the rest of its life. The row as it stands is
|
|
4771
|
+
* re-read and re-judged (`canReturnToDefault`: a ref a person named never
|
|
4772
|
+
* returns — that branch is theirs to sort out), the move stamped returned
|
|
4773
|
+
* so the thread may follow its next pull request, and the answer carries
|
|
4774
|
+
* the move back for the card. A row that no longer qualifies — another
|
|
4775
|
+
* attach moved it meanwhile — is answered as it stands, nothing written. */
|
|
4776
|
+
private async returnBindingToDefault(
|
|
4777
|
+
binding: ThreadBinding,
|
|
4778
|
+
defaultRef: string,
|
|
4779
|
+
): Promise<{ binding: ThreadBinding; returned?: Returned }> {
|
|
4780
|
+
const key = threadBindingKey(binding.threadKey);
|
|
4781
|
+
const current = (await this.ctx.storage.get<ThreadBinding>(key)) ?? binding;
|
|
4782
|
+
if (current.rebound === undefined || !canReturnToDefault(current, defaultRef)) return { binding: current };
|
|
4783
|
+
const back = returnToDefault(
|
|
4784
|
+
{ ...current, rebound: current.rebound },
|
|
4785
|
+
defaultRef,
|
|
4786
|
+
new Date(systemClock()).toISOString(),
|
|
4787
|
+
);
|
|
4788
|
+
await this.ctx.storage.put(key, back.binding);
|
|
4789
|
+
console.log(
|
|
4790
|
+
`attach ${binding.threadKey}: ${back.returned.from} is gone from the mirror (the thread's own pull request #${back.returned.pr}) — returned to ${back.returned.to}; the tree is provisioned there`,
|
|
4791
|
+
);
|
|
4792
|
+
return back;
|
|
4811
4793
|
}
|
|
4812
4794
|
|
|
4813
4795
|
/** The release's word on what the run pushed (`/detach` `pushed`, item
|
|
4814
|
-
* 16a), remembered on the binding BEFORE the detach decides anything:
|
|
4815
|
-
*
|
|
4816
|
-
*
|
|
4796
|
+
* 16a), remembered on the binding BEFORE the detach decides anything: the
|
|
4797
|
+
* tree evicted, a busy one kept, an already-evicted binding answered as
|
|
4798
|
+
* such — and in every case the fact must outlive the
|
|
4817
4799
|
* tree, since it is what a follow-up's rebind onto the thread's own pull
|
|
4818
4800
|
* request branch reads once the tree is gone. No binding → nothing to
|
|
4819
4801
|
* remember on (the detach answers its 404 next). */
|
|
@@ -4835,10 +4817,6 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4835
4817
|
refHint: string | null;
|
|
4836
4818
|
wantSha: string | null;
|
|
4837
4819
|
reuse: boolean;
|
|
4838
|
-
/** The rebind moved the record alone over a dirty tree already on the
|
|
4839
|
-
* branch (item 16) and promised not to touch it: the worktree step keeps
|
|
4840
|
-
* the tree as it stands instead of item 17's dirty wipe. */
|
|
4841
|
-
keepTree: boolean;
|
|
4842
4820
|
slug: string;
|
|
4843
4821
|
t0: number;
|
|
4844
4822
|
facts: RepoFacts;
|
|
@@ -4850,8 +4828,14 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4850
4828
|
rebound?: Rebound;
|
|
4851
4829
|
rebindRefused?: RebindRefused;
|
|
4852
4830
|
}): Promise<AttachOk | ThreadErr> {
|
|
4853
|
-
const { threadKey, refHint, wantSha, reuse,
|
|
4831
|
+
const { threadKey, refHint, wantSha, reuse, slug, t0, facts, record, mode, rollback } = input;
|
|
4854
4832
|
const { rebound, rebindRefused } = input;
|
|
4833
|
+
// Reassigned once, under the lock, when the bound ref turns out gone from
|
|
4834
|
+
// the mirror and the binding goes back to the default (item 16's second
|
|
4835
|
+
// movement): everything after the lock — deps, credentials, the row's
|
|
4836
|
+
// final write, the answer — then speaks of the returned binding.
|
|
4837
|
+
let binding = input.binding;
|
|
4838
|
+
let returned: Returned | undefined;
|
|
4855
4839
|
|
|
4856
4840
|
// Command-level token mint — before the lock so mint latency
|
|
4857
4841
|
// never holds the mutex, and failure never blocks the attach.
|
|
@@ -4882,7 +4866,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4882
4866
|
// The expected head applies to the ref it was resolved for; a sticky
|
|
4883
4867
|
// binding on another branch drops it (item 51) rather than fetching on
|
|
4884
4868
|
// every attach of a thread that can never be at that commit.
|
|
4885
|
-
|
|
4869
|
+
let want = wantShaForBinding({ boundRef: binding.ref, refHint, wantSha });
|
|
4886
4870
|
let fetchToken: string | null = token;
|
|
4887
4871
|
if (!fetchToken && githubAppConfigured(this.env) && (await this.mirrorNeedsFetchFor(binding.ref, want))) {
|
|
4888
4872
|
fetchToken = (await mintRepoScopedToken(this.env, slug).catch(() => null))?.token ?? null;
|
|
@@ -4912,11 +4896,27 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4912
4896
|
// for every missing ref, so only a fetched mirror needs the re-read.
|
|
4913
4897
|
// (Under the mirror mutex — the mirror cannot change in between.)
|
|
4914
4898
|
const refExists = !fetched || (await this.refExists(binding.ref));
|
|
4915
|
-
|
|
4899
|
+
let target = attachTarget({
|
|
4916
4900
|
refExists,
|
|
4917
4901
|
wantSha: want,
|
|
4918
4902
|
commitInMirror: !refExists && want !== null && (await this.commitInMirror(want)),
|
|
4919
4903
|
});
|
|
4904
|
+
if (target.kind === "unknown-ref" && canReturnToDefault(binding, facts.defaultRef)) {
|
|
4905
|
+
// The branch a rebind moved this thread onto is gone even after the
|
|
4906
|
+
// fetch — deleted once its pull request merged. The binding goes
|
|
4907
|
+
// back to the default (item 16's second movement) and the attach
|
|
4908
|
+
// goes on there: the default is always in the mirror, and the
|
|
4909
|
+
// worktree step below provisions the tree at its tip.
|
|
4910
|
+
const back = await this.returnBindingToDefault(binding, facts.defaultRef);
|
|
4911
|
+
binding = back.binding;
|
|
4912
|
+
returned = back.returned;
|
|
4913
|
+
want = wantShaForBinding({ boundRef: binding.ref, refHint, wantSha });
|
|
4914
|
+
target = attachTarget({
|
|
4915
|
+
refExists: await this.refExists(binding.ref),
|
|
4916
|
+
wantSha: want,
|
|
4917
|
+
commitInMirror: false,
|
|
4918
|
+
});
|
|
4919
|
+
}
|
|
4920
4920
|
if (target.kind === "unknown-ref") {
|
|
4921
4921
|
throw new StepError(
|
|
4922
4922
|
"unknown-ref",
|
|
@@ -4929,7 +4929,6 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
4929
4929
|
const recreated = await this.ensureThreadWorktree(binding, sha, mode.originUrl, mode.modeSwitch, {
|
|
4930
4930
|
detached: target.kind === "sha",
|
|
4931
4931
|
reuse,
|
|
4932
|
-
keepTree,
|
|
4933
4932
|
});
|
|
4934
4933
|
return { sha, threadLockKey, recreated };
|
|
4935
4934
|
}, ATTACH_MUTEX_WAIT_MS);
|
|
@@ -5027,20 +5026,20 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5027
5026
|
trace: this.currentSteps(),
|
|
5028
5027
|
...(rebound !== undefined ? { rebound } : {}),
|
|
5029
5028
|
...(rebindRefused !== undefined ? { rebindRefused } : {}),
|
|
5029
|
+
...(returned !== undefined ? { returned } : {}),
|
|
5030
5030
|
};
|
|
5031
5031
|
}
|
|
5032
5032
|
|
|
5033
5033
|
/** Ensure the worktree exists. A provisioning attach wipes dirty/stale
|
|
5034
|
-
* trees
|
|
5035
|
-
*
|
|
5036
|
-
*
|
|
5037
|
-
*
|
|
5038
|
-
*
|
|
5039
|
-
*
|
|
5040
|
-
*
|
|
5041
|
-
* the
|
|
5042
|
-
*
|
|
5043
|
-
* holding the mirror mutex: the clone reads the mirror.
|
|
5034
|
+
* trees — a run starts from a clean tree at the bound ref's tip (item 17),
|
|
5035
|
+
* and a rebind that moved the binding leaves a tree on the old branch that
|
|
5036
|
+
* this discipline recreates like any stale one; a reusing attach
|
|
5037
|
+
* (`opts.reuse`, item 66) keeps a readable tree exactly as it stands (the
|
|
5038
|
+
* dirt and the HEAD are the resumed run's own work) and throws
|
|
5039
|
+
* `ReuseRefusedError` for one it cannot keep, touching nothing. The
|
|
5040
|
+
* decision is the pure `decideWorktree`; this method measures the facts
|
|
5041
|
+
* and runs the verdict. Returns true when the tree was (re)created. MUST be
|
|
5042
|
+
* called holding the mirror mutex: the clone reads the mirror.
|
|
5044
5043
|
*
|
|
5045
5044
|
* "Dirty" is tracked-file dirt (`status --porcelain -uno`): untracked
|
|
5046
5045
|
* scratch files are the thread's own state and survive re-attach.
|
|
@@ -5063,11 +5062,8 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5063
5062
|
/** `detached`: the bound ref is gone from the mirror and `sha` is the
|
|
5064
5063
|
* expected commit it still holds (item 51) — the tree is checked out at
|
|
5065
5064
|
* that commit, detached, instead of at a branch. `reuse`: a resumed
|
|
5066
|
-
* run's attach (item 66): keep the tree as it stands, or refuse.
|
|
5067
|
-
|
|
5068
|
-
* tree already on the branch (item 16): keep a readable tree as it
|
|
5069
|
-
* stands, provision one that is not there. */
|
|
5070
|
-
opts: { detached: boolean; reuse: boolean; keepTree: boolean } = { detached: false, reuse: false, keepTree: false },
|
|
5065
|
+
* run's attach (item 66): keep the tree as it stands, or refuse. */
|
|
5066
|
+
opts: { detached: boolean; reuse: boolean } = { detached: false, reuse: false },
|
|
5071
5067
|
): Promise<boolean> {
|
|
5072
5068
|
const wt = binding.worktreePath;
|
|
5073
5069
|
const threadDir = parentDir(wt);
|
|
@@ -5104,14 +5100,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5104
5100
|
}
|
|
5105
5101
|
}
|
|
5106
5102
|
}
|
|
5107
|
-
const decision = decideWorktree({
|
|
5108
|
-
reuse: opts.reuse,
|
|
5109
|
-
keepTree: opts.keepTree,
|
|
5110
|
-
modeSwitch,
|
|
5111
|
-
sha,
|
|
5112
|
-
worktreePath: wt,
|
|
5113
|
-
facts,
|
|
5114
|
-
});
|
|
5103
|
+
const decision = decideWorktree({ reuse: opts.reuse, modeSwitch, sha, worktreePath: wt, facts });
|
|
5115
5104
|
if (decision.kind === "refuse") throw new ReuseRefusedError(decision.why);
|
|
5116
5105
|
if (decision.kind === "reuse") return false;
|
|
5117
5106
|
|
|
@@ -6044,13 +6033,21 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
6044
6033
|
|
|
6045
6034
|
/** Remove a thread's worktree (when the runtime is up — a slept container
|
|
6046
6035
|
* already lost it) and release its pool user; the binding is KEPT, marked
|
|
6047
|
-
* evicted
|
|
6048
|
-
*
|
|
6036
|
+
* evicted with the cause (`evictedWhy`) and with what the tree held
|
|
6037
|
+
* (`evictedLeftBehind`, or `evictedUnmeasured` when git could not read it
|
|
6038
|
+
* — item 17: never a reason to keep the tree, never discarded silently), so
|
|
6039
|
+
* the ref stays sticky and the next attach recreates the tree with it.
|
|
6040
|
+
* Shared by the inactivity sweep, the reclamation pass, the disk-pressure
|
|
6041
|
+
* path and /detach. */
|
|
6049
6042
|
private async evictBinding(
|
|
6050
6043
|
binding: ThreadBinding,
|
|
6051
6044
|
runtimeActive: boolean,
|
|
6052
6045
|
logCtx: string,
|
|
6053
6046
|
why: string,
|
|
6047
|
+
/** What the caller measured in the tree first (`measureTreeBeforeEviction`,
|
|
6048
|
+
* as the thread user with the runtime up); absent when the tree was clean,
|
|
6049
|
+
* already gone with the disk, or not measured (a force detach). */
|
|
6050
|
+
tree?: EvictedTree,
|
|
6054
6051
|
): Promise<boolean> {
|
|
6055
6052
|
const threadDir = parentDir(binding.worktreePath);
|
|
6056
6053
|
if (runtimeActive && threadDir.startsWith(`${THREADS_DIR}/`)) {
|
|
@@ -6086,25 +6083,30 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
6086
6083
|
evicted: true,
|
|
6087
6084
|
evictedAt: new Date(systemClock()).toISOString(),
|
|
6088
6085
|
evictedWhy: why,
|
|
6086
|
+
evictedLeftBehind: tree && "leftBehind" in tree ? tree.leftBehind : undefined,
|
|
6087
|
+
evictedUnmeasured: tree && "unmeasured" in tree ? tree.unmeasured : undefined,
|
|
6089
6088
|
} satisfies ThreadBinding);
|
|
6089
|
+
if (tree) console.log(`${logCtx}: ${binding.threadKey} evicted (${why}) — ${evictedTreeSentence(tree)}`);
|
|
6090
6090
|
return true;
|
|
6091
6091
|
}
|
|
6092
6092
|
|
|
6093
6093
|
/** POST /detach: a run has ended — give the thread's pool user back now
|
|
6094
6094
|
* instead of holding it until the TTL sweep (the pool is sized for
|
|
6095
|
-
* simultaneous runs).
|
|
6096
|
-
*
|
|
6097
|
-
*
|
|
6098
|
-
* the
|
|
6099
|
-
*
|
|
6100
|
-
*
|
|
6101
|
-
*
|
|
6095
|
+
* simultaneous runs). A run starts from a clean tree (item 17), so the tree
|
|
6096
|
+
* goes whatever it holds: what it held — uncommitted changes, unpushed
|
|
6097
|
+
* commits — is measured once, as the thread user, and named in the answer
|
|
6098
|
+
* (`leftBehind`) so the loss is never silent. The one thing that keeps a
|
|
6099
|
+
* tree is an op still in flight in it (`busy`) — `force` (read-only agents,
|
|
6100
|
+
* hard stops) KILLS that op first (the bot has already dropped its fetch,
|
|
6101
|
+
* the command would otherwise run on and hold the user until the sweep)
|
|
6102
|
+
* and measures nothing. No binding → 404-shaped error; already evicted → a
|
|
6103
|
+
* no-op success. Never flips lifecycle state. */
|
|
6102
6104
|
async detachThread(
|
|
6103
6105
|
threadKey: string,
|
|
6104
6106
|
force: boolean,
|
|
6105
6107
|
/** What the ending run pushed (item 16a): remembered on the binding first, whatever the detach then decides. */
|
|
6106
6108
|
pushed: readonly PushedBranch[] = [],
|
|
6107
|
-
): Promise<
|
|
6109
|
+
): Promise<DetachAnswer | ThreadErr> {
|
|
6108
6110
|
if (pushed.length > 0) await this.rememberOwnBranches(threadKey, pushed);
|
|
6109
6111
|
const binding = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
|
|
6110
6112
|
if (!binding) return { error: `no-binding: ${threadKey} has never attached to this resident`, status: 404 };
|
|
@@ -6125,11 +6127,14 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
6125
6127
|
if (left > 0) return { released: false, reason: busyAfterKillReason(left), user: binding.user };
|
|
6126
6128
|
}
|
|
6127
6129
|
const active = await this.isRuntimeActive().catch(() => false);
|
|
6128
|
-
|
|
6129
|
-
|
|
6130
|
-
|
|
6131
|
-
|
|
6132
|
-
//
|
|
6130
|
+
// What the tree still holds, for the answer and the eviction's record —
|
|
6131
|
+
// never a reason to keep it. Not measured on a force release: a read-only
|
|
6132
|
+
// tree holds nothing, and a hard stop's tree is whatever the killed
|
|
6133
|
+
// command left. A probe that fails names nothing in the answer (never a
|
|
6134
|
+
// guess); the record and the log say it could not be measured.
|
|
6135
|
+
let tree: EvictedTree | undefined;
|
|
6136
|
+
if (!force && active) tree = await this.measureTreeBeforeEviction(binding);
|
|
6137
|
+
// Re-check right before removal: the measurement above awaited (the DO
|
|
6133
6138
|
// yields at each await), so an exec that arrived mid-detach would otherwise
|
|
6134
6139
|
// have its tree removed under it.
|
|
6135
6140
|
const busyNow = this.threadOpsInFlight.get(threadKey) ?? 0;
|
|
@@ -6139,22 +6144,23 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
6139
6144
|
reason: `busy: ${busyNow} operation(s) started during detach — kept`,
|
|
6140
6145
|
user: binding.user,
|
|
6141
6146
|
};
|
|
6142
|
-
// Same re-read as the sweep: a re-attach during the
|
|
6147
|
+
// Same re-read as the sweep: a re-attach during the measurement means a
|
|
6143
6148
|
// fresh tree we must not remove from a stale snapshot.
|
|
6144
6149
|
const current = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
|
|
6145
6150
|
if (!current || current.evicted) return { released: false, reason: "already-evicted" };
|
|
6146
6151
|
if (current.lastAttachAt !== binding.lastAttachAt)
|
|
6147
|
-
return { released: false, reason: "re-attached during the
|
|
6152
|
+
return { released: false, reason: "re-attached during the detach — kept", user: current.user };
|
|
6148
6153
|
const user = current.user;
|
|
6149
|
-
// Same as the sweep: `active` was read before the
|
|
6154
|
+
// Same as the sweep: `active` was read before the measurement's awaits; a
|
|
6150
6155
|
// container that woke meanwhile must get the rm, not an orphaned tree.
|
|
6151
6156
|
const activeNow = await this.isRuntimeActive().catch(() => false);
|
|
6152
|
-
if (!(await this.evictBinding(current, activeNow, `detach`, "detach"))) {
|
|
6157
|
+
if (!(await this.evictBinding(current, activeNow, `detach`, "detach", tree))) {
|
|
6153
6158
|
return { released: false, reason: "re-attached during eviction — kept", user };
|
|
6154
6159
|
}
|
|
6155
6160
|
// Item 55: the tree is gone; the gauge catches up at the next refresh
|
|
6156
6161
|
// instance's `measure` step, and the admission's `df` sees the space now.
|
|
6157
|
-
|
|
6162
|
+
const leftBehind = tree && "leftBehind" in tree ? tree.leftBehind : undefined;
|
|
6163
|
+
return { released: true, user, ...(leftBehind !== undefined ? { leftBehind } : {}) };
|
|
6158
6164
|
}
|
|
6159
6165
|
|
|
6160
6166
|
/** Force-detach's kill: end every process owned by the pool user —
|
|
@@ -6191,20 +6197,22 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
6191
6197
|
}
|
|
6192
6198
|
}
|
|
6193
6199
|
|
|
6194
|
-
/**
|
|
6195
|
-
*
|
|
6196
|
-
*
|
|
6197
|
-
*
|
|
6198
|
-
*
|
|
6199
|
-
*
|
|
6200
|
-
*
|
|
6201
|
-
*
|
|
6200
|
+
/** What this thread's tree holds: the tracked changes and unpushed commits,
|
|
6201
|
+
* or that git could not read it. The git probes run AS THE THREAD USER
|
|
6202
|
+
* (su), never as root: the worktree is thread-owned, so root git in it
|
|
6203
|
+
* would be refused by safe.directory and would be the exact repo-local-
|
|
6204
|
+
* config execution vector safe.directory exists to block. Read by
|
|
6205
|
+
* `measureTreeBeforeEviction` for the eviction's record — never a reason
|
|
6206
|
+
* to keep a tree (item 17) — and by the disk-full recycle's guard (item
|
|
6207
|
+
* 54), where unknown (git failed) counts as NOT clean. A tree that no
|
|
6208
|
+
* longer exists (disk recycled by a sleep/wake) is clean with nothing
|
|
6209
|
+
* measured: there was nothing to discard.
|
|
6202
6210
|
*
|
|
6203
6211
|
* ONE spawn: the presence test and both git probes fold
|
|
6204
6212
|
* into the pure `worktreeCleanlinessScript` (test -d as root, both git
|
|
6205
6213
|
* commands inside a single privilege-dropped `su`, tagged lines out);
|
|
6206
6214
|
* `parseWorktreeCleanliness` encodes the exact decision table above. */
|
|
6207
|
-
private async worktreeCleanliness(binding: ThreadBinding): Promise<
|
|
6215
|
+
private async worktreeCleanliness(binding: ThreadBinding): Promise<WorktreeCleanliness> {
|
|
6208
6216
|
const injected = { GIT_TERMINAL_PROMPT: "0" }; // same injection as threadRun — fail fast, never prompt
|
|
6209
6217
|
validateEnvNames(injected);
|
|
6210
6218
|
const r = await this.run(["sh", "-c", worktreeCleanlinessScript(binding.worktreePath, binding.user)], {
|
|
@@ -6214,6 +6222,14 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
6214
6222
|
return parseWorktreeCleanliness(r);
|
|
6215
6223
|
}
|
|
6216
6224
|
|
|
6225
|
+
/** What a tree holds right before its eviction, for the record and the log
|
|
6226
|
+
* (item 17): the one-spawn probe as the thread user, read once into
|
|
6227
|
+
* `evictedTreeOf` — never into a keep decision. Callers skip it when the
|
|
6228
|
+
* runtime is down: the disk, and the tree with it, is already gone. */
|
|
6229
|
+
private async measureTreeBeforeEviction(binding: ThreadBinding): Promise<EvictedTree | undefined> {
|
|
6230
|
+
return evictedTreeOf(await this.worktreeCleanliness(binding));
|
|
6231
|
+
}
|
|
6232
|
+
|
|
6217
6233
|
/** Anything that must not be interrupted by a container stop or counted
|
|
6218
6234
|
* as idle: thread exec/read/write, disposable /op runs, and attaches past
|
|
6219
6235
|
* their own image check (mid clone/install under the mirror lock). */
|
|
@@ -6254,35 +6270,29 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
6254
6270
|
const ttlDays = record?.worktreeTtlDays ?? WORKTREE_TTL_DAYS_DEFAULT;
|
|
6255
6271
|
const cutoff = systemClock() - ttlDays * 86_400_000;
|
|
6256
6272
|
const all = await this.ctx.storage.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX });
|
|
6257
|
-
const active = await this.isRuntimeActive().catch(() => false);
|
|
6258
6273
|
const idleCutoff = systemClock() - CLEAN_IDLE_RELEASE_S * 1000;
|
|
6259
6274
|
for (const binding of all.values()) {
|
|
6260
6275
|
if (binding.evicted || !binding.user) continue;
|
|
6261
6276
|
const last = Date.parse(binding.lastAttachAt);
|
|
6262
6277
|
if (last >= cutoff) {
|
|
6263
|
-
// Not past the TTL. Still release it if it has been idle for an hour
|
|
6264
|
-
// nothing is running on it
|
|
6265
|
-
//
|
|
6266
|
-
//
|
|
6267
|
-
//
|
|
6268
|
-
//
|
|
6269
|
-
//
|
|
6270
|
-
//
|
|
6278
|
+
// Not past the TTL. Still release it if it has been idle for an hour
|
|
6279
|
+
// and nothing is running on it: the run that used it is over, and
|
|
6280
|
+
// whatever its tree holds has no future — the next attach provisions
|
|
6281
|
+
// a clean tree (item 17) — so there is nothing to keep it for. This is
|
|
6282
|
+
// what drains the bindings of runs whose release never came (a
|
|
6283
|
+
// resident that was sick at the run's end, a run older than
|
|
6284
|
+
// `/detach`). A live run is protected by its op in flight, not by its
|
|
6285
|
+
// dirt. A slept container has no tree any more anyway (sleep destroys
|
|
6286
|
+
// the disk).
|
|
6271
6287
|
const busy = this.threadOpsInFlight.get(binding.threadKey) ?? 0;
|
|
6272
|
-
|
|
6273
|
-
last < idleCutoff && busy === 0 && (!active || (await this.worktreeCleanliness(binding)).clean);
|
|
6274
|
-
// Re-read right before removal: the clean check awaited (the DO
|
|
6275
|
-
// yields), so an exec that arrived meanwhile would otherwise have
|
|
6276
|
-
// its tree removed under it — same guard as detachThread.
|
|
6277
|
-
const busyNow = this.threadOpsInFlight.get(binding.threadKey) ?? 0;
|
|
6278
|
-
if (!cleanIdle || busyNow > 0) {
|
|
6288
|
+
if (last >= idleCutoff || busy > 0) {
|
|
6279
6289
|
kept++;
|
|
6280
6290
|
continue;
|
|
6281
6291
|
}
|
|
6282
6292
|
}
|
|
6283
|
-
// Re-read the binding
|
|
6284
|
-
//
|
|
6285
|
-
// from this loop's stale snapshot would rm the fresh tree.
|
|
6293
|
+
// Re-read the binding: earlier iterations awaited (the DO yields), so a
|
|
6294
|
+
// re-attach that completed meanwhile bumped lastAttachAt and rebuilt the
|
|
6295
|
+
// tree — evicting from this loop's stale snapshot would rm the fresh tree.
|
|
6286
6296
|
const current = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(binding.threadKey));
|
|
6287
6297
|
if (!current || current.evicted || current.lastAttachAt !== binding.lastAttachAt) {
|
|
6288
6298
|
kept++;
|
|
@@ -6292,8 +6302,17 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
6292
6302
|
// attach), and an eviction decided on a stale "inactive" would skip the
|
|
6293
6303
|
// rm and orphan a real tree.
|
|
6294
6304
|
const activeNow = await this.isRuntimeActive().catch(() => false);
|
|
6305
|
+
// What the tree holds goes on the eviction's record (item 17), measured
|
|
6306
|
+
// while the runtime is up; a slept container has no tree to measure.
|
|
6307
|
+
const tree = activeNow ? await this.measureTreeBeforeEviction(current) : undefined;
|
|
6295
6308
|
if (
|
|
6296
|
-
await this.evictBinding(
|
|
6309
|
+
await this.evictBinding(
|
|
6310
|
+
current,
|
|
6311
|
+
activeNow,
|
|
6312
|
+
`worktree-sweep ${resource}`,
|
|
6313
|
+
last >= cutoff ? "clean-idle" : "ttl",
|
|
6314
|
+
tree,
|
|
6315
|
+
)
|
|
6297
6316
|
)
|
|
6298
6317
|
evicted.push(binding.threadKey);
|
|
6299
6318
|
else kept++;
|
|
@@ -6551,9 +6570,11 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
6551
6570
|
* mirror (the refresh cycle's `fetch --prune` just ran) or its PR was
|
|
6552
6571
|
* merged/closed. Runs inside the refresh cycle — a poll on the existing
|
|
6553
6572
|
* cadence, since the GitHub App has webhooks off — and via /debug
|
|
6554
|
-
* reclaim-now. Never touches the default branch
|
|
6555
|
-
*
|
|
6556
|
-
*
|
|
6573
|
+
* reclaim-now. Never touches the default branch or a busy thread
|
|
6574
|
+
* (reclaimDecision); every keep is named. What the tree holds is never a
|
|
6575
|
+
* reason to keep it (item 17) — it is measured once as the thread user and
|
|
6576
|
+
* recorded by the eviction, the sweep's `evictBinding` with the same
|
|
6577
|
+
* re-read guards. */
|
|
6557
6578
|
async reclaimFinishedRefs(
|
|
6558
6579
|
resource: string,
|
|
6559
6580
|
defaultRef: string,
|
|
@@ -6613,21 +6634,16 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
6613
6634
|
// by name), so its fate is never looked up.
|
|
6614
6635
|
const { fate, detail } = (!isDefaultRef && fates.get(binding.ref)) || { fate: "unknown" as RefFate, detail: "" };
|
|
6615
6636
|
const busy = this.threadOpsInFlight.get(binding.threadKey) ?? 0;
|
|
6616
|
-
|
|
6617
|
-
// anything: a finished ref, nothing running, runtime up (down → the tree
|
|
6618
|
-
// is already gone with the disk → null).
|
|
6619
|
-
const finished = fate === "gone" || fate === "merged" || fate === "closed";
|
|
6620
|
-
const clean = !active
|
|
6621
|
-
? null
|
|
6622
|
-
: finished && !isDefaultRef && busy === 0
|
|
6623
|
-
? (await this.worktreeCleanliness(binding)).clean
|
|
6624
|
-
: null;
|
|
6625
|
-
const decision = reclaimDecision({ fate, isDefaultRef, busy, clean });
|
|
6637
|
+
const decision = reclaimDecision({ fate, isDefaultRef, busy });
|
|
6626
6638
|
if (!decision.reclaim) {
|
|
6627
6639
|
kept.push({ threadKey: binding.threadKey, ref: binding.ref, why: decision.why });
|
|
6628
6640
|
continue;
|
|
6629
6641
|
}
|
|
6630
|
-
//
|
|
6642
|
+
// What the tree holds goes on the eviction's record (item 17), never
|
|
6643
|
+
// into the decision: measured as the thread user while the runtime is
|
|
6644
|
+
// up; down → the tree is already gone with the disk.
|
|
6645
|
+
const tree = active ? await this.measureTreeBeforeEviction(binding) : undefined;
|
|
6646
|
+
// Same guards as the sweep: the measurement awaited, so re-read the
|
|
6631
6647
|
// binding (a re-attach means a fresh tree) and the op counter.
|
|
6632
6648
|
const busyNow = this.threadOpsInFlight.get(binding.threadKey) ?? 0;
|
|
6633
6649
|
const current = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(binding.threadKey));
|
|
@@ -6641,7 +6657,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
6641
6657
|
}
|
|
6642
6658
|
const activeNow = await this.isRuntimeActive().catch(() => false);
|
|
6643
6659
|
const why = `${decision.why}${detail}`;
|
|
6644
|
-
if (await this.evictBinding(current, activeNow, `reclaim ${resource}`, why)) {
|
|
6660
|
+
if (await this.evictBinding(current, activeNow, `reclaim ${resource}`, why, tree)) {
|
|
6645
6661
|
reclaimed.push({ threadKey: binding.threadKey, ref: binding.ref, why });
|
|
6646
6662
|
console.log(`reclaim ${resource}: evicted ${binding.threadKey} on ${binding.ref} — ${why}`);
|
|
6647
6663
|
} else kept.push({ threadKey: binding.threadKey, ref: binding.ref, why: "re-attached" });
|
|
@@ -6752,18 +6768,35 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
6752
6768
|
// left out; nothing here is secret (credential files are never persisted).
|
|
6753
6769
|
const threads = [...bindings.values()]
|
|
6754
6770
|
.sort((a, b) => b.lastAttachAt.localeCompare(a.lastAttachAt))
|
|
6755
|
-
.map(
|
|
6756
|
-
|
|
6757
|
-
|
|
6758
|
-
|
|
6759
|
-
|
|
6760
|
-
|
|
6761
|
-
|
|
6762
|
-
|
|
6763
|
-
|
|
6764
|
-
|
|
6765
|
-
|
|
6766
|
-
|
|
6771
|
+
.map(
|
|
6772
|
+
({
|
|
6773
|
+
threadKey,
|
|
6774
|
+
ref,
|
|
6775
|
+
sha,
|
|
6776
|
+
user,
|
|
6777
|
+
deps,
|
|
6778
|
+
boundAt,
|
|
6779
|
+
lastAttachAt,
|
|
6780
|
+
evicted,
|
|
6781
|
+
evictedAt,
|
|
6782
|
+
evictedWhy,
|
|
6783
|
+
evictedLeftBehind,
|
|
6784
|
+
evictedUnmeasured,
|
|
6785
|
+
}) => ({
|
|
6786
|
+
threadKey,
|
|
6787
|
+
ref,
|
|
6788
|
+
sha: sha ?? null,
|
|
6789
|
+
user,
|
|
6790
|
+
deps: deps ?? null,
|
|
6791
|
+
boundAt,
|
|
6792
|
+
lastAttachAt,
|
|
6793
|
+
evicted: evicted ?? false,
|
|
6794
|
+
evictedAt: evictedAt ?? null,
|
|
6795
|
+
evictedWhy: evictedWhy ?? null,
|
|
6796
|
+
evictedLeftBehind: evictedLeftBehind ?? null,
|
|
6797
|
+
evictedUnmeasured: evictedUnmeasured ?? null,
|
|
6798
|
+
}),
|
|
6799
|
+
);
|
|
6767
6800
|
return {
|
|
6768
6801
|
resource: map.get(RESOURCE_KEY) ?? null,
|
|
6769
6802
|
state: map.get(STATE_KEY) ?? "down",
|