@codeyam-editor/codeyam-editor 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/npm/cloud.js +779 -116
- package/npm/container.js +11 -0
- package/npm/disk-pressure.js +89 -0
- package/npm/editor-dev.js +144 -52
- package/npm/editor.js +47 -17
- package/npm/fleet-branch.js +49 -0
- package/npm/fleet-broker-death.js +44 -0
- package/npm/fleet-broker-debounce.js +63 -0
- package/npm/fleet-broker-status.js +45 -0
- package/npm/fleet-build-data.js +81 -0
- package/npm/fleet-cleanup.js +216 -16
- package/npm/fleet-effect-status.js +179 -0
- package/npm/fleet-grouping.js +56 -0
- package/npm/fleet-job-badge.js +83 -0
- package/npm/fleet-memory.js +35 -5
- package/npm/fleet-plans.js +96 -0
- package/npm/fleet-reconcile.js +58 -1
- package/npm/fleet-reset.js +94 -16
- package/npm/launch-config.js +197 -14
- package/npm/preflight-guidance.js +9 -0
- package/npm/scenario-check.js +106 -1
- package/npm/scenario-handlers.js +16 -0
- package/npm/scenario-interactivity.js +53 -7
- package/npm/scenario-playwright.js +184 -5
- package/npm/utils.js +238 -53
- package/package.json +5 -5
- package/ui/dist/assets/EditorShell-tMMdarZD.js +187 -0
- package/ui/dist/assets/ScenarioDataPanel-xWYGZOpo.js +5 -0
- package/ui/dist/assets/{ScenarioDataPanelFullPage-C0YHQB-T.js → ScenarioDataPanelFullPage-5G6KuZgw.js} +1 -1
- package/ui/dist/assets/ScenarioRenderer-CSdhX1Ak.js +1 -0
- package/ui/dist/assets/index-BGqairFV.css +1 -0
- package/ui/dist/assets/{index-BUYNFqKK.js → index-BuK0lizt.js} +9 -9
- package/ui/dist/index.html +2 -2
- package/ui/dist/assets/EditorShell-DWaeBxgj.js +0 -186
- package/ui/dist/assets/ScenarioDataPanel-zJSXtXL4.js +0 -5
- package/ui/dist/assets/ScenarioRenderer-CgDxuigm.js +0 -1
- package/ui/dist/assets/index-DVr5rY2d.css +0 -1
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// Pure consecutive-failure debounce for the Fleet Dashboard's broker tri-state
|
|
3
|
+
// (scripts/operator/fleet-dashboard/server.js requires this). Given the raw
|
|
4
|
+
// single-poll status from npm/fleet-broker-status.js::deriveBrokerStatus and the
|
|
5
|
+
// per-VM debounce state from the previous poll, decide the status the card
|
|
6
|
+
// actually renders. No I/O — server.js does the polling and threads the state.
|
|
7
|
+
//
|
|
8
|
+
// The bug this fixes (observed live 2026-06-02, VM3): the dashboard polls each
|
|
9
|
+
// VM's broker every 3s, and the editor probes the broker with a single 200ms
|
|
10
|
+
// socket attempt. One transient timeout (IAP / SSH-tunnel congestion — the same
|
|
11
|
+
// class of hiccup behind the known false "Editor stopped") instantly flipped a
|
|
12
|
+
// healthy VM's card to 'down', which raised both the red broker-down dot and the
|
|
13
|
+
// 💀 death alarm, even though the broker was fine (the build tab opened and chat
|
|
14
|
+
// started immediately). The single probe is left as-is on the editor side; the
|
|
15
|
+
// right seam for absorbing a one-poll blip is here at the aggregation layer.
|
|
16
|
+
//
|
|
17
|
+
// Asymmetric by design: ONLY the flip to 'down' is debounced (it requires
|
|
18
|
+
// `threshold` consecutive 'down' polls). 'idle' / 'agent-live' / 'unknown' take
|
|
19
|
+
// effect immediately — we never want to DELAY clearing a false alarm, only
|
|
20
|
+
// delay raising one. And an unreachable editor ('unknown') breaks a pending
|
|
21
|
+
// down-streak rather than escalating it, mirroring fleet-broker-status.js's
|
|
22
|
+
// "absence of evidence is never down" rule.
|
|
23
|
+
//
|
|
24
|
+
// State shape carried across polls: { status, downStreak, lastConfirmed }.
|
|
25
|
+
// status — the debounced value the card renders this poll.
|
|
26
|
+
// downStreak — how many consecutive raw 'down' polls we've seen.
|
|
27
|
+
// lastConfirmed — the last raw 'idle'/'agent-live' we proved; held during a
|
|
28
|
+
// pending (sub-threshold) down-streak so the card stays calm.
|
|
29
|
+
|
|
30
|
+
// ~9s at the dashboard's 3s poll cadence: comfortably absorbs a transient probe
|
|
31
|
+
// timeout and the brief down-window of a routine restart-server (broker down for
|
|
32
|
+
// <3s between SIGTERM and respawn), while still surfacing a real outage quickly.
|
|
33
|
+
// Named so retuning is a one-line edit.
|
|
34
|
+
const DOWN_DEBOUNCE_THRESHOLD = 3;
|
|
35
|
+
|
|
36
|
+
function debounceBrokerStatus(prev, rawStatus, opts) {
|
|
37
|
+
const threshold = (opts && opts.threshold) || DOWN_DEBOUNCE_THRESHOLD;
|
|
38
|
+
const lastConfirmed = prev ? prev.lastConfirmed : undefined;
|
|
39
|
+
|
|
40
|
+
// Confirmed up: the daemon answered now. Render immediately, reset the streak,
|
|
41
|
+
// and remember this as the last-confirmed-up value.
|
|
42
|
+
if (rawStatus === 'idle' || rawStatus === 'agent-live') {
|
|
43
|
+
return { status: rawStatus, downStreak: 0, lastConfirmed: rawStatus };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// Editor unreachable: we proved nothing. Never down — reset any pending
|
|
47
|
+
// down-streak and render 'unknown' (the card shows nothing for it).
|
|
48
|
+
if (rawStatus !== 'down') {
|
|
49
|
+
return { status: 'unknown', downStreak: 0, lastConfirmed };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// Raw 'down': only a real outage if it persists. Count the streak.
|
|
53
|
+
const downStreak = (prev ? prev.downStreak : 0) + 1;
|
|
54
|
+
if (downStreak >= threshold) {
|
|
55
|
+
// Confirmed down — surface the alarm.
|
|
56
|
+
return { status: 'down', downStreak, lastConfirmed };
|
|
57
|
+
}
|
|
58
|
+
// Pending (sub-threshold) down: hold the last confirmed up status if we have
|
|
59
|
+
// one, else 'unknown'. The card stays calm during the blip.
|
|
60
|
+
return { status: lastConfirmed || 'unknown', downStreak, lastConfirmed };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
module.exports = { debounceBrokerStatus, DOWN_DEBOUNCE_THRESHOLD };
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// Pure tri-state derivation for the Fleet Dashboard's per-VM broker indicator
|
|
3
|
+
// (scripts/operator/fleet-dashboard/server.js requires this). Given the raw
|
|
4
|
+
// /api/session-info poll merge `v`, collapse it into the one label the card
|
|
5
|
+
// renders. No I/O — server.js does the polling.
|
|
6
|
+
//
|
|
7
|
+
// The bug this fixes (observed live 2026-06-01): the old indicator was
|
|
8
|
+
// `v.reachable ? !!v.agentBrokerLive : null`, where `agentBrokerLive` means
|
|
9
|
+
// "a build-mode agent session is live in the broker" — NOT "broker daemon up".
|
|
10
|
+
// So a perfectly IDLE VM (healthy broker daemon, no parked build agent) reported
|
|
11
|
+
// `agentBrokerLive=false`, indistinguishable from a VM whose broker was actually
|
|
12
|
+
// DOWN. An operator killed a working broker chasing that phantom. The editor now
|
|
13
|
+
// also reports `brokerDaemonReachable` (the daemon answered its socket) and a
|
|
14
|
+
// derived `brokerStatus`; this module layers the dashboard's own reachability on
|
|
15
|
+
// top so the four states stay distinct:
|
|
16
|
+
//
|
|
17
|
+
// 'unknown' — editor unreachable (can't probe the broker at all), OR an
|
|
18
|
+
// older VM whose editor predates `brokerDaemonReachable`. Never
|
|
19
|
+
// rendered as a problem: absence of evidence is not "down".
|
|
20
|
+
// 'down' — daemon unreachable → the real broker problem, the state that
|
|
21
|
+
// had NO signal before. A VM here cannot start an agent session.
|
|
22
|
+
// 'idle' — daemon up, no build agent → a normal idle VM, not an error.
|
|
23
|
+
// 'agent-live' — a live build-mode agent session in the broker's build slot.
|
|
24
|
+
const BROKER_STATUSES = ['unknown', 'down', 'idle', 'agent-live'];
|
|
25
|
+
|
|
26
|
+
function deriveBrokerStatus(v) {
|
|
27
|
+
// Editor unreachable: we polled nothing, so the broker state is unknowable.
|
|
28
|
+
// Reporting 'down' here is exactly the conflation that misled operators —
|
|
29
|
+
// an unreachable editor is not a dead broker.
|
|
30
|
+
if (!v || !v.reachable) return 'unknown';
|
|
31
|
+
// Prefer the editor's own tri-state when it reports one (newer VMs).
|
|
32
|
+
if (v.brokerStatus === 'down' || v.brokerStatus === 'idle' || v.brokerStatus === 'agent-live') {
|
|
33
|
+
return v.brokerStatus;
|
|
34
|
+
}
|
|
35
|
+
// Fallback derivation from the raw bits, for a VM reporting the bools but not
|
|
36
|
+
// the pre-derived `brokerStatus`. `agentBrokerLive` implies the daemon is up.
|
|
37
|
+
if (v.agentBrokerLive) return 'agent-live';
|
|
38
|
+
if (v.brokerDaemonReachable === true) return 'idle';
|
|
39
|
+
if (v.brokerDaemonReachable === false) return 'down';
|
|
40
|
+
// Older VM: reachable editor, but no broker-reachability field at all. Treat
|
|
41
|
+
// as 'unknown' (graceful fallback) — never falsely 'down'.
|
|
42
|
+
return 'unknown';
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
module.exports = { deriveBrokerStatus, BROKER_STATUSES };
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Assemble the per-VM wire object the dashboard's /data response carries to the
|
|
4
|
+
// browser. This is the single source of truth for a card's transport shape: the
|
|
5
|
+
// card UI (scripts/operator/fleet-dashboard/public/index.html) reads `v.<field>`
|
|
6
|
+
// off each entry, so every field the card renders MUST be emitted here.
|
|
7
|
+
//
|
|
8
|
+
// It was split out of server.js's buildData() so the shape is unit-testable.
|
|
9
|
+
// The regression that motivated the split: `currentProvider` (the VM's
|
|
10
|
+
// configured build agent) was merged onto the poll state AND read by the card to
|
|
11
|
+
// render the 🤖 agent badge, but it was silently missing from this envelope — so
|
|
12
|
+
// it lived in server memory and never reached the browser, and the badge never
|
|
13
|
+
// appeared. A hand-maintained object literal inside a 197KB monolith has no
|
|
14
|
+
// guard against that drift; a pure function does.
|
|
15
|
+
//
|
|
16
|
+
// `v` is the merged poll state (state.vms[n]). `ctx` carries the derived,
|
|
17
|
+
// cross-VM, and side-effecting values buildData computes per VM (status,
|
|
18
|
+
// debounced broker tri-state, reconciled job, queue position, launch cfg, …) —
|
|
19
|
+
// the things this pure function can't know on its own.
|
|
20
|
+
function buildVmEnvelope(v, ctx) {
|
|
21
|
+
return {
|
|
22
|
+
n: ctx.n,
|
|
23
|
+
url: v.url || null,
|
|
24
|
+
reachable: !!v.reachable,
|
|
25
|
+
hasSession: !!v.hasSession,
|
|
26
|
+
// Three-state: true = broker confirmed up, false = confirmed down, null =
|
|
27
|
+
// editor unreachable so broker state is unknown. Never conflate "down" with
|
|
28
|
+
// "unreachable" — that mismatch is what made the card show "broker down"
|
|
29
|
+
// while a direct pgrep proved the broker was up.
|
|
30
|
+
brokerLive: v.reachable ? !!v.agentBrokerLive : null,
|
|
31
|
+
// Broker tri-state: 'down' | 'idle' | 'agent-live' | 'unknown' (derived
|
|
32
|
+
// upstream, debounced). Richer than brokerLive's is-an-agent-live boolean.
|
|
33
|
+
brokerStatus: ctx.brokerStatus,
|
|
34
|
+
awaitingInput: !!v.awaitingUserInput,
|
|
35
|
+
feature: v.feature || null,
|
|
36
|
+
step: v.step != null ? v.step : null,
|
|
37
|
+
mode: v.mode || null,
|
|
38
|
+
label: v.label || null,
|
|
39
|
+
status: ctx.status,
|
|
40
|
+
orphan: ctx.orphan || null,
|
|
41
|
+
degradedReasons: ctx.degradedReasons || [],
|
|
42
|
+
isHead: !!ctx.isHead,
|
|
43
|
+
// True when this VM holds the queue head but is unreachable (operator
|
|
44
|
+
// decides manually whether to release it).
|
|
45
|
+
headHolderUnreachable: !!ctx.headHolderUnreachable,
|
|
46
|
+
queuePos: ctx.queuePos != null ? ctx.queuePos : null,
|
|
47
|
+
queueLen: ctx.queueLen,
|
|
48
|
+
job: ctx.job || null,
|
|
49
|
+
config: ctx.config || null,
|
|
50
|
+
project: ctx.project != null ? ctx.project : null,
|
|
51
|
+
variant: ctx.variant != null ? ctx.variant : null,
|
|
52
|
+
cardIdentity: ctx.cardIdentity,
|
|
53
|
+
brokerHistory: Array.isArray(v.brokerHistory) ? v.brokerHistory : [],
|
|
54
|
+
brokerDeath: ctx.brokerDeath,
|
|
55
|
+
// Auth state + reason + token expiry. Only surfaced when the editor is
|
|
56
|
+
// reachable (an unreachable VM's cached auth is stale and misleading).
|
|
57
|
+
providerAuth: v.reachable && v.providerAuth ? v.providerAuth : null,
|
|
58
|
+
effects: v.effects || null,
|
|
59
|
+
// The VM's live-configured build agent (lowercase provider), reported by
|
|
60
|
+
// /api/session-info and merged (sticky) onto the poll state. Drives the
|
|
61
|
+
// card's 🤖 agent badge. null for an older VM whose editor predates the
|
|
62
|
+
// field (the card omits the badge rather than rendering a blank chip).
|
|
63
|
+
currentProvider: v.currentProvider || null,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// The fields the card UI depends on being present in every /data VM entry. This
|
|
68
|
+
// is the contract `buildVmEnvelope` must satisfy: the regression that prompted
|
|
69
|
+
// this module was a field the card read but the envelope didn't emit, so the
|
|
70
|
+
// test asserts the envelope keys are a superset of this list. Adding a field the
|
|
71
|
+
// card renders means adding it here AND in buildVmEnvelope — the test fails loud
|
|
72
|
+
// if you do one without the other.
|
|
73
|
+
const REQUIRED_CARD_FIELDS = [
|
|
74
|
+
'n', 'url', 'reachable', 'hasSession', 'brokerLive', 'brokerStatus',
|
|
75
|
+
'awaitingInput', 'feature', 'step', 'mode', 'label', 'status', 'orphan',
|
|
76
|
+
'degradedReasons', 'isHead', 'headHolderUnreachable', 'queuePos', 'queueLen',
|
|
77
|
+
'job', 'config', 'project', 'variant', 'cardIdentity', 'brokerHistory',
|
|
78
|
+
'brokerDeath', 'providerAuth', 'effects', 'currentProvider',
|
|
79
|
+
];
|
|
80
|
+
|
|
81
|
+
module.exports = { buildVmEnvelope, REQUIRED_CARD_FIELDS };
|
package/npm/fleet-cleanup.js
CHANGED
|
@@ -76,31 +76,162 @@ function classifyDescribeError(err, stderr) {
|
|
|
76
76
|
}
|
|
77
77
|
|
|
78
78
|
// Decide what the startup reap should do for one sidecar. Pure: takes the
|
|
79
|
-
// decoded sidecar + GCP-existence verdict + (for adds) container-up verdict
|
|
80
|
-
// returns a tag the caller dispatches on.
|
|
81
|
-
// verdict when
|
|
82
|
-
// 'cleanup' destroy gone, OR add gone
|
|
83
|
-
// 'reconcile-add' add up
|
|
84
|
-
// '
|
|
85
|
-
// '
|
|
86
|
-
// '
|
|
87
|
-
//
|
|
88
|
-
//
|
|
79
|
+
// decoded sidecar + GCP-existence verdict + (for adds) container-up verdict +
|
|
80
|
+
// (for up adds) project-wiring verdict, returns a tag the caller dispatches on.
|
|
81
|
+
// verdict when dispatch
|
|
82
|
+
// 'cleanup' destroy gone, OR add gone runDestroyCleanup / runAddCleanup
|
|
83
|
+
// 'reconcile-add' add up, container up, project wired (or unknown) ensure roster + recordLaunch, then clear
|
|
84
|
+
// 'half-provisioned' add up, container up, but project NOT wired surface needs-recovery card (finish-wire from cfg)
|
|
85
|
+
// 'recover-bootstrap' add instance up but container missing/down re-run cloud:up to finish the half-done bootstrap
|
|
86
|
+
// 'clear-only' destroy sidecar but instance still exists just clear the sidecar
|
|
87
|
+
// 'leave' gcloud uncertain (exists === null) leave for the next boot
|
|
88
|
+
// The (exists, containerUp) split distinguishes a finished add from a container-
|
|
89
|
+
// down half-provisioned one: a dashboard SIGKILL'd mid-`cloud:up` leaves a GCP
|
|
89
90
|
// instance with no docker/container, which the old (exists-only) matrix wrongly
|
|
90
|
-
// classified 'reconcile-add' — adding an unreachable VM to the roster.
|
|
91
|
-
// `
|
|
92
|
-
//
|
|
93
|
-
//
|
|
94
|
-
|
|
91
|
+
// classified 'reconcile-add' — adding an unreachable VM to the roster. The
|
|
92
|
+
// `projectWired` axis closes the remaining gap (improve39 VM-7): an instance +
|
|
93
|
+
// container that ARE up but whose `/workspace` is unwired (no git remote, empty
|
|
94
|
+
// manifest) is NOT a finished add — it's half-provisioned and must surface a
|
|
95
|
+
// recoverable card, not be silently reconciled "done". Only a POSITIVE unwired
|
|
96
|
+
// signal (`projectWired === false`) downgrades to 'half-provisioned'; an
|
|
97
|
+
// unknown/unprobed wiring (`null`/`undefined`) stays 'reconcile-add' so a flaky
|
|
98
|
+
// probe can't strand a healthy VM. `containerUp`/`projectWired` are only consulted
|
|
99
|
+
// for an add whose instance exists; destroy and the instance-gone add path ignore
|
|
100
|
+
// them. Splitting the matrix out lets the table be exhaustively tested without any
|
|
101
|
+
// of the side-effects in server.js.
|
|
102
|
+
function reapVerdict({ action, exists, containerUp, projectWired }) {
|
|
95
103
|
if (exists === null) return 'leave';
|
|
96
104
|
if (action === 'destroy') return exists ? 'clear-only' : 'cleanup';
|
|
97
105
|
if (action === 'add') {
|
|
98
106
|
if (!exists) return 'cleanup';
|
|
99
|
-
|
|
107
|
+
if (!containerUp) return 'recover-bootstrap';
|
|
108
|
+
return projectWired === false ? 'half-provisioned' : 'reconcile-add';
|
|
100
109
|
}
|
|
101
110
|
return 'leave';
|
|
102
111
|
}
|
|
103
112
|
|
|
113
|
+
// Classify whether a VM's editor `/workspace` is actually wired to a client
|
|
114
|
+
// project, from a best-effort probe of two signals: the git remote URL and
|
|
115
|
+
// whether `package.json` carries a real `name`. Pure: server.js runs the SSH +
|
|
116
|
+
// `docker exec` probe and hands the raw stdout/err here so the decision is
|
|
117
|
+
// unit-testable without a live VM.
|
|
118
|
+
// - err present (ssh/docker blip, timeout) → null (unknown — do NOT
|
|
119
|
+
// strand a healthy VM on a flaky probe; reapVerdict treats null as wired)
|
|
120
|
+
// - a git remote OR a non-empty package.json name → true (wired)
|
|
121
|
+
// - neither (empty / no-git workspace) → false (half-provisioned —
|
|
122
|
+
// the VM-7 signature: `/workspace` with no remote and an empty manifest)
|
|
123
|
+
// The probe emits the two signals separated by the `---CY---` sentinel so this
|
|
124
|
+
// parser is independent of locale/line-ending noise in the ssh transport.
|
|
125
|
+
function classifyProjectWiring({ stdout, err } = {}) {
|
|
126
|
+
if (err) return null;
|
|
127
|
+
const parts = String(stdout == null ? '' : stdout).split('---CY---').map((s) => s.trim());
|
|
128
|
+
const remote = parts[0] || '';
|
|
129
|
+
const nameCount = Number((parts[1] || '0').trim()) || 0;
|
|
130
|
+
if (remote || nameCount > 0) return true;
|
|
131
|
+
return false;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// Durable exit marker for a detached `add` provision (improve40). The add's
|
|
135
|
+
// `cloud:up` shell is spawned in its OWN session (detached) so a dashboard
|
|
136
|
+
// SIGKILL — the 2GB launchd hard cap, exactly the kill that aborted the VM-7
|
|
137
|
+
// add mid-flight — no longer reaches it; the provision keeps running. When that
|
|
138
|
+
// detached shell finally exits it writes `add-<n>.exit` (its trailing exit code
|
|
139
|
+
// + an ISO timestamp) into PENDING_JOBS_DIR, alongside the `add-<n>.json`
|
|
140
|
+
// sidecar. The marker is the GROUND-TRUTH completion signal the next boot reads
|
|
141
|
+
// instead of inferring `in-flight → error` from the lost child: a marker means
|
|
142
|
+
// "the provision finished while the dashboard was away," its code says whether
|
|
143
|
+
// it finished OK. The name is derived here so the shell `trap` (server.js) and
|
|
144
|
+
// the JS reader build the exact same path.
|
|
145
|
+
function exitMarkerName(n) {
|
|
146
|
+
return `add-${n}.exit`;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// Parse an exit-marker file body into { exitCode, finishedAt }, or null when the
|
|
150
|
+
// body is missing/blank/malformed. Format is two lines — the numeric exit code
|
|
151
|
+
// then an ISO timestamp — written by the shell trap. A non-integer first line is
|
|
152
|
+
// treated as no-marker (null) so a half-written / corrupt marker falls back to
|
|
153
|
+
// the gcloud reconcile path rather than being read as a bogus exit code. Pure:
|
|
154
|
+
// takes the text so it's unit-testable without touching disk.
|
|
155
|
+
function parseExitMarker(text) {
|
|
156
|
+
if (text == null) return null;
|
|
157
|
+
const lines = String(text).split('\n').map((s) => s.trim()).filter(Boolean);
|
|
158
|
+
if (!lines.length) return null;
|
|
159
|
+
const code = Number(lines[0]);
|
|
160
|
+
if (!Number.isInteger(code)) return null;
|
|
161
|
+
return { exitCode: code, finishedAt: lines[1] || null };
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// Read VM-n's exit marker from `dir`, or null when it's absent/unreadable/
|
|
165
|
+
// malformed (the still-running and never-written cases both surface as null).
|
|
166
|
+
function readExitMarker(dir, n) {
|
|
167
|
+
try { return parseExitMarker(fs.readFileSync(path.join(dir, exitMarkerName(n)), 'utf8')); }
|
|
168
|
+
catch { return null; }
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// Remove a consumed exit marker; a missing file is the already-cleared path, not
|
|
172
|
+
// an error (mirrors clearPendingJob).
|
|
173
|
+
function clearExitMarker(dir, n) {
|
|
174
|
+
try { fs.unlinkSync(path.join(dir, exitMarkerName(n))); }
|
|
175
|
+
catch (_) { /* missing is fine */ }
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// Decide how a snapshot-restored `add` should be re-attached after a dashboard
|
|
179
|
+
// restart, BEFORE any gcloud probe. Pure: takes the durable signals the boot
|
|
180
|
+
// path gathers — whether the exit marker exists, its recorded code, and whether
|
|
181
|
+
// the detached provision child is still alive (recorded pid, `kill -0`).
|
|
182
|
+
// verdict when dispatch
|
|
183
|
+
// 'done' marker present, exit code 0 roster + recordLaunch, clear sidecar/marker
|
|
184
|
+
// 'error' marker present, non-zero exit code surface the real failure, clear sidecar/marker
|
|
185
|
+
// 'running' no marker yet, child still alive RE-ATTACH: keep polling, do NOT mark error
|
|
186
|
+
// 'reconcile' child gone, no usable marker fall through to fc.reapVerdict (gcloud probe)
|
|
187
|
+
// This is the durable fix's brain: a provision that survived the kill (still
|
|
188
|
+
// running, or finished while we were away) is classified from GROUND TRUTH, so
|
|
189
|
+
// it is never blanket-converted `in-flight → error` the way the snapshot-only
|
|
190
|
+
// path did (the improve39 VM-7 false `add-failed`). Only when the child is gone
|
|
191
|
+
// AND left no parseable marker do we fall back to inferring state from gcloud —
|
|
192
|
+
// `reconcile` hands that exact (exists, containerUp, projectWired) decision to
|
|
193
|
+
// fc.reapVerdict. A marker whose first line isn't an integer is treated as
|
|
194
|
+
// absent (parseExitMarker → null) so a corrupt marker reconciles rather than
|
|
195
|
+
// asserting a bogus code. The matrix is exhaustively tested without a live VM.
|
|
196
|
+
function reattachVerdict({ markerPresent, exitCode, childAlive } = {}) {
|
|
197
|
+
if (markerPresent) {
|
|
198
|
+
// Guard the null/undefined coercion BEFORE Number() — `Number(null)` is 0,
|
|
199
|
+
// which would masquerade as a clean exit. A marker with no parseable code
|
|
200
|
+
// (half-written / corrupt) must reconcile, not assert success.
|
|
201
|
+
const code = exitCode == null ? NaN : Number(exitCode);
|
|
202
|
+
if (Number.isInteger(code)) return code === 0 ? 'done' : 'error';
|
|
203
|
+
return 'reconcile'; // marker present but no parseable code — let gcloud decide
|
|
204
|
+
}
|
|
205
|
+
if (childAlive) return 'running';
|
|
206
|
+
return 'reconcile';
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// improve45 — map a reattachVerdict tag to the dashboard's NEXT action, now that
|
|
210
|
+
// a clean exit marker is NECESSARY BUT NOT SUFFICIENT to roster. A detached
|
|
211
|
+
// `cloud:up` can exit 0 AFTER the image build but BEFORE `docker compose up`
|
|
212
|
+
// ever runs — an ssh broken-pipe or dashboard SIGKILL landing in that window,
|
|
213
|
+
// with the wrapping EXIT trap still recording 0 (the live VM2 signature: built
|
|
214
|
+
// image, repo cloned, ZERO editor containers). So 'done' no longer means "roster
|
|
215
|
+
// it"; it means "confirm the container is actually up first." Both the clean-exit
|
|
216
|
+
// ('done') and the no-usable-marker ('reconcile') cases therefore route through
|
|
217
|
+
// the SAME gcloud container + wiring probe (reconcileViaGcloud → fc.reapVerdict),
|
|
218
|
+
// which rosters only a genuinely-up + wired add and routes a half-provisioned one
|
|
219
|
+
// to recover-bootstrap. 'error' (real non-zero exit) and 'running' (child still
|
|
220
|
+
// alive) are unchanged. reattachVerdict stays the honest classifier of the
|
|
221
|
+
// child's exit SIGNAL; this helper expresses how that signal is CONSUMED now that
|
|
222
|
+
// exit-0 no longer implies rostering. Pure: server.js maps the returned action to
|
|
223
|
+
// its side-effecting step, so the matrix is exhaustively testable without a VM.
|
|
224
|
+
// verdict next step meaning
|
|
225
|
+
// 'done' 'probe' confirm the container before rostering
|
|
226
|
+
// 'reconcile' 'probe' no usable marker — let the gcloud probe decide
|
|
227
|
+
// 'error' 'surface-error' real failure — show add-failed with the code
|
|
228
|
+
// 'running' 'watch' child still alive — keep polling for the marker
|
|
229
|
+
function reattachNextStep(verdict) {
|
|
230
|
+
if (verdict === 'error') return 'surface-error';
|
|
231
|
+
if (verdict === 'running') return 'watch';
|
|
232
|
+
return 'probe'; // 'done' and 'reconcile' both confirm via the gcloud probe
|
|
233
|
+
}
|
|
234
|
+
|
|
104
235
|
// Anti-loop budget for the `recover-bootstrap` path. Pure: given the count of
|
|
105
236
|
// recoveries ALREADY persisted on the sidecar, return the next attempt number,
|
|
106
237
|
// whether the reap should proceed (re-run cloud:up) or treat the VM as
|
|
@@ -113,6 +244,66 @@ function recoverBootstrapDecision({ recoverAttempts, max = 3 }) {
|
|
|
113
244
|
return { attempt, proceed: attempt <= max, max };
|
|
114
245
|
}
|
|
115
246
|
|
|
247
|
+
// improve40 — select the orphaned `add` jobs that must be reconciled from the
|
|
248
|
+
// VM's REAL provision state even though their per-job sidecar is GONE. The
|
|
249
|
+
// durable re-attach path (sidecar + exit marker + recorded pid) cannot fire for
|
|
250
|
+
// these — a restart left only the snapshot record — so the snapshot job's
|
|
251
|
+
// stashed `config` is the sole cfg source and a live gcloud probe is the only
|
|
252
|
+
// ground truth. Pure: takes the restored jobs map + the set of `<action>-<n>`
|
|
253
|
+
// sidecar keys the sidecar reap already owns; returns the [{ n, cfg }] the boot
|
|
254
|
+
// path hands to reconcileViaGcloud. A job is selected when ALL hold:
|
|
255
|
+
// - it is an `add`,
|
|
256
|
+
// - it is flagged `restartOrphaned` (restoreFromSnapshot downgraded a
|
|
257
|
+
// running add → error because the dashboard restarted before it finished —
|
|
258
|
+
// this is the ONLY error state we re-probe; an add that failed IN-PROCESS
|
|
259
|
+
// carries a plain terminal `error` with no flag and stays surfaced), AND
|
|
260
|
+
// - NO `add-<n>` sidecar exists (a sidecar means the sidecar reap already
|
|
261
|
+
// owns it — never double-reconcile).
|
|
262
|
+
// `cfg` rides from the job's `config`; null when the snapshot never carried one
|
|
263
|
+
// (older snapshot) — the caller still probes, but a roster reconcile then can't
|
|
264
|
+
// recordLaunch, exactly mirroring the sidecar path's `meta.cfg`-absent case.
|
|
265
|
+
function orphanedAddsToReconcile({ jobs, sidecarKeys } = {}) {
|
|
266
|
+
const sidecars = sidecarKeys instanceof Set ? sidecarKeys : new Set(sidecarKeys || []);
|
|
267
|
+
const src = (jobs && typeof jobs === 'object') ? jobs : {};
|
|
268
|
+
const out = [];
|
|
269
|
+
for (const [k, j] of Object.entries(src)) {
|
|
270
|
+
if (!j || typeof j !== 'object') continue;
|
|
271
|
+
if (j.action !== 'add') continue;
|
|
272
|
+
if (!j.restartOrphaned) continue;
|
|
273
|
+
if (sidecars.has(`add-${k}`)) continue;
|
|
274
|
+
out.push({ n: Number(k), cfg: j.config || null });
|
|
275
|
+
}
|
|
276
|
+
return out;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
// improve41 — the destroy-direction mirror of orphanedAddsToReconcile: decide
|
|
280
|
+
// what to do with a VM that is STILL in the roster but whose GCP instance state
|
|
281
|
+
// we have just re-probed. The bug this closes: a Destroy that races a dashboard
|
|
282
|
+
// restart can leave the instance deleted while the startup reap took the
|
|
283
|
+
// `clear-only` path (instance still present at reap time) and dropped the only
|
|
284
|
+
// sidecar — so nothing ever runs roster cleanup and the VM polls forever
|
|
285
|
+
// "unreachable." A sidecar-independent reconcile, keyed on REAL GCP existence
|
|
286
|
+
// rather than on a pending-job sidecar, catches it. Pure: server.js runs the
|
|
287
|
+
// gcloud describe (→ exists, via classifyDescribeError) and reads session /
|
|
288
|
+
// pending-add state, and hands the three facts here so the matrix is testable
|
|
289
|
+
// without a live VM.
|
|
290
|
+
// verdict when dispatch
|
|
291
|
+
// 'cleanup' instance gone AND the VM is idle runDestroyCleanup
|
|
292
|
+
// 'leave' exists uncertain (null), OR the VM is busy do nothing this pass
|
|
293
|
+
// 'keep' instance still exists do nothing (healthy)
|
|
294
|
+
// Fail-safe by construction: cleanup fires ONLY on a DEFINITIVE absence
|
|
295
|
+
// (exists === false) — a flaky/transient describe (null) is always 'leave', so a
|
|
296
|
+
// gcloud blip can never evict a healthy VM (mirroring reapVerdict's `leave`
|
|
297
|
+
// posture). "Busy" (a live session or an in-flight add to this number) also
|
|
298
|
+
// holds the entry for an operator rather than auto-removing — the shouldn't-
|
|
299
|
+
// happen-but-fail-safe case where a vanished instance still reports a session.
|
|
300
|
+
function reconcileRosteredVm({ exists, hasSession, hasPendingAdd } = {}) {
|
|
301
|
+
if (exists === null || exists === undefined) return "leave";
|
|
302
|
+
if (exists) return "keep";
|
|
303
|
+
if (hasSession || hasPendingAdd) return "leave";
|
|
304
|
+
return "cleanup";
|
|
305
|
+
}
|
|
306
|
+
|
|
116
307
|
module.exports = {
|
|
117
308
|
parsePendingJobFilename,
|
|
118
309
|
persistPendingJob,
|
|
@@ -120,5 +311,14 @@ module.exports = {
|
|
|
120
311
|
listPendingJobs,
|
|
121
312
|
classifyDescribeError,
|
|
122
313
|
reapVerdict,
|
|
314
|
+
reconcileRosteredVm,
|
|
315
|
+
orphanedAddsToReconcile,
|
|
316
|
+
classifyProjectWiring,
|
|
123
317
|
recoverBootstrapDecision,
|
|
318
|
+
exitMarkerName,
|
|
319
|
+
parseExitMarker,
|
|
320
|
+
readExitMarker,
|
|
321
|
+
clearExitMarker,
|
|
322
|
+
reattachVerdict,
|
|
323
|
+
reattachNextStep,
|
|
124
324
|
};
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// Pure predicate module for the Fleet Dashboard's per-button "will clicking
|
|
3
|
+
// this change the VM, or is it already in that state?" indicators
|
|
4
|
+
// (scripts/operator/fleet-dashboard/server.js requires this). Given an
|
|
5
|
+
// already-gathered FACTS object — no I/O, no git/docker exec; server.js does
|
|
6
|
+
// the impure fact-gathering — computeButtonEffects returns a per-button verdict
|
|
7
|
+
// map. Mirrors the pure-module-plus-colocated-vitest-test shape of
|
|
8
|
+
// npm/fleet-reset.js so the whole decision table is unit-testable without
|
|
9
|
+
// spawning a process.
|
|
10
|
+
//
|
|
11
|
+
// Each verdict is { effect, reason }:
|
|
12
|
+
// - 'changes' — clicking does something; `reason` says what.
|
|
13
|
+
// - 'noop' — already in that state; clicking changes nothing.
|
|
14
|
+
// - 'recommended' — soft signal (run-setup only): a reset/update landed after
|
|
15
|
+
// the last successful setup. NEVER a confident green no-op.
|
|
16
|
+
// - 'unknown' — not yet checked, VM unreachable, or a needed fact is
|
|
17
|
+
// missing. We never CLAIM a safe no-op we couldn't verify —
|
|
18
|
+
// silent failure here would be worse than no indicator.
|
|
19
|
+
//
|
|
20
|
+
// The predicate for each button is derived from what that button actually
|
|
21
|
+
// touches (its tooltip in public/index.html is the source of truth): the sync
|
|
22
|
+
// buttons key off a git commit comparison, the reset buttons off
|
|
23
|
+
// working-tree/session/`.codeyam` state, and rebuild off a build-commit
|
|
24
|
+
// comparison.
|
|
25
|
+
|
|
26
|
+
// Short sha for compact reason strings. Tolerates null / short input.
|
|
27
|
+
function short(sha) {
|
|
28
|
+
return sha ? String(sha).slice(0, 7) : '?';
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// "origin/<branch>" — or a bare "origin" when the branch is unknown.
|
|
32
|
+
function originRef(branch) {
|
|
33
|
+
return branch ? `origin/${branch}` : 'origin';
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// "N commit(s) behind origin/<branch>" with correct singular/plural.
|
|
37
|
+
function behindReason(behind, branch) {
|
|
38
|
+
return `${behind} commit${behind === 1 ? '' : 's'} behind ${originRef(branch)}`;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Compose a short reason listing which signals make a reset non-trivial, so the
|
|
42
|
+
// operator sees WHAT a Reset would discard rather than a bare "changes". The
|
|
43
|
+
// `agent` (a live build agent in the broker) leads the list — it's the most
|
|
44
|
+
// consequential thing a reset's `pty-broker stop` would silently discard.
|
|
45
|
+
function resetReason({ agent, session, dirty, behind, codeyam }) {
|
|
46
|
+
const parts = [];
|
|
47
|
+
if (agent) parts.push('running build agent');
|
|
48
|
+
if (dirty) parts.push('uncommitted changes');
|
|
49
|
+
if (behind) parts.push('source behind origin');
|
|
50
|
+
if (session) parts.push('active session');
|
|
51
|
+
if (codeyam) parts.push('.codeyam state');
|
|
52
|
+
return parts.length ? `will discard: ${parts.join(', ')}` : 'will reset state';
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// The action ids governed by an effect indicator — the same ids server.js /
|
|
56
|
+
// index.html use for the buttons. Reconfigure / Env / Destroy are deliberately
|
|
57
|
+
// absent (one-shots, not converge-to-a-state actions, so a no-op predicate is
|
|
58
|
+
// meaningless for them).
|
|
59
|
+
const ACTION_IDS = [
|
|
60
|
+
'update-client-source',
|
|
61
|
+
'update-editor-source',
|
|
62
|
+
'reset-editor-source',
|
|
63
|
+
'reset-client',
|
|
64
|
+
'rebuild-binary',
|
|
65
|
+
'update-reset-everything',
|
|
66
|
+
'run-setup',
|
|
67
|
+
];
|
|
68
|
+
|
|
69
|
+
// facts (a plain object gathered by server.js — no I/O):
|
|
70
|
+
// {
|
|
71
|
+
// reachable: boolean,
|
|
72
|
+
// hasSession: boolean,
|
|
73
|
+
// client: { headSha, originSha, behind, dirty } | null, // /workspace
|
|
74
|
+
// editor: { headSha, originSha, behind, dirty, buildCommit } | null,
|
|
75
|
+
// codeyamDirty: boolean,
|
|
76
|
+
// setupRanSince: boolean, // a setup completed after the last reset/update
|
|
77
|
+
// clientBranch: string | null, // for reason text only
|
|
78
|
+
// editorBranch: string | null, // for reason text only
|
|
79
|
+
// }
|
|
80
|
+
function computeButtonEffects(facts) {
|
|
81
|
+
const f = facts || {};
|
|
82
|
+
|
|
83
|
+
// Unreachable VM (or no facts at all): every governed button is unknown. We
|
|
84
|
+
// could not reach the editor to gather facts, so never render a green no-op.
|
|
85
|
+
if (!f.reachable) {
|
|
86
|
+
const out = {};
|
|
87
|
+
for (const id of ACTION_IDS) out[id] = { effect: 'unknown', reason: 'VM unreachable' };
|
|
88
|
+
return out;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const client = f.client || null;
|
|
92
|
+
const editor = f.editor || null;
|
|
93
|
+
const hasSession = !!f.hasSession;
|
|
94
|
+
const codeyamDirty = !!f.codeyamDirty;
|
|
95
|
+
// A live build agent in the broker. A reset's first step is `pty-broker stop`,
|
|
96
|
+
// which kills it regardless of whether a workflow session is registered — so
|
|
97
|
+
// this is discardable work even when hasSession is false. Explicit-true only:
|
|
98
|
+
// a null/absent value (older or unreachable VM) must NOT flip a genuine noop.
|
|
99
|
+
const brokerLive = f.brokerLive === true;
|
|
100
|
+
|
|
101
|
+
// (1) Update client — ff-advance /workspace to origin tip. Changes iff behind.
|
|
102
|
+
const updateClient = client == null
|
|
103
|
+
? { effect: 'unknown', reason: 'client state not checked' }
|
|
104
|
+
: client.behind > 0
|
|
105
|
+
? { effect: 'changes', reason: behindReason(client.behind, f.clientBranch) }
|
|
106
|
+
: { effect: 'noop', reason: 'up to date with ' + originRef(f.clientBranch) };
|
|
107
|
+
|
|
108
|
+
// (2) Update editor — advance the editor source the binary builds from.
|
|
109
|
+
const updateEditor = editor == null
|
|
110
|
+
? { effect: 'unknown', reason: 'editor state not checked' }
|
|
111
|
+
: editor.behind > 0
|
|
112
|
+
? { effect: 'changes', reason: behindReason(editor.behind, f.editorBranch) }
|
|
113
|
+
: { effect: 'noop', reason: 'up to date with ' + originRef(f.editorBranch) };
|
|
114
|
+
|
|
115
|
+
// (5) Rebuild binary — advances source then compiles. Changes iff the running
|
|
116
|
+
// binary's build commit differs from the editor origin tip. Unknown when we
|
|
117
|
+
// don't have both commits to compare.
|
|
118
|
+
const rebuild = (editor == null || !editor.buildCommit || !editor.originSha)
|
|
119
|
+
? { effect: 'unknown', reason: 'binary/source commit not checked' }
|
|
120
|
+
: editor.buildCommit !== editor.originSha
|
|
121
|
+
? { effect: 'changes', reason: `binary at ${short(editor.buildCommit)}, ${originRef(f.editorBranch)} at ${short(editor.originSha)}` }
|
|
122
|
+
: { effect: 'noop', reason: 'binary built from ' + originRef(f.editorBranch) + ' tip' };
|
|
123
|
+
|
|
124
|
+
// (3) Reset editor — discards uncommitted + session, advances source, wipes
|
|
125
|
+
// .codeyam. No-op ONLY when clean at origin tip with no session.
|
|
126
|
+
const resetEditor = editor == null
|
|
127
|
+
? { effect: 'unknown', reason: 'editor state not checked' }
|
|
128
|
+
: (brokerLive || hasSession || editor.dirty || editor.behind > 0 || codeyamDirty)
|
|
129
|
+
? { effect: 'changes', reason: resetReason({ agent: brokerLive, session: hasSession, dirty: editor.dirty, behind: editor.behind > 0, codeyam: codeyamDirty }) }
|
|
130
|
+
: { effect: 'noop', reason: 'already clean at origin tip, no session' };
|
|
131
|
+
|
|
132
|
+
// (3b) Reset client — hard-clean /workspace + .codeyam, drop the session.
|
|
133
|
+
// No source-advance, so `behind` is NOT part of this predicate.
|
|
134
|
+
const resetClient = client == null
|
|
135
|
+
? { effect: 'unknown', reason: 'client state not checked' }
|
|
136
|
+
: (brokerLive || client.dirty || codeyamDirty || hasSession)
|
|
137
|
+
? { effect: 'changes', reason: resetReason({ agent: brokerLive, session: hasSession, dirty: client.dirty, behind: false, codeyam: codeyamDirty }) }
|
|
138
|
+
: { effect: 'noop', reason: 'already clean, no session' };
|
|
139
|
+
|
|
140
|
+
// (4b) Update & Reset Everything — composite. Changes iff ANY component would
|
|
141
|
+
// change: rebuild-binary, reset-editor-source, or the client being
|
|
142
|
+
// behind/dirty. Reason summarizes which parts.
|
|
143
|
+
const everything = (() => {
|
|
144
|
+
if (editor == null || client == null) {
|
|
145
|
+
return { effect: 'unknown', reason: 'state not checked' };
|
|
146
|
+
}
|
|
147
|
+
const parts = [];
|
|
148
|
+
if (rebuild.effect === 'changes') parts.push('rebuild');
|
|
149
|
+
if (resetEditor.effect === 'changes') parts.push('reset editor');
|
|
150
|
+
if (client.behind > 0 || client.dirty) parts.push('reset client');
|
|
151
|
+
if (parts.length > 0) return { effect: 'changes', reason: parts.join(' + ') };
|
|
152
|
+
// No part would change. But if a component was itself unverifiable, don't
|
|
153
|
+
// claim a confident no-op — fall back to unknown.
|
|
154
|
+
if (rebuild.effect === 'unknown' || resetEditor.effect === 'unknown') {
|
|
155
|
+
return { effect: 'unknown', reason: 'state not checked' };
|
|
156
|
+
}
|
|
157
|
+
return { effect: 'noop', reason: 'everything already at origin tip, no session' };
|
|
158
|
+
})();
|
|
159
|
+
|
|
160
|
+
// (run-setup) Honest soft signal — there is no cheap upstream proof that
|
|
161
|
+
// `npm run setup` would change nothing (DB/migration state isn't
|
|
162
|
+
// introspected). `recommended` when a reset/update landed after the last
|
|
163
|
+
// successful setup; otherwise `unknown`. Never a confident green no-op.
|
|
164
|
+
const runSetup = f.setupRanSince
|
|
165
|
+
? { effect: 'unknown', reason: 'no setup-affecting change since last setup' }
|
|
166
|
+
: { effect: 'recommended', reason: 'a reset/update landed since the last setup' };
|
|
167
|
+
|
|
168
|
+
return {
|
|
169
|
+
'update-client-source': updateClient,
|
|
170
|
+
'update-editor-source': updateEditor,
|
|
171
|
+
'reset-editor-source': resetEditor,
|
|
172
|
+
'reset-client': resetClient,
|
|
173
|
+
'rebuild-binary': rebuild,
|
|
174
|
+
'update-reset-everything': everything,
|
|
175
|
+
'run-setup': runSetup,
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
module.exports = { computeButtonEffects, ACTION_IDS };
|