@codeyam-editor/codeyam-editor 0.1.15 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/npm/app-port-drift.js +23 -0
- package/npm/fleet-add-failure-evidence.js +309 -0
- package/npm/fleet-add-failure.js +51 -311
- package/npm/fleet-build-data.js +47 -20
- package/npm/fleet-preview-health.js +156 -0
- package/npm/fleet-revision.js +50 -0
- package/npm/fleet-sticky-fields.js +124 -0
- package/npm/launch-config.js +43 -142
- package/npm/scenario-check.js +68 -383
- package/npm/scenario-issues.js +47 -0
- package/npm/scenario-launch.js +349 -0
- package/npm/scenario-playwright.js +42 -321
- package/npm/scenario-targeting.js +338 -0
- package/npm/vm-forwards.js +158 -0
- package/package.json +5 -5
- package/ui/dist/assets/{ContainerInspector-Ceop11Fv.js → ContainerInspector-Cig2PDjT.js} +1 -1
- package/ui/dist/assets/{EditorShell-DjI5cEF_.js → EditorShell-DOuUJNSm.js} +76 -74
- package/ui/dist/assets/{ScenarioDataPanel-D-nWT7xD.js → ScenarioDataPanel-BRUYhl1o.js} +10 -10
- package/ui/dist/assets/{ScenarioDataPanelFullPage-ZNFZcTQU.js → ScenarioDataPanelFullPage-DjwRM8HJ.js} +1 -1
- package/ui/dist/assets/ScenarioRenderer-CcbccxCW.js +26 -0
- package/ui/dist/assets/{index-ClrdYzlw.js → index-Bkrhkuf5.js} +6 -6
- package/ui/dist/assets/index-Dw4EB88N.css +1 -0
- package/ui/dist/index.html +2 -2
- package/ui/dist/assets/ScenarioRenderer-BW6QLGFW.js +0 -26
- package/ui/dist/assets/index-B3SMxYUB.css +0 -1
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// The dashboard's honest preview verdict: does the path the load balancer uses
|
|
4
|
+
// to reach a VM's Live Preview actually answer?
|
|
5
|
+
//
|
|
6
|
+
// Every older preview signal is blind to a dead preview by construction.
|
|
7
|
+
// `previewServing` is the editor reporting on its OWN proxy, `previewIngress`
|
|
8
|
+
// can be answered by the LB itself, and the LB's backend health checks the
|
|
9
|
+
// editor port (14199), never the app. VM-15 (2026-10-17) read green on all of
|
|
10
|
+
// them while its host published 3000 and the editor's proxy bound 4321, so
|
|
11
|
+
// docker-proxy accepted every LB connection and reset it.
|
|
12
|
+
//
|
|
13
|
+
// Two detectors, because they answer different questions:
|
|
14
|
+
// • app-port drift (npm/app-port-drift.js) names the CAUSE — the published
|
|
15
|
+
// host port and the declared app port disagree;
|
|
16
|
+
// • the endpoint dial catches the whole accept-then-reset CLASS, including a
|
|
17
|
+
// dev server that died or a proxy wedged bound-but-not-accepting.
|
|
18
|
+
// Pure: server.js owns the gcloud read and the dial, and feeds the results here.
|
|
19
|
+
|
|
20
|
+
const { appPortDriftRemediations } = require('./app-port-drift');
|
|
21
|
+
|
|
22
|
+
const PORT_DRIFT_RUNBOOK = 'docs/preview-port-drift.md';
|
|
23
|
+
|
|
24
|
+
/** Parse `gcloud compute network-endpoint-groups list-network-endpoints
|
|
25
|
+
* --format='value(networkEndpoint.ipAddress,networkEndpoint.port)'` output
|
|
26
|
+
* into `[{ ip, port }]`. Lines that do not carry both an address and a valid
|
|
27
|
+
* port are dropped — an unreadable endpoint is never dialed as a guess. */
|
|
28
|
+
function parseNegEndpoints(stdout) {
|
|
29
|
+
if (typeof stdout !== 'string') return [];
|
|
30
|
+
const out = [];
|
|
31
|
+
for (const line of stdout.split('\n')) {
|
|
32
|
+
const [ip, portText] = line.trim().split(/\s+/);
|
|
33
|
+
const port = Number(portText);
|
|
34
|
+
if (!ip || !/^\d{1,3}(\.\d{1,3}){3}$/.test(ip)) continue;
|
|
35
|
+
if (!Number.isInteger(port) || port <= 0 || port >= 65536) continue;
|
|
36
|
+
out.push({ ip, port });
|
|
37
|
+
}
|
|
38
|
+
return out;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Classify one dial of a preview endpoint.
|
|
42
|
+
*
|
|
43
|
+
* ANY HTTP response — 200, 307, 404, 500 — means the app answered, so the LB
|
|
44
|
+
* can reach it: `answered`. A connection reset or refusal is the Envoy
|
|
45
|
+
* "upstream connect error … connection termination" shape: `reset`. A dial
|
|
46
|
+
* that hit its budget is `timeout`. Anything else is `unknown`, which never
|
|
47
|
+
* raises an alarm. */
|
|
48
|
+
function classifyPreviewDial({ statusCode, error, timedOut } = {}) {
|
|
49
|
+
if (Number.isInteger(statusCode)) return 'answered';
|
|
50
|
+
if (timedOut) return 'timeout';
|
|
51
|
+
const code = error && error.code;
|
|
52
|
+
if (code === 'ECONNRESET' || code === 'ECONNREFUSED' || code === 'EPIPE'
|
|
53
|
+
|| /socket hang up/i.test(String(error && error.message))) {
|
|
54
|
+
return 'reset';
|
|
55
|
+
}
|
|
56
|
+
return 'unknown';
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Fold the per-endpoint dial verdicts into one `{ verdict, endpoints }`.
|
|
60
|
+
* Every endpoint the LB may pick must answer, so the worst verdict wins:
|
|
61
|
+
* any `reset`/`timeout` is a dead preview; all `answered` is a live one; no
|
|
62
|
+
* endpoints or any `unknown` leaves the verdict `unknown`. */
|
|
63
|
+
function aggregatePreviewDials(dials) {
|
|
64
|
+
const list = Array.isArray(dials) ? dials : [];
|
|
65
|
+
const endpoints = list.map((d) => ({ ip: d.ip, port: d.port, verdict: d.verdict }));
|
|
66
|
+
const dead = list.find((d) => d.verdict === 'reset' || d.verdict === 'timeout');
|
|
67
|
+
if (dead) return { verdict: dead.verdict, endpoints };
|
|
68
|
+
if (list.length && list.every((d) => d.verdict === 'answered')) return { verdict: 'answered', endpoints };
|
|
69
|
+
return { verdict: 'unknown', endpoints };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Read a VM's preview NEG endpoints and dial every one, through injected I/O
|
|
73
|
+
* so it runs with no gcloud and no network in tests. `deps.listEndpoints()`
|
|
74
|
+
* resolves the gcloud list output (null on failure); `deps.dial(ep)` resolves
|
|
75
|
+
* a `classifyPreviewDial` outcome. No endpoints → `unknown`, never dead. */
|
|
76
|
+
async function collectPreviewEndpoint(deps) {
|
|
77
|
+
const endpoints = parseNegEndpoints(await deps.listEndpoints());
|
|
78
|
+
if (!endpoints.length) return { verdict: 'unknown', endpoints: [] };
|
|
79
|
+
const dials = await Promise.all(endpoints.map(async (ep) => (
|
|
80
|
+
{ ...ep, verdict: classifyPreviewDial(await deps.dial(ep)) }
|
|
81
|
+
)));
|
|
82
|
+
return aggregatePreviewDials(dials);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** The app-port-drift reason for the card's amber badge, or null. `drift` is a
|
|
86
|
+
* `collectAppPortDrift` finding (or any `{ published, declared }`); the
|
|
87
|
+
* remediations are recomputed when absent so a bare pair still names the fix. */
|
|
88
|
+
function appPortDriftReason(drift, vmNumber) {
|
|
89
|
+
if (!drift || !Number.isInteger(drift.published) || !Number.isInteger(drift.declared)) return null;
|
|
90
|
+
if (drift.published === drift.declared) return null;
|
|
91
|
+
const remediations = Array.isArray(drift.remediations) && drift.remediations.length
|
|
92
|
+
? drift.remediations
|
|
93
|
+
: appPortDriftRemediations({ vmNumber, declared: drift.declared, published: drift.published, idle: drift.idle || null });
|
|
94
|
+
return {
|
|
95
|
+
kind: 'app-port-drift',
|
|
96
|
+
published: drift.published,
|
|
97
|
+
declared: drift.declared,
|
|
98
|
+
fix: remediations[0].text,
|
|
99
|
+
runbook: PORT_DRIFT_RUNBOOK,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** The dead-endpoint reason for the card's amber badge, or null. Only a
|
|
104
|
+
* confirmed `reset`/`timeout` raises it; `answered` and `unknown` stay silent. */
|
|
105
|
+
function previewEndpointReason(previewEndpoint) {
|
|
106
|
+
if (!previewEndpoint) return null;
|
|
107
|
+
const { verdict, endpoints } = previewEndpoint;
|
|
108
|
+
if (verdict !== 'reset' && verdict !== 'timeout') return null;
|
|
109
|
+
const dead = (endpoints || []).find((e) => e.verdict === verdict) || {};
|
|
110
|
+
return {
|
|
111
|
+
kind: 'endpoint-dead',
|
|
112
|
+
verdict,
|
|
113
|
+
endpoint: dead.ip && dead.port ? `${dead.ip}:${dead.port}` : null,
|
|
114
|
+
runbook: PORT_DRIFT_RUNBOOK,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** Resolve the one `previewMisconfigured` reason the card renders, most
|
|
119
|
+
* specific first: the preview-origin guard (its own shape, unchanged), then
|
|
120
|
+
* the port drift that names the cause, then the bare dead endpoint. Null when
|
|
121
|
+
* the VM is unreachable — its last-known readings are stale, and an
|
|
122
|
+
* unreachable card has louder problems to report. */
|
|
123
|
+
function resolvePreviewMisconfigured(v, vmNumber) {
|
|
124
|
+
if (!v || !v.reachable) return null;
|
|
125
|
+
return v.previewMisconfigured
|
|
126
|
+
|| appPortDriftReason(v.appPortDrift, vmNumber)
|
|
127
|
+
|| previewEndpointReason(v.previewEndpoint)
|
|
128
|
+
|| null;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** Re-derive `backendHealth.preview` so it never claims a health no check
|
|
132
|
+
* established. The LB's own health check dials the editor port, so its
|
|
133
|
+
* preview verdict says nothing about the app: `healthy` survives only when
|
|
134
|
+
* the endpoint dial answered, becomes `unhealthy` when the dial was reset or
|
|
135
|
+
* timed out, and `unverified` when no dial has landed. Non-healthy LB verdicts
|
|
136
|
+
* (`absent`, `unhealthy`, `unknown`) pass through untouched, as does `editor`. */
|
|
137
|
+
function gatePreviewBackendHealth(backendHealth, previewEndpoint) {
|
|
138
|
+
if (!backendHealth || typeof backendHealth !== 'object') return backendHealth || null;
|
|
139
|
+
if (backendHealth.preview !== 'healthy') return backendHealth;
|
|
140
|
+
const verdict = previewEndpoint && previewEndpoint.verdict;
|
|
141
|
+
let preview = 'unverified';
|
|
142
|
+
if (verdict === 'answered') preview = 'healthy';
|
|
143
|
+
else if (verdict === 'reset' || verdict === 'timeout') preview = 'unhealthy';
|
|
144
|
+
return { ...backendHealth, preview };
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
module.exports = {
|
|
148
|
+
parseNegEndpoints,
|
|
149
|
+
classifyPreviewDial,
|
|
150
|
+
aggregatePreviewDials,
|
|
151
|
+
collectPreviewEndpoint,
|
|
152
|
+
appPortDriftReason,
|
|
153
|
+
previewEndpointReason,
|
|
154
|
+
resolvePreviewMisconfigured,
|
|
155
|
+
gatePreviewBackendHealth,
|
|
156
|
+
};
|
package/npm/fleet-revision.js
CHANGED
|
@@ -150,6 +150,55 @@ function projectBinaryRevision(memo = {}) {
|
|
|
150
150
|
return { ...memo, level: classifyBinaryStaleness(memo) };
|
|
151
151
|
}
|
|
152
152
|
|
|
153
|
+
/** The `/data` view of the pinned branch's PUBLISH state — whether new and
|
|
154
|
+
* refreshed VMs will boot the tip's binary, or an older one.
|
|
155
|
+
*
|
|
156
|
+
* `binaryRevision` above measures only this host's own CLI. The artifact every
|
|
157
|
+
* newly provisioned VM pulls is a different thing, published on demand, and it
|
|
158
|
+
* stayed invisible: on 2026-10-17 VMs 3 and 8 would have booted a build many
|
|
159
|
+
* commits behind the tip, with nothing on this page saying so. The hourly
|
|
160
|
+
* operator refresh (refresh-operator-repo.sh, step 9b) now publishes the tip
|
|
161
|
+
* itself and records what it found in `auto-publish.state.json`; this reads
|
|
162
|
+
* that record.
|
|
163
|
+
*
|
|
164
|
+
* `lockAlive` is whether the publish that record started is still running. A
|
|
165
|
+
* `publishing` record with a dead lock is a publish that ended between ticks —
|
|
166
|
+
* it may have succeeded or failed, and only the next tick knows, so it reads
|
|
167
|
+
* `behind` rather than claiming either.
|
|
168
|
+
*
|
|
169
|
+
* Levels: 'published' | 'publishing' | 'behind' | 'unchecked'. 'unchecked'
|
|
170
|
+
* covers no record yet and a record for a branch other than the pin (the pin
|
|
171
|
+
* moved; the next tick rewrites it) — both render nothing rather than a
|
|
172
|
+
* verdict about the wrong branch.
|
|
173
|
+
*
|
|
174
|
+
* Pure; no I/O. */
|
|
175
|
+
function projectBranchPublish({ doc, lockAlive = false, branch } = {}) {
|
|
176
|
+
const none = { level: 'unchecked', text: null };
|
|
177
|
+
if (!doc || typeof doc !== 'object' || typeof doc.state !== 'string') return none;
|
|
178
|
+
if (branch && doc.branch !== branch) return none;
|
|
179
|
+
const base = {
|
|
180
|
+
branch: doc.branch || null,
|
|
181
|
+
tip: doc.tip || null,
|
|
182
|
+
tag: doc.tag || null,
|
|
183
|
+
behind: Number.isInteger(doc.behind) ? doc.behind : null,
|
|
184
|
+
checkedAt: doc.checkedAt || null,
|
|
185
|
+
};
|
|
186
|
+
const gap = base.behind === null ? 'an older build' : `${base.behind} commit(s) behind`;
|
|
187
|
+
if (doc.state === 'published') {
|
|
188
|
+
return { ...base, level: 'published', text: 'branch binary published at tip' };
|
|
189
|
+
}
|
|
190
|
+
if (doc.state === 'code-identical') {
|
|
191
|
+
return { ...base, level: 'published', text: `branch binary current (${base.tag || 'newest artifact'} is code-identical to the tip)` };
|
|
192
|
+
}
|
|
193
|
+
if (doc.state === 'publishing' && lockAlive) {
|
|
194
|
+
return { ...base, level: 'publishing', text: `publishing the tip — new VMs boot ${base.tag || 'an older build'} (${gap}) until it lands` };
|
|
195
|
+
}
|
|
196
|
+
if (doc.state === 'publishing') {
|
|
197
|
+
return { ...base, level: 'behind', text: `tip binary not confirmed published — new VMs boot ${base.tag || 'an older build'} (${gap}); the next hourly refresh retries` };
|
|
198
|
+
}
|
|
199
|
+
return none;
|
|
200
|
+
}
|
|
201
|
+
|
|
153
202
|
/** Why did the CLI shell-out fail — and is it the nameable one?
|
|
154
203
|
*
|
|
155
204
|
* The 2026-08-21 outage surfaced ONLY as a truncated `queueFreshness.error`
|
|
@@ -207,6 +256,7 @@ module.exports = {
|
|
|
207
256
|
projectDashboardRevision,
|
|
208
257
|
classifyBinaryStaleness,
|
|
209
258
|
projectBinaryRevision,
|
|
259
|
+
projectBranchPublish,
|
|
210
260
|
classifyCliFailure,
|
|
211
261
|
projectCliHealth,
|
|
212
262
|
};
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fes = require('./fleet-effect-status');
|
|
4
|
+
const fstrand = require('./fleet-strand');
|
|
5
|
+
|
|
6
|
+
// npm/fleet-sticky-fields.js — the readings pollVMs (fleet-dashboard server.js)
|
|
7
|
+
// carries across its 3s hot-path rebuild of a VM's record.
|
|
8
|
+
//
|
|
9
|
+
// Every field here is refreshed somewhere SLOWER than the 3s poll — the
|
|
10
|
+
// EFFECT_POLL_MS gather, or an out-of-band action — so the rebuild must keep the
|
|
11
|
+
// last-known value alive or the card would blank and flicker back every cycle.
|
|
12
|
+
// Each fallback below is the one the field has always had, and they differ on
|
|
13
|
+
// purpose: a tri-state reading defaults to null ("unknown, never a false
|
|
14
|
+
// alarm"), a counter or timestamp to 0, an in-flight flag to false, and two
|
|
15
|
+
// credential readings stay `undefined` so a VM never gathered reads as unknown.
|
|
16
|
+
//
|
|
17
|
+
// Pure: takes the VM's previous record (or undefined) and returns the carried
|
|
18
|
+
// subset. pollVMs spreads it after the probe results, which is where these
|
|
19
|
+
// fields always sat.
|
|
20
|
+
function carriedSlowGatherFields(prev) {
|
|
21
|
+
const p = prev || {};
|
|
22
|
+
return {
|
|
23
|
+
// Per-button effect verdicts (improve39), refreshed in place by pollEffects.
|
|
24
|
+
effects: p.effects || null,
|
|
25
|
+
// Disk-pressure reading { usedPct, level }, folded into the effects gather.
|
|
26
|
+
disk: p.disk || null,
|
|
27
|
+
// Credential-integrity verdict (true | false | null) from the gather's CRED
|
|
28
|
+
// marker; pollVMs folds it into providerAuth, so donor selection and the
|
|
29
|
+
// card read it from the one record.
|
|
30
|
+
credentialIntact: prev ? prev.credentialIntact : undefined,
|
|
31
|
+
// Which Claude store is live (project|home), from the same slow gather.
|
|
32
|
+
claudeStore: prev ? prev.claudeStore : undefined,
|
|
33
|
+
// LB backend health { editor, preview } (vm: Card Readiness Display). Without
|
|
34
|
+
// the carry the readiness state would blank and the card flicker back to a
|
|
35
|
+
// clickable link every hot-path poll.
|
|
36
|
+
backendHealth: p.backendHealth || null,
|
|
37
|
+
// The two LB-path preview readings: `publishedAppPort` is the host port the
|
|
38
|
+
// editor container publishes (folded into the fact gather's SSH);
|
|
39
|
+
// `previewEndpoint` is the dial of the preview NEG endpoint the LB uses.
|
|
40
|
+
publishedAppPort: prev ? prev.publishedAppPort : null,
|
|
41
|
+
previewEndpoint: prev ? prev.previewEndpoint : null,
|
|
42
|
+
// IAP-access verdict ('granted'|'no-accessor'|'unknown'); without the carry
|
|
43
|
+
// the 'iap-access-missing' readiness state would blank every hot poll.
|
|
44
|
+
iapAccess: p.iapAccess || null,
|
|
45
|
+
// Live IAM member lists { editor, preview, all, partial }. null until the
|
|
46
|
+
// first successful gather and whenever a surface read errors — the card
|
|
47
|
+
// treats null as "not yet verified", never as "nobody has access".
|
|
48
|
+
iapMembers: p.iapMembers || null,
|
|
49
|
+
// Public-URL verdict 'wired'|'unwired'|'unknown' plus its auto-rewire
|
|
50
|
+
// debounce (vm: Dashboard validates the public ingress URL). The timestamp
|
|
51
|
+
// and in-flight flag MUST survive the rebuild or the re-wire would re-fire
|
|
52
|
+
// every 3s while a correct LB change is still propagating (VM5's correct
|
|
53
|
+
// rule 503'd for minutes before flipping).
|
|
54
|
+
publicUrlHealth: p.publicUrlHealth || null,
|
|
55
|
+
// Collapsed URL-wiring state (vm--add-time-url-wiring-converges-reliably),
|
|
56
|
+
// so the card's "wiring…" / clickable / "wiring-failed" read does not
|
|
57
|
+
// flicker between slow gathers. null in legacy quick-tunnel mode.
|
|
58
|
+
urlWiringState: p.urlWiringState || null,
|
|
59
|
+
lastIngressRewireAt: p.lastIngressRewireAt || 0,
|
|
60
|
+
ingressRewireInFlight: !!p.ingressRewireInFlight,
|
|
61
|
+
// "Editor BES exists but has zero NEG backends" (the backendless
|
|
62
|
+
// half-wire). Without the carry the auto-rewire gate reads undefined and
|
|
63
|
+
// suppresses the backendless self-heal until the next slow cycle.
|
|
64
|
+
backendEditorBackendless: !!p.backendEditorBackendless,
|
|
65
|
+
// Env-override verdict (applied/skipped + tagged repo), set out-of-band by
|
|
66
|
+
// reapplyVmEnv (recordEnvOverrideStatus), never on the 3s poll.
|
|
67
|
+
envOverride: p.envOverride || null,
|
|
68
|
+
// WS-usability signals (vm: Readiness Verifies Real Usability): the public
|
|
69
|
+
// WebSocket-upgrade verdict and the editor BES timeoutSec, so the 'ws-dead'
|
|
70
|
+
// readiness label does not flicker between slow cycles.
|
|
71
|
+
publicWsHealth: p.publicWsHealth || null,
|
|
72
|
+
backendTimeoutSec: p.backendTimeoutSec != null ? p.backendTimeoutSec : null,
|
|
73
|
+
// Launcher-strand verdict, its consecutive streak, and the recover debounce
|
|
74
|
+
// (vm: Fleet converges onto the launcher-strand fix). The debounce MUST
|
|
75
|
+
// survive the rebuild or the self-heal would re-fire every cycle while a
|
|
76
|
+
// heavy rebuild is still settling — the unreachableStreak pattern exactly.
|
|
77
|
+
strandVerdict: p.strandVerdict || null,
|
|
78
|
+
strandStreak: p.strandStreak || 0,
|
|
79
|
+
lastStrandRecoverAt: p.lastStrandRecoverAt || 0,
|
|
80
|
+
strandRecoverInFlight: !!p.strandRecoverInFlight,
|
|
81
|
+
// "Running binary predates the launcher-strand boot guard" { stale, reason }:
|
|
82
|
+
// the proactive signal for a VM serving fine now that will strand on its
|
|
83
|
+
// next re-exec.
|
|
84
|
+
binaryGuardStale: p.binaryGuardStale || null,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// The fields one in-container facts gather lands on a VM's record — the
|
|
89
|
+
// producer side of the carry above. A failed gather yields only all-unknown
|
|
90
|
+
// button effects; a successful one yields the effects, the disk reading, the
|
|
91
|
+
// three readings the gather carries only when it reached its FACT marker (so a
|
|
92
|
+
// fallback gather keeps the last-known verdict by omitting the key), and the
|
|
93
|
+
// launcher-strand binary-guard verdict (vm: Fleet converges onto the
|
|
94
|
+
// launcher-strand fix). `editorIdentity` is the VM's advertised identity, whose
|
|
95
|
+
// `bootGuards` the guard check reads; `unknownEffects(reason)` builds the
|
|
96
|
+
// all-unknown effects map. Pure: returns the patch, writes nothing.
|
|
97
|
+
function factsGatherFields({ err, facts, editorIdentity, unknownEffects }) {
|
|
98
|
+
if (err || !facts) {
|
|
99
|
+
return { effects: unknownEffects(`effect check unavailable: ${err || 'no facts'}`) };
|
|
100
|
+
}
|
|
101
|
+
const out = {};
|
|
102
|
+
try { out.effects = fes.computeButtonEffects(facts); }
|
|
103
|
+
catch (e) { out.effects = unknownEffects(`effect check error: ${(e && e.message) || 'unknown'}`); }
|
|
104
|
+
out.disk = facts.disk || null;
|
|
105
|
+
for (const key of ['credentialIntact', 'claudeStore', 'publishedAppPort']) {
|
|
106
|
+
if (facts[key] !== undefined) out[key] = facts[key];
|
|
107
|
+
}
|
|
108
|
+
// Read the `bootGuards` the editor advertises, falling back to the editor
|
|
109
|
+
// source's behind-count when an older binary predates the field. Requires
|
|
110
|
+
// BOTH guards: a binary with serve_cwd_directly that predates
|
|
111
|
+
// capture_session_auth (VM-13) must still be flagged so its component
|
|
112
|
+
// captures don't silently 401 on the non-loopback bind.
|
|
113
|
+
try {
|
|
114
|
+
const g = fstrand.classifyBinaryGuard({
|
|
115
|
+
bootGuards: (editorIdentity && editorIdentity.bootGuards) || undefined,
|
|
116
|
+
editorBehind: (facts.editor && facts.editor.behind) || 0,
|
|
117
|
+
requiredGuard: [fstrand.SERVE_CWD_GUARD, fstrand.CAPTURE_AUTH_GUARD],
|
|
118
|
+
});
|
|
119
|
+
out.binaryGuardStale = g.stale ? g : null;
|
|
120
|
+
} catch (e) { /* best effort — never block the gather */ }
|
|
121
|
+
return out;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
module.exports = { carriedSlowGatherFields, factsGatherFields };
|
package/npm/launch-config.js
CHANGED
|
@@ -15,6 +15,15 @@ const { normalizeRepoUrl } = require("./fleet-env");
|
|
|
15
15
|
const { API_BUDGET_DEFAULTS, wouldExceedApiBudget } = require("./fleet-api-budget");
|
|
16
16
|
const { posix: posixPath } = require("node:path");
|
|
17
17
|
const { hostDiskHeadroomCheck } = require("./operator-host-disk");
|
|
18
|
+
const {
|
|
19
|
+
editorTunnelCmd,
|
|
20
|
+
previewTunnelCmd,
|
|
21
|
+
editorForwardSweepCmd,
|
|
22
|
+
previewForwardSweepCmd,
|
|
23
|
+
cloudUpSweepCmd,
|
|
24
|
+
cloudflaredTunnelSweepCmd,
|
|
25
|
+
localVmReapCmd,
|
|
26
|
+
} = require("./vm-forwards");
|
|
18
27
|
|
|
19
28
|
/** The build agents a VM's editor terminal can run, as the keys of cloud.js's
|
|
20
29
|
* AGENT_ENV_VARS. Deriving the selectable set from that map keeps it a single
|
|
@@ -736,6 +745,39 @@ function resolveCardIdentity(live, historyCfg) {
|
|
|
736
745
|
};
|
|
737
746
|
}
|
|
738
747
|
|
|
748
|
+
const ADOPTION_STATES = ["adopted", "server-not-adopted", "aliases-diverged"];
|
|
749
|
+
|
|
750
|
+
/** Resolve the card's adoption badges from the editor's live `editorAdoption`
|
|
751
|
+
* — whether the deployment `resolveCardIdentity` reports is the binary the VM
|
|
752
|
+
* is actually RUNNING. The deployment record is written at install time,
|
|
753
|
+
* adopted or not, so on its own a half-applied refresh (VM 23, 2026-10-17)
|
|
754
|
+
* read as healthy. The VM compares artifact digests and alias inodes itself;
|
|
755
|
+
* this only normalizes its answer, so index.html renders it and never
|
|
756
|
+
* re-derives it. Returns:
|
|
757
|
+
* - adoptionState "adopted" | "server-not-adopted" | "aliases-diverged", or
|
|
758
|
+
* "unknown" when the editor reports none (older editor, no
|
|
759
|
+
* deployment record) or an unrecognized value. Unlike
|
|
760
|
+
* deployedState, both non-adopted values are real faults.
|
|
761
|
+
* - aliasesDiverged / aliasesChecked the alias counts, for the hover; null
|
|
762
|
+
* when unreported.
|
|
763
|
+
* - aliasesStale whether any CLI role alias is on a previous build, set even
|
|
764
|
+
* when adoptionState names the more urgent server fault — a
|
|
765
|
+
* server that never adopted the install usually left the
|
|
766
|
+
* aliases behind too (VM 23 had both), and the card shows each.
|
|
767
|
+
* Pure; no I/O. */
|
|
768
|
+
function resolveCardAdoption(live) {
|
|
769
|
+
const lea = live && live.editorAdoption;
|
|
770
|
+
const adoptionState = lea && ADOPTION_STATES.includes(lea.state) ? lea.state : "unknown";
|
|
771
|
+
const count = (n) => (Number.isInteger(n) ? n : null);
|
|
772
|
+
const aliasesDiverged = count(lea && lea.aliasesDiverged);
|
|
773
|
+
return {
|
|
774
|
+
adoptionState,
|
|
775
|
+
aliasesDiverged,
|
|
776
|
+
aliasesChecked: count(lea && lea.aliasesChecked),
|
|
777
|
+
aliasesStale: adoptionState === "aliases-diverged" || aliasesDiverged > 0,
|
|
778
|
+
};
|
|
779
|
+
}
|
|
780
|
+
|
|
739
781
|
/** Sticky-merge a VM poll's `clientProject` against the last-known-good value
|
|
740
782
|
* so a transient null poll (timeout / partial probe) does not clobber a
|
|
741
783
|
* populated identity. The live `clientProject.branch` is the fleet
|
|
@@ -1028,18 +1070,6 @@ function claudeAuthSeeded(mode) {
|
|
|
1028
1070
|
// the server.js helper so callers don't have to thread one through.
|
|
1029
1071
|
function _shq(s) { return `'${String(s).replace(/'/g, `'\\''`)}'`; }
|
|
1030
1072
|
|
|
1031
|
-
// Assemble a `pkill -f` whose ERE is ANCHORED at the target's own executable:
|
|
1032
|
-
// `^`, an optional directory prefix, then `exe`. Every sweep builder in this
|
|
1033
|
-
// module routes through here so the anchor is structural rather than remembered
|
|
1034
|
-
// — an unanchored `pkill -f` matches the FULL command line, which for the fleet
|
|
1035
|
-
// dashboard's `sh -c '<add script>'` wrapper includes the script text itself.
|
|
1036
|
-
// See the "Self-match-safe process sweeps" block below for the full failure
|
|
1037
|
-
// mode. `argv` is an ERE fragment matched after the executable; escape any
|
|
1038
|
-
// literal dots in it.
|
|
1039
|
-
function _anchoredSweep({ exe, argv } = {}) {
|
|
1040
|
-
return `pkill -f "^([^ ]*/)?${exe} ${argv}" 2>/dev/null`;
|
|
1041
|
-
}
|
|
1042
|
-
|
|
1043
1073
|
/** The ONE definition of "a whole Claude `.credentials.json`": a positive
|
|
1044
1074
|
* plausible `expiresAt` (13+ digit millis, first digit non-zero — so
|
|
1045
1075
|
* `0`/negatives/tiny values fail), plus non-empty access + refresh tokens
|
|
@@ -1382,136 +1412,6 @@ function buildEnvApplyCommand({ gcloud, instance, zone, envFile, container, edit
|
|
|
1382
1412
|
return `base64 < ${_shq(envFile)} | ${gcloud} compute ssh ${instance} --tunnel-through-iap --zone=${zone} --command=${_shq(sink)}`;
|
|
1383
1413
|
}
|
|
1384
1414
|
|
|
1385
|
-
/** Build the `nohup … gcloud compute ssh … -N -L …` shell string that opens the
|
|
1386
|
-
* EDITOR reachability forward (`editorPort` → VM:14199) as its OWN ssh process,
|
|
1387
|
-
* logging to `/tmp/ssh-tunnel-vm-<n>.log`. The editor forward is the dashboard's
|
|
1388
|
-
* reachability signal — the probe to `PORT_BASE+N` connects through it — so it
|
|
1389
|
-
* must have a lifetime INDEPENDENT of the preview forward (whose port legitimately
|
|
1390
|
-
* changes as the app's dev server is resolved). Splitting the two forwards into
|
|
1391
|
-
* separate ssh processes is what makes a preview re-wire non-disruptive: a
|
|
1392
|
-
* preview-port change restarts only the preview tunnel and never blips :14199, so
|
|
1393
|
-
* the reachability probe can't flip the VM to a false "unreachable" (the
|
|
1394
|
-
* VM-1/2/3 flapping observed 2026-06-22). Pairs with `previewTunnelCmd`; the two
|
|
1395
|
-
* builders are the single source of truth for the attach (addVm) and re-wire call
|
|
1396
|
-
* sites so neither forward's `-L` spec can drift back into the other's process.
|
|
1397
|
-
* Pure; no I/O. */
|
|
1398
|
-
function editorTunnelCmd({ gcloud, inst, zone, editorPort, n } = {}) {
|
|
1399
|
-
return `nohup ${gcloud} compute ssh ${inst} --tunnel-through-iap --zone=${zone} -- -N -L ${editorPort}:127.0.0.1:14199 > /tmp/ssh-tunnel-vm-${n}.log 2>&1 &`;
|
|
1400
|
-
}
|
|
1401
|
-
|
|
1402
|
-
/** Build the `nohup … gcloud compute ssh … -N -L …` shell string that opens the
|
|
1403
|
-
* LIVE-PREVIEW forward (`previewPort` → VM:<appPort>) as its OWN ssh process,
|
|
1404
|
-
* logging to `/tmp/ssh-preview-vm-<n>.log`. The preview forward's target port
|
|
1405
|
-
* changes whenever the app's dev-server port is (re)resolved, so it lives apart
|
|
1406
|
-
* from the editor reachability forward (`editorTunnelCmd`): a preview re-wire
|
|
1407
|
-
* pkills + respawns ONLY this process, leaving the editor :14199 forward — and
|
|
1408
|
-
* the reachability probe riding it — untouched. The preview local port is unique
|
|
1409
|
-
* per VM and never collides with the editor port. Sweep it through
|
|
1410
|
-
* `previewForwardSweepCmd` — the bare `pkill -f "<previewPort>:127.0.0.1:"` this
|
|
1411
|
-
* comment used to call "targets exactly this process" did NOT: it also matched
|
|
1412
|
-
* every `sh -c '<add script>'` wrapper carrying that literal. Pure; no I/O. */
|
|
1413
|
-
function previewTunnelCmd({ gcloud, inst, zone, previewPort, appPort, n } = {}) {
|
|
1414
|
-
return `nohup ${gcloud} compute ssh ${inst} --tunnel-through-iap --zone=${zone} -- -N -L ${previewPort}:127.0.0.1:${appPort} > /tmp/ssh-preview-vm-${n}.log 2>&1 &`;
|
|
1415
|
-
}
|
|
1416
|
-
|
|
1417
|
-
/* ── Self-match-safe process sweeps ─────────────────────────────────────────
|
|
1418
|
-
*
|
|
1419
|
-
* Every builder below emits a `pkill -f` whose match expression is ANCHORED at
|
|
1420
|
-
* the TARGET's executable — a leading `^`, an optional directory prefix, then
|
|
1421
|
-
* the program name (`ssh`, `node`, `cloudflared`). That anchor is the entire
|
|
1422
|
-
* point of these builders, not decoration.
|
|
1423
|
-
*
|
|
1424
|
-
* `pkill -f` matches the FULL command line. The fleet dashboard runs an Add as
|
|
1425
|
-
* `/bin/sh -c '<the entire add script text>'`, and that script text EMBEDS these
|
|
1426
|
-
* very sweep lines — so an unanchored pattern like `pkill -f
|
|
1427
|
-
* "14199:127.0.0.1:14199"` is a substring of the wrapper's own argv. It SIGTERMs
|
|
1428
|
-
* the wrapper that is running it, and every concurrent SIBLING add wrapper too.
|
|
1429
|
-
* The wrapper then dies from an uncaught signal, so its EXIT trap never writes
|
|
1430
|
-
* the `add-<n>.exit` marker that `fc.readExitMarker` / `fc.reattachVerdict` exist
|
|
1431
|
-
* to consume; `startJob`'s exec callback reports a red "Add failed" on a VM that
|
|
1432
|
-
* is fine; and the detached `cloud:up` it started — whose own argv does NOT carry
|
|
1433
|
-
* the pattern — is orphaned to init and keeps provisioning, after which
|
|
1434
|
-
* stranded-reconcile declares the VM half-provisioned and stacks a SECOND
|
|
1435
|
-
* `cloud:up` on top of it. That is the 2026-07-31 VM-3..8 batch, whose containers
|
|
1436
|
-
* were Up and answering /api/health 200 throughout.
|
|
1437
|
-
*
|
|
1438
|
-
* Excluding `$$` / `$PPID` does NOT fix this: the processes being wrongly killed
|
|
1439
|
-
* are SIBLINGS, not self or parent. The pattern itself must be incapable of
|
|
1440
|
-
* matching a `sh -c '<script>'` command line, and anchoring at the executable is
|
|
1441
|
-
* what guarantees that — a wrapper's argv always begins with the shell, never
|
|
1442
|
-
* with `ssh` / `node` / `cloudflared`.
|
|
1443
|
-
*
|
|
1444
|
-
* Same trap, same lesson as `npm/cloud.js` bootstrapStatusCommand (:1446) and
|
|
1445
|
-
* projectBootstrapStatusCommand (:1760), which document it explicitly ("the
|
|
1446
|
-
* status command's own argv contains the bootstrap path, so a `pgrep -f` on it
|
|
1447
|
-
* would match this very probe") and dodge it with a PID file + `kill -0`. A PID
|
|
1448
|
-
* file cannot replace THESE sweeps: they deliberately target STRAY processes left
|
|
1449
|
-
* by a prior add, a crashed watchdog respawn, or an earlier container generation
|
|
1450
|
-
* — processes this script never owned and holds no PID for. So the sweep stays
|
|
1451
|
-
* pattern-based; only the pattern's blast radius changes.
|
|
1452
|
-
*
|
|
1453
|
-
* Keep every sweep in this family behind one of these builders. Re-inlining a
|
|
1454
|
-
* bare `pkill -f` at a call site is exactly how the bug got in the first time. */
|
|
1455
|
-
|
|
1456
|
-
/** Sweep a stray EDITOR reachability forward — the `-L <localPort>:127.0.0.1:14199`
|
|
1457
|
-
* ssh process `editorTunnelCmd` opens.
|
|
1458
|
-
*
|
|
1459
|
-
* Targets the `ssh` CHILD, not the `gcloud compute ssh` python parent. The child
|
|
1460
|
-
* is what binds the local listening socket, so killing it is what actually
|
|
1461
|
-
* releases the port; the parent is only waiting on it and exits on its own once
|
|
1462
|
-
* it goes. Only the child's argv starts with `ssh`, which is also what makes the
|
|
1463
|
-
* anchor safe. Pure; no I/O. */
|
|
1464
|
-
function editorForwardSweepCmd({ localPort } = {}) {
|
|
1465
|
-
return _anchoredSweep({ exe: "ssh", argv: `.*-L ${localPort}:127\\.0\\.0\\.1:14199` });
|
|
1466
|
-
}
|
|
1467
|
-
|
|
1468
|
-
/** Sweep a stray LIVE-PREVIEW forward — the `-L <localPort>:127.0.0.1:<appPort>`
|
|
1469
|
-
* ssh process `previewTunnelCmd` opens. The remote port is left open because the
|
|
1470
|
-
* app's dev-server port legitimately drifts; the per-VM preview local port is the
|
|
1471
|
-
* identifying half and never collides with the editor port. Same ssh-child
|
|
1472
|
-
* targeting as `editorForwardSweepCmd`. Pure; no I/O. */
|
|
1473
|
-
function previewForwardSweepCmd({ localPort } = {}) {
|
|
1474
|
-
return _anchoredSweep({ exe: "ssh", argv: `.*-L ${localPort}:127\\.0\\.0\\.1:` });
|
|
1475
|
-
}
|
|
1476
|
-
|
|
1477
|
-
/** Sweep a wedged `cloud:up` provision for one instance.
|
|
1478
|
-
*
|
|
1479
|
-
* Targets the `node npm/cloud.js up --instance-name <inst>` process, NOT the
|
|
1480
|
-
* `npm run cloud:up` parent — call sites already hold the npm child's PID and
|
|
1481
|
-
* `kill` it directly; this reaps the node grandchild that PID does not cover.
|
|
1482
|
-
* `--instance-name` is the first pair `cloudUpArgs` emits, so the literal stays
|
|
1483
|
-
* adjacent to `up`. Pure; no I/O. */
|
|
1484
|
-
function cloudUpSweepCmd({ inst } = {}) {
|
|
1485
|
-
return _anchoredSweep({ exe: "node", argv: `.*cloud\\.js up --instance-name ${inst}` });
|
|
1486
|
-
}
|
|
1487
|
-
|
|
1488
|
-
/** Sweep the cloudflared quick-tunnel fronting a VM's editor port.
|
|
1489
|
-
*
|
|
1490
|
-
* Anchored for the same reason as its siblings, with one extra bite: this sweep
|
|
1491
|
-
* runs inside a `sh -c` compound built by `exec()`, so unanchored it matched its
|
|
1492
|
-
* OWN parent shell — killing the compound before the editor/preview sweeps that
|
|
1493
|
-
* follow it on the same line ever ran. Pure; no I/O. */
|
|
1494
|
-
function cloudflaredTunnelSweepCmd({ localPort } = {}) {
|
|
1495
|
-
return _anchoredSweep({ exe: "cloudflared", argv: `tunnel --url http://localhost:${localPort}` });
|
|
1496
|
-
}
|
|
1497
|
-
|
|
1498
|
-
/** Build the destroy-time local reap compound for one VM: free the editor port,
|
|
1499
|
-
* then sweep the cloudflared quick-tunnel and BOTH split ssh forwards.
|
|
1500
|
-
*
|
|
1501
|
-
* Lives here rather than inline in `cleanLocalVmArtifacts` because the ORDERING
|
|
1502
|
-
* is load-bearing and was previously untestable: the whole compound runs as one
|
|
1503
|
-
* `sh -c`, so the unanchored cloudflared sweep used to match that shell's own
|
|
1504
|
-
* argv and kill it BEFORE the two forward sweeps behind it ever ran — meaning a
|
|
1505
|
-
* destroy silently orphaned both forwards. Pure; no I/O. */
|
|
1506
|
-
function localVmReapCmd({ port, previewPort } = {}) {
|
|
1507
|
-
return [
|
|
1508
|
-
`lsof -ti:${port} 2>/dev/null | xargs -r kill`,
|
|
1509
|
-
cloudflaredTunnelSweepCmd({ localPort: port }),
|
|
1510
|
-
editorForwardSweepCmd({ localPort: port }),
|
|
1511
|
-
previewForwardSweepCmd({ localPort: previewPort }),
|
|
1512
|
-
].join("; ");
|
|
1513
|
-
}
|
|
1514
|
-
|
|
1515
1415
|
/** Poll cadence for the Add's container gate, as `[{ untilSec, everySec }, …]`
|
|
1516
1416
|
* bands over elapsed time; the last band (no `untilSec`) runs to the deadline.
|
|
1517
1417
|
* A cold image build takes 15-25 min, so the old flat 10s cadence asked "is it
|
|
@@ -1717,6 +1617,7 @@ module.exports = {
|
|
|
1717
1617
|
latestLaunchTs,
|
|
1718
1618
|
recentlyLaunchedWithin,
|
|
1719
1619
|
resolveCardIdentity,
|
|
1620
|
+
resolveCardAdoption,
|
|
1720
1621
|
stickyClientProject,
|
|
1721
1622
|
stickyAppPort,
|
|
1722
1623
|
stickyPreviewReachable,
|