@codeyam-editor/codeyam-editor 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/npm/cli.js +5 -1
- package/npm/cloud-branch-switch.js +1 -1
- package/npm/cloud.js +3166 -150
- package/npm/codesign-plan.js +68 -0
- package/npm/container.js +3 -0
- package/npm/disk-pressure.js +78 -0
- package/npm/editor-dev.js +96 -15
- package/npm/editor.js +85 -37
- package/npm/empty-project-dir.js +92 -7
- package/npm/fleet-add-failure.js +252 -0
- package/npm/fleet-asset-version.js +37 -0
- package/npm/fleet-auth-health.js +213 -0
- package/npm/fleet-auth.js +6 -1
- package/npm/fleet-build-data.js +109 -5
- package/npm/fleet-cleanup.js +232 -16
- package/npm/fleet-destroy-safety.js +272 -0
- package/npm/fleet-early-roster.js +87 -0
- package/npm/fleet-effect-poll.js +110 -0
- package/npm/fleet-effect-status.js +33 -32
- package/npm/fleet-env.js +86 -0
- package/npm/fleet-ghcr-secret.js +129 -0
- package/npm/fleet-git-maintenance.js +142 -0
- package/npm/fleet-host-op-limiter.js +68 -0
- package/npm/fleet-host.js +74 -0
- package/npm/fleet-iap-token.js +165 -0
- package/npm/fleet-job-badge.js +1 -1
- package/npm/fleet-launchd.js +171 -0
- package/npm/fleet-live-status.js +155 -0
- package/npm/fleet-memory.js +245 -9
- package/npm/fleet-plans.js +233 -4
- package/npm/fleet-pool.js +75 -16
- package/npm/fleet-prewarm.js +241 -0
- package/npm/fleet-project-drift.js +58 -0
- package/npm/fleet-public-url.js +394 -0
- package/npm/fleet-queue.js +95 -1
- package/npm/fleet-reachability-debounce.js +69 -0
- package/npm/fleet-reachability.js +80 -0
- package/npm/fleet-reconcile.js +43 -1
- package/npm/fleet-reset.js +330 -13
- package/npm/fleet-roster.js +6 -8
- package/npm/fleet-running-plan.js +85 -0
- package/npm/fleet-self-update.js +142 -0
- package/npm/fleet-session-token-cache.js +65 -0
- package/npm/fleet-setup-steps.js +112 -0
- package/npm/fleet-sleep-status.js +33 -0
- package/npm/fleet-status-refs.js +96 -0
- package/npm/fleet-strand.js +181 -0
- package/npm/fleet-stuck-status.js +117 -0
- package/npm/fleet-vm-access.js +404 -0
- package/npm/fleet-vm-probe-target.js +282 -0
- package/npm/fleet-vm-readiness.js +237 -0
- package/npm/fleet-vm-slugs.js +116 -0
- package/npm/google-ingress.js +226 -0
- package/npm/ingress-invoke.js +770 -0
- package/npm/launch-config.js +386 -9
- package/npm/scenario-check.js +880 -37
- package/npm/scenario-handlers.js +154 -30
- package/npm/scenario-interactivity.js +145 -6
- package/npm/scenario-issues.js +13 -1
- package/npm/scenario-metrics.js +116 -1
- package/npm/scenario-mocks.js +144 -21
- package/npm/scenario-playwright.js +649 -38
- package/npm/script-drift.js +152 -0
- package/npm/utils.js +365 -52
- package/npm/vm-urls.js +69 -8
- package/package.json +5 -5
- package/ui/dist/assets/ContainerInspector-DgM0pTAx.js +2 -0
- package/ui/dist/assets/EditorShell-B7Q06uJ_.js +252 -0
- package/ui/dist/assets/EditorShell-DP5UY20G.css +1 -0
- package/ui/dist/assets/{Markdown-BHBqz-Mj.js → Markdown-Chgdf3PW.js} +1 -1
- package/ui/dist/assets/ProjectLauncherScreen-BRrEr8Ng.js +1 -0
- package/ui/dist/assets/ScenarioDataPanel-BeQfa28g.js +9 -0
- package/ui/dist/assets/{ScenarioDataPanel-DYP7pi_n.css → ScenarioDataPanel-C1LukYI6.css} +1 -1
- package/ui/dist/assets/ScenarioDataPanelFullPage-BLqRCxQX.js +1 -0
- package/ui/dist/assets/ScenarioRenderer-CIAxfHPh.js +21 -0
- package/ui/dist/assets/StepsSimulator-CymGgfbu.js +1 -0
- package/ui/dist/assets/index-BygCiJSb.css +1 -0
- package/ui/dist/assets/{index-BuK0lizt.js → index-DCHqRrwO.js} +9 -9
- package/ui/dist/assets/{markdown-v0F0UEt3.js → markdown-C9tbsVHX.js} +1 -1
- package/ui/dist/assets/react-nrLBr15I.js +28 -0
- package/ui/dist/assets/useEvents-S3Qk1Y_d.js +1 -0
- package/ui/dist/assets/useServerIdentity-B7rI0u3O.js +1 -0
- package/ui/dist/codeyam-mark.svg +4 -0
- package/ui/dist/codeyam-wordmark.svg +5 -0
- package/ui/dist/index.html +4 -4
- package/ui/dist/open-source/codeyam-counter.png +0 -0
- package/ui/dist/open-source/el-carot.png +0 -0
- package/ui/dist/open-source/tabcommand.png +0 -0
- package/ui/dist/assets/ContainerInspector-C8EkH4PP.js +0 -2
- package/ui/dist/assets/EditorShell-tMMdarZD.js +0 -187
- package/ui/dist/assets/ScenarioDataPanel-xWYGZOpo.js +0 -5
- package/ui/dist/assets/ScenarioDataPanelFullPage-5G6KuZgw.js +0 -1
- package/ui/dist/assets/ScenarioRenderer-CSdhX1Ak.js +0 -1
- package/ui/dist/assets/StepsSimulator-CRtmlyNZ.js +0 -1
- package/ui/dist/assets/index-BGqairFV.css +0 -1
- package/ui/dist/assets/react-CSS0HapR.js +0 -28
- package/ui/dist/assets/useEvents-DggWrtHe.js +0 -1
package/npm/fleet-build-data.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
+
const fr = require('./fleet-reachability');
|
|
4
|
+
|
|
3
5
|
// Assemble the per-VM wire object the dashboard's /data response carries to the
|
|
4
6
|
// browser. This is the single source of truth for a card's transport shape: the
|
|
5
7
|
// card UI (scripts/operator/fleet-dashboard/public/index.html) reads `v.<field>`
|
|
@@ -21,7 +23,37 @@ function buildVmEnvelope(v, ctx) {
|
|
|
21
23
|
return {
|
|
22
24
|
n: ctx.n,
|
|
23
25
|
url: v.url || null,
|
|
24
|
-
|
|
26
|
+
// The VM's public Live Preview URL (<slug>.preview.<domain> in named-tunnel
|
|
27
|
+
// mode, else the local forward). The card reads it as `previewSub` to render
|
|
28
|
+
// the 👁 preview link. Server-computed in server.js and merged onto the poll
|
|
29
|
+
// state — but it was dropped here, so the link was silently absent on EVERY
|
|
30
|
+
// card fleet-wide even when the VM's preview served fine. The exact
|
|
31
|
+
// currentProvider/vmAccess envelope-drift class this module guards against.
|
|
32
|
+
previewUrl: v.previewUrl || null,
|
|
33
|
+
// The app dev-server port the VM serves on (Vite 5173, Next 3000, …),
|
|
34
|
+
// resolved stickily upstream. The card reads it as `appUp` to gate the
|
|
35
|
+
// preview link live. Dropped alongside previewUrl — same silent-omit bug.
|
|
36
|
+
appPort: v.appPort != null ? v.appPort : null,
|
|
37
|
+
// Sticky preview-SERVING signal — "is the VM's dev server up?" — derived
|
|
38
|
+
// from the previewState the editor emits on /api/session-info (forwarding →
|
|
39
|
+
// true, placeholder → false, else null). Reaches every VM over the IAP-LB
|
|
40
|
+
// seam on the GCE operator host, where the old loopback `previewReachable`
|
|
41
|
+
// read false fleet-wide. The card gates the clickable 👁 link on `=== true`.
|
|
42
|
+
// Three-state, never conflating "down" with "unknown".
|
|
43
|
+
previewServing: v.previewServing === true ? true : (v.previewServing === false ? false : null),
|
|
44
|
+
// Sticky preview-INGRESS signal — "is the VM's public preview URL wired?" —
|
|
45
|
+
// from a reachability probe of <slug>.preview.<domain> over the LB, DISTINCT
|
|
46
|
+
// from previewServing. true = wired, false = confirmed dead (→ muted 'public
|
|
47
|
+
// URL not wired' hint), null = unknown (skipped / IAP audience mismatch — a
|
|
48
|
+
// null must never gate the link off). Three-state, same discipline.
|
|
49
|
+
previewIngress: v.previewIngress === true ? true : (v.previewIngress === false ? false : null),
|
|
50
|
+
// `reachable` (three-state) + `reachableSource` (which signal decided, or
|
|
51
|
+
// 'unpolled'). Same tri-state discipline as previewServing/previewIngress
|
|
52
|
+
// above, but delegated to npm/fleet-reachability.js because FIVE consumers
|
|
53
|
+
// ask the same question and the old `!!v.reachable` coercion here is what
|
|
54
|
+
// manufactured the 2026-07-30 fleet-wide phantom outage — 13 cards claiming
|
|
55
|
+
// a definite `false` that no probe had ever established.
|
|
56
|
+
...fr.normalizeReachability(v),
|
|
25
57
|
hasSession: !!v.hasSession,
|
|
26
58
|
// Three-state: true = broker confirmed up, false = confirmed down, null =
|
|
27
59
|
// editor unreachable so broker state is unknown. Never conflate "down" with
|
|
@@ -32,13 +64,38 @@ function buildVmEnvelope(v, ctx) {
|
|
|
32
64
|
// upstream, debounced). Richer than brokerLive's is-an-agent-live boolean.
|
|
33
65
|
brokerStatus: ctx.brokerStatus,
|
|
34
66
|
awaitingInput: !!v.awaitingUserInput,
|
|
67
|
+
// True when the editor reports the build workflow reached its terminal step
|
|
68
|
+
// and is parked awaiting cleanup (session_info_response.rs emits it at
|
|
69
|
+
// step >= total). Flows onto poll state via the `...info` spread in server.js
|
|
70
|
+
// exactly like `feature`/`awaitingUserInput`; defaulted false here so a legacy
|
|
71
|
+
// editor that predates the field degrades to today's behavior. Consumed by
|
|
72
|
+
// fleet-live-status.js to read a parked-but-done agent as calm 'session-done'
|
|
73
|
+
// instead of active-green 'building'.
|
|
74
|
+
needsCleanup: !!v.needsCleanup,
|
|
35
75
|
feature: v.feature || null,
|
|
76
|
+
// The VM's live-claimed plan slug, reported by /api/session-info and merged
|
|
77
|
+
// (sticky) onto the poll state. The card renders it as the what-it's-working-on
|
|
78
|
+
// line when `feature` is null — a VM that has claimed a plan but not yet named
|
|
79
|
+
// a feature (early plan/explore steps) previously showed a blank "no feature
|
|
80
|
+
// (idle)" card. Preferred over the origin claim-ref because this live source
|
|
81
|
+
// never goes stale under a failed claim-ref push. null for a legacy editor
|
|
82
|
+
// that predates the field, or a VM with no claimed plan.
|
|
83
|
+
selectedPlan: v.selectedPlan || null,
|
|
36
84
|
step: v.step != null ? v.step : null,
|
|
37
85
|
mode: v.mode || null,
|
|
38
86
|
label: v.label || null,
|
|
39
87
|
status: ctx.status,
|
|
40
88
|
orphan: ctx.orphan || null,
|
|
41
89
|
degradedReasons: ctx.degradedReasons || [],
|
|
90
|
+
// Derived presentation split of degradedReasons (vm: Dashboard 'Finishing
|
|
91
|
+
// setup' state): `setupSteps` are expected first-run wiring rendered as a
|
|
92
|
+
// calm blue checklist, `attentionReasons` are genuine problems kept in the
|
|
93
|
+
// amber ⚠ treatment. `degradedReasons` is left intact for the many places
|
|
94
|
+
// that still consume the flat list (filters, sort, reconcile); these two are
|
|
95
|
+
// purely additive. Both default to [] so a legacy/healthy VM renders neither
|
|
96
|
+
// block. See npm/fleet-setup-steps.js::classifyReasons.
|
|
97
|
+
setupSteps: Array.isArray(ctx.setupSteps) ? ctx.setupSteps : [],
|
|
98
|
+
attentionReasons: Array.isArray(ctx.attentionReasons) ? ctx.attentionReasons : [],
|
|
42
99
|
isHead: !!ctx.isHead,
|
|
43
100
|
// True when this VM holds the queue head but is unreachable (operator
|
|
44
101
|
// decides manually whether to release it).
|
|
@@ -55,12 +112,56 @@ function buildVmEnvelope(v, ctx) {
|
|
|
55
112
|
// Auth state + reason + token expiry. Only surfaced when the editor is
|
|
56
113
|
// reachable (an unreachable VM's cached auth is stale and misleading).
|
|
57
114
|
providerAuth: v.reachable && v.providerAuth ? v.providerAuth : null,
|
|
115
|
+
// The silently-broken Live Preview guard: { previewOrigin, expectedOrigin }
|
|
116
|
+
// when the VM's own /api/config proves its preview can never load in a
|
|
117
|
+
// remote browser (a not-same-origin-safe stack stuck on a loopback/subpath
|
|
118
|
+
// origin), null when healthy or unknown. Drives the card's amber "Live
|
|
119
|
+
// Preview unreachable" badge. Gated on `reachable` like providerAuth — an
|
|
120
|
+
// unreachable VM's last-known config is stale, and an unreachable card has
|
|
121
|
+
// louder problems to report than its preview origin.
|
|
122
|
+
previewMisconfigured: (v.reachable && v.previewMisconfigured) || null,
|
|
58
123
|
effects: v.effects || null,
|
|
59
124
|
// The VM's live-configured build agent (lowercase provider), reported by
|
|
60
125
|
// /api/session-info and merged (sticky) onto the poll state. Drives the
|
|
61
126
|
// card's 🤖 agent badge. null for an older VM whose editor predates the
|
|
62
127
|
// field (the card omits the badge rather than rendering a blank chip).
|
|
63
128
|
currentProvider: v.currentProvider || null,
|
|
129
|
+
// Raw workflow timestamps (ISO) surfaced by /api/session-info and merged onto
|
|
130
|
+
// the poll state via the `...info` spread, exactly like feature/step/label.
|
|
131
|
+
// The card live-ticks the elapsed minutes off these between polls. null for a
|
|
132
|
+
// legacy editor that predates the fields, or a VM with no active step.
|
|
133
|
+
startedAt: v.startedAt || null,
|
|
134
|
+
featureStartedAt: v.featureStartedAt || null,
|
|
135
|
+
// The server-computed stuck / needs-you verdict (npm/fleet-stuck-status.js):
|
|
136
|
+
// { stepElapsedMs, level: 'ok'|'slow'|'stuck', needsYou, needsYouElapsedMs }.
|
|
137
|
+
// The authoritative level only changes on poll; the card colors the step
|
|
138
|
+
// tenure badge by it. null when buildData couldn't compute one.
|
|
139
|
+
attention: ctx.attention || null,
|
|
140
|
+
// Per-VM card readiness derived upstream (npm/fleet-vm-readiness.js): one of
|
|
141
|
+
// READINESS_STATES. The card colors the border and GATES the editor link on
|
|
142
|
+
// it — a non-'ready' VM never presents its public URL as live, because the
|
|
143
|
+
// LB would 503. Computed in buildData where the broker tri-state / job / LB
|
|
144
|
+
// mode are on hand, then carried here so the browser needs no extra request.
|
|
145
|
+
readiness: ctx.readiness || null,
|
|
146
|
+
// The raw LB backend-health map { editor, preview } the readiness derivation
|
|
147
|
+
// consumed, surfaced so the card can show a precise tooltip (which surface
|
|
148
|
+
// is unwired). null in legacy mode or before the first slow probe lands.
|
|
149
|
+
backendHealth: v.backendHealth || null,
|
|
150
|
+
// The per-VM IAP access whitelist (the principals explicitly granted
|
|
151
|
+
// resource-level access to THIS VM, from the fleet-vm-access sidecar). Merged
|
|
152
|
+
// onto the poll state at server.js, it drives the card's 🔐 access line and
|
|
153
|
+
// the Manage access modal's "this VM only" list. It was previously dropped
|
|
154
|
+
// here — the exact currentProvider-style envelope drift this module guards
|
|
155
|
+
// against — so the line and list rendered empty even when the sidecar held
|
|
156
|
+
// principals. Defaults to [] for a VM with no list.
|
|
157
|
+
vmAccess: Array.isArray(v.vmAccess) ? v.vmAccess : [],
|
|
158
|
+
// Per-slot persisted-env override verdict (set by reapplyVmEnv, sticky on
|
|
159
|
+
// poll state): { present, skipped, reason, taggedRepo, setAt, keys }. Drives
|
|
160
|
+
// the card's env-override badge — an "applied" chip listing the masked keys +
|
|
161
|
+
// tagged repo, or a warning when a stale/untagged override is NOT replayed so
|
|
162
|
+
// a months-old override is never invisible again. null when no env is
|
|
163
|
+
// persisted for the slot (or the slot has never been reapplied).
|
|
164
|
+
envOverride: v.envOverride || null,
|
|
64
165
|
};
|
|
65
166
|
}
|
|
66
167
|
|
|
@@ -71,11 +172,14 @@ function buildVmEnvelope(v, ctx) {
|
|
|
71
172
|
// card renders means adding it here AND in buildVmEnvelope — the test fails loud
|
|
72
173
|
// if you do one without the other.
|
|
73
174
|
const REQUIRED_CARD_FIELDS = [
|
|
74
|
-
'n', 'url', 'reachable', 'hasSession', 'brokerLive', 'brokerStatus',
|
|
75
|
-
'awaitingInput', 'feature', 'step', 'mode', 'label', 'status', 'orphan',
|
|
76
|
-
'degradedReasons', '
|
|
175
|
+
'n', 'url', 'previewUrl', 'appPort', 'previewServing', 'previewIngress', 'reachable', 'reachableSource', 'hasSession', 'brokerLive', 'brokerStatus',
|
|
176
|
+
'awaitingInput', 'needsCleanup', 'feature', 'selectedPlan', 'step', 'mode', 'label', 'status', 'orphan',
|
|
177
|
+
'degradedReasons', 'setupSteps', 'attentionReasons',
|
|
178
|
+
'isHead', 'headHolderUnreachable', 'queuePos', 'queueLen',
|
|
77
179
|
'job', 'config', 'project', 'variant', 'cardIdentity', 'brokerHistory',
|
|
78
|
-
'brokerDeath', 'providerAuth', 'effects', 'currentProvider',
|
|
180
|
+
'brokerDeath', 'providerAuth', 'previewMisconfigured', 'effects', 'currentProvider',
|
|
181
|
+
'readiness', 'backendHealth', 'vmAccess',
|
|
182
|
+
'startedAt', 'featureStartedAt', 'attention', 'envOverride',
|
|
79
183
|
];
|
|
80
184
|
|
|
81
185
|
module.exports = { buildVmEnvelope, REQUIRED_CARD_FIELDS };
|
package/npm/fleet-cleanup.js
CHANGED
|
@@ -78,13 +78,19 @@ function classifyDescribeError(err, stderr) {
|
|
|
78
78
|
// Decide what the startup reap should do for one sidecar. Pure: takes the
|
|
79
79
|
// decoded sidecar + GCP-existence verdict + (for adds) container-up verdict +
|
|
80
80
|
// (for up adds) project-wiring verdict, returns a tag the caller dispatches on.
|
|
81
|
-
// verdict
|
|
82
|
-
// 'cleanup'
|
|
83
|
-
// 'reconcile-add'
|
|
84
|
-
// 'half-provisioned'
|
|
85
|
-
// '
|
|
86
|
-
//
|
|
87
|
-
// '
|
|
81
|
+
// verdict when dispatch
|
|
82
|
+
// 'cleanup' destroy gone, OR add gone runDestroyCleanup / runAddCleanup
|
|
83
|
+
// 'reconcile-add' add up, container up, project wired, agent usable ensure roster + recordLaunch, then clear
|
|
84
|
+
// 'half-provisioned' add up, container up, but project NOT wired surface needs-recovery card (finish-wire from cfg)
|
|
85
|
+
// 'provision-incomplete' add up + container up + wired, but agent NOT surface needs-recovery card (the agent
|
|
86
|
+
// usable (CLI missing / auth wedged at unknown) isn't installed/authed — VM5 false-green)
|
|
87
|
+
// 'recover-bootstrap' add instance up but container missing/down re-run cloud:up to finish the half-done bootstrap
|
|
88
|
+
// (runAddRecoverBootstrap first probes the editor
|
|
89
|
+
// directly and diverts to rostering when the
|
|
90
|
+
// bootstrap is actually done and only the LB
|
|
91
|
+
// ingress is missing)
|
|
92
|
+
// 'clear-only' destroy sidecar but instance still exists just clear the sidecar
|
|
93
|
+
// 'leave' gcloud uncertain (exists === null) leave for the next boot
|
|
88
94
|
// The (exists, containerUp) split distinguishes a finished add from a container-
|
|
89
95
|
// down half-provisioned one: a dashboard SIGKILL'd mid-`cloud:up` leaves a GCP
|
|
90
96
|
// instance with no docker/container, which the old (exists-only) matrix wrongly
|
|
@@ -99,17 +105,82 @@ function classifyDescribeError(err, stderr) {
|
|
|
99
105
|
// for an add whose instance exists; destroy and the instance-gone add path ignore
|
|
100
106
|
// them. Splitting the matrix out lets the table be exhaustively tested without any
|
|
101
107
|
// of the side-effects in server.js.
|
|
102
|
-
|
|
108
|
+
//
|
|
109
|
+
// The `agentReady` axis (vm: Readiness Verifies Real Usability) closes the VM5
|
|
110
|
+
// false-green: an instance + container that ARE up and wired, but whose build
|
|
111
|
+
// agent isn't usable — the CLI isn't installed, or the auth probe is wedged at
|
|
112
|
+
// `unknown` — is NOT a finished add. Mirrors `projectWired`'s posture EXACTLY:
|
|
113
|
+
// only a POSITIVE not-ready signal (`agentReady === false`) downgrades, to
|
|
114
|
+
// 'provision-incomplete'; `null`/`undefined` (the probe couldn't decide, or an
|
|
115
|
+
// older editor doesn't report it) stays 'reconcile-add' so a flaky probe never
|
|
116
|
+
// strands a healthy VM. Checked AFTER `projectWired` — an unwired box is the more
|
|
117
|
+
// fundamental fault and keeps its own verdict.
|
|
118
|
+
//
|
|
119
|
+
// NOT an axis here (vm--batch-add-must-roster-and-wire-each-vm): "the editor
|
|
120
|
+
// answers on its own port but has no LB ingress". That state is a finished
|
|
121
|
+
// bootstrap missing its wiring, and re-running cloud:up cannot create the
|
|
122
|
+
// missing ingress — so it used to burn the 3-attempt budget and land on
|
|
123
|
+
// 'bootstrap-recovery-exhausted', because the signal being retried on
|
|
124
|
+
// (`lbReachable`) can never pass while the ingress is absent:
|
|
125
|
+
//
|
|
126
|
+
// no ingress → LB 503 → not reachable → recover → still no ingress → exhausted
|
|
127
|
+
//
|
|
128
|
+
// The rescue deliberately lives in server.js's runAddRecoverBootstrap rather
|
|
129
|
+
// than as a verdict here, because it needs a live IAP probe and is only worth
|
|
130
|
+
// paying for on the branch that was about to spend a recovery attempt. Adding it
|
|
131
|
+
// as a `reapVerdict` axis would force every container-down reap to run that
|
|
132
|
+
// probe eagerly just to compute a verdict, for a state that is rare.
|
|
133
|
+
function reapVerdict({ action, exists, containerUp, projectWired, agentReady, lbReachable }) {
|
|
103
134
|
if (exists === null) return 'leave';
|
|
104
135
|
if (action === 'destroy') return exists ? 'clear-only' : 'cleanup';
|
|
105
136
|
if (action === 'add') {
|
|
106
137
|
if (!exists) return 'cleanup';
|
|
138
|
+
// The `lbReachable` axis (vm--gce-reachable-flag-from-lb-probe) closes the
|
|
139
|
+
// false-orphan: a VM whose editor ANSWERS over the LB is provisioned by
|
|
140
|
+
// definition — its add finished, whatever a stale `add-<n>.json` sidecar
|
|
141
|
+
// left lying around implies. Without this, the container probe below could
|
|
142
|
+
// report "container down" for a VM that is demonstrably up and serving,
|
|
143
|
+
// driving recover-bootstrap → "exceeded 3 attempts" → a
|
|
144
|
+
// needs-manual-recovery card for a perfectly healthy box.
|
|
145
|
+
// Inverted posture vs projectWired/agentReady, and deliberately so: those
|
|
146
|
+
// axes let only a NEGATIVE signal downgrade, this one lets only a POSITIVE
|
|
147
|
+
// signal promote. `false`/`null`/`undefined` (probe said no, couldn't
|
|
148
|
+
// decide, or never ran) all fall through to the unchanged ladder below, so
|
|
149
|
+
// this can never strand a VM — it can only rescue one.
|
|
150
|
+
if (lbReachable === true) return 'reconcile-add';
|
|
107
151
|
if (!containerUp) return 'recover-bootstrap';
|
|
108
|
-
|
|
152
|
+
if (projectWired === false) return 'half-provisioned';
|
|
153
|
+
if (agentReady === false) return 'provision-incomplete';
|
|
154
|
+
return 'reconcile-add';
|
|
109
155
|
}
|
|
110
156
|
return 'leave';
|
|
111
157
|
}
|
|
112
158
|
|
|
159
|
+
// Classify the startup reap's LB rescue signal from a best-effort LB+IAP editor
|
|
160
|
+
// probe. Pure: server.js runs the HTTP probe and hands the raw outcome here, the
|
|
161
|
+
// same split classifyProjectWiring follows, so the rule is unit-testable without
|
|
162
|
+
// a live VM or an LB.
|
|
163
|
+
//
|
|
164
|
+
// Deliberately tri-state and ASYMMETRIC — it returns `true` or `null`, NEVER
|
|
165
|
+
// `false`:
|
|
166
|
+
// - probeViaLb false (a laptop has no LB to ask) → null (no opinion)
|
|
167
|
+
// - err present (threw, auth fault, timeout) → null (no opinion)
|
|
168
|
+
// - the editor answered → true (add is finished)
|
|
169
|
+
// - the editor did not answer → null (NOT false)
|
|
170
|
+
//
|
|
171
|
+
// That last line is the load-bearing one. A silent probe is not evidence a VM is
|
|
172
|
+
// dead — the LB may be unwired mid-add, or the IAP token not yet minted. Since
|
|
173
|
+
// reapVerdict promotes on `true` and ignores everything else, returning null
|
|
174
|
+
// leaves the pre-existing container/wiring ladder in charge. Returning `false`
|
|
175
|
+
// here would be indistinguishable to reapVerdict, but it would be a lie about
|
|
176
|
+
// what the probe proved, and the next person to add an axis would read it as
|
|
177
|
+
// evidence. The invariant this pins: this signal can rescue a VM, never strand one.
|
|
178
|
+
function classifyLbReachableForReap({ probeViaLb, reachable, err } = {}) {
|
|
179
|
+
if (!probeViaLb) return null;
|
|
180
|
+
if (err) return null;
|
|
181
|
+
return reachable ? true : null;
|
|
182
|
+
}
|
|
183
|
+
|
|
113
184
|
// Classify whether a VM's editor `/workspace` is actually wired to a client
|
|
114
185
|
// project, from a best-effort probe of two signals: the git remote URL and
|
|
115
186
|
// whether `package.json` carries a real `name`. Pure: server.js runs the SSH +
|
|
@@ -175,13 +246,28 @@ function clearExitMarker(dir, n) {
|
|
|
175
246
|
catch (_) { /* missing is fine */ }
|
|
176
247
|
}
|
|
177
248
|
|
|
249
|
+
// True when an integer exit code is the shell's "killed by signal N" encoding,
|
|
250
|
+
// i.e. 128 + N for N in 1..37 — the realistic kills being 143 (SIGTERM, the
|
|
251
|
+
// `launchctl bootout` graceful teardown), 137 (SIGKILL), 130 (SIGINT) and 129
|
|
252
|
+
// (SIGHUP). Pure boundary check: 128 itself is NOT a signal kill (it's `exit
|
|
253
|
+
// 128`, an ordinary failure code), so the range is the OPEN interval
|
|
254
|
+
// (128, 165]. Anything outside it — a small `exit 1` from the addVm script's own
|
|
255
|
+
// failure branches, 0, NaN, null — is not a signal kill. The split that matters:
|
|
256
|
+
// "did a signal kill the provision shell" (interrupted → reconcile) vs "did the
|
|
257
|
+
// provision logic itself decide it failed" (real error). Exported so the matrix
|
|
258
|
+
// is unit-testable directly.
|
|
259
|
+
function isSignalKillExit(code) {
|
|
260
|
+
return Number.isInteger(code) && code > 128 && code <= 165;
|
|
261
|
+
}
|
|
262
|
+
|
|
178
263
|
// Decide how a snapshot-restored `add` should be re-attached after a dashboard
|
|
179
264
|
// restart, BEFORE any gcloud probe. Pure: takes the durable signals the boot
|
|
180
265
|
// path gathers — whether the exit marker exists, its recorded code, and whether
|
|
181
266
|
// the detached provision child is still alive (recorded pid, `kill -0`).
|
|
182
267
|
// verdict when dispatch
|
|
183
268
|
// 'done' marker present, exit code 0 roster + recordLaunch, clear sidecar/marker
|
|
184
|
-
// '
|
|
269
|
+
// 'reconcile' marker present, SIGNAL-kill code (128<c≤165) interrupted, not failed — gcloud probe decides
|
|
270
|
+
// 'error' marker present, other non-zero exit code surface the real failure, clear sidecar/marker
|
|
185
271
|
// 'running' no marker yet, child still alive RE-ATTACH: keep polling, do NOT mark error
|
|
186
272
|
// 'reconcile' child gone, no usable marker fall through to fc.reapVerdict (gcloud probe)
|
|
187
273
|
// This is the durable fix's brain: a provision that survived the kill (still
|
|
@@ -192,14 +278,33 @@ function clearExitMarker(dir, n) {
|
|
|
192
278
|
// `reconcile` hands that exact (exists, containerUp, projectWired) decision to
|
|
193
279
|
// fc.reapVerdict. A marker whose first line isn't an integer is treated as
|
|
194
280
|
// absent (parseExitMarker → null) so a corrupt marker reconciles rather than
|
|
195
|
-
// asserting a bogus code.
|
|
281
|
+
// asserting a bogus code.
|
|
282
|
+
//
|
|
283
|
+
// SIGNAL-kill codes (improve46): `setsid` shields the detached provision from a
|
|
284
|
+
// dashboard-process-GROUP SIGKILL (the 2GB launchd hard cap), but NOT from a
|
|
285
|
+
// `launchctl bootout`, which tears down the whole launchd job tree regardless of
|
|
286
|
+
// session/process-group. A graceful SIGTERM then reaches the "detached" child and
|
|
287
|
+
// its EXIT trap records 143 (128+SIGTERM) into the marker. That is an INTERRUPTED
|
|
288
|
+
// add, not a failed one — the manual re-add succeeds in seconds off the image the
|
|
289
|
+
// interrupted build already left on the VM. So a signal-kill code routes to
|
|
290
|
+
// 'reconcile' (let the gcloud probe read ground truth and roster / recover) rather
|
|
291
|
+
// than 'error' (a false add-failed that throws away a resumable build). A genuine
|
|
292
|
+
// provision failure exits with a small non-zero code (e.g. `exit 1`), which STILL
|
|
293
|
+
// classifies 'error'. The matrix is exhaustively tested without a live VM.
|
|
196
294
|
function reattachVerdict({ markerPresent, exitCode, childAlive } = {}) {
|
|
197
295
|
if (markerPresent) {
|
|
198
296
|
// Guard the null/undefined coercion BEFORE Number() — `Number(null)` is 0,
|
|
199
297
|
// which would masquerade as a clean exit. A marker with no parseable code
|
|
200
298
|
// (half-written / corrupt) must reconcile, not assert success.
|
|
201
299
|
const code = exitCode == null ? NaN : Number(exitCode);
|
|
202
|
-
if (Number.isInteger(code))
|
|
300
|
+
if (Number.isInteger(code)) {
|
|
301
|
+
if (code === 0) return 'done';
|
|
302
|
+
// A signal killed the provision shell (e.g. bootout SIGTERM → 143): the add
|
|
303
|
+
// was interrupted, not failed — reconcile against real VM state instead of
|
|
304
|
+
// surfacing a false add-failed and discarding a resumable build.
|
|
305
|
+
if (isSignalKillExit(code)) return 'reconcile';
|
|
306
|
+
return 'error';
|
|
307
|
+
}
|
|
203
308
|
return 'reconcile'; // marker present but no parseable code — let gcloud decide
|
|
204
309
|
}
|
|
205
310
|
if (childAlive) return 'running';
|
|
@@ -297,11 +402,119 @@ function orphanedAddsToReconcile({ jobs, sidecarKeys } = {}) {
|
|
|
297
402
|
// posture). "Busy" (a live session or an in-flight add to this number) also
|
|
298
403
|
// holds the entry for an operator rather than auto-removing — the shouldn't-
|
|
299
404
|
// happen-but-fail-safe case where a vanished instance still reports a session.
|
|
300
|
-
|
|
405
|
+
// verdict when dispatch
|
|
406
|
+
// 'cleanup' instance gone AND the VM is idle runDestroyCleanup
|
|
407
|
+
// 'leave' exists uncertain (null), OR the VM is busy do nothing this pass
|
|
408
|
+
// 'keep' instance still exists do nothing (healthy)
|
|
409
|
+
// 'reset' existing idle instance unreachable too long trigger auto-reset
|
|
410
|
+
// Fail-safe by construction: cleanup fires ONLY on a DEFINITIVE absence
|
|
411
|
+
// (exists === false) — a flaky/transient describe (null) is always 'leave', so a
|
|
412
|
+
// gcloud blip can never evict a healthy VM (mirroring reapVerdict's `leave`
|
|
413
|
+
// posture). "Busy" (a live session or an in-flight add to this number) also
|
|
414
|
+
// holds the entry for an operator rather than auto-removing — the shouldn't-
|
|
415
|
+
// happen-but-fail-safe case where a vanished instance still reports a session.
|
|
416
|
+
// auto-reset escalation (vm: OOM Wedge Prevention):
|
|
417
|
+
// When the VM has been unreachable for at least RESET_UNREACHABLE_THRESHOLD
|
|
418
|
+
// (e.g. 200, which is roughly 10 minutes at a 3s poll rate), we issue a 'reset'
|
|
419
|
+
// verdict. This auto-reset is guarded and bounded by a cooldown window (in ms)
|
|
420
|
+
// and a max attempts limit to prevent infinite reboot loops.
|
|
421
|
+
//
|
|
422
|
+
// false-unreachable reboot safety (2026-06-25): "unreachable" as the dashboard
|
|
423
|
+
// sees it means the LOCALLY-FORWARDED port is dark — which is ALSO the signature
|
|
424
|
+
// of an operator-side fault (the laptop's gcloud auth expired, the ssh -L tunnel
|
|
425
|
+
// zombied, a laptop network blip) where the VM itself is perfectly healthy.
|
|
426
|
+
// Rebooting a healthy production VM off that signal interrupts a live session.
|
|
427
|
+
// So three operator-fault guards are layered BEFORE the cooldown/attempts budget
|
|
428
|
+
// (a reset is a last resort, never the response to a dark local port). Each is
|
|
429
|
+
// OPT-IN: it only changes the verdict when the caller supplies the fact, so
|
|
430
|
+
// legacy callers (and the OOM-wedge tests) that pass none keep the old behavior.
|
|
431
|
+
// operatorAuthDegraded our own gcloud auth is broken → EVERY VM reads
|
|
432
|
+
// unreachable; rebooting them all is exactly wrong → keep
|
|
433
|
+
// fleetWideUnreachable a simultaneous N-VM "outage" is the operator (auth/
|
|
434
|
+
// tunnel/network), not N dead VMs → keep
|
|
435
|
+
// gcpReachable GCP-side confirmation: true ⇒ the VM answers from
|
|
436
|
+
// GCP's vantage, so the dark local port is a laptop
|
|
437
|
+
// tunnel fault → 'respawn' (never reset); null ⇒ the
|
|
438
|
+
// GCP-side probe was inconclusive → keep (don't escalate
|
|
439
|
+
// a destructive reset on absence of evidence); false ⇒
|
|
440
|
+
// genuinely unreachable from GCP too → fall through to
|
|
441
|
+
// the cooldown/attempts budget and reset as before.
|
|
442
|
+
function reconcileRosteredVm({ exists, hasSession, hasPendingAdd, streak, resetHistory, threshold, cooldownMs, maxAttempts, operatorAuthDegraded, fleetWideUnreachable, gcpReachable } = {}) {
|
|
301
443
|
if (exists === null || exists === undefined) return "leave";
|
|
302
|
-
if (exists)
|
|
303
|
-
|
|
304
|
-
|
|
444
|
+
if (!exists) {
|
|
445
|
+
if (hasSession || hasPendingAdd) return "leave";
|
|
446
|
+
return "cleanup";
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
// If the VM exists, check if it's dead-unreachable and eligible for auto-reset.
|
|
450
|
+
if (streak !== undefined && threshold !== undefined) {
|
|
451
|
+
if (streak >= threshold) {
|
|
452
|
+
if (hasSession || hasPendingAdd) {
|
|
453
|
+
// Do NOT auto-reset a VM that is busy (holding a live session or an in-flight add)
|
|
454
|
+
return "keep";
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// Operator-fault guards — layered before the cooldown/attempts budget so a
|
|
458
|
+
// laptop-side fault never even consumes a reset attempt.
|
|
459
|
+
if (operatorAuthDegraded) return "keep";
|
|
460
|
+
if (fleetWideUnreachable) return "keep";
|
|
461
|
+
if (gcpReachable === true) return "respawn";
|
|
462
|
+
if (gcpReachable === null) return "keep";
|
|
463
|
+
|
|
464
|
+
const history = resetHistory || { count: 0, lastResetAt: 0 };
|
|
465
|
+
const attempts = history.count || 0;
|
|
466
|
+
const limit = maxAttempts !== undefined ? maxAttempts : 3;
|
|
467
|
+
|
|
468
|
+
if (attempts >= limit) {
|
|
469
|
+
// Budget exhausted (VM wedges repeatedly) -> park (needs-manual-recovery status)
|
|
470
|
+
return "keep";
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
const lastReset = history.lastResetAt || 0;
|
|
474
|
+
const cooldown = cooldownMs !== undefined ? cooldownMs : 600000; // default 10 minutes
|
|
475
|
+
const now = Date.now();
|
|
476
|
+
if (now - lastReset < cooldown) {
|
|
477
|
+
// Within cooldown window -> do not reset again yet
|
|
478
|
+
return "keep";
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
return "reset";
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
return "keep";
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
// The bootstrap-recovery ladder step the startup reap runs for a VM whose first
|
|
489
|
+
// cloud:up did not finish: decide (via recoverBootstrapDecision) whether another
|
|
490
|
+
// attempt is allowed, then either flag the VM exhausted (leave the sidecar for
|
|
491
|
+
// manual action) or clear any stale orphan flag and re-run cloud:up via `addVm`.
|
|
492
|
+
// The dashboard's `state` and `addVm` are injected so this stays a pure,
|
|
493
|
+
// importable unit — `log`/`err` default to the console but are overridable in
|
|
494
|
+
// tests. Behavior mirrors the previous inline server.js definition exactly.
|
|
495
|
+
function continueBootstrapRecovery(
|
|
496
|
+
n,
|
|
497
|
+
meta,
|
|
498
|
+
cfg,
|
|
499
|
+
{ state, addVm, log = console.log, err = console.error } = {},
|
|
500
|
+
) {
|
|
501
|
+
const { attempt, proceed, max } = recoverBootstrapDecision({
|
|
502
|
+
recoverAttempts: meta.recoverAttempts,
|
|
503
|
+
});
|
|
504
|
+
if (!proceed) {
|
|
505
|
+
err(
|
|
506
|
+
`startup-reap: VM-${n} exceeded ${max} bootstrap-recovery attempts — leaving sidecar for manual action`,
|
|
507
|
+
);
|
|
508
|
+
state.orphans[n] = { reason: "bootstrap-recovery-exhausted", attempts: attempt - 1 };
|
|
509
|
+
return;
|
|
510
|
+
}
|
|
511
|
+
log(
|
|
512
|
+
`startup-reap: VM-${n} re-running cloud:up to finish bootstrap (recovery attempt ${attempt}/${max})`,
|
|
513
|
+
);
|
|
514
|
+
// A prior exhausted/no-cfg run may have flagged this VM; clear it now that a
|
|
515
|
+
// fresh attempt is starting so the card moves to `recovering`, not stale red.
|
|
516
|
+
delete state.orphans[n];
|
|
517
|
+
addVm(n, cfg, { recovering: true, recoverAttempt: attempt });
|
|
305
518
|
}
|
|
306
519
|
|
|
307
520
|
module.exports = {
|
|
@@ -314,11 +527,14 @@ module.exports = {
|
|
|
314
527
|
reconcileRosteredVm,
|
|
315
528
|
orphanedAddsToReconcile,
|
|
316
529
|
classifyProjectWiring,
|
|
530
|
+
classifyLbReachableForReap,
|
|
317
531
|
recoverBootstrapDecision,
|
|
532
|
+
continueBootstrapRecovery,
|
|
318
533
|
exitMarkerName,
|
|
319
534
|
parseExitMarker,
|
|
320
535
|
readExitMarker,
|
|
321
536
|
clearExitMarker,
|
|
537
|
+
isSignalKillExit,
|
|
322
538
|
reattachVerdict,
|
|
323
539
|
reattachNextStep,
|
|
324
540
|
};
|