@codeyam-editor/codeyam-editor 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/npm/cli.js +5 -1
- package/npm/cloud-branch-switch.js +1 -1
- package/npm/cloud.js +3166 -150
- package/npm/codesign-plan.js +68 -0
- package/npm/container.js +3 -0
- package/npm/disk-pressure.js +78 -0
- package/npm/editor-dev.js +96 -15
- package/npm/editor.js +85 -37
- package/npm/empty-project-dir.js +92 -7
- package/npm/fleet-add-failure.js +252 -0
- package/npm/fleet-asset-version.js +37 -0
- package/npm/fleet-auth-health.js +213 -0
- package/npm/fleet-auth.js +6 -1
- package/npm/fleet-build-data.js +109 -5
- package/npm/fleet-cleanup.js +232 -16
- package/npm/fleet-destroy-safety.js +272 -0
- package/npm/fleet-early-roster.js +87 -0
- package/npm/fleet-effect-poll.js +110 -0
- package/npm/fleet-effect-status.js +33 -32
- package/npm/fleet-env.js +86 -0
- package/npm/fleet-ghcr-secret.js +129 -0
- package/npm/fleet-git-maintenance.js +142 -0
- package/npm/fleet-host-op-limiter.js +68 -0
- package/npm/fleet-host.js +74 -0
- package/npm/fleet-iap-token.js +165 -0
- package/npm/fleet-job-badge.js +1 -1
- package/npm/fleet-launchd.js +171 -0
- package/npm/fleet-live-status.js +155 -0
- package/npm/fleet-memory.js +245 -9
- package/npm/fleet-plans.js +233 -4
- package/npm/fleet-pool.js +75 -16
- package/npm/fleet-prewarm.js +241 -0
- package/npm/fleet-project-drift.js +58 -0
- package/npm/fleet-public-url.js +394 -0
- package/npm/fleet-queue.js +95 -1
- package/npm/fleet-reachability-debounce.js +69 -0
- package/npm/fleet-reachability.js +80 -0
- package/npm/fleet-reconcile.js +43 -1
- package/npm/fleet-reset.js +330 -13
- package/npm/fleet-roster.js +6 -8
- package/npm/fleet-running-plan.js +85 -0
- package/npm/fleet-self-update.js +142 -0
- package/npm/fleet-session-token-cache.js +65 -0
- package/npm/fleet-setup-steps.js +112 -0
- package/npm/fleet-sleep-status.js +33 -0
- package/npm/fleet-status-refs.js +96 -0
- package/npm/fleet-strand.js +181 -0
- package/npm/fleet-stuck-status.js +117 -0
- package/npm/fleet-vm-access.js +404 -0
- package/npm/fleet-vm-probe-target.js +282 -0
- package/npm/fleet-vm-readiness.js +237 -0
- package/npm/fleet-vm-slugs.js +116 -0
- package/npm/google-ingress.js +226 -0
- package/npm/ingress-invoke.js +770 -0
- package/npm/launch-config.js +386 -9
- package/npm/scenario-check.js +880 -37
- package/npm/scenario-handlers.js +154 -30
- package/npm/scenario-interactivity.js +145 -6
- package/npm/scenario-issues.js +13 -1
- package/npm/scenario-metrics.js +116 -1
- package/npm/scenario-mocks.js +144 -21
- package/npm/scenario-playwright.js +649 -38
- package/npm/script-drift.js +152 -0
- package/npm/utils.js +365 -52
- package/npm/vm-urls.js +69 -8
- package/package.json +5 -5
- package/ui/dist/assets/ContainerInspector-DgM0pTAx.js +2 -0
- package/ui/dist/assets/EditorShell-B7Q06uJ_.js +252 -0
- package/ui/dist/assets/EditorShell-DP5UY20G.css +1 -0
- package/ui/dist/assets/{Markdown-BHBqz-Mj.js → Markdown-Chgdf3PW.js} +1 -1
- package/ui/dist/assets/ProjectLauncherScreen-BRrEr8Ng.js +1 -0
- package/ui/dist/assets/ScenarioDataPanel-BeQfa28g.js +9 -0
- package/ui/dist/assets/{ScenarioDataPanel-DYP7pi_n.css → ScenarioDataPanel-C1LukYI6.css} +1 -1
- package/ui/dist/assets/ScenarioDataPanelFullPage-BLqRCxQX.js +1 -0
- package/ui/dist/assets/ScenarioRenderer-CIAxfHPh.js +21 -0
- package/ui/dist/assets/StepsSimulator-CymGgfbu.js +1 -0
- package/ui/dist/assets/index-BygCiJSb.css +1 -0
- package/ui/dist/assets/{index-BuK0lizt.js → index-DCHqRrwO.js} +9 -9
- package/ui/dist/assets/{markdown-v0F0UEt3.js → markdown-C9tbsVHX.js} +1 -1
- package/ui/dist/assets/react-nrLBr15I.js +28 -0
- package/ui/dist/assets/useEvents-S3Qk1Y_d.js +1 -0
- package/ui/dist/assets/useServerIdentity-B7rI0u3O.js +1 -0
- package/ui/dist/codeyam-mark.svg +4 -0
- package/ui/dist/codeyam-wordmark.svg +5 -0
- package/ui/dist/index.html +4 -4
- package/ui/dist/open-source/codeyam-counter.png +0 -0
- package/ui/dist/open-source/el-carot.png +0 -0
- package/ui/dist/open-source/tabcommand.png +0 -0
- package/ui/dist/assets/ContainerInspector-C8EkH4PP.js +0 -2
- package/ui/dist/assets/EditorShell-tMMdarZD.js +0 -187
- package/ui/dist/assets/ScenarioDataPanel-xWYGZOpo.js +0 -5
- package/ui/dist/assets/ScenarioDataPanelFullPage-5G6KuZgw.js +0 -1
- package/ui/dist/assets/ScenarioRenderer-CSdhX1Ak.js +0 -1
- package/ui/dist/assets/StepsSimulator-CRtmlyNZ.js +0 -1
- package/ui/dist/assets/index-BGqairFV.css +0 -1
- package/ui/dist/assets/react-CSS0HapR.js +0 -28
- package/ui/dist/assets/useEvents-DggWrtHe.js +0 -1
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// Pure decision + formatting helpers for the fleet dashboard's destroy/deroster
|
|
3
|
+
// false-unreachable safety (2026-06-26 incident, where a reauth-triggered
|
|
4
|
+
// false-unreachable cascade permanently deleted VM2/3/4). The dashboard's
|
|
5
|
+
// server.js gathers the live operator-fault facts (state.operatorAuthDegraded +
|
|
6
|
+
// fleetWideUnreachableNow) and runs the side effects (spawn the delete, log,
|
|
7
|
+
// stop+deroster); the DECISIONS those facts drive — "is this destroy blocked,
|
|
8
|
+
// and why" and "what does the attribution log line read" — live here so they
|
|
9
|
+
// are unit-testable in isolation, the same pure/IO split npm/fleet-auth-health.js
|
|
10
|
+
// (classifyOperatorAuth) and npm/fleet-cleanup.js (reconcileRosteredVm) follow.
|
|
11
|
+
//
|
|
12
|
+
// Pure logic only — no I/O, no env reads, no clock, no global state.
|
|
13
|
+
|
|
14
|
+
// Concurrent-gcloud count above which a pile-up is itself evidence that the
|
|
15
|
+
// operator plane is resource-throttled rather than credential-broken. Measured
|
|
16
|
+
// 2026-07-29: 16 concurrent gcloud processes at load 8.49 with the cgroup near
|
|
17
|
+
// MemoryHigh, against a steady state of 0-2. Set well above steady state so a
|
|
18
|
+
// couple of ordinary in-flight polls never read as a pile-up.
|
|
19
|
+
const GCLOUD_PILEUP_THRESHOLD = 8;
|
|
20
|
+
|
|
21
|
+
// Fraction of the cgroup's memory.high at which memory counts as throttling
|
|
22
|
+
// pressure. MUST stay below the 0.874 that actually occurred on 2026-07-29, or
|
|
23
|
+
// the incident goes undetected and this whole change is inert — there is a
|
|
24
|
+
// regression test pinning exactly that.
|
|
25
|
+
//
|
|
26
|
+
// Deliberately NOT fleet-memory.decideMemoryDrain, which looks superficially like
|
|
27
|
+
// the same decision and is the wrong signal twice over:
|
|
28
|
+
// 1. WRONG QUANTITY. It reads process.memoryUsage().rss — the node process
|
|
29
|
+
// alone. memory.current is the whole CGROUP, including the 16 gcloud
|
|
30
|
+
// subprocesses. Under a fork storm the pressure lives precisely in what RSS
|
|
31
|
+
// excludes, so RSS systematically under-reads this condition.
|
|
32
|
+
// 2. WRONG THRESHOLD. Its soft cap is 1500 MB = 97.7% of the observed
|
|
33
|
+
// memory.high, but the incident sat at 87.4% — it returns false at the
|
|
34
|
+
// measured value.
|
|
35
|
+
// The two coexist because they answer different questions: decideMemoryDrain
|
|
36
|
+
// asks "should I admit another heavy job?", this asks "is the plane too starved
|
|
37
|
+
// to trust its own probes?".
|
|
38
|
+
const MEMORY_PRESSURE_RATIO = 0.85;
|
|
39
|
+
|
|
40
|
+
// The recovery for a resource-throttled operator plane. Named once here so the
|
|
41
|
+
// refusal message, the log line, and the tests can never drift apart.
|
|
42
|
+
const THROTTLE_NEXT_ACTION = 'systemctl restart codeyam-dashboard';
|
|
43
|
+
|
|
44
|
+
// Classify whether the operator plane is RESOURCE-THROTTLED (starved) rather
|
|
45
|
+
// than credential-broken, from the cheap local pressure signals server.js reads.
|
|
46
|
+
// Neither input spawns a subprocess — which matters, because spawning one is
|
|
47
|
+
// exactly what fails when this condition holds.
|
|
48
|
+
// memoryCurrent the cgroup's memory.current, in bytes
|
|
49
|
+
// memoryHigh the cgroup's memory.high, in bytes ('max' when the cgroup
|
|
50
|
+
// is unlimited, which yields no ratio and so no signal)
|
|
51
|
+
// gcloudProcessCount concurrent gcloud processes on the host
|
|
52
|
+
// Returns { throttled, signals }: `signals` names the evidence, so the refusal
|
|
53
|
+
// message can cite WHY it concluded throttling instead of asserting it. An absent
|
|
54
|
+
// or unreadable signal contributes nothing rather than defaulting to pressure —
|
|
55
|
+
// fabricating pressure would relabel a REAL auth break as a memory problem, which
|
|
56
|
+
// is the same class of misdiagnosis in the opposite direction.
|
|
57
|
+
function classifyOperatorThrottle({ memoryCurrent, memoryHigh, gcloudProcessCount } = {}) {
|
|
58
|
+
const signals = [];
|
|
59
|
+
const cur = Number(memoryCurrent);
|
|
60
|
+
const high = Number(memoryHigh);
|
|
61
|
+
if (Number.isFinite(cur) && Number.isFinite(high) && high > 0) {
|
|
62
|
+
const ratio = cur / high;
|
|
63
|
+
if (ratio >= MEMORY_PRESSURE_RATIO) {
|
|
64
|
+
signals.push(`${Math.round(ratio * 100)}% of the cgroup memory throttle`);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
const procs = Number(gcloudProcessCount);
|
|
68
|
+
if (Number.isFinite(procs) && procs >= GCLOUD_PILEUP_THRESHOLD) {
|
|
69
|
+
signals.push(`${procs} concurrent gcloud processes`);
|
|
70
|
+
}
|
|
71
|
+
return { throttled: signals.length > 0, signals };
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Classify whether a destructive removal (hard destroy or --shrink) must be
|
|
75
|
+
// REFUSED because the operator's own side is faulted, from the facts server.js
|
|
76
|
+
// computes. Returns { blocked, kind, reason, nextAction }: blocked is true when
|
|
77
|
+
// the operator's plane is faulted (auth degraded) OR a fleet-wide unreachable
|
|
78
|
+
// spike is in progress — the signature of a host-side fault (which makes EVERY VM
|
|
79
|
+
// read unreachable), NOT N genuinely-dead VMs.
|
|
80
|
+
//
|
|
81
|
+
// The `kind` split exists because a throttled plane and a broken credential have
|
|
82
|
+
// OPPOSITE recoveries, and reporting the wrong one is the 2026-07-29 defect: the
|
|
83
|
+
// banner sent the operator to fix an SA binding that was perfectly healthy, or to
|
|
84
|
+
// reach for override-safety — punching through the one guard that exists because
|
|
85
|
+
// a false-unreachable cascade permanently deleted VM2/3/4 on 2026-06-26. The
|
|
86
|
+
// correct action, restarting the dashboard, was named nowhere.
|
|
87
|
+
//
|
|
88
|
+
// A throttled verdict requires BOTH halves of the evidence: the auth probe
|
|
89
|
+
// produced no answer (authProbeFailed — a failed invocation, not a failed
|
|
90
|
+
// credential) AND local pressure corroborates it. Empty probe output on its own
|
|
91
|
+
// stays inconclusive and keeps the auth reason, so a transient hiccup is never
|
|
92
|
+
// relabeled a memory problem; a genuine auth ERROR is never reclassified at all,
|
|
93
|
+
// no matter how loaded the host is.
|
|
94
|
+
//
|
|
95
|
+
// What this does NOT do is let anything through. Every degraded classification
|
|
96
|
+
// returns blocked: true — the refusal was correct, only its diagnosis was wrong.
|
|
97
|
+
// authDegraded state.operatorAuthDegraded — the operator plane is faulted
|
|
98
|
+
// authProbeFailed the auth verdict's kind was 'probe-failed' (empty output,
|
|
99
|
+
// no error message) rather than 'auth'
|
|
100
|
+
// memoryCurrent the cgroup's memory.current in bytes (see
|
|
101
|
+
// memoryHigh the cgroup's memory.high in bytes classifyOperatorThrottle)
|
|
102
|
+
// gcloudProcessCount concurrent gcloud processes on the host
|
|
103
|
+
// fleetWide fleetWideUnreachableNow() — N+ VMs unreachable at once
|
|
104
|
+
function classifyDestroyBlock({ authDegraded, authProbeFailed, memoryCurrent, memoryHigh, gcloudProcessCount, fleetWide } = {}) {
|
|
105
|
+
if (authDegraded) {
|
|
106
|
+
const pressure = classifyOperatorThrottle({ memoryCurrent, memoryHigh, gcloudProcessCount });
|
|
107
|
+
if (authProbeFailed && pressure.throttled) {
|
|
108
|
+
return {
|
|
109
|
+
blocked: true,
|
|
110
|
+
kind: 'throttled',
|
|
111
|
+
reason: `operator plane resource-exhausted (${pressure.signals.join(', ')}) — the auth probe was starved, not denied`,
|
|
112
|
+
nextAction: THROTTLE_NEXT_ACTION,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
return {
|
|
116
|
+
blocked: true,
|
|
117
|
+
kind: 'auth',
|
|
118
|
+
reason: 'operator gcloud auth degraded',
|
|
119
|
+
nextAction: 'resolve auth (SA binding) — scripts/operator/setup-service-account.sh',
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
if (fleetWide) {
|
|
123
|
+
return {
|
|
124
|
+
blocked: true,
|
|
125
|
+
kind: 'fleet-wide',
|
|
126
|
+
reason: 'fleet-wide unreachable spike in progress (operator-side fault signature)',
|
|
127
|
+
nextAction: 'wait out the spike',
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
// The NOT-blocked shape stays exactly { blocked, reason } — `kind` and
|
|
131
|
+
// `nextAction` describe a fault, and there is no fault here to describe.
|
|
132
|
+
return { blocked: false, reason: null };
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// Classify a non-Force destroy's unpushed-commit INTENT decision from the WARM
|
|
136
|
+
// ahead-count the dashboard reads off /api/session-info, mirroring the pure
|
|
137
|
+
// decision shape of classifyDestroyBlock. The three outcomes:
|
|
138
|
+
// ahead number > 0 → REFUSE: unpushed work exists (Force-recoverable)
|
|
139
|
+
// ahead number === 0 → PROCEED: verified clean
|
|
140
|
+
// ahead null / undefined → REFUSE: could NOT verify (editor unreachable /
|
|
141
|
+
// field absent) — a DISTINCT refusal, never conflated
|
|
142
|
+
// with "VM down", and Force-recoverable
|
|
143
|
+
// Returns { refuse, forceable, outcome, msg }: `outcome` is the
|
|
144
|
+
// logDestroyDecision attribution token (null when proceeding); `msg` is the
|
|
145
|
+
// operator-facing refusal text (null when proceeding). Pure — the caller owns the
|
|
146
|
+
// warm probe, the cached fallback, the logging, and the return; this only maps
|
|
147
|
+
// ahead → decision so it is unit-testable in isolation (the way the cold-SSH
|
|
148
|
+
// version never was).
|
|
149
|
+
function classifyUnpushedDestroy({ n, ahead } = {}) {
|
|
150
|
+
if (typeof ahead === 'number' && ahead > 0) {
|
|
151
|
+
return {
|
|
152
|
+
refuse: true,
|
|
153
|
+
forceable: true,
|
|
154
|
+
outcome: 'REFUSED:unpushed',
|
|
155
|
+
msg: `VM-${n} has ${ahead} unpushed commit(s) — not destroying. Use Force to override.`,
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
if (ahead === 0) {
|
|
159
|
+
return { refuse: false, forceable: false, outcome: null, msg: null };
|
|
160
|
+
}
|
|
161
|
+
return {
|
|
162
|
+
refuse: true,
|
|
163
|
+
forceable: true,
|
|
164
|
+
outcome: 'REFUSED:unverifiable',
|
|
165
|
+
msg: `VM-${n} — could not verify unpushed commits (editor unreachable). Use Force to destroy anyway.`,
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// Build the single attributable destroy-decision log line. Mirrors the
|
|
170
|
+
// performAutoReset "confirmed … not fleet-wide" template so every destroy —
|
|
171
|
+
// proceed OR refused, across every trigger source — is diagnosable from one
|
|
172
|
+
// grep. `remote` ({ ip, ua }) is the caller attribution whose absence made the
|
|
173
|
+
// 2026-06-26 delete impossible to trace back to its trigger; null/absent renders
|
|
174
|
+
// as `n/a` (e.g. the shrink / reap paths have no HTTP caller). All the flag
|
|
175
|
+
// fields are normalized to real booleans so a missing field never reads as
|
|
176
|
+
// `undefined` in the line.
|
|
177
|
+
function formatDestroyDecision(info = {}) {
|
|
178
|
+
const remote = info.remote ? `${info.remote.ip || '?'}/${info.remote.ua || '?'}` : 'n/a';
|
|
179
|
+
return (
|
|
180
|
+
`destroy-decision: VM-${info.n} source=${info.source} force=${!!info.force} ` +
|
|
181
|
+
`override=${!!info.overrideSafety} remote=${remote} hasSession=${!!info.hasSession} ` +
|
|
182
|
+
`authDegraded=${!!info.authDegraded} fleetWide=${!!info.fleetWide} → ${info.outcome}`
|
|
183
|
+
);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// Build the ENTRY line for a destroy request, emitted before any probe runs —
|
|
187
|
+
// the third sibling of formatDestroyDecision (the up-front DECISION) and
|
|
188
|
+
// formatDestroyOutcome (the terminal JOB fate). Without it a request that died
|
|
189
|
+
// mid-flight left NO record at all: on 2026-07-29 two destroy POSTs both ended in
|
|
190
|
+
// HTTP 000 and the log's last entry was an earlier refusal, so the operator could
|
|
191
|
+
// not tell whether a delete had started against their VM — the worst possible
|
|
192
|
+
// answer to "did you delete it?".
|
|
193
|
+
//
|
|
194
|
+
// It keeps the `destroy-decision:` prefix so ONE grep still finds a request's
|
|
195
|
+
// whole life, and carries only what is known at entry. It deliberately does NOT
|
|
196
|
+
// render authDegraded/fleetWide/hasSession: those are not measured yet, and
|
|
197
|
+
// printing `false` for an unmeasured verdict would read as "auth was fine" —
|
|
198
|
+
// exactly the false diagnosis this change exists to remove.
|
|
199
|
+
function formatDestroyAttempt(info = {}) {
|
|
200
|
+
const remote = info.remote ? `${info.remote.ip || '?'}/${info.remote.ua || '?'}` : 'n/a';
|
|
201
|
+
return (
|
|
202
|
+
`destroy-decision: VM-${info.n} attempt source=${info.source} force=${!!info.force} ` +
|
|
203
|
+
`override=${!!info.overrideSafety} remote=${remote}`
|
|
204
|
+
);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// Render the operator-facing REFUSAL for a blocked destructive removal, pairing
|
|
208
|
+
// the verdict's reason with its verdict-specific recovery. Extracted from
|
|
209
|
+
// server.js so the "a throttled refusal never sends the operator at auth"
|
|
210
|
+
// property is assertable at the MESSAGE level, not just on the reason field.
|
|
211
|
+
//
|
|
212
|
+
// The recovery renders as the greppable `Next valid action:` contract (CLAUDE.md
|
|
213
|
+
// CLI error conventions) rather than as prose. override-safety is mentioned LAST
|
|
214
|
+
// and never as the primary fix: it bypasses the guard that exists because a
|
|
215
|
+
// false-unreachable cascade permanently deleted VM2/3/4 on 2026-06-26, and
|
|
216
|
+
// putting it beside the real recovery is what made two operators reach for it
|
|
217
|
+
// within one session. `detail` lets the shrink path swap the wording while
|
|
218
|
+
// keeping an identical recovery contract, so the two destructive paths that share
|
|
219
|
+
// operatorFaultVerdict can never give conflicting advice for one fault.
|
|
220
|
+
function formatDestroyRefusal({ n, reason, nextAction, detail } = {}) {
|
|
221
|
+
const what = detail || `refusing to destroy VM-${n}`;
|
|
222
|
+
return (
|
|
223
|
+
`REFUSED: ${reason} — ${what}; a false-unreachable storm can delete healthy VMs.\n` +
|
|
224
|
+
`Next valid action: ${nextAction}\n` +
|
|
225
|
+
'If this destructive action is genuinely intended right now, pass override-safety.'
|
|
226
|
+
);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
// Render the refusal for a destroy whose DECISION exceeded its wall-clock budget.
|
|
230
|
+
// HTTP 000 is the worst possible answer to "did you delete my VM?" — it is
|
|
231
|
+
// indistinguishable from a delete that started and lost its response, and it is
|
|
232
|
+
// what two destroy POSTs returned on 2026-07-29 after 150s and 240s.
|
|
233
|
+
//
|
|
234
|
+
// The wording is deliberately "no verdict" and "no delete CONFIRMED dispatched",
|
|
235
|
+
// not "nothing happened": the in-flight decision may still resolve after the
|
|
236
|
+
// bound elapses, so claiming otherwise would be a second false certainty. It
|
|
237
|
+
// points at the `destroy-decision:` grep instead, which the entry-time attempt
|
|
238
|
+
// line guarantees is populated.
|
|
239
|
+
function formatDestroyBoundExceeded({ n, budgetMs } = {}) {
|
|
240
|
+
return (
|
|
241
|
+
`REFUSED: no destroy verdict for VM-${n} within ${Math.round(budgetMs / 1000)}s — ` +
|
|
242
|
+
'the dashboard is not answering, which usually means it is resource-throttled.\n' +
|
|
243
|
+
`Next valid action: ${THROTTLE_NEXT_ACTION}\n` +
|
|
244
|
+
`No delete was confirmed dispatched; grep the log for \`destroy-decision: VM-${n}\` to see how far this request got.`
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// Build the single TERMINAL-OUTCOME journal line for a completed destroy JOB —
|
|
249
|
+
// the sibling of formatDestroyDecision, which records the up-front DECISION. The
|
|
250
|
+
// decision line alone was a blind spot: diagnosing the VM-8 silent refusal
|
|
251
|
+
// surfaced only `destroy-decision … → proceed:delete`, nothing about how the job
|
|
252
|
+
// itself ended, so a job that refused/errored after the decision was invisible.
|
|
253
|
+
// With this line a completed destroy's fate is one `grep destroy-outcome` away.
|
|
254
|
+
// `outcome` is the caller-computed terminal string: `done`, `refused:<kind>`, or
|
|
255
|
+
// `error code=<n>`. Pure format only — the caller owns the classification.
|
|
256
|
+
function formatDestroyOutcome(info = {}) {
|
|
257
|
+
return `destroy-outcome: VM-${info.n} ${info.outcome}`;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
module.exports = {
|
|
261
|
+
GCLOUD_PILEUP_THRESHOLD,
|
|
262
|
+
MEMORY_PRESSURE_RATIO,
|
|
263
|
+
THROTTLE_NEXT_ACTION,
|
|
264
|
+
classifyDestroyBlock,
|
|
265
|
+
classifyOperatorThrottle,
|
|
266
|
+
classifyUnpushedDestroy,
|
|
267
|
+
formatDestroyAttempt,
|
|
268
|
+
formatDestroyBoundExceeded,
|
|
269
|
+
formatDestroyDecision,
|
|
270
|
+
formatDestroyOutcome,
|
|
271
|
+
formatDestroyRefusal,
|
|
272
|
+
};
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// npm/fleet-early-roster.js — decides WHEN an in-flight `add` earns its roster
|
|
4
|
+
// entry.
|
|
5
|
+
//
|
|
6
|
+
// Background (vm--batch-add-must-roster-and-wire-each-vm): rostering used to
|
|
7
|
+
// happen in exactly one place on the live add path — `startJob`'s onSuccess
|
|
8
|
+
// callback — which only fires when the whole ~28-minute add script exits 0. A
|
|
9
|
+
// VM whose container came up but whose add later failed (wiring, reachability,
|
|
10
|
+
// a dashboard SIGKILL) was therefore never rostered, and an unrostered VM is
|
|
11
|
+
// invisible to `reconcile-ingress`, which skips it. That produced a deadlock:
|
|
12
|
+
//
|
|
13
|
+
// not rostered → reconcile skips → no backend service → LB 503 →
|
|
14
|
+
// not reachable → bootstrap recovery retries → exhausted → still not rostered
|
|
15
|
+
//
|
|
16
|
+
// Observed live 2026-07-20 provisioning VM-4..VM-11: the roster stayed `[1,2,3]`
|
|
17
|
+
// for a `count=8` add even though every container was `Up` and serving 200
|
|
18
|
+
// locally. The only recovery was to destroy and re-add one at a time.
|
|
19
|
+
//
|
|
20
|
+
// The fix is to roster EARLY — as soon as the instance exists and the container
|
|
21
|
+
// gate has passed — so a later-stage failure leaves a *rostered, reconcilable*
|
|
22
|
+
// VM that the periodic sweep can finish wiring, rather than an invisible one.
|
|
23
|
+
// The add script announces that moment by printing a marker line; `startJob`
|
|
24
|
+
// taps the child's stdout and hands the accumulated text here.
|
|
25
|
+
//
|
|
26
|
+
// This module is PURE (no fs, no network, no roster writes) so the "has this VM
|
|
27
|
+
// earned its roster entry yet" rule is unit-testable without a live VM. The
|
|
28
|
+
// actual `addToRoster` write stays in server.js.
|
|
29
|
+
|
|
30
|
+
// The marker the add script prints once its container gate passes. Chosen to
|
|
31
|
+
// match the existing `CY_DEGRADED` marker convention that `startJob` already
|
|
32
|
+
// scans for, so both live in the same namespace and are equally greppable in a
|
|
33
|
+
// job log.
|
|
34
|
+
const ROSTER_MARKER = 'CY_ROSTER';
|
|
35
|
+
|
|
36
|
+
// Build the marker line the add script emits. Kept here (rather than inlined in
|
|
37
|
+
// server.js) so the emitter and the parser can never drift apart — a test that
|
|
38
|
+
// round-trips `parseRosterMarkers(rosterMarkerLine(n))` pins them together.
|
|
39
|
+
function rosterMarkerLine(n) {
|
|
40
|
+
return `${ROSTER_MARKER} ${n}`;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// Extract every VM number announced by a roster marker in `text`.
|
|
44
|
+
//
|
|
45
|
+
// Scans the FULL accumulated buffer rather than a single chunk: child stdout
|
|
46
|
+
// arrives in arbitrary chunks that can split a line mid-marker, so a
|
|
47
|
+
// chunk-local parse would miss (or double-count) a marker straddling a
|
|
48
|
+
// boundary. Re-parsing the whole buffer is idempotent and cheap relative to the
|
|
49
|
+
// ~28-minute job it rides along with.
|
|
50
|
+
//
|
|
51
|
+
// Returns unique numbers in ascending order. Junk (`CY_ROSTER abc`,
|
|
52
|
+
// `CY_ROSTER 1.5`, a bare `CY_ROSTER`) is dropped — the roster is a set of
|
|
53
|
+
// VM-number integers, so a malformed marker must never poison it.
|
|
54
|
+
function parseRosterMarkers(text) {
|
|
55
|
+
if (typeof text !== 'string' || text === '') return [];
|
|
56
|
+
const out = new Set();
|
|
57
|
+
for (const line of text.split('\n')) {
|
|
58
|
+
const t = line.trim();
|
|
59
|
+
if (!t.startsWith(`${ROSTER_MARKER} `)) continue;
|
|
60
|
+
const rest = t.slice(ROSTER_MARKER.length + 1).trim();
|
|
61
|
+
// Reject anything that is not a plain positive integer. `Number()` alone
|
|
62
|
+
// would accept '1.5', '0x2', and ' 3 '; the regex gate keeps it strict.
|
|
63
|
+
if (!/^\d+$/.test(rest)) continue;
|
|
64
|
+
const n = Number(rest);
|
|
65
|
+
if (Number.isInteger(n) && n > 0) out.add(n);
|
|
66
|
+
}
|
|
67
|
+
return [...out].sort((a, b) => a - b);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Should VM `n` be rostered right now, given the job output so far?
|
|
71
|
+
//
|
|
72
|
+
// `alreadyRostered` is the caller's own latch — server.js flips it after the
|
|
73
|
+
// first successful write so a marker sitting in the buffer for the rest of the
|
|
74
|
+
// job doesn't re-trigger a write on every subsequent chunk. (The roster write
|
|
75
|
+
// is itself idempotent, so this is an efficiency guard, not a correctness one.)
|
|
76
|
+
function shouldEarlyRoster({ text, vmNumber, alreadyRostered } = {}) {
|
|
77
|
+
if (alreadyRostered) return false;
|
|
78
|
+
if (!Number.isInteger(vmNumber) || vmNumber <= 0) return false;
|
|
79
|
+
return parseRosterMarkers(text).includes(vmNumber);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
module.exports = {
|
|
83
|
+
ROSTER_MARKER,
|
|
84
|
+
rosterMarkerLine,
|
|
85
|
+
parseRosterMarkers,
|
|
86
|
+
shouldEarlyRoster,
|
|
87
|
+
};
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// Self-healing cycle guard for the Fleet Dashboard's slow effect poll
|
|
3
|
+
// (scripts/operator/fleet-dashboard/server.js requires this). No gcloud/SSH
|
|
4
|
+
// I/O — this module owns the control-flow guarantee ONLY; the server keeps its
|
|
5
|
+
// gather edges and worker pools.
|
|
6
|
+
//
|
|
7
|
+
// The bug this fixes: `pollEffects` guards against overlap with a
|
|
8
|
+
// `effectPollRunning` flag (two concurrent full cycles would double the gcloud
|
|
9
|
+
// load), releasing it in a `finally` that runs only AFTER
|
|
10
|
+
// `await Promise.all(...)` of the reachable + launcher-strand sweeps. On
|
|
11
|
+
// 2026-07-09 a single probe's callback never fired (an SSH/docker-exec against
|
|
12
|
+
// VM-8, whose GCP instance had been deleted mid-session), so that awaited
|
|
13
|
+
// promise never resolved, the `finally` never ran, and `effectPollRunning`
|
|
14
|
+
// stayed `true` forever — every subsequent 45s tick early-returned and fleet
|
|
15
|
+
// readiness (iapAccess, backendHealth, readiness, launcher-strand self-heal)
|
|
16
|
+
// froze fleet-wide until the dashboard was restarted.
|
|
17
|
+
//
|
|
18
|
+
// Two guarantees make that freeze impossible:
|
|
19
|
+
// makeGuardedCycle — the no-overlap guard is kept (it's correct), but its
|
|
20
|
+
// release is made unconditional: the returned promise ALWAYS settles, either
|
|
21
|
+
// when the tasks finish or when a whole-cycle watchdog fires, and the guard
|
|
22
|
+
// flag is cleared in a `finally` either way. A hung task can no longer strand
|
|
23
|
+
// the next tick.
|
|
24
|
+
// withTimeout — wraps a single probe promise so a never-firing callback
|
|
25
|
+
// resolves to a safe fallback verdict instead of hanging the sweep.
|
|
26
|
+
|
|
27
|
+
// Wrap a promise so it can never hang the caller: resolve to `fallback` if
|
|
28
|
+
// `promise` has not settled within `ms`, and also on rejection (a probe that
|
|
29
|
+
// throws must degrade gracefully, never reject the whole sweep). `ms <= 0`
|
|
30
|
+
// disables the timeout and just awaits the promise (rejections still fold to
|
|
31
|
+
// `fallback`), so a caller can opt a leg out without branching. Whichever of
|
|
32
|
+
// (settle, timeout) happens first wins; the other is a no-op.
|
|
33
|
+
function withTimeout(promise, ms, fallback) {
|
|
34
|
+
const settled = Promise.resolve(promise).catch(() => fallback);
|
|
35
|
+
if (!(ms > 0)) return settled;
|
|
36
|
+
return new Promise((resolve) => {
|
|
37
|
+
let done = false;
|
|
38
|
+
const finish = (val) => { if (!done) { done = true; resolve(val); } };
|
|
39
|
+
// NOT unref'd: this timer is load-bearing — it resolves the promise the
|
|
40
|
+
// caller awaits, so letting the event loop exit before it fires would leave
|
|
41
|
+
// the await hung. It is cleared the instant the real promise settles, so it
|
|
42
|
+
// holds the loop open only for the (short) timeout window.
|
|
43
|
+
const timer = setTimeout(() => finish(fallback), ms);
|
|
44
|
+
settled.then((val) => { clearTimeout(timer); finish(val); });
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// Build a guarded cycle runner. `run(tasks)` executes an array of zero-arg async
|
|
49
|
+
// task factories (or bare promises) under a no-overlap guard:
|
|
50
|
+
// - while a cycle is in flight, a second `run(...)` returns `{ skipped: true }`
|
|
51
|
+
// immediately without launching the tasks — the no-overlap contract.
|
|
52
|
+
// - the returned promise ALWAYS settles: when the tasks finish, or — if
|
|
53
|
+
// `watchdogMs > 0` — when the whole-cycle watchdog fires first (returning
|
|
54
|
+
// `{ wedged: true }`). Either way the guard flag is cleared in a `finally`,
|
|
55
|
+
// so the NEXT tick can always run. A never-resolving task therefore cannot
|
|
56
|
+
// permanently wedge the loop (the 2026-07-09 freeze).
|
|
57
|
+
// `onWedge(elapsedMs)` (optional) is called when the watchdog fires, so the
|
|
58
|
+
// caller can log a wedge warning. A task that rejects is swallowed (one bad probe
|
|
59
|
+
// must not fail the whole cycle); the caller composes concurrency/ordering.
|
|
60
|
+
function makeGuardedCycle(opts) {
|
|
61
|
+
const o = opts || {};
|
|
62
|
+
const watchdogMs = o.watchdogMs > 0 ? o.watchdogMs : 0;
|
|
63
|
+
const onWedge = typeof o.onWedge === 'function' ? o.onWedge : null;
|
|
64
|
+
let running = false;
|
|
65
|
+
const run = async (tasks) => {
|
|
66
|
+
if (running) return { skipped: true };
|
|
67
|
+
running = true;
|
|
68
|
+
try {
|
|
69
|
+
const work = Promise.all((tasks || []).map((t) => (typeof t === 'function' ? t() : t)))
|
|
70
|
+
.then(() => false, () => false); // settle to `wedged=false`; swallow a rejecting probe
|
|
71
|
+
let wedged = false;
|
|
72
|
+
if (watchdogMs > 0) {
|
|
73
|
+
wedged = await new Promise((resolve) => {
|
|
74
|
+
// NOT unref'd: this watchdog resolves the promise the caller awaits, so
|
|
75
|
+
// it must keep the loop alive until it fires (or the work settles first
|
|
76
|
+
// and clears it). Unref'ing it would let a standalone await hang.
|
|
77
|
+
const timer = setTimeout(() => resolve(true), watchdogMs);
|
|
78
|
+
work.then((w) => { clearTimeout(timer); resolve(w); });
|
|
79
|
+
});
|
|
80
|
+
} else {
|
|
81
|
+
await work;
|
|
82
|
+
}
|
|
83
|
+
if (wedged && onWedge) { try { onWedge(watchdogMs); } catch (e) { /* logging must never re-wedge */ } }
|
|
84
|
+
return { skipped: false, wedged };
|
|
85
|
+
} finally {
|
|
86
|
+
// Unconditional release — the whole point. Whether the tasks finished or
|
|
87
|
+
// the watchdog fired, the guard is cleared so the next tick proceeds. Any
|
|
88
|
+
// still-pending task keeps running detached, but holds no guard.
|
|
89
|
+
running = false;
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
return { run };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// A roster entry whose Add errored out is an `add-failed` ghost: its GCP instance
|
|
96
|
+
// may be half-created or already reconciled away, so an SSH/docker-exec strand
|
|
97
|
+
// probe against it is the exact call that hung the 2026-07-09 sweep (VM-8). The
|
|
98
|
+
// strand selection skips it — a failed Add is owned by the reachable / vanish
|
|
99
|
+
// reconcilers, not the launcher-strand self-heal. `withTimeout` (above) is the
|
|
100
|
+
// general backstop for any hung probe; this removes the doomed probe at the
|
|
101
|
+
// source for the named trigger.
|
|
102
|
+
function isAddFailedGhost(job) {
|
|
103
|
+
return !!(job && job.state === 'error' && job.action === 'add');
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
module.exports = {
|
|
107
|
+
withTimeout,
|
|
108
|
+
makeGuardedCycle,
|
|
109
|
+
isAddFailedGhost,
|
|
110
|
+
};
|
|
@@ -58,12 +58,10 @@ function resetReason({ agent, session, dirty, behind, codeyam }) {
|
|
|
58
58
|
// meaningless for them).
|
|
59
59
|
const ACTION_IDS = [
|
|
60
60
|
'update-client-source',
|
|
61
|
-
'update-editor-source',
|
|
62
|
-
'reset-editor-source',
|
|
63
61
|
'reset-client',
|
|
62
|
+
'end-session',
|
|
64
63
|
'rebuild-binary',
|
|
65
64
|
'update-reset-everything',
|
|
66
|
-
'run-setup',
|
|
67
65
|
];
|
|
68
66
|
|
|
69
67
|
// facts (a plain object gathered by server.js — no I/O):
|
|
@@ -105,13 +103,6 @@ function computeButtonEffects(facts) {
|
|
|
105
103
|
? { effect: 'changes', reason: behindReason(client.behind, f.clientBranch) }
|
|
106
104
|
: { effect: 'noop', reason: 'up to date with ' + originRef(f.clientBranch) };
|
|
107
105
|
|
|
108
|
-
// (2) Update editor — advance the editor source the binary builds from.
|
|
109
|
-
const updateEditor = editor == null
|
|
110
|
-
? { effect: 'unknown', reason: 'editor state not checked' }
|
|
111
|
-
: editor.behind > 0
|
|
112
|
-
? { effect: 'changes', reason: behindReason(editor.behind, f.editorBranch) }
|
|
113
|
-
: { effect: 'noop', reason: 'up to date with ' + originRef(f.editorBranch) };
|
|
114
|
-
|
|
115
106
|
// (5) Rebuild binary — advances source then compiles. Changes iff the running
|
|
116
107
|
// binary's build commit differs from the editor origin tip. Unknown when we
|
|
117
108
|
// don't have both commits to compare.
|
|
@@ -121,25 +112,38 @@ function computeButtonEffects(facts) {
|
|
|
121
112
|
? { effect: 'changes', reason: `binary at ${short(editor.buildCommit)}, ${originRef(f.editorBranch)} at ${short(editor.originSha)}` }
|
|
122
113
|
: { effect: 'noop', reason: 'binary built from ' + originRef(f.editorBranch) + ' tip' };
|
|
123
114
|
|
|
124
|
-
//
|
|
125
|
-
// .codeyam. No-op ONLY when clean at origin tip with no session.
|
|
115
|
+
// Reset editor (editor-state reset) — discards uncommitted + session, advances
|
|
116
|
+
// source, wipes .codeyam. No-op ONLY when clean at origin tip with no session.
|
|
117
|
+
// INTERNAL-ONLY: the standalone "Reset editor" button was removed; this
|
|
118
|
+
// predicate survives purely as an input to the "Reset all + rebuild" composite
|
|
119
|
+
// below (which still runs resetEditorStateScript), so it is NOT returned/keyed.
|
|
126
120
|
const resetEditor = editor == null
|
|
127
121
|
? { effect: 'unknown', reason: 'editor state not checked' }
|
|
128
122
|
: (brokerLive || hasSession || editor.dirty || editor.behind > 0 || codeyamDirty)
|
|
129
123
|
? { effect: 'changes', reason: resetReason({ agent: brokerLive, session: hasSession, dirty: editor.dirty, behind: editor.behind > 0, codeyam: codeyamDirty }) }
|
|
130
124
|
: { effect: 'noop', reason: 'already clean at origin tip, no session' };
|
|
131
125
|
|
|
132
|
-
// (
|
|
133
|
-
//
|
|
126
|
+
// (2) Reset client — hard-clean /workspace + .codeyam, drop the session, now
|
|
127
|
+
// ADVANCING to origin tip. Since the reset advances source, a VM behind origin
|
|
128
|
+
// would be changed by a re-run, so `behind` IS part of this predicate.
|
|
134
129
|
const resetClient = client == null
|
|
135
130
|
? { effect: 'unknown', reason: 'client state not checked' }
|
|
136
|
-
: (brokerLive || client.dirty || codeyamDirty || hasSession)
|
|
137
|
-
? { effect: 'changes', reason: resetReason({ agent: brokerLive, session: hasSession, dirty: client.dirty, behind:
|
|
138
|
-
: { effect: 'noop', reason: 'already clean, no session' };
|
|
131
|
+
: (brokerLive || client.dirty || client.behind > 0 || codeyamDirty || hasSession)
|
|
132
|
+
? { effect: 'changes', reason: resetReason({ agent: brokerLive, session: hasSession, dirty: client.dirty, behind: client.behind > 0, codeyam: codeyamDirty }) }
|
|
133
|
+
: { effect: 'noop', reason: 'already clean at origin tip, no session' };
|
|
139
134
|
|
|
140
|
-
// (
|
|
141
|
-
//
|
|
142
|
-
//
|
|
135
|
+
// (2b) End session — kills the live agent + clears session state. The ONLY
|
|
136
|
+
// thing it touches is the running session, so it changes the VM iff a live
|
|
137
|
+
// agent or a registered session exists; otherwise it's a clean no-op. Unlike
|
|
138
|
+
// the resets it ignores dirty/behind/.codeyam — it neither advances source nor
|
|
139
|
+
// wipes state.
|
|
140
|
+
const endSession = (brokerLive || hasSession)
|
|
141
|
+
? { effect: 'changes', reason: resetReason({ agent: brokerLive, session: hasSession }) }
|
|
142
|
+
: { effect: 'noop', reason: 'no active session' };
|
|
143
|
+
|
|
144
|
+
// (4) Reset all + rebuild — composite. Changes iff ANY component would
|
|
145
|
+
// change: rebuild-binary, the editor-state reset (internal resetEditor
|
|
146
|
+
// predicate above), or the client being behind/dirty. Reason summarizes parts.
|
|
143
147
|
const everything = (() => {
|
|
144
148
|
if (editor == null || client == null) {
|
|
145
149
|
return { effect: 'unknown', reason: 'state not checked' };
|
|
@@ -157,23 +161,20 @@ function computeButtonEffects(facts) {
|
|
|
157
161
|
return { effect: 'noop', reason: 'everything already at origin tip, no session' };
|
|
158
162
|
})();
|
|
159
163
|
|
|
160
|
-
// (run-setup) Honest soft signal — there is no cheap upstream proof that
|
|
161
|
-
// `npm run setup` would change nothing (DB/migration state isn't
|
|
162
|
-
// introspected). `recommended` when a reset/update landed after the last
|
|
163
|
-
// successful setup; otherwise `unknown`. Never a confident green no-op.
|
|
164
|
-
const runSetup = f.setupRanSince
|
|
165
|
-
? { effect: 'unknown', reason: 'no setup-affecting change since last setup' }
|
|
166
|
-
: { effect: 'recommended', reason: 'a reset/update landed since the last setup' };
|
|
167
|
-
|
|
168
164
|
return {
|
|
169
165
|
'update-client-source': updateClient,
|
|
170
|
-
'update-editor-source': updateEditor,
|
|
171
|
-
'reset-editor-source': resetEditor,
|
|
172
166
|
'reset-client': resetClient,
|
|
167
|
+
'end-session': endSession,
|
|
173
168
|
'rebuild-binary': rebuild,
|
|
174
169
|
'update-reset-everything': everything,
|
|
175
|
-
'run-setup': runSetup,
|
|
176
170
|
};
|
|
177
171
|
}
|
|
178
172
|
|
|
179
|
-
|
|
173
|
+
// Decide whether a Live Preview SSH forward needs to be re-wired (improve41)
|
|
174
|
+
function needsRewire(forwardAppPort, polledAppPort) {
|
|
175
|
+
if (typeof polledAppPort !== 'number' || isNaN(polledAppPort)) return false;
|
|
176
|
+
if (!forwardAppPort) return false;
|
|
177
|
+
return forwardAppPort !== polledAppPort;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
module.exports = { computeButtonEffects, ACTION_IDS, needsRewire };
|