@codeyam-editor/codeyam-editor 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/npm/cloud.js +377 -21
- package/npm/editor-dev.js +24 -2
- package/npm/editor.js +15 -5
- package/npm/fleet-add-failure.js +355 -21
- package/npm/fleet-auth-account.js +61 -0
- package/npm/fleet-auth.js +69 -1
- package/npm/fleet-branch.js +124 -1
- package/npm/fleet-build-data.js +12 -1
- package/npm/fleet-cleanup.js +84 -3
- package/npm/fleet-datastore-health.js +107 -0
- package/npm/fleet-public-url.js +81 -12
- package/npm/fleet-reconcile.js +46 -11
- package/npm/fleet-revision.js +108 -0
- package/npm/fleet-run-setup.js +66 -0
- package/npm/fleet-session-activity.js +162 -0
- package/npm/fleet-vm-readiness.js +57 -16
- package/npm/group-bounded-spawn.js +111 -0
- package/npm/launch-config.js +216 -23
- package/npm/orphan-ssh-reaper.js +81 -0
- package/npm/scenario-check.js +697 -34
- package/npm/scenario-interactivity.js +477 -23
- package/npm/scenario-issues.js +61 -0
- package/npm/scenario-playwright.js +174 -20
- package/npm/utils.js +262 -22
- package/package.json +5 -5
- package/ui/dist/assets/{ContainerInspector-DSVZvEkB.js → ContainerInspector-DiN91foX.js} +1 -1
- package/ui/dist/assets/EditorShell-BEIdHnHd.css +1 -0
- package/ui/dist/assets/EditorShell-CPV5vhjo.js +330 -0
- package/ui/dist/assets/ScenarioDataPanel-DHAHyraq.js +18 -0
- package/ui/dist/assets/{ScenarioDataPanelFullPage-Cyk6efiI.js → ScenarioDataPanelFullPage-CHIWip3n.js} +1 -1
- package/ui/dist/assets/ScenarioRenderer-D9a91iwS.js +21 -0
- package/ui/dist/assets/index--cE0hUJw.css +1 -0
- package/ui/dist/assets/{index-BPLKgwon.js → index-sW8wcg7v.js} +9 -9
- package/ui/dist/index.html +2 -2
- package/ui/dist/preview-subdir-demo/index.html +62 -0
- package/ui/dist/preview-subdir-demo/sibling.html +48 -0
- package/ui/dist/assets/EditorShell-DnXdHrar.js +0 -256
- package/ui/dist/assets/EditorShell-FvbXyqqD.css +0 -1
- package/ui/dist/assets/ScenarioDataPanel-6qJE-0lA.js +0 -15
- package/ui/dist/assets/ScenarioRenderer-Uyt54vHI.js +0 -21
- package/ui/dist/assets/index-DKpgJLo3.css +0 -1
package/npm/fleet-auth.js
CHANGED
|
@@ -108,9 +108,77 @@ function deriveDefaultAuthAccount(vms, lastAuthAt) {
|
|
|
108
108
|
return best ? best.account : null;
|
|
109
109
|
}
|
|
110
110
|
|
|
111
|
+
// Enumerate the Claude accounts the fleet can seed a new VM from: one row per
|
|
112
|
+
// distinct account email held by an authenticated VM, carrying the account
|
|
113
|
+
// object and the VM numbers holding it (so the Add dialog can show
|
|
114
|
+
// "jared@codeyam.com — VM-14"). Pure over the polled VM snapshot — the same
|
|
115
|
+
// array deriveDefaultAuthAccount reads. Emails compare case-insensitively; rows
|
|
116
|
+
// sort by email and each row's VM numbers ascend, so the result is deterministic.
|
|
117
|
+
function listAvailableAccounts(vms) {
|
|
118
|
+
const byEmail = new Map();
|
|
119
|
+
for (const v of vms || []) {
|
|
120
|
+
const pa = v && v.providerAuth;
|
|
121
|
+
const acct = pa && pa.account;
|
|
122
|
+
if (!acct || !acct.email || pa.state !== 'authenticated') continue;
|
|
123
|
+
const key = String(acct.email).toLowerCase();
|
|
124
|
+
const row = byEmail.get(key) || { account: acct, vms: [] };
|
|
125
|
+
if (!row.vms.includes(v.n)) row.vms.push(v.n);
|
|
126
|
+
byEmail.set(key, row);
|
|
127
|
+
}
|
|
128
|
+
return [...byEmail.values()]
|
|
129
|
+
.map((r) => ({ account: r.account, vms: [...r.vms].sort((a, b) => a - b) }))
|
|
130
|
+
.sort((a, b) => String(a.account.email).localeCompare(String(b.account.email)));
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// Pick the credential donor for a chosen account: the lowest-numbered VM in
|
|
134
|
+
// `reachableDonors` whose authenticated account is `email`, or null when no
|
|
135
|
+
// reachable VM holds it (or no email was asked for). `vms` is the per-VM poll
|
|
136
|
+
// state keyed by VM number (`state.vms`). Pure.
|
|
137
|
+
function selectDonorForAccount(reachableDonors, vms, email) {
|
|
138
|
+
if (!email) return null;
|
|
139
|
+
const want = String(email).toLowerCase();
|
|
140
|
+
for (const n of [...(reachableDonors || [])].sort((a, b) => a - b)) {
|
|
141
|
+
const pa = vms && vms[n] && vms[n].providerAuth;
|
|
142
|
+
const got = pa && pa.state === 'authenticated' && pa.account && pa.account.email;
|
|
143
|
+
if (got && String(got).toLowerCase() === want) return n;
|
|
144
|
+
}
|
|
145
|
+
return null;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// A plausible Claude account email: one "@", a dotted domain, no whitespace or
|
|
149
|
+
// shell-significant characters. The shared guard for every place an operator
|
|
150
|
+
// can name an account (the Add form's per-launch override, the set-default
|
|
151
|
+
// action, the persisted sidecar). Deliberately NOT checked against the fleet,
|
|
152
|
+
// whose roster can change between naming an account and provisioning with it.
|
|
153
|
+
const ACCOUNT_EMAIL_RE = /^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$/;
|
|
154
|
+
function isAccountEmail(s) {
|
|
155
|
+
return typeof s === 'string' && ACCOUNT_EMAIL_RE.test(s.trim());
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// Resolve the fleet's chosen Claude account email: the operator's persisted
|
|
159
|
+
// choice (`saved`, read from npm/fleet-auth-account.js by the caller) when it is
|
|
160
|
+
// a plausible email, else the derived default — the most recently auth-updated
|
|
161
|
+
// VM's account — so a fleet that has never chosen behaves as it did before the
|
|
162
|
+
// picker existed. Pure. Returns an email string, or null when neither exists.
|
|
163
|
+
function resolveChosenAccount(saved, vms, lastAuthAt) {
|
|
164
|
+
if (isAccountEmail(saved)) return saved.trim();
|
|
165
|
+
const derived = deriveDefaultAuthAccount(vms, lastAuthAt);
|
|
166
|
+
return (derived && derived.email) || null;
|
|
167
|
+
}
|
|
168
|
+
|
|
111
169
|
// Return true if this one VM is eligible for an auth-start action now.
|
|
112
170
|
function isAuthStartable(v, job) {
|
|
113
171
|
return classifyAuthVm(v, job).eligible;
|
|
114
172
|
}
|
|
115
173
|
|
|
116
|
-
module.exports = {
|
|
174
|
+
module.exports = {
|
|
175
|
+
classifyAuthVm,
|
|
176
|
+
selectAuthTargets,
|
|
177
|
+
summarizeAuthBatch,
|
|
178
|
+
deriveDefaultAuthAccount,
|
|
179
|
+
listAvailableAccounts,
|
|
180
|
+
selectDonorForAccount,
|
|
181
|
+
isAccountEmail,
|
|
182
|
+
resolveChosenAccount,
|
|
183
|
+
isAuthStartable,
|
|
184
|
+
};
|
package/npm/fleet-branch.js
CHANGED
|
@@ -46,4 +46,127 @@ function resolveFleetBranch({ envBranch, gitBranch, fallback } = {}) {
|
|
|
46
46
|
return cleanBranch(fallback) || UNKNOWN_BRANCH;
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
-
|
|
49
|
+
// ---------- Dead-pin detection ----------
|
|
50
|
+
// `fleet-advance-to.sh` refuses to WRITE a pin for a branch absent from origin,
|
|
51
|
+
// but that guard runs once and nothing re-checks the pin already in place —
|
|
52
|
+
// while the branch it names can be deleted by anyone merging a PR with the
|
|
53
|
+
// delete-branch box ticked. Observed 2026-08-15: PR #104 merged
|
|
54
|
+
// `editor-improvements-72` and deleted it; hours later the operator host still
|
|
55
|
+
// read `FLEET_BRANCH=editor-improvements-72` and nothing said so.
|
|
56
|
+
//
|
|
57
|
+
// The probe itself (`git ls-remote`) is I/O and belongs to server.js, on the
|
|
58
|
+
// SLOW memo cadence rather than the ~3s /data poll. What lives here is the pure
|
|
59
|
+
// classification of its outcome — which is the part a test can actually reach,
|
|
60
|
+
// since a function defined in server.js can never be executed by a test (that
|
|
61
|
+
// file has top-level side effects on require).
|
|
62
|
+
//
|
|
63
|
+
// The tri-state mirrors scripts/operator/lib/branch-on-origin.sh exactly, and
|
|
64
|
+
// the mirroring is the point: `gone` is a DEFINITIVE absence, `indeterminate`
|
|
65
|
+
// means absence was NOT proven (no network, no remote, auth trouble). Widening
|
|
66
|
+
// `indeterminate` into `gone` would turn every offline operator host into a
|
|
67
|
+
// fleet-wide "your pin is dead" alarm.
|
|
68
|
+
|
|
69
|
+
/** Map a `git ls-remote --exit-code --heads` exit code to the shared tri-state.
|
|
70
|
+
* 0 -> 'live', 2 -> 'gone', anything else (including a non-numeric or absent
|
|
71
|
+
* code) -> 'indeterminate'. Pure. */
|
|
72
|
+
function lsRemoteStatus(exitCode) {
|
|
73
|
+
if (exitCode === 0) return 'live';
|
|
74
|
+
if (exitCode === 2) return 'gone';
|
|
75
|
+
return 'indeterminate';
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Classify the health of the fleet's branch pin.
|
|
79
|
+
*
|
|
80
|
+
* Returns `{ state, branch, pinned, dead, blocks, recovery }` where `state` is
|
|
81
|
+
* 'live' | 'gone' | 'indeterminate' | 'unknown'.
|
|
82
|
+
*
|
|
83
|
+
* 'unknown' is the un-probeable case — no branch, or the UNKNOWN_BRANCH
|
|
84
|
+
* sentinel that `resolveFleetBranch` yields when there is neither a pin nor a
|
|
85
|
+
* readable git HEAD. There is no ref to ask origin about, so reporting it as
|
|
86
|
+
* `gone` would invent a deletion that never happened.
|
|
87
|
+
*
|
|
88
|
+
* `dead` is true ONLY for 'gone'. It is the single field a caller should gate a
|
|
89
|
+
* degradation banner on, so no consumer has to re-derive which states are
|
|
90
|
+
* actionable.
|
|
91
|
+
*
|
|
92
|
+
* `blocks` names what a dead pin is actively breaking — both consequences,
|
|
93
|
+
* because neither announces itself: the operator-repo refresh cannot fetch a
|
|
94
|
+
* deleted ref, and the pin also drives `cloud.json editorBranch`, so newly
|
|
95
|
+
* provisioned VMs check out a ref that is not there. Pure; no I/O. */
|
|
96
|
+
function classifyBranchPin({ branch, lsRemoteExitCode, pinned = false } = {}) {
|
|
97
|
+
const name = cleanBranch(branch);
|
|
98
|
+
const probeable = name !== null && name !== UNKNOWN_BRANCH;
|
|
99
|
+
const state = probeable ? lsRemoteStatus(lsRemoteExitCode) : 'unknown';
|
|
100
|
+
const dead = state === 'gone';
|
|
101
|
+
return {
|
|
102
|
+
state,
|
|
103
|
+
branch: name,
|
|
104
|
+
pinned: Boolean(pinned),
|
|
105
|
+
dead,
|
|
106
|
+
blocks: dead
|
|
107
|
+
? [
|
|
108
|
+
'refresh-operator-repo.sh cannot fetch the pinned branch, so this host keeps serving whatever its clone already had',
|
|
109
|
+
'cloud.json editorBranch reads the same pin, so newly provisioned VMs check out a ref that does not exist',
|
|
110
|
+
]
|
|
111
|
+
: [],
|
|
112
|
+
recovery: dead
|
|
113
|
+
? `scripts/operator/fleet-advance-to.sh <BRANCH> — move the fleet to a live branch (it refuses a branch that is itself absent from origin)`
|
|
114
|
+
: null,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** Decide whether the fleet pin should override a stored per-VM branch when an
|
|
119
|
+
* add is RE-ISSUED, and report every override so the caller can log it.
|
|
120
|
+
*
|
|
121
|
+
* Returns `{ config, overrides }` — a NEW config (the input is never mutated)
|
|
122
|
+
* and `[{ field, from, to }]`, empty when nothing changed.
|
|
123
|
+
*
|
|
124
|
+
* The problem this solves: a per-VM config that recorded `editor-improvements-75`
|
|
125
|
+
* was not wrong when it was written — it EXPIRED, silently, the moment that
|
|
126
|
+
* branch merged and origin pruned it. This repo prunes on merge, so every stored
|
|
127
|
+
* feature-branch name has that property. The pin does not expire: it is the
|
|
128
|
+
* durable statement of fleet-wide policy, and on 2026-08-25 it was correct
|
|
129
|
+
* throughout while four retried cards each failed on their stale snapshot (a
|
|
130
|
+
* fresh pool add read the pin and succeeded at the same moment).
|
|
131
|
+
*
|
|
132
|
+
* Three guards, each declining to act rather than acting wrongly:
|
|
133
|
+
* - `selfHost` false ⇒ untouched. A client VM's branch is a deliberate
|
|
134
|
+
* per-project choice; overwriting it with a fleet pin would silently
|
|
135
|
+
* retarget someone else's project. Only the self-host path, where the branch
|
|
136
|
+
* IS fleet policy, falls back to the pin.
|
|
137
|
+
* - no pin, or the UNKNOWN_BRANCH sentinel ⇒ untouched. There is no branch to
|
|
138
|
+
* prefer, and stamping the sentinel would guarantee the failure this exists
|
|
139
|
+
* to prevent.
|
|
140
|
+
* - `pinState === 'gone'` ⇒ untouched. A pin PROVEN absent from origin is not
|
|
141
|
+
* an improvement on a stale-but-possibly-live stored branch; swapping one
|
|
142
|
+
* dead ref for another just moves the failure. Note the deliberate
|
|
143
|
+
* asymmetry with 'indeterminate', which does NOT decline: absence was not
|
|
144
|
+
* proven there, and refusing to converge every time the network blips would
|
|
145
|
+
* strand the fleet on stale branches for the sake of a maybe.
|
|
146
|
+
*
|
|
147
|
+
* Pure; no I/O. */
|
|
148
|
+
function pinCorrectedBranches({ cfg, pin, selfHost = false, pinState } = {}) {
|
|
149
|
+
const config = { ...(cfg || {}) };
|
|
150
|
+
const overrides = [];
|
|
151
|
+
const name = cleanBranch(pin);
|
|
152
|
+
if (!selfHost || !name || name === UNKNOWN_BRANCH || pinState === 'gone') {
|
|
153
|
+
return { config, overrides };
|
|
154
|
+
}
|
|
155
|
+
for (const field of ['clientBranch', 'editorBranch']) {
|
|
156
|
+
const from = cleanBranch(config[field]);
|
|
157
|
+
if (from && from !== name) {
|
|
158
|
+
config[field] = name;
|
|
159
|
+
overrides.push({ field, from, to: name });
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
return { config, overrides };
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
module.exports = {
|
|
166
|
+
UNKNOWN_BRANCH,
|
|
167
|
+
cleanBranch,
|
|
168
|
+
resolveFleetBranch,
|
|
169
|
+
lsRemoteStatus,
|
|
170
|
+
classifyBranchPin,
|
|
171
|
+
pinCorrectedBranches,
|
|
172
|
+
};
|
package/npm/fleet-build-data.js
CHANGED
|
@@ -120,6 +120,17 @@ function buildVmEnvelope(v, ctx) {
|
|
|
120
120
|
// unreachable VM's last-known config is stale, and an unreachable card has
|
|
121
121
|
// louder problems to report than its preview origin.
|
|
122
122
|
previewMisconfigured: (v.reachable && v.previewMisconfigured) || null,
|
|
123
|
+
// The project's DECLARED datastore and whether it is answering:
|
|
124
|
+
// { type, target, local, reachable, lastError }, or null when the project
|
|
125
|
+
// declares none (a Rust CLI, a static site) — which is the common case and
|
|
126
|
+
// must render nothing at all. `reachable` is tri-state exactly like
|
|
127
|
+
// previewServing/previewIngress above: true = probe reached it, false =
|
|
128
|
+
// confirmed down, null = unknown (editor just booted, or the probe went
|
|
129
|
+
// stale). Never conflate "down" with "unknown" — only a confirmed false
|
|
130
|
+
// raises the amber reason. Gated on `reachable` like providerAuth: an
|
|
131
|
+
// unreachable VM's last-known datastore verdict is stale, and it has louder
|
|
132
|
+
// problems to report.
|
|
133
|
+
datastore: (v.reachable && v.datastore) || null,
|
|
123
134
|
effects: v.effects || null,
|
|
124
135
|
// The VM's live-configured build agent (lowercase provider), reported by
|
|
125
136
|
// /api/session-info and merged (sticky) onto the poll state. Drives the
|
|
@@ -177,7 +188,7 @@ const REQUIRED_CARD_FIELDS = [
|
|
|
177
188
|
'degradedReasons', 'setupSteps', 'attentionReasons',
|
|
178
189
|
'isHead', 'headHolderUnreachable', 'queuePos', 'queueLen',
|
|
179
190
|
'job', 'config', 'project', 'variant', 'cardIdentity', 'brokerHistory',
|
|
180
|
-
'brokerDeath', 'providerAuth', 'previewMisconfigured', 'effects', 'currentProvider',
|
|
191
|
+
'brokerDeath', 'providerAuth', 'previewMisconfigured', 'datastore', 'effects', 'currentProvider',
|
|
181
192
|
'readiness', 'backendHealth', 'vmAccess',
|
|
182
193
|
'startedAt', 'featureStartedAt', 'attention', 'envOverride',
|
|
183
194
|
];
|
package/npm/fleet-cleanup.js
CHANGED
|
@@ -618,20 +618,37 @@ function reconcileRosteredVm({ exists, hasSession, hasPendingAdd, streak, resetH
|
|
|
618
618
|
// The dashboard's `state` and `addVm` are injected so this stays a pure,
|
|
619
619
|
// importable unit — `log`/`err` default to the console but are overridable in
|
|
620
620
|
// tests. Behavior mirrors the previous inline server.js definition exactly.
|
|
621
|
+
//
|
|
622
|
+
// The exhausted branch records a TERMINAL DISPOSITION, not just a flag. The flag
|
|
623
|
+
// alone was write-only: nothing destroys the instance and the only code that
|
|
624
|
+
// clears the flag is a *subsequent* recovery attempt, which by definition never
|
|
625
|
+
// comes once the budget is spent. VM-10 therefore ran empty and unrostered for a
|
|
626
|
+
// full day on 2026-08-29, billing, while the dashboard rendered a generic
|
|
627
|
+
// needs-manual-recovery card and clearing it needed hand-edited JSON.
|
|
628
|
+
// `instancePresumedRunning` says the GCE instance is believed to still exist —
|
|
629
|
+
// PRESUMED, because this ladder step never probes GCP; `exhaustedOrphanDisposition`
|
|
630
|
+
// is what reconciles the presumption against reality. `exhaustedAt` anchors the
|
|
631
|
+
// accrued-cost clock the card renders. `addVm` is still never called: that is the
|
|
632
|
+
// safety property that stops a wedged VM looping cloud:up forever.
|
|
621
633
|
function continueBootstrapRecovery(
|
|
622
634
|
n,
|
|
623
635
|
meta,
|
|
624
636
|
cfg,
|
|
625
|
-
{ state, addVm, log = console.log, err = console.error } = {},
|
|
637
|
+
{ state, addVm, log = console.log, err = console.error, now = Date.now } = {},
|
|
626
638
|
) {
|
|
627
639
|
const { attempt, proceed, max } = recoverBootstrapDecision({
|
|
628
640
|
recoverAttempts: meta.recoverAttempts,
|
|
629
641
|
});
|
|
630
642
|
if (!proceed) {
|
|
631
643
|
err(
|
|
632
|
-
`startup-reap: VM-${n} exceeded ${max} bootstrap-recovery attempts — leaving sidecar for manual action`,
|
|
644
|
+
`startup-reap: VM-${n} exceeded ${max} bootstrap-recovery attempts — instance presumed still running, leaving sidecar for manual action`,
|
|
633
645
|
);
|
|
634
|
-
state.orphans[n] = {
|
|
646
|
+
state.orphans[n] = {
|
|
647
|
+
reason: "bootstrap-recovery-exhausted",
|
|
648
|
+
attempts: attempt - 1,
|
|
649
|
+
instancePresumedRunning: true,
|
|
650
|
+
exhaustedAt: now(),
|
|
651
|
+
};
|
|
635
652
|
return;
|
|
636
653
|
}
|
|
637
654
|
log(
|
|
@@ -643,6 +660,68 @@ function continueBootstrapRecovery(
|
|
|
643
660
|
addVm(n, cfg, { recovering: true, recoverAttempt: attempt });
|
|
644
661
|
}
|
|
645
662
|
|
|
663
|
+
// The exit `continueBootstrapRecovery`'s exhausted flag never had. That flag is
|
|
664
|
+
// written once and, until this verdict existed, could only be cleared by a
|
|
665
|
+
// subsequent recovery attempt the spent budget guarantees will never run — so it
|
|
666
|
+
// outlived the instance it described and had to be removed by hand-editing
|
|
667
|
+
// `.codeyam/logs/dashboard-state.json`. This reconciles the flag's PRESUMPTION
|
|
668
|
+
// (`instancePresumedRunning`) against two live facts the sweep supplies:
|
|
669
|
+
//
|
|
670
|
+
// instanceExists true ⇒ the presumption holds, the box is still billing
|
|
671
|
+
// false ⇒ the instance is gone (destroyed out-of-band) → 'clear',
|
|
672
|
+
// so the card self-heals with no operator action
|
|
673
|
+
// null ⇒ the describe probe was INCONCLUSIVE. Never act on
|
|
674
|
+
// absence of evidence — same discipline reapVerdict
|
|
675
|
+
// applies to its own `exists === null`.
|
|
676
|
+
// rostered false ⇒ flagged, billing, and in nobody's roster: invisible to
|
|
677
|
+
// every roster-driven tool yet costing money → 'cost',
|
|
678
|
+
// the distinct ownerless surface, NOT a generic
|
|
679
|
+
// needs-manual-recovery card.
|
|
680
|
+
// true ⇒ still owned, so it is recoverable → 'keep'.
|
|
681
|
+
// null ⇒ roster unknown; keep rather than raise a cost alarm
|
|
682
|
+
// that may be false (positive-signal-only, mirroring
|
|
683
|
+
// how projectWired/agentReady downgrade).
|
|
684
|
+
//
|
|
685
|
+
// Deliberately NOT a destroy verdict. Destroying a VM is irreversible and a
|
|
686
|
+
// half-provisioned box occasionally holds the clue that explains the failure, so
|
|
687
|
+
// the terminal disposition is "surface it loudly and offer the button" — the
|
|
688
|
+
// operator presses it.
|
|
689
|
+
function exhaustedOrphanDisposition({ reason, instanceExists, rostered } = {}) {
|
|
690
|
+
if (reason !== 'bootstrap-recovery-exhausted') return 'leave';
|
|
691
|
+
if (instanceExists === null || instanceExists === undefined) return 'leave';
|
|
692
|
+
if (!instanceExists) return 'clear';
|
|
693
|
+
return rostered === false ? 'cost' : 'keep';
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
// Fold a 'cost'/'keep' verdict into the orphan entry the card renders. Split out
|
|
697
|
+
// of the dashboard's poller because the stickiness rule below is the only real
|
|
698
|
+
// logic in that branch, and inlined in server.js it was unreachable by any test:
|
|
699
|
+
// the dashboard has top-level side effects on require, so its pollers can only be
|
|
700
|
+
// covered by a source-scan wiring guard that cannot evaluate behavior.
|
|
701
|
+
//
|
|
702
|
+
// `ownerlessSince` dates the accrued cost from when the VM was FIRST confirmed
|
|
703
|
+
// ownerless, not from the latest probe — re-stamping it every tick would peg the
|
|
704
|
+
// card's elapsed time near zero forever and hide exactly the multi-day cost the
|
|
705
|
+
// surface exists to reveal. It is dropped (not zeroed, not left stale) the moment
|
|
706
|
+
// a VM stops being ownerless — an operator adopting it back into the roster flips
|
|
707
|
+
// the verdict to 'keep', and a leftover `ownerlessSince` would let the cost card
|
|
708
|
+
// resurrect itself with a fabricated start time if the VM ever went ownerless
|
|
709
|
+
// again. `instanceConfirmedAt` records that the presumption was checked live.
|
|
710
|
+
//
|
|
711
|
+
// Returns a NEW entry; the caller decides where to store it. Both verdicts keep
|
|
712
|
+
// the flag — the instance is real and only an operator may destroy it.
|
|
713
|
+
function annotateOwnerlessCost(orphan, { verdict, now = Date.now() } = {}) {
|
|
714
|
+
const next = { ...orphan, instancePresumedRunning: true, instanceConfirmedAt: now };
|
|
715
|
+
if (verdict === 'cost') {
|
|
716
|
+
next.ownerlessCost = true;
|
|
717
|
+
next.ownerlessSince = (orphan && orphan.ownerlessSince) || now;
|
|
718
|
+
return next;
|
|
719
|
+
}
|
|
720
|
+
next.ownerlessCost = false;
|
|
721
|
+
delete next.ownerlessSince;
|
|
722
|
+
return next;
|
|
723
|
+
}
|
|
724
|
+
|
|
646
725
|
module.exports = {
|
|
647
726
|
parsePendingJobFilename,
|
|
648
727
|
persistPendingJob,
|
|
@@ -656,6 +735,8 @@ module.exports = {
|
|
|
656
735
|
classifyLbReachableForReap,
|
|
657
736
|
recoverBootstrapDecision,
|
|
658
737
|
continueBootstrapRecovery,
|
|
738
|
+
exhaustedOrphanDisposition,
|
|
739
|
+
annotateOwnerlessCost,
|
|
659
740
|
exitMarkerName,
|
|
660
741
|
parseExitMarker,
|
|
661
742
|
readExitMarker,
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// npm/fleet-datastore-health.js — pure classifier for the fleet dashboard's
|
|
4
|
+
// "declared datastore unreachable" surface: is the datastore a VM's project
|
|
5
|
+
// DECLARES actually answering?
|
|
6
|
+
//
|
|
7
|
+
// Nothing anywhere reported this, so a VM whose database was down read
|
|
8
|
+
// perfectly green. On VM-12 (2026-08-24) the dashboard, the LB backend health,
|
|
9
|
+
// the IAP edge and the editor's own server-identity all reported healthy while
|
|
10
|
+
// every DB-backed route returned 500 — the only evidence was `ECONNREFUSED
|
|
11
|
+
// 127.0.0.1:5432` buried in the container log, which nobody polls. The
|
|
12
|
+
// 2026-08-14 occurrence on the same VM presented as previewServing=false with an
|
|
13
|
+
// EMPTY degradedReasons list: the dashboard knew something was wrong and had
|
|
14
|
+
// nothing to say about it. This module is the "something to say".
|
|
15
|
+
//
|
|
16
|
+
// The editor probes (it is the only party that knows what the project declares,
|
|
17
|
+
// and the resolution already exists in-crate) and carries the result on the
|
|
18
|
+
// existing session-info poll; this module decides, and server.js renders. Pure
|
|
19
|
+
// like fleet-project-drift.js: all the decision logic lives here and unit-tests
|
|
20
|
+
// without a VM, while server.js stays the thin I/O shell.
|
|
21
|
+
//
|
|
22
|
+
// A new `datastore-unreachable` code needs NO change to
|
|
23
|
+
// fleet-setup-steps.js::classifyReasons — it routes unknown codes to
|
|
24
|
+
// attentionReasons by design ("an unrecognized signal is surfaced loudly, never
|
|
25
|
+
// silently hidden"), so this lands amber. It must NOT be added to SETUP_CODES: a
|
|
26
|
+
// dead datastore is never expected first-run wiring.
|
|
27
|
+
|
|
28
|
+
// Decide the datastore-health reasons for one VM.
|
|
29
|
+
//
|
|
30
|
+
// declared truthy when the project declares a datastore at all. A project
|
|
31
|
+
// with none (a Rust CLI, a static site) must never raise a reason.
|
|
32
|
+
// reachable TRI-STATE, matching the editor's own discipline: true = the probe
|
|
33
|
+
// reached it, false = the probe failed, null/undefined = unknown
|
|
34
|
+
// (not probed yet, or the probe went stale). Only an explicit
|
|
35
|
+
// `false` raises a reason — collapsing unknown into false would
|
|
36
|
+
// light up every VM during the window between editor boot and the
|
|
37
|
+
// first probe, which is the failure mode that trains an operator to
|
|
38
|
+
// ignore the surface.
|
|
39
|
+
// target credential-free rendering of where it lives ("127.0.0.1:5432/app",
|
|
40
|
+
// or a file path). Never the DSN — that carries credentials.
|
|
41
|
+
// shape 'file' (a database file on disk) or 'network' (a server on a
|
|
42
|
+
// socket). Published by the editor rather than inferred from `type`
|
|
43
|
+
// here, because "sqlite means file-shaped" is a per-stack fact this
|
|
44
|
+
// module would otherwise have to learn for every engine. It decides
|
|
45
|
+
// WHICH recovery applies at all: a down server gets started, a
|
|
46
|
+
// missing file gets restored.
|
|
47
|
+
// local whether the datastore is something this VM is expected to host
|
|
48
|
+
// (loopback host, or a file database). Decides where the recovery
|
|
49
|
+
// points; a remote/managed datastore is still reported, but telling
|
|
50
|
+
// an operator to start it here would send them somewhere there is
|
|
51
|
+
// nothing to do.
|
|
52
|
+
// type the declared engine ('postgresql' | 'sqlite' | 'mysql' | …), used
|
|
53
|
+
// only to name what is down.
|
|
54
|
+
// lastError the probe's own failure string ("connection refused: …"), so the
|
|
55
|
+
// card says WHY rather than just that.
|
|
56
|
+
// n the VM number, so a local recovery can name the VM to act on.
|
|
57
|
+
//
|
|
58
|
+
// Returns { unreachable, reasons }. `reasons` is [] whenever healthy, unknown, or
|
|
59
|
+
// nothing is declared, so the caller spreads it unconditionally.
|
|
60
|
+
function classifyDatastoreHealth({ declared, reachable, target, shape, local, type, lastError, n } = {}) {
|
|
61
|
+
// Explicit `false` only. `null`/`undefined` is unknown and never a fault —
|
|
62
|
+
// the same absence-of-evidence-is-not-a-fault rule the reachability and
|
|
63
|
+
// backend-health classifiers already follow.
|
|
64
|
+
const unreachable = !!declared && reachable === false;
|
|
65
|
+
if (!unreachable) return { unreachable: false, reasons: [] };
|
|
66
|
+
return {
|
|
67
|
+
unreachable: true,
|
|
68
|
+
reasons: [datastoreUnreachableReason({ type, target, shape, local, lastError, n })],
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// Which recovery even applies, given what KIND of datastore this is and where
|
|
73
|
+
// it lives. Its own function because "is it down?" and "what do you do about
|
|
74
|
+
// it?" are separate decisions, and this is the one that grows a branch every
|
|
75
|
+
// time a new datastore shape shows up.
|
|
76
|
+
//
|
|
77
|
+
// It follows the ingress-unwired precedent — the whole point of that reason was
|
|
78
|
+
// carrying the exact fix instead of a bare complaint — but deliberately stops
|
|
79
|
+
// short of naming a COMMAND. What an operator runs to start a datastore is a
|
|
80
|
+
// property of the VM image, not of this module; naming one this module cannot
|
|
81
|
+
// know is installed would be a confident lie on an amber card. What IS knowable,
|
|
82
|
+
// and what actually unblocks someone, is which recovery applies and where to act.
|
|
83
|
+
//
|
|
84
|
+
// The `file` branch is not cosmetic: a file database has no server to start, so
|
|
85
|
+
// routing it through the local branch would print the nonsensical "start it".
|
|
86
|
+
function recoveryClause({ shape, local, n } = {}) {
|
|
87
|
+
if (shape === 'file') {
|
|
88
|
+
return 'Recover: this is a file database and the file is missing — restore or recreate it on the VM.';
|
|
89
|
+
}
|
|
90
|
+
if (local) {
|
|
91
|
+
return `Recover: start it on VM-${n} — it is declared local, so nothing outside the VM can be at fault.`;
|
|
92
|
+
}
|
|
93
|
+
return 'Recover: this is a remote/managed datastore — check the datastore service and the VM\'s network path. There is nothing local to start.';
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// The single source for the `datastore-unreachable` reason string, mirroring
|
|
97
|
+
// fleet-public-url.js::ingressUnwiredReason. Named and exported so the card
|
|
98
|
+
// reason and any future log line cannot drift apart — the drift that a
|
|
99
|
+
// template literal inlined at its one call site invites the moment a second
|
|
100
|
+
// caller appears.
|
|
101
|
+
function datastoreUnreachableReason({ type, target, shape, local, lastError, n } = {}) {
|
|
102
|
+
const what = `${type || 'declared'} datastore${target ? ` at ${target}` : ''}`;
|
|
103
|
+
const why = lastError ? ` (${lastError})` : '';
|
|
104
|
+
return `datastore-unreachable: the ${what} is not answering${why}. ${recoveryClause({ shape, local, n })}`;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
module.exports = { classifyDatastoreHealth, recoveryClause, datastoreUnreachableReason };
|
package/npm/fleet-public-url.js
CHANGED
|
@@ -302,10 +302,32 @@ function ingressUnwiredReason({ url, domain, slug, n } = {}) {
|
|
|
302
302
|
* to all GFEs (VM5's CORRECT rule 503'd for ~minutes before flipping), so a
|
|
303
303
|
* per-cycle re-fire would storm the LB while a good change is still
|
|
304
304
|
* propagating. A missing/0 `lastRewireAt` means "never re-wired" → the window
|
|
305
|
-
* is satisfied.
|
|
305
|
+
* is satisfied.
|
|
306
|
+
*
|
|
307
|
+
* FOURTH CASE — `routed === 'unrouted'` BYPASSES both `reachable` and the
|
|
308
|
+
* `rewirable` backend test above, and that bypass is the whole point. A VM the
|
|
309
|
+
* url-map carries no host rule for has no route from its public hostname to
|
|
310
|
+
* anything, so it is unreachable BY CONSTRUCTION — gating its self-heal on
|
|
311
|
+
* `reachable` gates it on the absence of the very symptom it exists to repair,
|
|
312
|
+
* and it can therefore never fire. Observed 2026-09-08 on 3 of 3 freshly-added
|
|
313
|
+
* self-host VMs: NEGs created, backend services created, NEGs ATTACHED, and no
|
|
314
|
+
* host rule — the add's wiring tail died between the NEG attach and
|
|
315
|
+
* `reconcile_host_rule_once`. The backend test misses it for an independent
|
|
316
|
+
* reason: the backends WERE attached, so this reads as a genuinely-converging
|
|
317
|
+
* 'unhealthy' that the gate deliberately excludes. Both exclusions are correct
|
|
318
|
+
* for a VM that answers and is merely mis-routed, and both are exactly inverted
|
|
319
|
+
* for a VM with no route at all — hence a distinct entry condition rather than
|
|
320
|
+
* a relaxation of the existing one. Do NOT "restore" the `reachable`
|
|
321
|
+
* precondition on this branch as an obvious safety check; it is what made three
|
|
322
|
+
* plainly-broken VMs invisible to the automation for a day.
|
|
323
|
+
* 'unknown' (the url-map read errored) is NOT 'unrouted' — absence of evidence
|
|
324
|
+
* never authorizes a re-wire, matching classifyBackendAttachment's error case.
|
|
325
|
+
* The add's own wiring tail is still excluded here by `inFlight`: a VM mid-
|
|
326
|
+
* wiring is unrouted-so-far, not broken. */
|
|
306
327
|
function shouldAutoRewireIngress({
|
|
307
328
|
verdict,
|
|
308
329
|
reachable,
|
|
330
|
+
routed,
|
|
309
331
|
backendEditorHealth,
|
|
310
332
|
backendEditorBackendless,
|
|
311
333
|
inFlight,
|
|
@@ -314,21 +336,67 @@ function shouldAutoRewireIngress({
|
|
|
314
336
|
debounceMs,
|
|
315
337
|
} = {}) {
|
|
316
338
|
if (verdict !== 'unwired') return false;
|
|
317
|
-
|
|
318
|
-
//
|
|
319
|
-
//
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
339
|
+
// An unrouted VM skips the reachability + backend-health tests below (see the
|
|
340
|
+
// FOURTH CASE note above), but never the in-flight and debounce guards — those
|
|
341
|
+
// are what keep one re-wire per VM per propagation window.
|
|
342
|
+
const unrouted = routed === 'unrouted';
|
|
343
|
+
if (!unrouted) {
|
|
344
|
+
if (!reachable) return false;
|
|
345
|
+
// 'healthy' = host-rule drift (VM5); 'absent' = backend never created
|
|
346
|
+
// (interrupted-add VM-4); 'unhealthy' + backendless = BES exists with zero NEG
|
|
347
|
+
// backends (concurrent-add editor-band race) — it can't converge unaided, so it
|
|
348
|
+
// re-wires too. A genuinely-converging 'unhealthy' (backend attached, health
|
|
349
|
+
// flipping → backendless false) and 'unknown' stay excluded.
|
|
350
|
+
const rewirable =
|
|
351
|
+
backendEditorHealth === 'healthy' ||
|
|
352
|
+
backendEditorHealth === 'absent' ||
|
|
353
|
+
(backendEditorHealth === 'unhealthy' && backendEditorBackendless === true);
|
|
354
|
+
if (!rewirable) return false;
|
|
355
|
+
}
|
|
328
356
|
if (inFlight) return false;
|
|
329
357
|
return (Number(now) - (Number(lastRewireAt) || 0)) >= Number(debounceMs);
|
|
330
358
|
}
|
|
331
359
|
|
|
360
|
+
/** Classify whether the shared fleet url-map routes `host` at all — the input
|
|
361
|
+
* `shouldAutoRewireIngress`'s unrouted branch turns on, and a signal no backend
|
|
362
|
+
* probe can give. `backendEditorHealth` describes the backend SERVICE; none of
|
|
363
|
+
* its four values can express "nothing routes here", which is why all three of
|
|
364
|
+
* the 2026-09-08 stranded VMs classified as a converging 'unhealthy' while
|
|
365
|
+
* carrying no host rule at all. Reads:
|
|
366
|
+
*
|
|
367
|
+
* 'routed' — `host` appears verbatim among the url-map's host rules.
|
|
368
|
+
* 'unrouted' — the read SUCCEEDED, listed at least one host rule, and `host`
|
|
369
|
+
* is not among them. This is the only reading that authorizes a
|
|
370
|
+
* re-wire, and it requires positive evidence of a populated map.
|
|
371
|
+
* 'unknown' — the read errored, produced nothing, or no host was asked for.
|
|
372
|
+
*
|
|
373
|
+
* EMPTY OUTPUT IS 'unknown', NOT 'unrouted', and the distinction is load-
|
|
374
|
+
* bearing. A url-map that lists zero host rules is indistinguishable from a
|
|
375
|
+
* read that silently returned nothing, and reading it as 'unrouted' would
|
|
376
|
+
* authorize a simultaneous re-wire for EVERY VM in the fleet off a single
|
|
377
|
+
* unreliable read — the re-wire storm classifyBackendAttachment's error case
|
|
378
|
+
* exists to prevent, at fleet scale. Absence of evidence never manufactures a
|
|
379
|
+
* fault.
|
|
380
|
+
*
|
|
381
|
+
* Matching is by exact token, never substring: gcloud flattens
|
|
382
|
+
* `value(hostRules.hosts)` with whitespace and `;` separators, and a substring
|
|
383
|
+
* test would read `vm-2.editor.<domain>` as routed on the strength of an
|
|
384
|
+
* unrelated `staging.vm-2.editor.<domain>` rule.
|
|
385
|
+
*
|
|
386
|
+
* Takes the raw (err, stdout) pair rather than a VM number so it stays PURE and
|
|
387
|
+
* needs no domain/slug derivation — the same I/O-edge / pure-half split
|
|
388
|
+
* classifyBackendAttachment follows. The caller derives `host` from
|
|
389
|
+
* google-ingress's `urlMapHostRules` / vmUrls and does the gcloud read. */
|
|
390
|
+
function classifyUrlMapRouting(err, stdout, host) {
|
|
391
|
+
if (err) return 'unknown';
|
|
392
|
+
if (!host) return 'unknown';
|
|
393
|
+
const hosts = String(stdout == null ? '' : stdout)
|
|
394
|
+
.split(/[\s;,]+/)
|
|
395
|
+
.filter(Boolean);
|
|
396
|
+
if (hosts.length === 0) return 'unknown';
|
|
397
|
+
return hosts.includes(String(host)) ? 'routed' : 'unrouted';
|
|
398
|
+
}
|
|
399
|
+
|
|
332
400
|
/** Classify a `gcloud compute backend-services describe --format=value(backends[].group)`
|
|
333
401
|
* result into the attachment vocabulary — the signal `get-health` cannot give,
|
|
334
402
|
* because an empty health result is ambiguous between "no backend attached" and
|
|
@@ -390,5 +458,6 @@ module.exports = {
|
|
|
390
458
|
ingressUnwiredReason,
|
|
391
459
|
shouldAutoRewireIngress,
|
|
392
460
|
classifyBackendAttachment,
|
|
461
|
+
classifyUrlMapRouting,
|
|
393
462
|
classifyUrlWiringState,
|
|
394
463
|
};
|