@codeyam-editor/codeyam-editor 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/npm/cloud.js +477 -41
- package/npm/editor-dev.js +24 -2
- package/npm/editor.js +15 -5
- package/npm/fleet-add-failure.js +406 -25
- package/npm/fleet-branch.js +124 -1
- package/npm/fleet-build-data.js +12 -1
- package/npm/fleet-cleanup.js +84 -3
- package/npm/fleet-datastore-health.js +107 -0
- package/npm/fleet-memory.js +29 -4
- package/npm/fleet-queue.js +141 -0
- package/npm/fleet-revision.js +108 -0
- package/npm/fleet-run-setup.js +66 -0
- package/npm/fleet-session-activity.js +162 -0
- package/npm/fleet-vm-access.js +162 -0
- package/npm/fleet-vm-readiness.js +57 -16
- package/npm/ingress-invoke.js +46 -2
- package/npm/launch-config.js +169 -2
- package/npm/scenario-check.js +708 -35
- package/npm/scenario-handlers.js +18 -0
- package/npm/scenario-interactivity.js +477 -23
- package/npm/scenario-issues.js +72 -0
- package/npm/scenario-mocks.js +245 -39
- package/npm/scenario-playwright.js +137 -26
- package/npm/utils.js +262 -22
- package/package.json +5 -5
- package/ui/dist/assets/{ContainerInspector-Cu1tSN-o.js → ContainerInspector-C7I9wctz.js} +1 -1
- package/ui/dist/assets/EditorShell-9aYMeOdZ.js +330 -0
- package/ui/dist/assets/EditorShell-CE8iG_EM.css +1 -0
- package/ui/dist/assets/{ProjectLauncherScreen-Dmm6Z0mv.js → ProjectLauncherScreen-E0NYYiwy.js} +1 -1
- package/ui/dist/assets/ScenarioDataPanel-CLwkrex0.js +18 -0
- package/ui/dist/assets/{ScenarioDataPanelFullPage-BqMxWVKW.js → ScenarioDataPanelFullPage-DT0TZC4M.js} +1 -1
- package/ui/dist/assets/ScenarioRenderer-Bsarhcrl.js +21 -0
- package/ui/dist/assets/index-BRoAenc1.css +1 -0
- package/ui/dist/assets/index-iMR5siiG.js +27 -0
- package/ui/dist/assets/useServerIdentity-CVuHTzjg.js +1 -0
- package/ui/dist/index.html +2 -2
- package/ui/dist/preview-subdir-demo/index.html +62 -0
- package/ui/dist/preview-subdir-demo/sibling.html +48 -0
- package/ui/dist/assets/EditorShell--V2ExYQ6.js +0 -260
- package/ui/dist/assets/EditorShell-DP5UY20G.css +0 -1
- package/ui/dist/assets/ScenarioDataPanel-DjprZq2L.js +0 -9
- package/ui/dist/assets/ScenarioRenderer-DqLVVMYS.js +0 -21
- package/ui/dist/assets/index-CjxWhkSy.css +0 -1
- package/ui/dist/assets/index-VuyqBbGJ.js +0 -27
- package/ui/dist/assets/useServerIdentity-BdXy8TQv.js +0 -1
package/npm/fleet-branch.js
CHANGED
|
@@ -46,4 +46,127 @@ function resolveFleetBranch({ envBranch, gitBranch, fallback } = {}) {
|
|
|
46
46
|
return cleanBranch(fallback) || UNKNOWN_BRANCH;
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
-
|
|
49
|
+
// ---------- Dead-pin detection ----------
|
|
50
|
+
// `fleet-advance-to.sh` refuses to WRITE a pin for a branch absent from origin,
|
|
51
|
+
// but that guard runs once and nothing re-checks the pin already in place —
|
|
52
|
+
// while the branch it names can be deleted by anyone merging a PR with the
|
|
53
|
+
// delete-branch box ticked. Observed 2026-08-15: PR #104 merged
|
|
54
|
+
// `editor-improvements-72` and deleted it; hours later the operator host still
|
|
55
|
+
// read `FLEET_BRANCH=editor-improvements-72` and nothing said so.
|
|
56
|
+
//
|
|
57
|
+
// The probe itself (`git ls-remote`) is I/O and belongs to server.js, on the
|
|
58
|
+
// SLOW memo cadence rather than the ~3s /data poll. What lives here is the pure
|
|
59
|
+
// classification of its outcome — which is the part a test can actually reach,
|
|
60
|
+
// since a function defined in server.js can never be executed by a test (that
|
|
61
|
+
// file has top-level side effects on require).
|
|
62
|
+
//
|
|
63
|
+
// The tri-state mirrors scripts/operator/lib/branch-on-origin.sh exactly, and
|
|
64
|
+
// the mirroring is the point: `gone` is a DEFINITIVE absence, `indeterminate`
|
|
65
|
+
// means absence was NOT proven (no network, no remote, auth trouble). Widening
|
|
66
|
+
// `indeterminate` into `gone` would turn every offline operator host into a
|
|
67
|
+
// fleet-wide "your pin is dead" alarm.
|
|
68
|
+
|
|
69
|
+
/** Map a `git ls-remote --exit-code --heads` exit code to the shared tri-state.
|
|
70
|
+
* 0 -> 'live', 2 -> 'gone', anything else (including a non-numeric or absent
|
|
71
|
+
* code) -> 'indeterminate'. Pure. */
|
|
72
|
+
function lsRemoteStatus(exitCode) {
|
|
73
|
+
if (exitCode === 0) return 'live';
|
|
74
|
+
if (exitCode === 2) return 'gone';
|
|
75
|
+
return 'indeterminate';
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Classify the health of the fleet's branch pin.
|
|
79
|
+
*
|
|
80
|
+
* Returns `{ state, branch, pinned, dead, blocks, recovery }` where `state` is
|
|
81
|
+
* 'live' | 'gone' | 'indeterminate' | 'unknown'.
|
|
82
|
+
*
|
|
83
|
+
* 'unknown' is the un-probeable case — no branch, or the UNKNOWN_BRANCH
|
|
84
|
+
* sentinel that `resolveFleetBranch` yields when there is neither a pin nor a
|
|
85
|
+
* readable git HEAD. There is no ref to ask origin about, so reporting it as
|
|
86
|
+
* `gone` would invent a deletion that never happened.
|
|
87
|
+
*
|
|
88
|
+
* `dead` is true ONLY for 'gone'. It is the single field a caller should gate a
|
|
89
|
+
* degradation banner on, so no consumer has to re-derive which states are
|
|
90
|
+
* actionable.
|
|
91
|
+
*
|
|
92
|
+
* `blocks` names what a dead pin is actively breaking — both consequences,
|
|
93
|
+
* because neither announces itself: the operator-repo refresh cannot fetch a
|
|
94
|
+
* deleted ref, and the pin also drives `cloud.json editorBranch`, so newly
|
|
95
|
+
* provisioned VMs check out a ref that is not there. Pure; no I/O. */
|
|
96
|
+
function classifyBranchPin({ branch, lsRemoteExitCode, pinned = false } = {}) {
|
|
97
|
+
const name = cleanBranch(branch);
|
|
98
|
+
const probeable = name !== null && name !== UNKNOWN_BRANCH;
|
|
99
|
+
const state = probeable ? lsRemoteStatus(lsRemoteExitCode) : 'unknown';
|
|
100
|
+
const dead = state === 'gone';
|
|
101
|
+
return {
|
|
102
|
+
state,
|
|
103
|
+
branch: name,
|
|
104
|
+
pinned: Boolean(pinned),
|
|
105
|
+
dead,
|
|
106
|
+
blocks: dead
|
|
107
|
+
? [
|
|
108
|
+
'refresh-operator-repo.sh cannot fetch the pinned branch, so this host keeps serving whatever its clone already had',
|
|
109
|
+
'cloud.json editorBranch reads the same pin, so newly provisioned VMs check out a ref that does not exist',
|
|
110
|
+
]
|
|
111
|
+
: [],
|
|
112
|
+
recovery: dead
|
|
113
|
+
? `scripts/operator/fleet-advance-to.sh <BRANCH> — move the fleet to a live branch (it refuses a branch that is itself absent from origin)`
|
|
114
|
+
: null,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** Decide whether the fleet pin should override a stored per-VM branch when an
|
|
119
|
+
* add is RE-ISSUED, and report every override so the caller can log it.
|
|
120
|
+
*
|
|
121
|
+
* Returns `{ config, overrides }` — a NEW config (the input is never mutated)
|
|
122
|
+
* and `[{ field, from, to }]`, empty when nothing changed.
|
|
123
|
+
*
|
|
124
|
+
* The problem this solves: a per-VM config that recorded `editor-improvements-75`
|
|
125
|
+
* was not wrong when it was written — it EXPIRED, silently, the moment that
|
|
126
|
+
* branch merged and origin pruned it. This repo prunes on merge, so every stored
|
|
127
|
+
* feature-branch name has that property. The pin does not expire: it is the
|
|
128
|
+
* durable statement of fleet-wide policy, and on 2026-08-25 it was correct
|
|
129
|
+
* throughout while four retried cards each failed on their stale snapshot (a
|
|
130
|
+
* fresh pool add read the pin and succeeded at the same moment).
|
|
131
|
+
*
|
|
132
|
+
* Three guards, each declining to act rather than acting wrongly:
|
|
133
|
+
* - `selfHost` false ⇒ untouched. A client VM's branch is a deliberate
|
|
134
|
+
* per-project choice; overwriting it with a fleet pin would silently
|
|
135
|
+
* retarget someone else's project. Only the self-host path, where the branch
|
|
136
|
+
* IS fleet policy, falls back to the pin.
|
|
137
|
+
* - no pin, or the UNKNOWN_BRANCH sentinel ⇒ untouched. There is no branch to
|
|
138
|
+
* prefer, and stamping the sentinel would guarantee the failure this exists
|
|
139
|
+
* to prevent.
|
|
140
|
+
* - `pinState === 'gone'` ⇒ untouched. A pin PROVEN absent from origin is not
|
|
141
|
+
* an improvement on a stale-but-possibly-live stored branch; swapping one
|
|
142
|
+
* dead ref for another just moves the failure. Note the deliberate
|
|
143
|
+
* asymmetry with 'indeterminate', which does NOT decline: absence was not
|
|
144
|
+
* proven there, and refusing to converge every time the network blips would
|
|
145
|
+
* strand the fleet on stale branches for the sake of a maybe.
|
|
146
|
+
*
|
|
147
|
+
* Pure; no I/O. */
|
|
148
|
+
function pinCorrectedBranches({ cfg, pin, selfHost = false, pinState } = {}) {
|
|
149
|
+
const config = { ...(cfg || {}) };
|
|
150
|
+
const overrides = [];
|
|
151
|
+
const name = cleanBranch(pin);
|
|
152
|
+
if (!selfHost || !name || name === UNKNOWN_BRANCH || pinState === 'gone') {
|
|
153
|
+
return { config, overrides };
|
|
154
|
+
}
|
|
155
|
+
for (const field of ['clientBranch', 'editorBranch']) {
|
|
156
|
+
const from = cleanBranch(config[field]);
|
|
157
|
+
if (from && from !== name) {
|
|
158
|
+
config[field] = name;
|
|
159
|
+
overrides.push({ field, from, to: name });
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
return { config, overrides };
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
module.exports = {
|
|
166
|
+
UNKNOWN_BRANCH,
|
|
167
|
+
cleanBranch,
|
|
168
|
+
resolveFleetBranch,
|
|
169
|
+
lsRemoteStatus,
|
|
170
|
+
classifyBranchPin,
|
|
171
|
+
pinCorrectedBranches,
|
|
172
|
+
};
|
package/npm/fleet-build-data.js
CHANGED
|
@@ -120,6 +120,17 @@ function buildVmEnvelope(v, ctx) {
|
|
|
120
120
|
// unreachable VM's last-known config is stale, and an unreachable card has
|
|
121
121
|
// louder problems to report than its preview origin.
|
|
122
122
|
previewMisconfigured: (v.reachable && v.previewMisconfigured) || null,
|
|
123
|
+
// The project's DECLARED datastore and whether it is answering:
|
|
124
|
+
// { type, target, local, reachable, lastError }, or null when the project
|
|
125
|
+
// declares none (a Rust CLI, a static site) — which is the common case and
|
|
126
|
+
// must render nothing at all. `reachable` is tri-state exactly like
|
|
127
|
+
// previewServing/previewIngress above: true = probe reached it, false =
|
|
128
|
+
// confirmed down, null = unknown (editor just booted, or the probe went
|
|
129
|
+
// stale). Never conflate "down" with "unknown" — only a confirmed false
|
|
130
|
+
// raises the amber reason. Gated on `reachable` like providerAuth: an
|
|
131
|
+
// unreachable VM's last-known datastore verdict is stale, and it has louder
|
|
132
|
+
// problems to report.
|
|
133
|
+
datastore: (v.reachable && v.datastore) || null,
|
|
123
134
|
effects: v.effects || null,
|
|
124
135
|
// The VM's live-configured build agent (lowercase provider), reported by
|
|
125
136
|
// /api/session-info and merged (sticky) onto the poll state. Drives the
|
|
@@ -177,7 +188,7 @@ const REQUIRED_CARD_FIELDS = [
|
|
|
177
188
|
'degradedReasons', 'setupSteps', 'attentionReasons',
|
|
178
189
|
'isHead', 'headHolderUnreachable', 'queuePos', 'queueLen',
|
|
179
190
|
'job', 'config', 'project', 'variant', 'cardIdentity', 'brokerHistory',
|
|
180
|
-
'brokerDeath', 'providerAuth', 'previewMisconfigured', 'effects', 'currentProvider',
|
|
191
|
+
'brokerDeath', 'providerAuth', 'previewMisconfigured', 'datastore', 'effects', 'currentProvider',
|
|
181
192
|
'readiness', 'backendHealth', 'vmAccess',
|
|
182
193
|
'startedAt', 'featureStartedAt', 'attention', 'envOverride',
|
|
183
194
|
];
|
package/npm/fleet-cleanup.js
CHANGED
|
@@ -618,20 +618,37 @@ function reconcileRosteredVm({ exists, hasSession, hasPendingAdd, streak, resetH
|
|
|
618
618
|
// The dashboard's `state` and `addVm` are injected so this stays a pure,
|
|
619
619
|
// importable unit — `log`/`err` default to the console but are overridable in
|
|
620
620
|
// tests. Behavior mirrors the previous inline server.js definition exactly.
|
|
621
|
+
//
|
|
622
|
+
// The exhausted branch records a TERMINAL DISPOSITION, not just a flag. The flag
|
|
623
|
+
// alone was write-only: nothing destroys the instance and the only code that
|
|
624
|
+
// clears the flag is a *subsequent* recovery attempt, which by definition never
|
|
625
|
+
// comes once the budget is spent. VM-10 therefore ran empty and unrostered for a
|
|
626
|
+
// full day on 2026-08-29, billing, while the dashboard rendered a generic
|
|
627
|
+
// needs-manual-recovery card and clearing it needed hand-edited JSON.
|
|
628
|
+
// `instancePresumedRunning` says the GCE instance is believed to still exist —
|
|
629
|
+
// PRESUMED, because this ladder step never probes GCP; `exhaustedOrphanDisposition`
|
|
630
|
+
// is what reconciles the presumption against reality. `exhaustedAt` anchors the
|
|
631
|
+
// accrued-cost clock the card renders. `addVm` is still never called: that is the
|
|
632
|
+
// safety property that stops a wedged VM looping cloud:up forever.
|
|
621
633
|
function continueBootstrapRecovery(
|
|
622
634
|
n,
|
|
623
635
|
meta,
|
|
624
636
|
cfg,
|
|
625
|
-
{ state, addVm, log = console.log, err = console.error } = {},
|
|
637
|
+
{ state, addVm, log = console.log, err = console.error, now = Date.now } = {},
|
|
626
638
|
) {
|
|
627
639
|
const { attempt, proceed, max } = recoverBootstrapDecision({
|
|
628
640
|
recoverAttempts: meta.recoverAttempts,
|
|
629
641
|
});
|
|
630
642
|
if (!proceed) {
|
|
631
643
|
err(
|
|
632
|
-
`startup-reap: VM-${n} exceeded ${max} bootstrap-recovery attempts — leaving sidecar for manual action`,
|
|
644
|
+
`startup-reap: VM-${n} exceeded ${max} bootstrap-recovery attempts — instance presumed still running, leaving sidecar for manual action`,
|
|
633
645
|
);
|
|
634
|
-
state.orphans[n] = {
|
|
646
|
+
state.orphans[n] = {
|
|
647
|
+
reason: "bootstrap-recovery-exhausted",
|
|
648
|
+
attempts: attempt - 1,
|
|
649
|
+
instancePresumedRunning: true,
|
|
650
|
+
exhaustedAt: now(),
|
|
651
|
+
};
|
|
635
652
|
return;
|
|
636
653
|
}
|
|
637
654
|
log(
|
|
@@ -643,6 +660,68 @@ function continueBootstrapRecovery(
|
|
|
643
660
|
addVm(n, cfg, { recovering: true, recoverAttempt: attempt });
|
|
644
661
|
}
|
|
645
662
|
|
|
663
|
+
// The exit `continueBootstrapRecovery`'s exhausted flag never had. That flag is
|
|
664
|
+
// written once and, until this verdict existed, could only be cleared by a
|
|
665
|
+
// subsequent recovery attempt the spent budget guarantees will never run — so it
|
|
666
|
+
// outlived the instance it described and had to be removed by hand-editing
|
|
667
|
+
// `.codeyam/logs/dashboard-state.json`. This reconciles the flag's PRESUMPTION
|
|
668
|
+
// (`instancePresumedRunning`) against two live facts the sweep supplies:
|
|
669
|
+
//
|
|
670
|
+
// instanceExists true ⇒ the presumption holds, the box is still billing
|
|
671
|
+
// false ⇒ the instance is gone (destroyed out-of-band) → 'clear',
|
|
672
|
+
// so the card self-heals with no operator action
|
|
673
|
+
// null ⇒ the describe probe was INCONCLUSIVE. Never act on
|
|
674
|
+
// absence of evidence — same discipline reapVerdict
|
|
675
|
+
// applies to its own `exists === null`.
|
|
676
|
+
// rostered false ⇒ flagged, billing, and in nobody's roster: invisible to
|
|
677
|
+
// every roster-driven tool yet costing money → 'cost',
|
|
678
|
+
// the distinct ownerless surface, NOT a generic
|
|
679
|
+
// needs-manual-recovery card.
|
|
680
|
+
// true ⇒ still owned, so it is recoverable → 'keep'.
|
|
681
|
+
// null ⇒ roster unknown; keep rather than raise a cost alarm
|
|
682
|
+
// that may be false (positive-signal-only, mirroring
|
|
683
|
+
// how projectWired/agentReady downgrade).
|
|
684
|
+
//
|
|
685
|
+
// Deliberately NOT a destroy verdict. Destroying a VM is irreversible and a
|
|
686
|
+
// half-provisioned box occasionally holds the clue that explains the failure, so
|
|
687
|
+
// the terminal disposition is "surface it loudly and offer the button" — the
|
|
688
|
+
// operator presses it.
|
|
689
|
+
function exhaustedOrphanDisposition({ reason, instanceExists, rostered } = {}) {
|
|
690
|
+
if (reason !== 'bootstrap-recovery-exhausted') return 'leave';
|
|
691
|
+
if (instanceExists === null || instanceExists === undefined) return 'leave';
|
|
692
|
+
if (!instanceExists) return 'clear';
|
|
693
|
+
return rostered === false ? 'cost' : 'keep';
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
// Fold a 'cost'/'keep' verdict into the orphan entry the card renders. Split out
|
|
697
|
+
// of the dashboard's poller because the stickiness rule below is the only real
|
|
698
|
+
// logic in that branch, and inlined in server.js it was unreachable by any test:
|
|
699
|
+
// the dashboard has top-level side effects on require, so its pollers can only be
|
|
700
|
+
// covered by a source-scan wiring guard that cannot evaluate behavior.
|
|
701
|
+
//
|
|
702
|
+
// `ownerlessSince` dates the accrued cost from when the VM was FIRST confirmed
|
|
703
|
+
// ownerless, not from the latest probe — re-stamping it every tick would peg the
|
|
704
|
+
// card's elapsed time near zero forever and hide exactly the multi-day cost the
|
|
705
|
+
// surface exists to reveal. It is dropped (not zeroed, not left stale) the moment
|
|
706
|
+
// a VM stops being ownerless — an operator adopting it back into the roster flips
|
|
707
|
+
// the verdict to 'keep', and a leftover `ownerlessSince` would let the cost card
|
|
708
|
+
// resurrect itself with a fabricated start time if the VM ever went ownerless
|
|
709
|
+
// again. `instanceConfirmedAt` records that the presumption was checked live.
|
|
710
|
+
//
|
|
711
|
+
// Returns a NEW entry; the caller decides where to store it. Both verdicts keep
|
|
712
|
+
// the flag — the instance is real and only an operator may destroy it.
|
|
713
|
+
function annotateOwnerlessCost(orphan, { verdict, now = Date.now() } = {}) {
|
|
714
|
+
const next = { ...orphan, instancePresumedRunning: true, instanceConfirmedAt: now };
|
|
715
|
+
if (verdict === 'cost') {
|
|
716
|
+
next.ownerlessCost = true;
|
|
717
|
+
next.ownerlessSince = (orphan && orphan.ownerlessSince) || now;
|
|
718
|
+
return next;
|
|
719
|
+
}
|
|
720
|
+
next.ownerlessCost = false;
|
|
721
|
+
delete next.ownerlessSince;
|
|
722
|
+
return next;
|
|
723
|
+
}
|
|
724
|
+
|
|
646
725
|
module.exports = {
|
|
647
726
|
parsePendingJobFilename,
|
|
648
727
|
persistPendingJob,
|
|
@@ -656,6 +735,8 @@ module.exports = {
|
|
|
656
735
|
classifyLbReachableForReap,
|
|
657
736
|
recoverBootstrapDecision,
|
|
658
737
|
continueBootstrapRecovery,
|
|
738
|
+
exhaustedOrphanDisposition,
|
|
739
|
+
annotateOwnerlessCost,
|
|
659
740
|
exitMarkerName,
|
|
660
741
|
parseExitMarker,
|
|
661
742
|
readExitMarker,
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// npm/fleet-datastore-health.js — pure classifier for the fleet dashboard's
|
|
4
|
+
// "declared datastore unreachable" surface: is the datastore a VM's project
|
|
5
|
+
// DECLARES actually answering?
|
|
6
|
+
//
|
|
7
|
+
// Nothing anywhere reported this, so a VM whose database was down read
|
|
8
|
+
// perfectly green. On VM-12 (2026-08-24) the dashboard, the LB backend health,
|
|
9
|
+
// the IAP edge and the editor's own server-identity all reported healthy while
|
|
10
|
+
// every DB-backed route returned 500 — the only evidence was `ECONNREFUSED
|
|
11
|
+
// 127.0.0.1:5432` buried in the container log, which nobody polls. The
|
|
12
|
+
// 2026-08-14 occurrence on the same VM presented as previewServing=false with an
|
|
13
|
+
// EMPTY degradedReasons list: the dashboard knew something was wrong and had
|
|
14
|
+
// nothing to say about it. This module is the "something to say".
|
|
15
|
+
//
|
|
16
|
+
// The editor probes (it is the only party that knows what the project declares,
|
|
17
|
+
// and the resolution already exists in-crate) and carries the result on the
|
|
18
|
+
// existing session-info poll; this module decides, and server.js renders. Pure
|
|
19
|
+
// like fleet-project-drift.js: all the decision logic lives here and unit-tests
|
|
20
|
+
// without a VM, while server.js stays the thin I/O shell.
|
|
21
|
+
//
|
|
22
|
+
// A new `datastore-unreachable` code needs NO change to
|
|
23
|
+
// fleet-setup-steps.js::classifyReasons — it routes unknown codes to
|
|
24
|
+
// attentionReasons by design ("an unrecognized signal is surfaced loudly, never
|
|
25
|
+
// silently hidden"), so this lands amber. It must NOT be added to SETUP_CODES: a
|
|
26
|
+
// dead datastore is never expected first-run wiring.
|
|
27
|
+
|
|
28
|
+
// Decide the datastore-health reasons for one VM.
|
|
29
|
+
//
|
|
30
|
+
// declared truthy when the project declares a datastore at all. A project
|
|
31
|
+
// with none (a Rust CLI, a static site) must never raise a reason.
|
|
32
|
+
// reachable TRI-STATE, matching the editor's own discipline: true = the probe
|
|
33
|
+
// reached it, false = the probe failed, null/undefined = unknown
|
|
34
|
+
// (not probed yet, or the probe went stale). Only an explicit
|
|
35
|
+
// `false` raises a reason — collapsing unknown into false would
|
|
36
|
+
// light up every VM during the window between editor boot and the
|
|
37
|
+
// first probe, which is the failure mode that trains an operator to
|
|
38
|
+
// ignore the surface.
|
|
39
|
+
// target credential-free rendering of where it lives ("127.0.0.1:5432/app",
|
|
40
|
+
// or a file path). Never the DSN — that carries credentials.
|
|
41
|
+
// shape 'file' (a database file on disk) or 'network' (a server on a
|
|
42
|
+
// socket). Published by the editor rather than inferred from `type`
|
|
43
|
+
// here, because "sqlite means file-shaped" is a per-stack fact this
|
|
44
|
+
// module would otherwise have to learn for every engine. It decides
|
|
45
|
+
// WHICH recovery applies at all: a down server gets started, a
|
|
46
|
+
// missing file gets restored.
|
|
47
|
+
// local whether the datastore is something this VM is expected to host
|
|
48
|
+
// (loopback host, or a file database). Decides where the recovery
|
|
49
|
+
// points; a remote/managed datastore is still reported, but telling
|
|
50
|
+
// an operator to start it here would send them somewhere there is
|
|
51
|
+
// nothing to do.
|
|
52
|
+
// type the declared engine ('postgresql' | 'sqlite' | 'mysql' | …), used
|
|
53
|
+
// only to name what is down.
|
|
54
|
+
// lastError the probe's own failure string ("connection refused: …"), so the
|
|
55
|
+
// card says WHY rather than just that.
|
|
56
|
+
// n the VM number, so a local recovery can name the VM to act on.
|
|
57
|
+
//
|
|
58
|
+
// Returns { unreachable, reasons }. `reasons` is [] whenever healthy, unknown, or
|
|
59
|
+
// nothing is declared, so the caller spreads it unconditionally.
|
|
60
|
+
function classifyDatastoreHealth({ declared, reachable, target, shape, local, type, lastError, n } = {}) {
|
|
61
|
+
// Explicit `false` only. `null`/`undefined` is unknown and never a fault —
|
|
62
|
+
// the same absence-of-evidence-is-not-a-fault rule the reachability and
|
|
63
|
+
// backend-health classifiers already follow.
|
|
64
|
+
const unreachable = !!declared && reachable === false;
|
|
65
|
+
if (!unreachable) return { unreachable: false, reasons: [] };
|
|
66
|
+
return {
|
|
67
|
+
unreachable: true,
|
|
68
|
+
reasons: [datastoreUnreachableReason({ type, target, shape, local, lastError, n })],
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// Which recovery even applies, given what KIND of datastore this is and where
|
|
73
|
+
// it lives. Its own function because "is it down?" and "what do you do about
|
|
74
|
+
// it?" are separate decisions, and this is the one that grows a branch every
|
|
75
|
+
// time a new datastore shape shows up.
|
|
76
|
+
//
|
|
77
|
+
// It follows the ingress-unwired precedent — the whole point of that reason was
|
|
78
|
+
// carrying the exact fix instead of a bare complaint — but deliberately stops
|
|
79
|
+
// short of naming a COMMAND. What an operator runs to start a datastore is a
|
|
80
|
+
// property of the VM image, not of this module; naming one this module cannot
|
|
81
|
+
// know is installed would be a confident lie on an amber card. What IS knowable,
|
|
82
|
+
// and what actually unblocks someone, is which recovery applies and where to act.
|
|
83
|
+
//
|
|
84
|
+
// The `file` branch is not cosmetic: a file database has no server to start, so
|
|
85
|
+
// routing it through the local branch would print the nonsensical "start it".
|
|
86
|
+
function recoveryClause({ shape, local, n } = {}) {
|
|
87
|
+
if (shape === 'file') {
|
|
88
|
+
return 'Recover: this is a file database and the file is missing — restore or recreate it on the VM.';
|
|
89
|
+
}
|
|
90
|
+
if (local) {
|
|
91
|
+
return `Recover: start it on VM-${n} — it is declared local, so nothing outside the VM can be at fault.`;
|
|
92
|
+
}
|
|
93
|
+
return 'Recover: this is a remote/managed datastore — check the datastore service and the VM\'s network path. There is nothing local to start.';
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// The single source for the `datastore-unreachable` reason string, mirroring
|
|
97
|
+
// fleet-public-url.js::ingressUnwiredReason. Named and exported so the card
|
|
98
|
+
// reason and any future log line cannot drift apart — the drift that a
|
|
99
|
+
// template literal inlined at its one call site invites the moment a second
|
|
100
|
+
// caller appears.
|
|
101
|
+
function datastoreUnreachableReason({ type, target, shape, local, lastError, n } = {}) {
|
|
102
|
+
const what = `${type || 'declared'} datastore${target ? ` at ${target}` : ''}`;
|
|
103
|
+
const why = lastError ? ` (${lastError})` : '';
|
|
104
|
+
return `datastore-unreachable: the ${what} is not answering${why}. ${recoveryClause({ shape, local, n })}`;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
module.exports = { classifyDatastoreHealth, recoveryClause, datastoreUnreachableReason };
|
package/npm/fleet-memory.js
CHANGED
|
@@ -290,8 +290,16 @@ function restoreFromSnapshot({ snapshot, maxAgeMs, nowIso, sidecarKeys } = {}) {
|
|
|
290
290
|
* Degrades to 'cgroup=unavailable' rather than printing zeros off-cgroup, so a
|
|
291
291
|
* laptop log cannot be misread as a cgroup at 0MB.
|
|
292
292
|
*
|
|
293
|
+
* `detachedWiringCount` — when supplied — appends `tails=N`, which is what makes
|
|
294
|
+
* the RSS/cgroup gap ATTRIBUTABLE rather than merely visible. On 2026-08-11 the
|
|
295
|
+
* cgroup read 2154MB against a 96MB RSS and the whole diagnosis was "those ~2GB
|
|
296
|
+
* are this process's own transient add children, not a leak". `RSS=96MB
|
|
297
|
+
* cgroup=2154MB tails=4` reads as a batch draining on its own; `RSS=1300MB
|
|
298
|
+
* cgroup=1400MB tails=0` reads as a genuine leak. Omitted (the pre-existing
|
|
299
|
+
* callers, and any host with no tail accounting) prints exactly as before.
|
|
300
|
+
*
|
|
293
301
|
* Pure; no I/O. */
|
|
294
|
-
function formatMemoryText({ rssBytes, memoryCurrent, memoryHigh } = {}) {
|
|
302
|
+
function formatMemoryText({ rssBytes, memoryCurrent, memoryHigh, detachedWiringCount } = {}) {
|
|
295
303
|
const mb = (b) => (Number(b) / 1024 / 1024).toFixed(0);
|
|
296
304
|
const current = Number(memoryCurrent);
|
|
297
305
|
const high = Number(memoryHigh);
|
|
@@ -308,6 +316,8 @@ function formatMemoryText({ rssBytes, memoryCurrent, memoryHigh } = {}) {
|
|
|
308
316
|
} else {
|
|
309
317
|
parts.push('cgroup=unavailable');
|
|
310
318
|
}
|
|
319
|
+
const tails = Number(detachedWiringCount);
|
|
320
|
+
if (Number.isFinite(tails) && tails >= 0) parts.push(`tails=${tails}`);
|
|
311
321
|
return parts.join(' ');
|
|
312
322
|
}
|
|
313
323
|
|
|
@@ -323,11 +333,26 @@ function formatMemoryText({ rssBytes, memoryCurrent, memoryHigh } = {}) {
|
|
|
323
333
|
* protecting. The dashboard's own self-update counts, being a heavy job by any
|
|
324
334
|
* other name.
|
|
325
335
|
*
|
|
326
|
-
*
|
|
327
|
-
|
|
336
|
+
* `detachedWiringCount` is the THIRD source of busy, and the one this guard was
|
|
337
|
+
* blind to: an Add settles at container-healthy while its detached wire-add-tail
|
|
338
|
+
* keeps spawning gcloud children for another 5-10 min in the SAME cgroup. Judging
|
|
339
|
+
* "safe to restart" off `state.jobs` alone reported no job in flight while that
|
|
340
|
+
* work was live — on 2026-08-11 the escalation crossed the restart threshold at
|
|
341
|
+
* 86% of memory.high and the only thing that held it back was an unrelated job
|
|
342
|
+
* happening to still be `running`. Had it settled thirty seconds earlier the
|
|
343
|
+
* dashboard would have restarted itself mid-batch and orphaned four adds, which
|
|
344
|
+
* is precisely the failure class the detached-provision work exists to prevent.
|
|
345
|
+
* Omitting the key behaves exactly as before, so every current caller is
|
|
346
|
+
* unaffected.
|
|
347
|
+
*
|
|
348
|
+
* Pure; no I/O — the caller passes state.jobs / state.dashboardUpdate and the
|
|
349
|
+
* live-tail count. */
|
|
350
|
+
function hasJobInFlight({ jobs, dashboardUpdate, detachedWiringCount } = {}) {
|
|
328
351
|
const busy = (j) => !!j && (j.state === 'running' || j.state === 'recovering');
|
|
329
352
|
const anyJob = Object.values((jobs && typeof jobs === 'object') ? jobs : {}).some(busy);
|
|
330
|
-
|
|
353
|
+
const tails = Number(detachedWiringCount);
|
|
354
|
+
const anyTail = Number.isFinite(tails) && tails > 0;
|
|
355
|
+
return anyJob || anyTail || !!(dashboardUpdate && dashboardUpdate.state === 'running');
|
|
331
356
|
}
|
|
332
357
|
|
|
333
358
|
/** Should this escalation level be announced to the journal?
|
package/npm/fleet-queue.js
CHANGED
|
@@ -126,9 +126,150 @@ function reconcileQueueDraft(draftOrder, liveOrder) {
|
|
|
126
126
|
return out;
|
|
127
127
|
}
|
|
128
128
|
|
|
129
|
+
// Consecutive failed reads before the panel stops trusting its mirror. At the
|
|
130
|
+
// default QUEUE_POLL_MS of 5000 this is ~15s of sustained failure — long enough
|
|
131
|
+
// that the ordinary 20s-timeout blip the next tick clears only ever reaches
|
|
132
|
+
// 'degraded', short enough that a real credential outage is flagged in seconds.
|
|
133
|
+
// Named rather than inlined so the verdict and the tooltip copy read the same
|
|
134
|
+
// number from one place.
|
|
135
|
+
const QUEUE_STALE_AFTER_FAILURES = 3;
|
|
136
|
+
|
|
137
|
+
/** Merge one commit-queue poll result over the previous memo.
|
|
138
|
+
*
|
|
139
|
+
* THE RULE: a failed read RETAINS the previous `entries` AND the previous
|
|
140
|
+
* `readAt`, and records `error`. Retaining the entries is deliberate — a stale
|
|
141
|
+
* order beats a blank panel for an operator mid-triage. Leaving `readAt`
|
|
142
|
+
* untouched is what makes the staleness legible: it is the ONLY signal that
|
|
143
|
+
* distinguishes a frozen view from a live one.
|
|
144
|
+
*
|
|
145
|
+
* Advancing `readAt` on a failure is precisely the bug this exists to prevent.
|
|
146
|
+
* On 2026-08-14 /etc/codeyam/git-credentials on the operator host went to 0
|
|
147
|
+
* bytes and every origin git operation began failing; the queue panel went on
|
|
148
|
+
* naming VM-5 as HEAD for 3.5 hours after VM-5 had released, while the real
|
|
149
|
+
* queue was VM-2 -> VM-8 -> VM-4. Nothing on the page distinguished it from
|
|
150
|
+
* live. This is the same lesson mergeDashboardRevision records from
|
|
151
|
+
* 2026-07-31, applied to the second poller.
|
|
152
|
+
*
|
|
153
|
+
* A previous `readAt` of null (no successful read has EVER happened) stays
|
|
154
|
+
* null and is not coerced to a timestamp — "never read" and "read a moment
|
|
155
|
+
* ago" must stay distinguishable, the same reason mergeDashboardRevision
|
|
156
|
+
* refuses to push a null `behind` through Number().
|
|
157
|
+
*
|
|
158
|
+
* Success is the absence of `error`. Pure; no I/O. */
|
|
159
|
+
function mergeQueuePoll({ prev, entries, readAt, error } = {}) {
|
|
160
|
+
const base = (prev && typeof prev === 'object') ? prev : {};
|
|
161
|
+
const str = (v) => ((typeof v === 'string' && v) ? v : null);
|
|
162
|
+
const prevEntries = Array.isArray(base.entries) ? base.entries : [];
|
|
163
|
+
const prevReadAt = str(base.readAt);
|
|
164
|
+
const prevFailures = (typeof base.consecutiveFailures === 'number'
|
|
165
|
+
&& Number.isFinite(base.consecutiveFailures) && base.consecutiveFailures >= 0)
|
|
166
|
+
? base.consecutiveFailures : 0;
|
|
167
|
+
const err = str(error);
|
|
168
|
+
if (err) {
|
|
169
|
+
return {
|
|
170
|
+
entries: prevEntries,
|
|
171
|
+
readAt: prevReadAt,
|
|
172
|
+
error: err,
|
|
173
|
+
consecutiveFailures: prevFailures + 1,
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
return {
|
|
177
|
+
entries: Array.isArray(entries) ? entries : [],
|
|
178
|
+
readAt: str(readAt) || prevReadAt,
|
|
179
|
+
error: null,
|
|
180
|
+
consecutiveFailures: 0,
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** What does the current queue memo mean for the operator?
|
|
185
|
+
*
|
|
186
|
+
* Levels:
|
|
187
|
+
* 'fresh' — the last read succeeded. Render exactly as before; every
|
|
188
|
+
* control stays live.
|
|
189
|
+
* 'degraded' — erroring, but under QUEUE_STALE_AFTER_FAILURES consecutive
|
|
190
|
+
* failures. A blip. Noted quietly, controls stay live — a
|
|
191
|
+
* single 20s-timeout hiccup that the next tick clears must not
|
|
192
|
+
* cry wolf or lock the operator out of their own queue.
|
|
193
|
+
* 'stale' — erroring at or past the threshold. The mirror is not to be
|
|
194
|
+
* trusted and writes computed against it are blocked.
|
|
195
|
+
* 'unknown' — erroring and NO read has ever succeeded (the boot window with
|
|
196
|
+
* broken credentials). Worth saying out loud: an empty-looking
|
|
197
|
+
* queue here means "we cannot see the queue", not "go".
|
|
198
|
+
*
|
|
199
|
+
* AGE IS DELIBERATELY NOT AN INPUT. A genuinely quiet queue is not a broken
|
|
200
|
+
* one, so escalation keys on consecutive read FAILURES only; age is carried by
|
|
201
|
+
* projectQueueFreshness purely to phrase the message ("last read 3h 12m ago").
|
|
202
|
+
* An hours-old `readAt` with no error is 'fresh' and must stay that way. Taking
|
|
203
|
+
* a clock here would imply otherwise, so the signature omits one — the same
|
|
204
|
+
* take-only-what-you-use shape as classifyDashboardStaleness.
|
|
205
|
+
*
|
|
206
|
+
* Pure; no I/O. */
|
|
207
|
+
function classifyQueueFreshness({ readAt, error, consecutiveFailures } = {}) {
|
|
208
|
+
if (!(typeof error === 'string' && error)) return 'fresh';
|
|
209
|
+
if (!(typeof readAt === 'string' && readAt)) return 'unknown';
|
|
210
|
+
const failures = (typeof consecutiveFailures === 'number' && Number.isFinite(consecutiveFailures))
|
|
211
|
+
? consecutiveFailures : 0;
|
|
212
|
+
return failures >= QUEUE_STALE_AFTER_FAILURES ? 'stale' : 'degraded';
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/** The `/data` view of the queue memo, with the freshness VERDICT resolved
|
|
216
|
+
* server-side.
|
|
217
|
+
*
|
|
218
|
+
* The panel is then a pure renderer. Two consumers re-deriving "is this stale"
|
|
219
|
+
* from a raw timestamp would drift, and `public/index.html` is an inline
|
|
220
|
+
* browser script with no module boundary — any decision left in it could only
|
|
221
|
+
* ever be source-scan tested, which is how it ships bugs green. Same split, and
|
|
222
|
+
* same reason, as projectDashboardRevision.
|
|
223
|
+
*
|
|
224
|
+
* `now` is injected (epoch ms) rather than read from the clock so the
|
|
225
|
+
* projection stays pure and its tests deterministic. `ageSec` is null when no
|
|
226
|
+
* successful read has happened or no clock was supplied.
|
|
227
|
+
*
|
|
228
|
+
* Pure; no I/O. */
|
|
229
|
+
function projectQueueFreshness(memo = {}, now = null) {
|
|
230
|
+
const at = (typeof memo.readAt === 'string' && memo.readAt) ? Date.parse(memo.readAt) : NaN;
|
|
231
|
+
const clock = (typeof now === 'number' && Number.isFinite(now)) ? now : null;
|
|
232
|
+
const ageSec = (Number.isFinite(at) && clock !== null)
|
|
233
|
+
? Math.max(0, Math.floor((clock - at) / 1000))
|
|
234
|
+
: null;
|
|
235
|
+
return { ...memo, level: classifyQueueFreshness(memo), ageSec };
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/** The operator-facing reason a queue WRITE must be refused, or null when writes
|
|
239
|
+
* are allowed.
|
|
240
|
+
*
|
|
241
|
+
* A write computed against a mirror known to be frozen is never the right
|
|
242
|
+
* action, and the silent-no-op outcome is worse than a refusal: a Release
|
|
243
|
+
* fires at a uuid that no longer holds the head, reports "released head
|
|
244
|
+
* bb65addf", changes nothing, and leaves the real head held. A reorder is
|
|
245
|
+
* worse still — computeReorderTargets validates the posted order against the
|
|
246
|
+
* very mirror that is wrong, so a doomed reorder looks perfectly valid.
|
|
247
|
+
*
|
|
248
|
+
* The verbatim git error is interpolated rather than summarised to "poll
|
|
249
|
+
* failed". On 2026-08-14 the entire recovery was legible from one line —
|
|
250
|
+
* `could not read Username for 'https://github.com'` — and truncating it would
|
|
251
|
+
* have cost the same 3.5 hours.
|
|
252
|
+
*
|
|
253
|
+
* Lives here rather than in server.js so the refusal COPY is directly
|
|
254
|
+
* assertable: server.js has top-level side effects on require, so a message
|
|
255
|
+
* built there could only ever be source-scan tested. Pure; no I/O. */
|
|
256
|
+
function queueWriteBlockReason(memo = {}) {
|
|
257
|
+
const level = classifyQueueFreshness(memo);
|
|
258
|
+
if (level !== 'stale' && level !== 'unknown') return null;
|
|
259
|
+
const why = (typeof memo.error === 'string' && memo.error) ? ` (${memo.error})` : '';
|
|
260
|
+
return level === 'unknown'
|
|
261
|
+
? `commit queue has never been read successfully${why} — cannot act on an unknown queue`
|
|
262
|
+
: `commit queue read is failing${why} — the displayed order cannot be trusted; fix the read before changing the queue`;
|
|
263
|
+
}
|
|
264
|
+
|
|
129
265
|
module.exports = {
|
|
130
266
|
decideAutoReleaseGoneHead,
|
|
131
267
|
computeReorderTargets,
|
|
132
268
|
nudgeQueueOrder,
|
|
133
269
|
reconcileQueueDraft,
|
|
270
|
+
mergeQueuePoll,
|
|
271
|
+
classifyQueueFreshness,
|
|
272
|
+
projectQueueFreshness,
|
|
273
|
+
queueWriteBlockReason,
|
|
274
|
+
QUEUE_STALE_AFTER_FAILURES,
|
|
134
275
|
};
|