@codeyam-editor/codeyam-editor 0.1.7 → 0.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/npm/cloud.js +477 -41
  2. package/npm/editor-dev.js +24 -2
  3. package/npm/editor.js +15 -5
  4. package/npm/fleet-add-failure.js +406 -25
  5. package/npm/fleet-branch.js +124 -1
  6. package/npm/fleet-build-data.js +12 -1
  7. package/npm/fleet-cleanup.js +84 -3
  8. package/npm/fleet-datastore-health.js +107 -0
  9. package/npm/fleet-memory.js +29 -4
  10. package/npm/fleet-queue.js +141 -0
  11. package/npm/fleet-revision.js +108 -0
  12. package/npm/fleet-run-setup.js +66 -0
  13. package/npm/fleet-session-activity.js +162 -0
  14. package/npm/fleet-vm-access.js +162 -0
  15. package/npm/fleet-vm-readiness.js +57 -16
  16. package/npm/ingress-invoke.js +46 -2
  17. package/npm/launch-config.js +169 -2
  18. package/npm/scenario-check.js +708 -35
  19. package/npm/scenario-handlers.js +18 -0
  20. package/npm/scenario-interactivity.js +477 -23
  21. package/npm/scenario-issues.js +72 -0
  22. package/npm/scenario-mocks.js +245 -39
  23. package/npm/scenario-playwright.js +137 -26
  24. package/npm/utils.js +262 -22
  25. package/package.json +5 -5
  26. package/ui/dist/assets/{ContainerInspector-Cu1tSN-o.js → ContainerInspector-C7I9wctz.js} +1 -1
  27. package/ui/dist/assets/EditorShell-9aYMeOdZ.js +330 -0
  28. package/ui/dist/assets/EditorShell-CE8iG_EM.css +1 -0
  29. package/ui/dist/assets/{ProjectLauncherScreen-Dmm6Z0mv.js → ProjectLauncherScreen-E0NYYiwy.js} +1 -1
  30. package/ui/dist/assets/ScenarioDataPanel-CLwkrex0.js +18 -0
  31. package/ui/dist/assets/{ScenarioDataPanelFullPage-BqMxWVKW.js → ScenarioDataPanelFullPage-DT0TZC4M.js} +1 -1
  32. package/ui/dist/assets/ScenarioRenderer-Bsarhcrl.js +21 -0
  33. package/ui/dist/assets/index-BRoAenc1.css +1 -0
  34. package/ui/dist/assets/index-iMR5siiG.js +27 -0
  35. package/ui/dist/assets/useServerIdentity-CVuHTzjg.js +1 -0
  36. package/ui/dist/index.html +2 -2
  37. package/ui/dist/preview-subdir-demo/index.html +62 -0
  38. package/ui/dist/preview-subdir-demo/sibling.html +48 -0
  39. package/ui/dist/assets/EditorShell--V2ExYQ6.js +0 -260
  40. package/ui/dist/assets/EditorShell-DP5UY20G.css +0 -1
  41. package/ui/dist/assets/ScenarioDataPanel-DjprZq2L.js +0 -9
  42. package/ui/dist/assets/ScenarioRenderer-DqLVVMYS.js +0 -21
  43. package/ui/dist/assets/index-CjxWhkSy.css +0 -1
  44. package/ui/dist/assets/index-VuyqBbGJ.js +0 -27
  45. package/ui/dist/assets/useServerIdentity-BdXy8TQv.js +0 -1
@@ -46,4 +46,127 @@ function resolveFleetBranch({ envBranch, gitBranch, fallback } = {}) {
46
46
  return cleanBranch(fallback) || UNKNOWN_BRANCH;
47
47
  }
48
48
 
49
- module.exports = { UNKNOWN_BRANCH, cleanBranch, resolveFleetBranch };
49
+ // ---------- Dead-pin detection ----------
50
+ // `fleet-advance-to.sh` refuses to WRITE a pin for a branch absent from origin,
51
+ // but that guard runs once and nothing re-checks the pin already in place —
52
+ // while the branch it names can be deleted by anyone merging a PR with the
53
+ // delete-branch box ticked. Observed 2026-08-15: PR #104 merged
54
+ // `editor-improvements-72` and deleted it; hours later the operator host still
55
+ // read `FLEET_BRANCH=editor-improvements-72` and nothing said so.
56
+ //
57
+ // The probe itself (`git ls-remote`) is I/O and belongs to server.js, on the
58
+ // SLOW memo cadence rather than the ~3s /data poll. What lives here is the pure
59
+ // classification of its outcome — which is the part a test can actually reach,
60
+ // since a function defined in server.js can never be executed by a test (that
61
+ // file has top-level side effects on require).
62
+ //
63
+ // The tri-state mirrors scripts/operator/lib/branch-on-origin.sh exactly, and
64
+ // the mirroring is the point: `gone` is a DEFINITIVE absence, `indeterminate`
65
+ // means absence was NOT proven (no network, no remote, auth trouble). Widening
66
+ // `indeterminate` into `gone` would turn every offline operator host into a
67
+ // fleet-wide "your pin is dead" alarm.
68
+
69
+ /** Map a `git ls-remote --exit-code --heads` exit code to the shared tri-state.
70
+ * 0 -> 'live', 2 -> 'gone', anything else (including a non-numeric or absent
71
+ * code) -> 'indeterminate'. Pure. */
72
+ function lsRemoteStatus(exitCode) {
73
+ if (exitCode === 0) return 'live';
74
+ if (exitCode === 2) return 'gone';
75
+ return 'indeterminate';
76
+ }
77
+
78
+ /** Classify the health of the fleet's branch pin.
79
+ *
80
+ * Returns `{ state, branch, pinned, dead, blocks, recovery }` where `state` is
81
+ * 'live' | 'gone' | 'indeterminate' | 'unknown'.
82
+ *
83
+ * 'unknown' is the un-probeable case — no branch, or the UNKNOWN_BRANCH
84
+ * sentinel that `resolveFleetBranch` yields when there is neither a pin nor a
85
+ * readable git HEAD. There is no ref to ask origin about, so reporting it as
86
+ * `gone` would invent a deletion that never happened.
87
+ *
88
+ * `dead` is true ONLY for 'gone'. It is the single field a caller should gate a
89
+ * degradation banner on, so no consumer has to re-derive which states are
90
+ * actionable.
91
+ *
92
+ * `blocks` names what a dead pin is actively breaking — both consequences,
93
+ * because neither announces itself: the operator-repo refresh cannot fetch a
94
+ * deleted ref, and the pin also drives `cloud.json editorBranch`, so newly
95
+ * provisioned VMs check out a ref that is not there. Pure; no I/O. */
96
+ function classifyBranchPin({ branch, lsRemoteExitCode, pinned = false } = {}) {
97
+ const name = cleanBranch(branch);
98
+ const probeable = name !== null && name !== UNKNOWN_BRANCH;
99
+ const state = probeable ? lsRemoteStatus(lsRemoteExitCode) : 'unknown';
100
+ const dead = state === 'gone';
101
+ return {
102
+ state,
103
+ branch: name,
104
+ pinned: Boolean(pinned),
105
+ dead,
106
+ blocks: dead
107
+ ? [
108
+ 'refresh-operator-repo.sh cannot fetch the pinned branch, so this host keeps serving whatever its clone already had',
109
+ 'cloud.json editorBranch reads the same pin, so newly provisioned VMs check out a ref that does not exist',
110
+ ]
111
+ : [],
112
+ recovery: dead
113
+ ? `scripts/operator/fleet-advance-to.sh <BRANCH> — move the fleet to a live branch (it refuses a branch that is itself absent from origin)`
114
+ : null,
115
+ };
116
+ }
117
+
118
+ /** Decide whether the fleet pin should override a stored per-VM branch when an
119
+ * add is RE-ISSUED, and report every override so the caller can log it.
120
+ *
121
+ * Returns `{ config, overrides }` — a NEW config (the input is never mutated)
122
+ * and `[{ field, from, to }]`, empty when nothing changed.
123
+ *
124
+ * The problem this solves: a per-VM config that recorded `editor-improvements-75`
125
+ * was not wrong when it was written — it EXPIRED, silently, the moment that
126
+ * branch merged and origin pruned it. This repo prunes on merge, so every stored
127
+ * feature-branch name has that property. The pin does not expire: it is the
128
+ * durable statement of fleet-wide policy, and on 2026-08-25 it was correct
129
+ * throughout while four retried cards each failed on their stale snapshot (a
130
+ * fresh pool add read the pin and succeeded at the same moment).
131
+ *
132
+ * Three guards, each declining to act rather than acting wrongly:
133
+ * - `selfHost` false ⇒ untouched. A client VM's branch is a deliberate
134
+ * per-project choice; overwriting it with a fleet pin would silently
135
+ * retarget someone else's project. Only the self-host path, where the branch
136
+ * IS fleet policy, falls back to the pin.
137
+ * - no pin, or the UNKNOWN_BRANCH sentinel ⇒ untouched. There is no branch to
138
+ * prefer, and stamping the sentinel would guarantee the failure this exists
139
+ * to prevent.
140
+ * - `pinState === 'gone'` ⇒ untouched. A pin PROVEN absent from origin is not
141
+ * an improvement on a stale-but-possibly-live stored branch; swapping one
142
+ * dead ref for another just moves the failure. Note the deliberate
143
+ * asymmetry with 'indeterminate', which does NOT decline: absence was not
144
+ * proven there, and refusing to converge every time the network blips would
145
+ * strand the fleet on stale branches for the sake of a maybe.
146
+ *
147
+ * Pure; no I/O. */
148
+ function pinCorrectedBranches({ cfg, pin, selfHost = false, pinState } = {}) {
149
+ const config = { ...(cfg || {}) };
150
+ const overrides = [];
151
+ const name = cleanBranch(pin);
152
+ if (!selfHost || !name || name === UNKNOWN_BRANCH || pinState === 'gone') {
153
+ return { config, overrides };
154
+ }
155
+ for (const field of ['clientBranch', 'editorBranch']) {
156
+ const from = cleanBranch(config[field]);
157
+ if (from && from !== name) {
158
+ config[field] = name;
159
+ overrides.push({ field, from, to: name });
160
+ }
161
+ }
162
+ return { config, overrides };
163
+ }
164
+
165
+ module.exports = {
166
+ UNKNOWN_BRANCH,
167
+ cleanBranch,
168
+ resolveFleetBranch,
169
+ lsRemoteStatus,
170
+ classifyBranchPin,
171
+ pinCorrectedBranches,
172
+ };
@@ -120,6 +120,17 @@ function buildVmEnvelope(v, ctx) {
120
120
  // unreachable VM's last-known config is stale, and an unreachable card has
121
121
  // louder problems to report than its preview origin.
122
122
  previewMisconfigured: (v.reachable && v.previewMisconfigured) || null,
123
+ // The project's DECLARED datastore and whether it is answering:
124
+ // { type, target, local, reachable, lastError }, or null when the project
125
+ // declares none (a Rust CLI, a static site) — which is the common case and
126
+ // must render nothing at all. `reachable` is tri-state exactly like
127
+ // previewServing/previewIngress above: true = probe reached it, false =
128
+ // confirmed down, null = unknown (editor just booted, or the probe went
129
+ // stale). Never conflate "down" with "unknown" — only a confirmed false
130
+ // raises the amber reason. Gated on `reachable` like providerAuth: an
131
+ // unreachable VM's last-known datastore verdict is stale, and it has louder
132
+ // problems to report.
133
+ datastore: (v.reachable && v.datastore) || null,
123
134
  effects: v.effects || null,
124
135
  // The VM's live-configured build agent (lowercase provider), reported by
125
136
  // /api/session-info and merged (sticky) onto the poll state. Drives the
@@ -177,7 +188,7 @@ const REQUIRED_CARD_FIELDS = [
177
188
  'degradedReasons', 'setupSteps', 'attentionReasons',
178
189
  'isHead', 'headHolderUnreachable', 'queuePos', 'queueLen',
179
190
  'job', 'config', 'project', 'variant', 'cardIdentity', 'brokerHistory',
180
- 'brokerDeath', 'providerAuth', 'previewMisconfigured', 'effects', 'currentProvider',
191
+ 'brokerDeath', 'providerAuth', 'previewMisconfigured', 'datastore', 'effects', 'currentProvider',
181
192
  'readiness', 'backendHealth', 'vmAccess',
182
193
  'startedAt', 'featureStartedAt', 'attention', 'envOverride',
183
194
  ];
@@ -618,20 +618,37 @@ function reconcileRosteredVm({ exists, hasSession, hasPendingAdd, streak, resetH
618
618
  // The dashboard's `state` and `addVm` are injected so this stays a pure,
619
619
  // importable unit — `log`/`err` default to the console but are overridable in
620
620
  // tests. Behavior mirrors the previous inline server.js definition exactly.
621
+ //
622
+ // The exhausted branch records a TERMINAL DISPOSITION, not just a flag. The flag
623
+ // alone was write-only: nothing destroys the instance and the only code that
624
+ // clears the flag is a *subsequent* recovery attempt, which by definition never
625
+ // comes once the budget is spent. VM-10 therefore ran empty and unrostered for a
626
+ // full day on 2026-08-29, billing, while the dashboard rendered a generic
627
+ // needs-manual-recovery card and clearing it needed hand-edited JSON.
628
+ // `instancePresumedRunning` says the GCE instance is believed to still exist —
629
+ // PRESUMED, because this ladder step never probes GCP; `exhaustedOrphanDisposition`
630
+ // is what reconciles the presumption against reality. `exhaustedAt` anchors the
631
+ // accrued-cost clock the card renders. `addVm` is still never called: that is the
632
+ // safety property that stops a wedged VM looping cloud:up forever.
621
633
  function continueBootstrapRecovery(
622
634
  n,
623
635
  meta,
624
636
  cfg,
625
- { state, addVm, log = console.log, err = console.error } = {},
637
+ { state, addVm, log = console.log, err = console.error, now = Date.now } = {},
626
638
  ) {
627
639
  const { attempt, proceed, max } = recoverBootstrapDecision({
628
640
  recoverAttempts: meta.recoverAttempts,
629
641
  });
630
642
  if (!proceed) {
631
643
  err(
632
- `startup-reap: VM-${n} exceeded ${max} bootstrap-recovery attempts — leaving sidecar for manual action`,
644
+ `startup-reap: VM-${n} exceeded ${max} bootstrap-recovery attempts — instance presumed still running, leaving sidecar for manual action`,
633
645
  );
634
- state.orphans[n] = { reason: "bootstrap-recovery-exhausted", attempts: attempt - 1 };
646
+ state.orphans[n] = {
647
+ reason: "bootstrap-recovery-exhausted",
648
+ attempts: attempt - 1,
649
+ instancePresumedRunning: true,
650
+ exhaustedAt: now(),
651
+ };
635
652
  return;
636
653
  }
637
654
  log(
@@ -643,6 +660,68 @@ function continueBootstrapRecovery(
643
660
  addVm(n, cfg, { recovering: true, recoverAttempt: attempt });
644
661
  }
645
662
 
663
+ // The exit `continueBootstrapRecovery`'s exhausted flag never had. That flag is
664
+ // written once and, until this verdict existed, could only be cleared by a
665
+ // subsequent recovery attempt the spent budget guarantees will never run — so it
666
+ // outlived the instance it described and had to be removed by hand-editing
667
+ // `.codeyam/logs/dashboard-state.json`. This reconciles the flag's PRESUMPTION
668
+ // (`instancePresumedRunning`) against two live facts the sweep supplies:
669
+ //
670
+ // instanceExists true ⇒ the presumption holds, the box is still billing
671
+ // false ⇒ the instance is gone (destroyed out-of-band) → 'clear',
672
+ // so the card self-heals with no operator action
673
+ // null ⇒ the describe probe was INCONCLUSIVE. Never act on
674
+ // absence of evidence — same discipline reapVerdict
675
+ // applies to its own `exists === null`.
676
+ // rostered false ⇒ flagged, billing, and in nobody's roster: invisible to
677
+ // every roster-driven tool yet costing money → 'cost',
678
+ // the distinct ownerless surface, NOT a generic
679
+ // needs-manual-recovery card.
680
+ // true ⇒ still owned, so it is recoverable → 'keep'.
681
+ // null ⇒ roster unknown; keep rather than raise a cost alarm
682
+ // that may be false (positive-signal-only, mirroring
683
+ // how projectWired/agentReady downgrade).
684
+ //
685
+ // Deliberately NOT a destroy verdict. Destroying a VM is irreversible and a
686
+ // half-provisioned box occasionally holds the clue that explains the failure, so
687
+ // the terminal disposition is "surface it loudly and offer the button" — the
688
+ // operator presses it.
689
+ function exhaustedOrphanDisposition({ reason, instanceExists, rostered } = {}) {
690
+ if (reason !== 'bootstrap-recovery-exhausted') return 'leave';
691
+ if (instanceExists === null || instanceExists === undefined) return 'leave';
692
+ if (!instanceExists) return 'clear';
693
+ return rostered === false ? 'cost' : 'keep';
694
+ }
695
+
696
+ // Fold a 'cost'/'keep' verdict into the orphan entry the card renders. Split out
697
+ // of the dashboard's poller because the stickiness rule below is the only real
698
+ // logic in that branch, and inlined in server.js it was unreachable by any test:
699
+ // the dashboard has top-level side effects on require, so its pollers can only be
700
+ // covered by a source-scan wiring guard that cannot evaluate behavior.
701
+ //
702
+ // `ownerlessSince` dates the accrued cost from when the VM was FIRST confirmed
703
+ // ownerless, not from the latest probe — re-stamping it every tick would peg the
704
+ // card's elapsed time near zero forever and hide exactly the multi-day cost the
705
+ // surface exists to reveal. It is dropped (not zeroed, not left stale) the moment
706
+ // a VM stops being ownerless — an operator adopting it back into the roster flips
707
+ // the verdict to 'keep', and a leftover `ownerlessSince` would let the cost card
708
+ // resurrect itself with a fabricated start time if the VM ever went ownerless
709
+ // again. `instanceConfirmedAt` records that the presumption was checked live.
710
+ //
711
+ // Returns a NEW entry; the caller decides where to store it. Both verdicts keep
712
+ // the flag — the instance is real and only an operator may destroy it.
713
+ function annotateOwnerlessCost(orphan, { verdict, now = Date.now() } = {}) {
714
+ const next = { ...orphan, instancePresumedRunning: true, instanceConfirmedAt: now };
715
+ if (verdict === 'cost') {
716
+ next.ownerlessCost = true;
717
+ next.ownerlessSince = (orphan && orphan.ownerlessSince) || now;
718
+ return next;
719
+ }
720
+ next.ownerlessCost = false;
721
+ delete next.ownerlessSince;
722
+ return next;
723
+ }
724
+
646
725
  module.exports = {
647
726
  parsePendingJobFilename,
648
727
  persistPendingJob,
@@ -656,6 +735,8 @@ module.exports = {
656
735
  classifyLbReachableForReap,
657
736
  recoverBootstrapDecision,
658
737
  continueBootstrapRecovery,
738
+ exhaustedOrphanDisposition,
739
+ annotateOwnerlessCost,
659
740
  exitMarkerName,
660
741
  parseExitMarker,
661
742
  readExitMarker,
@@ -0,0 +1,107 @@
1
+ 'use strict';
2
+
3
+ // npm/fleet-datastore-health.js — pure classifier for the fleet dashboard's
4
+ // "declared datastore unreachable" surface: is the datastore a VM's project
5
+ // DECLARES actually answering?
6
+ //
7
+ // Nothing anywhere reported this, so a VM whose database was down read
8
+ // perfectly green. On VM-12 (2026-08-24) the dashboard, the LB backend health,
9
+ // the IAP edge and the editor's own server-identity all reported healthy while
10
+ // every DB-backed route returned 500 — the only evidence was `ECONNREFUSED
11
+ // 127.0.0.1:5432` buried in the container log, which nobody polls. The
12
+ // 2026-08-14 occurrence on the same VM presented as previewServing=false with an
13
+ // EMPTY degradedReasons list: the dashboard knew something was wrong and had
14
+ // nothing to say about it. This module is the "something to say".
15
+ //
16
+ // The editor probes (it is the only party that knows what the project declares,
17
+ // and the resolution already exists in-crate) and carries the result on the
18
+ // existing session-info poll; this module decides, and server.js renders. Pure
19
+ // like fleet-project-drift.js: all the decision logic lives here and unit-tests
20
+ // without a VM, while server.js stays the thin I/O shell.
21
+ //
22
+ // A new `datastore-unreachable` code needs NO change to
23
+ // fleet-setup-steps.js::classifyReasons — it routes unknown codes to
24
+ // attentionReasons by design ("an unrecognized signal is surfaced loudly, never
25
+ // silently hidden"), so this lands amber. It must NOT be added to SETUP_CODES: a
26
+ // dead datastore is never expected first-run wiring.
27
+
28
+ // Decide the datastore-health reasons for one VM.
29
+ //
30
+ // declared truthy when the project declares a datastore at all. A project
31
+ // with none (a Rust CLI, a static site) must never raise a reason.
32
+ // reachable TRI-STATE, matching the editor's own discipline: true = the probe
33
+ // reached it, false = the probe failed, null/undefined = unknown
34
+ // (not probed yet, or the probe went stale). Only an explicit
35
+ // `false` raises a reason — collapsing unknown into false would
36
+ // light up every VM during the window between editor boot and the
37
+ // first probe, which is the failure mode that trains an operator to
38
+ // ignore the surface.
39
+ // target credential-free rendering of where it lives ("127.0.0.1:5432/app",
40
+ // or a file path). Never the DSN — that carries credentials.
41
+ // shape 'file' (a database file on disk) or 'network' (a server on a
42
+ // socket). Published by the editor rather than inferred from `type`
43
+ // here, because "sqlite means file-shaped" is a per-stack fact this
44
+ // module would otherwise have to learn for every engine. It decides
45
+ // WHICH recovery applies at all: a down server gets started, a
46
+ // missing file gets restored.
47
+ // local whether the datastore is something this VM is expected to host
48
+ // (loopback host, or a file database). Decides where the recovery
49
+ // points; a remote/managed datastore is still reported, but telling
50
+ // an operator to start it here would send them somewhere there is
51
+ // nothing to do.
52
+ // type the declared engine ('postgresql' | 'sqlite' | 'mysql' | …), used
53
+ // only to name what is down.
54
+ // lastError the probe's own failure string ("connection refused: …"), so the
55
+ // card says WHY rather than just that.
56
+ // n the VM number, so a local recovery can name the VM to act on.
57
+ //
58
+ // Returns { unreachable, reasons }. `reasons` is [] whenever healthy, unknown, or
59
+ // nothing is declared, so the caller spreads it unconditionally.
60
+ function classifyDatastoreHealth({ declared, reachable, target, shape, local, type, lastError, n } = {}) {
61
+ // Explicit `false` only. `null`/`undefined` is unknown and never a fault —
62
+ // the same absence-of-evidence-is-not-a-fault rule the reachability and
63
+ // backend-health classifiers already follow.
64
+ const unreachable = !!declared && reachable === false;
65
+ if (!unreachable) return { unreachable: false, reasons: [] };
66
+ return {
67
+ unreachable: true,
68
+ reasons: [datastoreUnreachableReason({ type, target, shape, local, lastError, n })],
69
+ };
70
+ }
71
+
72
+ // Which recovery even applies, given what KIND of datastore this is and where
73
+ // it lives. Its own function because "is it down?" and "what do you do about
74
+ // it?" are separate decisions, and this is the one that grows a branch every
75
+ // time a new datastore shape shows up.
76
+ //
77
+ // It follows the ingress-unwired precedent — the whole point of that reason was
78
+ // carrying the exact fix instead of a bare complaint — but deliberately stops
79
+ // short of naming a COMMAND. What an operator runs to start a datastore is a
80
+ // property of the VM image, not of this module; naming one this module cannot
81
+ // know is installed would be a confident lie on an amber card. What IS knowable,
82
+ // and what actually unblocks someone, is which recovery applies and where to act.
83
+ //
84
+ // The `file` branch is not cosmetic: a file database has no server to start, so
85
+ // routing it through the local branch would print the nonsensical "start it".
86
+ function recoveryClause({ shape, local, n } = {}) {
87
+ if (shape === 'file') {
88
+ return 'Recover: this is a file database and the file is missing — restore or recreate it on the VM.';
89
+ }
90
+ if (local) {
91
+ return `Recover: start it on VM-${n} — it is declared local, so nothing outside the VM can be at fault.`;
92
+ }
93
+ return 'Recover: this is a remote/managed datastore — check the datastore service and the VM\'s network path. There is nothing local to start.';
94
+ }
95
+
96
+ // The single source for the `datastore-unreachable` reason string, mirroring
97
+ // fleet-public-url.js::ingressUnwiredReason. Named and exported so the card
98
+ // reason and any future log line cannot drift apart — the drift that a
99
+ // template literal inlined at its one call site invites the moment a second
100
+ // caller appears.
101
+ function datastoreUnreachableReason({ type, target, shape, local, lastError, n } = {}) {
102
+ const what = `${type || 'declared'} datastore${target ? ` at ${target}` : ''}`;
103
+ const why = lastError ? ` (${lastError})` : '';
104
+ return `datastore-unreachable: the ${what} is not answering${why}. ${recoveryClause({ shape, local, n })}`;
105
+ }
106
+
107
+ module.exports = { classifyDatastoreHealth, recoveryClause, datastoreUnreachableReason };
@@ -290,8 +290,16 @@ function restoreFromSnapshot({ snapshot, maxAgeMs, nowIso, sidecarKeys } = {}) {
290
290
  * Degrades to 'cgroup=unavailable' rather than printing zeros off-cgroup, so a
291
291
  * laptop log cannot be misread as a cgroup at 0MB.
292
292
  *
293
+ * `detachedWiringCount` — when supplied — appends `tails=N`, which is what makes
294
+ * the RSS/cgroup gap ATTRIBUTABLE rather than merely visible. On 2026-08-11 the
295
+ * cgroup read 2154MB against a 96MB RSS and the whole diagnosis was "those ~2GB
296
+ * are this process's own transient add children, not a leak". `RSS=96MB
297
+ * cgroup=2154MB tails=4` reads as a batch draining on its own; `RSS=1300MB
298
+ * cgroup=1400MB tails=0` reads as a genuine leak. Omitted (the pre-existing
299
+ * callers, and any host with no tail accounting) prints exactly as before.
300
+ *
293
301
  * Pure; no I/O. */
294
- function formatMemoryText({ rssBytes, memoryCurrent, memoryHigh } = {}) {
302
+ function formatMemoryText({ rssBytes, memoryCurrent, memoryHigh, detachedWiringCount } = {}) {
295
303
  const mb = (b) => (Number(b) / 1024 / 1024).toFixed(0);
296
304
  const current = Number(memoryCurrent);
297
305
  const high = Number(memoryHigh);
@@ -308,6 +316,8 @@ function formatMemoryText({ rssBytes, memoryCurrent, memoryHigh } = {}) {
308
316
  } else {
309
317
  parts.push('cgroup=unavailable');
310
318
  }
319
+ const tails = Number(detachedWiringCount);
320
+ if (Number.isFinite(tails) && tails >= 0) parts.push(`tails=${tails}`);
311
321
  return parts.join(' ');
312
322
  }
313
323
 
@@ -323,11 +333,26 @@ function formatMemoryText({ rssBytes, memoryCurrent, memoryHigh } = {}) {
323
333
  * protecting. The dashboard's own self-update counts, being a heavy job by any
324
334
  * other name.
325
335
  *
326
- * Pure; no I/O — the caller passes state.jobs / state.dashboardUpdate. */
327
- function hasJobInFlight({ jobs, dashboardUpdate } = {}) {
336
+ * `detachedWiringCount` is the THIRD source of busy, and the one this guard was
337
+ * blind to: an Add settles at container-healthy while its detached wire-add-tail
338
+ * keeps spawning gcloud children for another 5-10 min in the SAME cgroup. Judging
339
+ * "safe to restart" off `state.jobs` alone reported no job in flight while that
340
+ * work was live — on 2026-08-11 the escalation crossed the restart threshold at
341
+ * 86% of memory.high and the only thing that held it back was an unrelated job
342
+ * happening to still be `running`. Had it settled thirty seconds earlier the
343
+ * dashboard would have restarted itself mid-batch and orphaned four adds, which
344
+ * is precisely the failure class the detached-provision work exists to prevent.
345
+ * Omitting the key behaves exactly as before, so every current caller is
346
+ * unaffected.
347
+ *
348
+ * Pure; no I/O — the caller passes state.jobs / state.dashboardUpdate and the
349
+ * live-tail count. */
350
+ function hasJobInFlight({ jobs, dashboardUpdate, detachedWiringCount } = {}) {
328
351
  const busy = (j) => !!j && (j.state === 'running' || j.state === 'recovering');
329
352
  const anyJob = Object.values((jobs && typeof jobs === 'object') ? jobs : {}).some(busy);
330
- return anyJob || !!(dashboardUpdate && dashboardUpdate.state === 'running');
353
+ const tails = Number(detachedWiringCount);
354
+ const anyTail = Number.isFinite(tails) && tails > 0;
355
+ return anyJob || anyTail || !!(dashboardUpdate && dashboardUpdate.state === 'running');
331
356
  }
332
357
 
333
358
  /** Should this escalation level be announced to the journal?
@@ -126,9 +126,150 @@ function reconcileQueueDraft(draftOrder, liveOrder) {
126
126
  return out;
127
127
  }
128
128
 
129
+ // Consecutive failed reads before the panel stops trusting its mirror. At the
130
+ // default QUEUE_POLL_MS of 5000 this is ~15s of sustained failure — long enough
131
+ // that the ordinary 20s-timeout blip the next tick clears only ever reaches
132
+ // 'degraded', short enough that a real credential outage is flagged in seconds.
133
+ // Named rather than inlined so the verdict and the tooltip copy read the same
134
+ // number from one place.
135
+ const QUEUE_STALE_AFTER_FAILURES = 3;
136
+
137
+ /** Merge one commit-queue poll result over the previous memo.
138
+ *
139
+ * THE RULE: a failed read RETAINS the previous `entries` AND the previous
140
+ * `readAt`, and records `error`. Retaining the entries is deliberate — a stale
141
+ * order beats a blank panel for an operator mid-triage. Leaving `readAt`
142
+ * untouched is what makes the staleness legible: it is the ONLY signal that
143
+ * distinguishes a frozen view from a live one.
144
+ *
145
+ * Advancing `readAt` on a failure is precisely the bug this exists to prevent.
146
+ * On 2026-08-14 /etc/codeyam/git-credentials on the operator host went to 0
147
+ * bytes and every origin git operation began failing; the queue panel went on
148
+ * naming VM-5 as HEAD for 3.5 hours after VM-5 had released, while the real
149
+ * queue was VM-2 -> VM-8 -> VM-4. Nothing on the page distinguished it from
150
+ * live. This is the same lesson mergeDashboardRevision records from
151
+ * 2026-07-31, applied to the second poller.
152
+ *
153
+ * A previous `readAt` of null (no successful read has EVER happened) stays
154
+ * null and is not coerced to a timestamp — "never read" and "read a moment
155
+ * ago" must stay distinguishable, the same reason mergeDashboardRevision
156
+ * refuses to push a null `behind` through Number().
157
+ *
158
+ * Success is the absence of `error`. Pure; no I/O. */
159
+ function mergeQueuePoll({ prev, entries, readAt, error } = {}) {
160
+ const base = (prev && typeof prev === 'object') ? prev : {};
161
+ const str = (v) => ((typeof v === 'string' && v) ? v : null);
162
+ const prevEntries = Array.isArray(base.entries) ? base.entries : [];
163
+ const prevReadAt = str(base.readAt);
164
+ const prevFailures = (typeof base.consecutiveFailures === 'number'
165
+ && Number.isFinite(base.consecutiveFailures) && base.consecutiveFailures >= 0)
166
+ ? base.consecutiveFailures : 0;
167
+ const err = str(error);
168
+ if (err) {
169
+ return {
170
+ entries: prevEntries,
171
+ readAt: prevReadAt,
172
+ error: err,
173
+ consecutiveFailures: prevFailures + 1,
174
+ };
175
+ }
176
+ return {
177
+ entries: Array.isArray(entries) ? entries : [],
178
+ readAt: str(readAt) || prevReadAt,
179
+ error: null,
180
+ consecutiveFailures: 0,
181
+ };
182
+ }
183
+
184
+ /** What does the current queue memo mean for the operator?
185
+ *
186
+ * Levels:
187
+ * 'fresh' — the last read succeeded. Render exactly as before; every
188
+ * control stays live.
189
+ * 'degraded' — erroring, but under QUEUE_STALE_AFTER_FAILURES consecutive
190
+ * failures. A blip. Noted quietly, controls stay live — a
191
+ * single 20s-timeout hiccup that the next tick clears must not
192
+ * cry wolf or lock the operator out of their own queue.
193
+ * 'stale' — erroring at or past the threshold. The mirror is not to be
194
+ * trusted and writes computed against it are blocked.
195
+ * 'unknown' — erroring and NO read has ever succeeded (the boot window with
196
+ * broken credentials). Worth saying out loud: an empty-looking
197
+ * queue here means "we cannot see the queue", not "go".
198
+ *
199
+ * AGE IS DELIBERATELY NOT AN INPUT. A genuinely quiet queue is not a broken
200
+ * one, so escalation keys on consecutive read FAILURES only; age is carried by
201
+ * projectQueueFreshness purely to phrase the message ("last read 3h 12m ago").
202
+ * An hours-old `readAt` with no error is 'fresh' and must stay that way. Taking
203
+ * a clock here would imply otherwise, so the signature omits one — the same
204
+ * take-only-what-you-use shape as classifyDashboardStaleness.
205
+ *
206
+ * Pure; no I/O. */
207
+ function classifyQueueFreshness({ readAt, error, consecutiveFailures } = {}) {
208
+ if (!(typeof error === 'string' && error)) return 'fresh';
209
+ if (!(typeof readAt === 'string' && readAt)) return 'unknown';
210
+ const failures = (typeof consecutiveFailures === 'number' && Number.isFinite(consecutiveFailures))
211
+ ? consecutiveFailures : 0;
212
+ return failures >= QUEUE_STALE_AFTER_FAILURES ? 'stale' : 'degraded';
213
+ }
214
+
215
+ /** The `/data` view of the queue memo, with the freshness VERDICT resolved
216
+ * server-side.
217
+ *
218
+ * The panel is then a pure renderer. Two consumers re-deriving "is this stale"
219
+ * from a raw timestamp would drift, and `public/index.html` is an inline
220
+ * browser script with no module boundary — any decision left in it could only
221
+ * ever be source-scan tested, which is how it ships bugs green. Same split, and
222
+ * same reason, as projectDashboardRevision.
223
+ *
224
+ * `now` is injected (epoch ms) rather than read from the clock so the
225
+ * projection stays pure and its tests deterministic. `ageSec` is null when no
226
+ * successful read has happened or no clock was supplied.
227
+ *
228
+ * Pure; no I/O. */
229
+ function projectQueueFreshness(memo = {}, now = null) {
230
+ const at = (typeof memo.readAt === 'string' && memo.readAt) ? Date.parse(memo.readAt) : NaN;
231
+ const clock = (typeof now === 'number' && Number.isFinite(now)) ? now : null;
232
+ const ageSec = (Number.isFinite(at) && clock !== null)
233
+ ? Math.max(0, Math.floor((clock - at) / 1000))
234
+ : null;
235
+ return { ...memo, level: classifyQueueFreshness(memo), ageSec };
236
+ }
237
+
238
+ /** The operator-facing reason a queue WRITE must be refused, or null when writes
239
+ * are allowed.
240
+ *
241
+ * A write computed against a mirror known to be frozen is never the right
242
+ * action, and the silent-no-op outcome is worse than a refusal: a Release
243
+ * fires at a uuid that no longer holds the head, reports "released head
244
+ * bb65addf", changes nothing, and leaves the real head held. A reorder is
245
+ * worse still — computeReorderTargets validates the posted order against the
246
+ * very mirror that is wrong, so a doomed reorder looks perfectly valid.
247
+ *
248
+ * The verbatim git error is interpolated rather than summarised to "poll
249
+ * failed". On 2026-08-14 the entire recovery was legible from one line —
250
+ * `could not read Username for 'https://github.com'` — and truncating it would
251
+ * have cost the same 3.5 hours.
252
+ *
253
+ * Lives here rather than in server.js so the refusal COPY is directly
254
+ * assertable: server.js has top-level side effects on require, so a message
255
+ * built there could only ever be source-scan tested. Pure; no I/O. */
256
+ function queueWriteBlockReason(memo = {}) {
257
+ const level = classifyQueueFreshness(memo);
258
+ if (level !== 'stale' && level !== 'unknown') return null;
259
+ const why = (typeof memo.error === 'string' && memo.error) ? ` (${memo.error})` : '';
260
+ return level === 'unknown'
261
+ ? `commit queue has never been read successfully${why} — cannot act on an unknown queue`
262
+ : `commit queue read is failing${why} — the displayed order cannot be trusted; fix the read before changing the queue`;
263
+ }
264
+
129
265
  module.exports = {
130
266
  decideAutoReleaseGoneHead,
131
267
  computeReorderTargets,
132
268
  nudgeQueueOrder,
133
269
  reconcileQueueDraft,
270
+ mergeQueuePoll,
271
+ classifyQueueFreshness,
272
+ projectQueueFreshness,
273
+ queueWriteBlockReason,
274
+ QUEUE_STALE_AFTER_FAILURES,
134
275
  };