@codeyam-editor/codeyam-editor 0.1.4 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/npm/cli.js +5 -1
  2. package/npm/cloud-branch-switch.js +1 -1
  3. package/npm/cloud.js +3166 -150
  4. package/npm/codesign-plan.js +68 -0
  5. package/npm/container.js +3 -0
  6. package/npm/disk-pressure.js +78 -0
  7. package/npm/editor-dev.js +96 -15
  8. package/npm/editor.js +85 -37
  9. package/npm/empty-project-dir.js +92 -7
  10. package/npm/fleet-add-failure.js +252 -0
  11. package/npm/fleet-asset-version.js +37 -0
  12. package/npm/fleet-auth-health.js +213 -0
  13. package/npm/fleet-auth.js +6 -1
  14. package/npm/fleet-build-data.js +109 -5
  15. package/npm/fleet-cleanup.js +232 -16
  16. package/npm/fleet-destroy-safety.js +272 -0
  17. package/npm/fleet-early-roster.js +87 -0
  18. package/npm/fleet-effect-poll.js +110 -0
  19. package/npm/fleet-effect-status.js +33 -32
  20. package/npm/fleet-env.js +86 -0
  21. package/npm/fleet-ghcr-secret.js +129 -0
  22. package/npm/fleet-git-maintenance.js +142 -0
  23. package/npm/fleet-host-op-limiter.js +68 -0
  24. package/npm/fleet-host.js +74 -0
  25. package/npm/fleet-iap-token.js +165 -0
  26. package/npm/fleet-job-badge.js +1 -1
  27. package/npm/fleet-launchd.js +171 -0
  28. package/npm/fleet-live-status.js +155 -0
  29. package/npm/fleet-memory.js +245 -9
  30. package/npm/fleet-plans.js +233 -4
  31. package/npm/fleet-pool.js +75 -16
  32. package/npm/fleet-prewarm.js +241 -0
  33. package/npm/fleet-project-drift.js +58 -0
  34. package/npm/fleet-public-url.js +394 -0
  35. package/npm/fleet-queue.js +95 -1
  36. package/npm/fleet-reachability-debounce.js +69 -0
  37. package/npm/fleet-reachability.js +80 -0
  38. package/npm/fleet-reconcile.js +43 -1
  39. package/npm/fleet-reset.js +330 -13
  40. package/npm/fleet-roster.js +6 -8
  41. package/npm/fleet-running-plan.js +85 -0
  42. package/npm/fleet-self-update.js +142 -0
  43. package/npm/fleet-session-token-cache.js +65 -0
  44. package/npm/fleet-setup-steps.js +112 -0
  45. package/npm/fleet-sleep-status.js +33 -0
  46. package/npm/fleet-status-refs.js +96 -0
  47. package/npm/fleet-strand.js +181 -0
  48. package/npm/fleet-stuck-status.js +117 -0
  49. package/npm/fleet-vm-access.js +404 -0
  50. package/npm/fleet-vm-probe-target.js +282 -0
  51. package/npm/fleet-vm-readiness.js +237 -0
  52. package/npm/fleet-vm-slugs.js +116 -0
  53. package/npm/google-ingress.js +226 -0
  54. package/npm/ingress-invoke.js +770 -0
  55. package/npm/launch-config.js +386 -9
  56. package/npm/scenario-check.js +880 -37
  57. package/npm/scenario-handlers.js +154 -30
  58. package/npm/scenario-interactivity.js +145 -6
  59. package/npm/scenario-issues.js +13 -1
  60. package/npm/scenario-metrics.js +116 -1
  61. package/npm/scenario-mocks.js +144 -21
  62. package/npm/scenario-playwright.js +649 -38
  63. package/npm/script-drift.js +152 -0
  64. package/npm/utils.js +365 -52
  65. package/npm/vm-urls.js +69 -8
  66. package/package.json +5 -5
  67. package/ui/dist/assets/ContainerInspector-DgM0pTAx.js +2 -0
  68. package/ui/dist/assets/EditorShell-B7Q06uJ_.js +252 -0
  69. package/ui/dist/assets/EditorShell-DP5UY20G.css +1 -0
  70. package/ui/dist/assets/{Markdown-BHBqz-Mj.js → Markdown-Chgdf3PW.js} +1 -1
  71. package/ui/dist/assets/ProjectLauncherScreen-BRrEr8Ng.js +1 -0
  72. package/ui/dist/assets/ScenarioDataPanel-BeQfa28g.js +9 -0
  73. package/ui/dist/assets/{ScenarioDataPanel-DYP7pi_n.css → ScenarioDataPanel-C1LukYI6.css} +1 -1
  74. package/ui/dist/assets/ScenarioDataPanelFullPage-BLqRCxQX.js +1 -0
  75. package/ui/dist/assets/ScenarioRenderer-CIAxfHPh.js +21 -0
  76. package/ui/dist/assets/StepsSimulator-CymGgfbu.js +1 -0
  77. package/ui/dist/assets/index-BygCiJSb.css +1 -0
  78. package/ui/dist/assets/{index-BuK0lizt.js → index-DCHqRrwO.js} +9 -9
  79. package/ui/dist/assets/{markdown-v0F0UEt3.js → markdown-C9tbsVHX.js} +1 -1
  80. package/ui/dist/assets/react-nrLBr15I.js +28 -0
  81. package/ui/dist/assets/useEvents-S3Qk1Y_d.js +1 -0
  82. package/ui/dist/assets/useServerIdentity-B7rI0u3O.js +1 -0
  83. package/ui/dist/codeyam-mark.svg +4 -0
  84. package/ui/dist/codeyam-wordmark.svg +5 -0
  85. package/ui/dist/index.html +4 -4
  86. package/ui/dist/open-source/codeyam-counter.png +0 -0
  87. package/ui/dist/open-source/el-carot.png +0 -0
  88. package/ui/dist/open-source/tabcommand.png +0 -0
  89. package/ui/dist/assets/ContainerInspector-C8EkH4PP.js +0 -2
  90. package/ui/dist/assets/EditorShell-tMMdarZD.js +0 -187
  91. package/ui/dist/assets/ScenarioDataPanel-xWYGZOpo.js +0 -5
  92. package/ui/dist/assets/ScenarioDataPanelFullPage-5G6KuZgw.js +0 -1
  93. package/ui/dist/assets/ScenarioRenderer-CSdhX1Ak.js +0 -1
  94. package/ui/dist/assets/StepsSimulator-CRtmlyNZ.js +0 -1
  95. package/ui/dist/assets/index-BGqairFV.css +0 -1
  96. package/ui/dist/assets/react-CSS0HapR.js +0 -28
  97. package/ui/dist/assets/useEvents-DggWrtHe.js +0 -1
@@ -1,5 +1,7 @@
1
1
  'use strict';
2
2
 
3
+ const fr = require('./fleet-reachability');
4
+
3
5
  // Assemble the per-VM wire object the dashboard's /data response carries to the
4
6
  // browser. This is the single source of truth for a card's transport shape: the
5
7
  // card UI (scripts/operator/fleet-dashboard/public/index.html) reads `v.<field>`
@@ -21,7 +23,37 @@ function buildVmEnvelope(v, ctx) {
21
23
  return {
22
24
  n: ctx.n,
23
25
  url: v.url || null,
24
- reachable: !!v.reachable,
26
+ // The VM's public Live Preview URL (<slug>.preview.<domain> in named-tunnel
27
+ // mode, else the local forward). The card reads it as `previewSub` to render
28
+ // the 👁 preview link. Server-computed in server.js and merged onto the poll
29
+ // state — but it was dropped here, so the link was silently absent on EVERY
30
+ // card fleet-wide even when the VM's preview served fine. The exact
31
+ // currentProvider/vmAccess envelope-drift class this module guards against.
32
+ previewUrl: v.previewUrl || null,
33
+ // The app dev-server port the VM serves on (Vite 5173, Next 3000, …),
34
+ // resolved stickily upstream. The card reads it as `appUp` to gate the
35
+ // preview link live. Dropped alongside previewUrl — same silent-omit bug.
36
+ appPort: v.appPort != null ? v.appPort : null,
37
+ // Sticky preview-SERVING signal — "is the VM's dev server up?" — derived
38
+ // from the previewState the editor emits on /api/session-info (forwarding →
39
+ // true, placeholder → false, else null). Reaches every VM over the IAP-LB
40
+ // seam on the GCE operator host, where the old loopback `previewReachable`
41
+ // read false fleet-wide. The card gates the clickable 👁 link on `=== true`.
42
+ // Three-state, never conflating "down" with "unknown".
43
+ previewServing: v.previewServing === true ? true : (v.previewServing === false ? false : null),
44
+ // Sticky preview-INGRESS signal — "is the VM's public preview URL wired?" —
45
+ // from a reachability probe of <slug>.preview.<domain> over the LB, DISTINCT
46
+ // from previewServing. true = wired, false = confirmed dead (→ muted 'public
47
+ // URL not wired' hint), null = unknown (skipped / IAP audience mismatch — a
48
+ // null must never gate the link off). Three-state, same discipline.
49
+ previewIngress: v.previewIngress === true ? true : (v.previewIngress === false ? false : null),
50
+ // `reachable` (three-state) + `reachableSource` (which signal decided, or
51
+ // 'unpolled'). Same tri-state discipline as previewServing/previewIngress
52
+ // above, but delegated to npm/fleet-reachability.js because FIVE consumers
53
+ // ask the same question and the old `!!v.reachable` coercion here is what
54
+ // manufactured the 2026-07-30 fleet-wide phantom outage — 13 cards claiming
55
+ // a definite `false` that no probe had ever established.
56
+ ...fr.normalizeReachability(v),
25
57
  hasSession: !!v.hasSession,
26
58
  // Three-state: true = broker confirmed up, false = confirmed down, null =
27
59
  // editor unreachable so broker state is unknown. Never conflate "down" with
@@ -32,13 +64,38 @@ function buildVmEnvelope(v, ctx) {
32
64
  // upstream, debounced). Richer than brokerLive's is-an-agent-live boolean.
33
65
  brokerStatus: ctx.brokerStatus,
34
66
  awaitingInput: !!v.awaitingUserInput,
67
+ // True when the editor reports the build workflow reached its terminal step
68
+ // and is parked awaiting cleanup (session_info_response.rs emits it at
69
+ // step >= total). Flows onto poll state via the `...info` spread in server.js
70
+ // exactly like `feature`/`awaitingUserInput`; defaulted false here so a legacy
71
+ // editor that predates the field degrades to today's behavior. Consumed by
72
+ // fleet-live-status.js to read a parked-but-done agent as calm 'session-done'
73
+ // instead of active-green 'building'.
74
+ needsCleanup: !!v.needsCleanup,
35
75
  feature: v.feature || null,
76
+ // The VM's live-claimed plan slug, reported by /api/session-info and merged
77
+ // (sticky) onto the poll state. The card renders it as the what-it's-working-on
78
+ // line when `feature` is null — a VM that has claimed a plan but not yet named
79
+ // a feature (early plan/explore steps) previously showed a blank "no feature
80
+ // (idle)" card. Preferred over the origin claim-ref because this live source
81
+ // never goes stale under a failed claim-ref push. null for a legacy editor
82
+ // that predates the field, or a VM with no claimed plan.
83
+ selectedPlan: v.selectedPlan || null,
36
84
  step: v.step != null ? v.step : null,
37
85
  mode: v.mode || null,
38
86
  label: v.label || null,
39
87
  status: ctx.status,
40
88
  orphan: ctx.orphan || null,
41
89
  degradedReasons: ctx.degradedReasons || [],
90
+ // Derived presentation split of degradedReasons (vm: Dashboard 'Finishing
91
+ // setup' state): `setupSteps` are expected first-run wiring rendered as a
92
+ // calm blue checklist, `attentionReasons` are genuine problems kept in the
93
+ // amber ⚠ treatment. `degradedReasons` is left intact for the many places
94
+ // that still consume the flat list (filters, sort, reconcile); these two are
95
+ // purely additive. Both default to [] so a legacy/healthy VM renders neither
96
+ // block. See npm/fleet-setup-steps.js::classifyReasons.
97
+ setupSteps: Array.isArray(ctx.setupSteps) ? ctx.setupSteps : [],
98
+ attentionReasons: Array.isArray(ctx.attentionReasons) ? ctx.attentionReasons : [],
42
99
  isHead: !!ctx.isHead,
43
100
  // True when this VM holds the queue head but is unreachable (operator
44
101
  // decides manually whether to release it).
@@ -55,12 +112,56 @@ function buildVmEnvelope(v, ctx) {
55
112
  // Auth state + reason + token expiry. Only surfaced when the editor is
56
113
  // reachable (an unreachable VM's cached auth is stale and misleading).
57
114
  providerAuth: v.reachable && v.providerAuth ? v.providerAuth : null,
115
+ // The silently-broken Live Preview guard: { previewOrigin, expectedOrigin }
116
+ // when the VM's own /api/config proves its preview can never load in a
117
+ // remote browser (a not-same-origin-safe stack stuck on a loopback/subpath
118
+ // origin), null when healthy or unknown. Drives the card's amber "Live
119
+ // Preview unreachable" badge. Gated on `reachable` like providerAuth — an
120
+ // unreachable VM's last-known config is stale, and an unreachable card has
121
+ // louder problems to report than its preview origin.
122
+ previewMisconfigured: (v.reachable && v.previewMisconfigured) || null,
58
123
  effects: v.effects || null,
59
124
  // The VM's live-configured build agent (lowercase provider), reported by
60
125
  // /api/session-info and merged (sticky) onto the poll state. Drives the
61
126
  // card's 🤖 agent badge. null for an older VM whose editor predates the
62
127
  // field (the card omits the badge rather than rendering a blank chip).
63
128
  currentProvider: v.currentProvider || null,
129
+ // Raw workflow timestamps (ISO) surfaced by /api/session-info and merged onto
130
+ // the poll state via the `...info` spread, exactly like feature/step/label.
131
+ // The card live-ticks the elapsed minutes off these between polls. null for a
132
+ // legacy editor that predates the fields, or a VM with no active step.
133
+ startedAt: v.startedAt || null,
134
+ featureStartedAt: v.featureStartedAt || null,
135
+ // The server-computed stuck / needs-you verdict (npm/fleet-stuck-status.js):
136
+ // { stepElapsedMs, level: 'ok'|'slow'|'stuck', needsYou, needsYouElapsedMs }.
137
+ // The authoritative level only changes on poll; the card colors the step
138
+ // tenure badge by it. null when buildData couldn't compute one.
139
+ attention: ctx.attention || null,
140
+ // Per-VM card readiness derived upstream (npm/fleet-vm-readiness.js): one of
141
+ // READINESS_STATES. The card colors the border and GATES the editor link on
142
+ // it — a non-'ready' VM never presents its public URL as live, because the
143
+ // LB would 503. Computed in buildData where the broker tri-state / job / LB
144
+ // mode are on hand, then carried here so the browser needs no extra request.
145
+ readiness: ctx.readiness || null,
146
+ // The raw LB backend-health map { editor, preview } the readiness derivation
147
+ // consumed, surfaced so the card can show a precise tooltip (which surface
148
+ // is unwired). null in legacy mode or before the first slow probe lands.
149
+ backendHealth: v.backendHealth || null,
150
+ // The per-VM IAP access whitelist (the principals explicitly granted
151
+ // resource-level access to THIS VM, from the fleet-vm-access sidecar). Merged
152
+ // onto the poll state at server.js, it drives the card's 🔐 access line and
153
+ // the Manage access modal's "this VM only" list. It was previously dropped
154
+ // here — the exact currentProvider-style envelope drift this module guards
155
+ // against — so the line and list rendered empty even when the sidecar held
156
+ // principals. Defaults to [] for a VM with no list.
157
+ vmAccess: Array.isArray(v.vmAccess) ? v.vmAccess : [],
158
+ // Per-slot persisted-env override verdict (set by reapplyVmEnv, sticky on
159
+ // poll state): { present, skipped, reason, taggedRepo, setAt, keys }. Drives
160
+ // the card's env-override badge — an "applied" chip listing the masked keys +
161
+ // tagged repo, or a warning when a stale/untagged override is NOT replayed so
162
+ // a months-old override is never invisible again. null when no env is
163
+ // persisted for the slot (or the slot has never been reapplied).
164
+ envOverride: v.envOverride || null,
64
165
  };
65
166
  }
66
167
 
@@ -71,11 +172,14 @@ function buildVmEnvelope(v, ctx) {
71
172
  // card renders means adding it here AND in buildVmEnvelope — the test fails loud
72
173
  // if you do one without the other.
73
174
  const REQUIRED_CARD_FIELDS = [
74
- 'n', 'url', 'reachable', 'hasSession', 'brokerLive', 'brokerStatus',
75
- 'awaitingInput', 'feature', 'step', 'mode', 'label', 'status', 'orphan',
76
- 'degradedReasons', 'isHead', 'headHolderUnreachable', 'queuePos', 'queueLen',
175
+ 'n', 'url', 'previewUrl', 'appPort', 'previewServing', 'previewIngress', 'reachable', 'reachableSource', 'hasSession', 'brokerLive', 'brokerStatus',
176
+ 'awaitingInput', 'needsCleanup', 'feature', 'selectedPlan', 'step', 'mode', 'label', 'status', 'orphan',
177
+ 'degradedReasons', 'setupSteps', 'attentionReasons',
178
+ 'isHead', 'headHolderUnreachable', 'queuePos', 'queueLen',
77
179
  'job', 'config', 'project', 'variant', 'cardIdentity', 'brokerHistory',
78
- 'brokerDeath', 'providerAuth', 'effects', 'currentProvider',
180
+ 'brokerDeath', 'providerAuth', 'previewMisconfigured', 'effects', 'currentProvider',
181
+ 'readiness', 'backendHealth', 'vmAccess',
182
+ 'startedAt', 'featureStartedAt', 'attention', 'envOverride',
79
183
  ];
80
184
 
81
185
  module.exports = { buildVmEnvelope, REQUIRED_CARD_FIELDS };
@@ -78,13 +78,19 @@ function classifyDescribeError(err, stderr) {
78
78
  // Decide what the startup reap should do for one sidecar. Pure: takes the
79
79
  // decoded sidecar + GCP-existence verdict + (for adds) container-up verdict +
80
80
  // (for up adds) project-wiring verdict, returns a tag the caller dispatches on.
81
- // verdict when dispatch
82
- // 'cleanup' destroy gone, OR add gone runDestroyCleanup / runAddCleanup
83
- // 'reconcile-add' add up, container up, project wired (or unknown) ensure roster + recordLaunch, then clear
84
- // 'half-provisioned' add up, container up, but project NOT wired surface needs-recovery card (finish-wire from cfg)
85
- // 'recover-bootstrap' add instance up but container missing/down re-run cloud:up to finish the half-done bootstrap
86
- // 'clear-only' destroy sidecar but instance still exists just clear the sidecar
87
- // 'leave' gcloud uncertain (exists === null) leave for the next boot
81
+ // verdict when dispatch
82
+ // 'cleanup' destroy gone, OR add gone runDestroyCleanup / runAddCleanup
83
+ // 'reconcile-add' add up, container up, project wired, agent usable ensure roster + recordLaunch, then clear
84
+ // 'half-provisioned' add up, container up, but project NOT wired surface needs-recovery card (finish-wire from cfg)
85
+ // 'provision-incomplete' add up + container up + wired, but agent NOT surface needs-recovery card (the agent
86
+ // usable (CLI missing / auth wedged at unknown) isn't installed/authed — VM5 false-green)
87
+ // 'recover-bootstrap' add instance up but container missing/down re-run cloud:up to finish the half-done bootstrap
88
+ // (runAddRecoverBootstrap first probes the editor
89
+ // directly and diverts to rostering when the
90
+ // bootstrap is actually done and only the LB
91
+ // ingress is missing)
92
+ // 'clear-only' destroy sidecar but instance still exists just clear the sidecar
93
+ // 'leave' gcloud uncertain (exists === null) leave for the next boot
88
94
  // The (exists, containerUp) split distinguishes a finished add from a container-
89
95
  // down half-provisioned one: a dashboard SIGKILL'd mid-`cloud:up` leaves a GCP
90
96
  // instance with no docker/container, which the old (exists-only) matrix wrongly
@@ -99,17 +105,82 @@ function classifyDescribeError(err, stderr) {
99
105
  // for an add whose instance exists; destroy and the instance-gone add path ignore
100
106
  // them. Splitting the matrix out lets the table be exhaustively tested without any
101
107
  // of the side-effects in server.js.
102
- function reapVerdict({ action, exists, containerUp, projectWired }) {
108
+ //
109
+ // The `agentReady` axis (vm: Readiness Verifies Real Usability) closes the VM5
110
+ // false-green: an instance + container that ARE up and wired, but whose build
111
+ // agent isn't usable — the CLI isn't installed, or the auth probe is wedged at
112
+ // `unknown` — is NOT a finished add. Mirrors `projectWired`'s posture EXACTLY:
113
+ // only a POSITIVE not-ready signal (`agentReady === false`) downgrades, to
114
+ // 'provision-incomplete'; `null`/`undefined` (the probe couldn't decide, or an
115
+ // older editor doesn't report it) stays 'reconcile-add' so a flaky probe never
116
+ // strands a healthy VM. Checked AFTER `projectWired` — an unwired box is the more
117
+ // fundamental fault and keeps its own verdict.
118
+ //
119
+ // NOT an axis here (vm--batch-add-must-roster-and-wire-each-vm): "the editor
120
+ // answers on its own port but has no LB ingress". That state is a finished
121
+ // bootstrap missing its wiring, and re-running cloud:up cannot create the
122
+ // missing ingress — so it used to burn the 3-attempt budget and land on
123
+ // 'bootstrap-recovery-exhausted', because the signal being retried on
124
+ // (`lbReachable`) can never pass while the ingress is absent:
125
+ //
126
+ // no ingress → LB 503 → not reachable → recover → still no ingress → exhausted
127
+ //
128
+ // The rescue deliberately lives in server.js's runAddRecoverBootstrap rather
129
+ // than as a verdict here, because it needs a live IAP probe and is only worth
130
+ // paying for on the branch that was about to spend a recovery attempt. Adding it
131
+ // as a `reapVerdict` axis would force every container-down reap to run that
132
+ // probe eagerly just to compute a verdict, for a state that is rare.
133
+ function reapVerdict({ action, exists, containerUp, projectWired, agentReady, lbReachable }) {
103
134
  if (exists === null) return 'leave';
104
135
  if (action === 'destroy') return exists ? 'clear-only' : 'cleanup';
105
136
  if (action === 'add') {
106
137
  if (!exists) return 'cleanup';
138
+ // The `lbReachable` axis (vm--gce-reachable-flag-from-lb-probe) closes the
139
+ // false-orphan: a VM whose editor ANSWERS over the LB is provisioned by
140
+ // definition — its add finished, whatever a stale `add-<n>.json` sidecar
141
+ // left lying around implies. Without this, the container probe below could
142
+ // report "container down" for a VM that is demonstrably up and serving,
143
+ // driving recover-bootstrap → "exceeded 3 attempts" → a
144
+ // needs-manual-recovery card for a perfectly healthy box.
145
+ // Inverted posture vs projectWired/agentReady, and deliberately so: those
146
+ // axes let only a NEGATIVE signal downgrade, this one lets only a POSITIVE
147
+ // signal promote. `false`/`null`/`undefined` (probe said no, couldn't
148
+ // decide, or never ran) all fall through to the unchanged ladder below, so
149
+ // this can never strand a VM — it can only rescue one.
150
+ if (lbReachable === true) return 'reconcile-add';
107
151
  if (!containerUp) return 'recover-bootstrap';
108
- return projectWired === false ? 'half-provisioned' : 'reconcile-add';
152
+ if (projectWired === false) return 'half-provisioned';
153
+ if (agentReady === false) return 'provision-incomplete';
154
+ return 'reconcile-add';
109
155
  }
110
156
  return 'leave';
111
157
  }
112
158
 
159
+ // Classify the startup reap's LB rescue signal from a best-effort LB+IAP editor
160
+ // probe. Pure: server.js runs the HTTP probe and hands the raw outcome here, the
161
+ // same split classifyProjectWiring follows, so the rule is unit-testable without
162
+ // a live VM or an LB.
163
+ //
164
+ // Deliberately tri-state and ASYMMETRIC — it returns `true` or `null`, NEVER
165
+ // `false`:
166
+ // - probeViaLb false (a laptop has no LB to ask) → null (no opinion)
167
+ // - err present (threw, auth fault, timeout) → null (no opinion)
168
+ // - the editor answered → true (add is finished)
169
+ // - the editor did not answer → null (NOT false)
170
+ //
171
+ // That last line is the load-bearing one. A silent probe is not evidence a VM is
172
+ // dead — the LB may be unwired mid-add, or the IAP token not yet minted. Since
173
+ // reapVerdict promotes on `true` and ignores everything else, returning null
174
+ // leaves the pre-existing container/wiring ladder in charge. Returning `false`
175
+ // here would be indistinguishable to reapVerdict, but it would be a lie about
176
+ // what the probe proved, and the next person to add an axis would read it as
177
+ // evidence. The invariant this pins: this signal can rescue a VM, never strand one.
178
+ function classifyLbReachableForReap({ probeViaLb, reachable, err } = {}) {
179
+ if (!probeViaLb) return null;
180
+ if (err) return null;
181
+ return reachable ? true : null;
182
+ }
183
+
113
184
  // Classify whether a VM's editor `/workspace` is actually wired to a client
114
185
  // project, from a best-effort probe of two signals: the git remote URL and
115
186
  // whether `package.json` carries a real `name`. Pure: server.js runs the SSH +
@@ -175,13 +246,28 @@ function clearExitMarker(dir, n) {
175
246
  catch (_) { /* missing is fine */ }
176
247
  }
177
248
 
249
+ // True when an integer exit code is the shell's "killed by signal N" encoding,
250
+ // i.e. 128 + N for N in 1..37 — the realistic kills being 143 (SIGTERM, the
251
+ // `launchctl bootout` graceful teardown), 137 (SIGKILL), 130 (SIGINT) and 129
252
+ // (SIGHUP). Pure boundary check: 128 itself is NOT a signal kill (it's `exit
253
+ // 128`, an ordinary failure code), so the range is the OPEN interval
254
+ // (128, 165]. Anything outside it — a small `exit 1` from the addVm script's own
255
+ // failure branches, 0, NaN, null — is not a signal kill. The split that matters:
256
+ // "did a signal kill the provision shell" (interrupted → reconcile) vs "did the
257
+ // provision logic itself decide it failed" (real error). Exported so the matrix
258
+ // is unit-testable directly.
259
+ function isSignalKillExit(code) {
260
+ return Number.isInteger(code) && code > 128 && code <= 165;
261
+ }
262
+
178
263
  // Decide how a snapshot-restored `add` should be re-attached after a dashboard
179
264
  // restart, BEFORE any gcloud probe. Pure: takes the durable signals the boot
180
265
  // path gathers — whether the exit marker exists, its recorded code, and whether
181
266
  // the detached provision child is still alive (recorded pid, `kill -0`).
182
267
  // verdict when dispatch
183
268
  // 'done' marker present, exit code 0 roster + recordLaunch, clear sidecar/marker
184
- // 'error' marker present, non-zero exit code surface the real failure, clear sidecar/marker
269
+ // 'reconcile' marker present, SIGNAL-kill code (128<c≤165) interrupted, not failed — gcloud probe decides
270
+ // 'error' marker present, other non-zero exit code surface the real failure, clear sidecar/marker
185
271
  // 'running' no marker yet, child still alive RE-ATTACH: keep polling, do NOT mark error
186
272
  // 'reconcile' child gone, no usable marker fall through to fc.reapVerdict (gcloud probe)
187
273
  // This is the durable fix's brain: a provision that survived the kill (still
@@ -192,14 +278,33 @@ function clearExitMarker(dir, n) {
192
278
  // `reconcile` hands that exact (exists, containerUp, projectWired) decision to
193
279
  // fc.reapVerdict. A marker whose first line isn't an integer is treated as
194
280
  // absent (parseExitMarker → null) so a corrupt marker reconciles rather than
195
- // asserting a bogus code. The matrix is exhaustively tested without a live VM.
281
+ // asserting a bogus code.
282
+ //
283
+ // SIGNAL-kill codes (improve46): `setsid` shields the detached provision from a
284
+ // dashboard-process-GROUP SIGKILL (the 2GB launchd hard cap), but NOT from a
285
+ // `launchctl bootout`, which tears down the whole launchd job tree regardless of
286
+ // session/process-group. A graceful SIGTERM then reaches the "detached" child and
287
+ // its EXIT trap records 143 (128+SIGTERM) into the marker. That is an INTERRUPTED
288
+ // add, not a failed one — the manual re-add succeeds in seconds off the image the
289
+ // interrupted build already left on the VM. So a signal-kill code routes to
290
+ // 'reconcile' (let the gcloud probe read ground truth and roster / recover) rather
291
+ // than 'error' (a false add-failed that throws away a resumable build). A genuine
292
+ // provision failure exits with a small non-zero code (e.g. `exit 1`), which STILL
293
+ // classifies 'error'. The matrix is exhaustively tested without a live VM.
196
294
  function reattachVerdict({ markerPresent, exitCode, childAlive } = {}) {
197
295
  if (markerPresent) {
198
296
  // Guard the null/undefined coercion BEFORE Number() — `Number(null)` is 0,
199
297
  // which would masquerade as a clean exit. A marker with no parseable code
200
298
  // (half-written / corrupt) must reconcile, not assert success.
201
299
  const code = exitCode == null ? NaN : Number(exitCode);
202
- if (Number.isInteger(code)) return code === 0 ? 'done' : 'error';
300
+ if (Number.isInteger(code)) {
301
+ if (code === 0) return 'done';
302
+ // A signal killed the provision shell (e.g. bootout SIGTERM → 143): the add
303
+ // was interrupted, not failed — reconcile against real VM state instead of
304
+ // surfacing a false add-failed and discarding a resumable build.
305
+ if (isSignalKillExit(code)) return 'reconcile';
306
+ return 'error';
307
+ }
203
308
  return 'reconcile'; // marker present but no parseable code — let gcloud decide
204
309
  }
205
310
  if (childAlive) return 'running';
@@ -297,11 +402,119 @@ function orphanedAddsToReconcile({ jobs, sidecarKeys } = {}) {
297
402
  // posture). "Busy" (a live session or an in-flight add to this number) also
298
403
  // holds the entry for an operator rather than auto-removing — the shouldn't-
299
404
  // happen-but-fail-safe case where a vanished instance still reports a session.
300
- function reconcileRosteredVm({ exists, hasSession, hasPendingAdd } = {}) {
405
+ // verdict when dispatch
406
+ // 'cleanup' instance gone AND the VM is idle runDestroyCleanup
407
+ // 'leave' exists uncertain (null), OR the VM is busy do nothing this pass
408
+ // 'keep' instance still exists do nothing (healthy)
409
+ // 'reset' existing idle instance unreachable too long trigger auto-reset
410
+ // Fail-safe by construction: cleanup fires ONLY on a DEFINITIVE absence
411
+ // (exists === false) — a flaky/transient describe (null) is always 'leave', so a
412
+ // gcloud blip can never evict a healthy VM (mirroring reapVerdict's `leave`
413
+ // posture). "Busy" (a live session or an in-flight add to this number) also
414
+ // holds the entry for an operator rather than auto-removing — the shouldn't-
415
+ // happen-but-fail-safe case where a vanished instance still reports a session.
416
+ // auto-reset escalation (vm: OOM Wedge Prevention):
417
+ // When the VM has been unreachable for at least RESET_UNREACHABLE_THRESHOLD
418
+ // (e.g. 200, which is roughly 10 minutes at a 3s poll rate), we issue a 'reset'
419
+ // verdict. This auto-reset is guarded and bounded by a cooldown window (in ms)
420
+ // and a max attempts limit to prevent infinite reboot loops.
421
+ //
422
+ // false-unreachable reboot safety (2026-06-25): "unreachable" as the dashboard
423
+ // sees it means the LOCALLY-FORWARDED port is dark — which is ALSO the signature
424
+ // of an operator-side fault (the laptop's gcloud auth expired, the ssh -L tunnel
425
+ // zombied, a laptop network blip) where the VM itself is perfectly healthy.
426
+ // Rebooting a healthy production VM off that signal interrupts a live session.
427
+ // So three operator-fault guards are layered BEFORE the cooldown/attempts budget
428
+ // (a reset is a last resort, never the response to a dark local port). Each is
429
+ // OPT-IN: it only changes the verdict when the caller supplies the fact, so
430
+ // legacy callers (and the OOM-wedge tests) that pass none keep the old behavior.
431
+ // operatorAuthDegraded our own gcloud auth is broken → EVERY VM reads
432
+ // unreachable; rebooting them all is exactly wrong → keep
433
+ // fleetWideUnreachable a simultaneous N-VM "outage" is the operator (auth/
434
+ // tunnel/network), not N dead VMs → keep
435
+ // gcpReachable GCP-side confirmation: true ⇒ the VM answers from
436
+ // GCP's vantage, so the dark local port is a laptop
437
+ // tunnel fault → 'respawn' (never reset); null ⇒ the
438
+ // GCP-side probe was inconclusive → keep (don't escalate
439
+ // a destructive reset on absence of evidence); false ⇒
440
+ // genuinely unreachable from GCP too → fall through to
441
+ // the cooldown/attempts budget and reset as before.
442
+ function reconcileRosteredVm({ exists, hasSession, hasPendingAdd, streak, resetHistory, threshold, cooldownMs, maxAttempts, operatorAuthDegraded, fleetWideUnreachable, gcpReachable } = {}) {
301
443
  if (exists === null || exists === undefined) return "leave";
302
- if (exists) return "keep";
303
- if (hasSession || hasPendingAdd) return "leave";
304
- return "cleanup";
444
+ if (!exists) {
445
+ if (hasSession || hasPendingAdd) return "leave";
446
+ return "cleanup";
447
+ }
448
+
449
+ // If the VM exists, check if it's dead-unreachable and eligible for auto-reset.
450
+ if (streak !== undefined && threshold !== undefined) {
451
+ if (streak >= threshold) {
452
+ if (hasSession || hasPendingAdd) {
453
+ // Do NOT auto-reset a VM that is busy (holding a live session or an in-flight add)
454
+ return "keep";
455
+ }
456
+
457
+ // Operator-fault guards — layered before the cooldown/attempts budget so a
458
+ // laptop-side fault never even consumes a reset attempt.
459
+ if (operatorAuthDegraded) return "keep";
460
+ if (fleetWideUnreachable) return "keep";
461
+ if (gcpReachable === true) return "respawn";
462
+ if (gcpReachable === null) return "keep";
463
+
464
+ const history = resetHistory || { count: 0, lastResetAt: 0 };
465
+ const attempts = history.count || 0;
466
+ const limit = maxAttempts !== undefined ? maxAttempts : 3;
467
+
468
+ if (attempts >= limit) {
469
+ // Budget exhausted (VM wedges repeatedly) -> park (needs-manual-recovery status)
470
+ return "keep";
471
+ }
472
+
473
+ const lastReset = history.lastResetAt || 0;
474
+ const cooldown = cooldownMs !== undefined ? cooldownMs : 600000; // default 10 minutes
475
+ const now = Date.now();
476
+ if (now - lastReset < cooldown) {
477
+ // Within cooldown window -> do not reset again yet
478
+ return "keep";
479
+ }
480
+
481
+ return "reset";
482
+ }
483
+ }
484
+
485
+ return "keep";
486
+ }
487
+
488
+ // The bootstrap-recovery ladder step the startup reap runs for a VM whose first
489
+ // cloud:up did not finish: decide (via recoverBootstrapDecision) whether another
490
+ // attempt is allowed, then either flag the VM exhausted (leave the sidecar for
491
+ // manual action) or clear any stale orphan flag and re-run cloud:up via `addVm`.
492
+ // The dashboard's `state` and `addVm` are injected so this stays a pure,
493
+ // importable unit — `log`/`err` default to the console but are overridable in
494
+ // tests. Behavior mirrors the previous inline server.js definition exactly.
495
+ function continueBootstrapRecovery(
496
+ n,
497
+ meta,
498
+ cfg,
499
+ { state, addVm, log = console.log, err = console.error } = {},
500
+ ) {
501
+ const { attempt, proceed, max } = recoverBootstrapDecision({
502
+ recoverAttempts: meta.recoverAttempts,
503
+ });
504
+ if (!proceed) {
505
+ err(
506
+ `startup-reap: VM-${n} exceeded ${max} bootstrap-recovery attempts — leaving sidecar for manual action`,
507
+ );
508
+ state.orphans[n] = { reason: "bootstrap-recovery-exhausted", attempts: attempt - 1 };
509
+ return;
510
+ }
511
+ log(
512
+ `startup-reap: VM-${n} re-running cloud:up to finish bootstrap (recovery attempt ${attempt}/${max})`,
513
+ );
514
+ // A prior exhausted/no-cfg run may have flagged this VM; clear it now that a
515
+ // fresh attempt is starting so the card moves to `recovering`, not stale red.
516
+ delete state.orphans[n];
517
+ addVm(n, cfg, { recovering: true, recoverAttempt: attempt });
305
518
  }
306
519
 
307
520
  module.exports = {
@@ -314,11 +527,14 @@ module.exports = {
314
527
  reconcileRosteredVm,
315
528
  orphanedAddsToReconcile,
316
529
  classifyProjectWiring,
530
+ classifyLbReachableForReap,
317
531
  recoverBootstrapDecision,
532
+ continueBootstrapRecovery,
318
533
  exitMarkerName,
319
534
  parseExitMarker,
320
535
  readExitMarker,
321
536
  clearExitMarker,
537
+ isSignalKillExit,
322
538
  reattachVerdict,
323
539
  reattachNextStep,
324
540
  };