@codeyam-editor/codeyam-editor 0.1.4 → 0.1.6

package/npm/ingress-invoke.js

@@ -0,0 +1,770 @@
+'use strict';
+// Pure builders for the LOCAL shell invocations that wire the fleet's Google
+// HTTPS Load Balancer ingress: the shared, VM-independent infra
+// (scripts/gcp/google-ingress.sh) and one VM's per-surface backends
+// (scripts/gcp/vm-ingress.sh). cloud.js calls these to decide WHEN to wire and
+// to build the exact `{ command, args, env }` it spawns — keeping the decision
+// and the argv/env shape unit-testable without ever spawning a process, exactly
+// like npm/vm-urls.js and npm/google-ingress.js single-source their derivations.
+//
+// Pure logic only — no I/O, no env reads beyond the explicit `env`/`cfg`
+// arguments callers pass in. The fleet domain governing ingress comes from the
+// project's `.codeyam/cloud.json` (`fleetDomain`, persisted by cloud.js) with a
+// `FLEET_DOMAIN` env fallback; when neither is set every builder reports the
+// legacy quick-tunnel mode (resolveIngressDomain → null) and cloud.js leaves the
+// Add/Destroy path byte-for-byte unchanged.
+//
+// Stack note: this is fleet-operator infrastructure (the cloud GCE fleet that
+// runs codeyam-editor itself), GCP-specific by design — the same posture as the
+// scripts it invokes. A project that never joins the fleet never sets
+// fleetDomain and never reaches a builder here.
+
+const { resolveFleetDomain } = require('./vm-urls');
+
+// Cloud DNS managed-zone name google-ingress.sh defaults to. Mirrored here so a
+// cloud.json without an explicit `dnsZone` resolves to the same zone the script
+// would, rather than passing an empty DNS_ZONE that shadows the script default.
+const DEFAULT_DNS_ZONE = 'fleet-zone';
+
+// The backend-service timeout (seconds) the editor's WebSocket needs to survive a
+// normal Build session. On an External HTTPS LB this value is the MAX lifetime a
+// WebSocket may stay open; 86400 (24h) clears the longest plausible session while
+// GCP still tears the socket down on client disconnect. Mirrors the literal
+// vm-ingress.sh creates BES with. See plan vm--lb-backend-service-websocket-timeout.
+const BES_WEBSOCKET_TIMEOUT_SEC = 86400;
+
+/** Resolve the fleet domain that governs ingress wiring, or null when none is
+ *  configured (legacy quick-tunnel mode). `cfg.fleetDomain` (persisted in
+ *  cloud.json) wins; the `FLEET_DOMAIN` env value is the fallback. Both paths go
+ *  through vm-urls.js::resolveFleetDomain, so a present-but-malformed value
+ *  throws here (a typo'd domain must fail loud, never bake a dead host into the
+ *  URL map) and a blank value reads as "unset". */
+function resolveIngressDomain(cfg = {}, env = {}) {
+  const fromCfg = cfg.fleetDomain;
+  if (fromCfg != null && String(fromCfg).trim() !== '') {
+    return resolveFleetDomain({ FLEET_DOMAIN: fromCfg });
+  }
+  return resolveFleetDomain(env);
+}
+
+/** Resolve the Cloud DNS managed-zone name: `cfg.dnsZone`, then a `DNS_ZONE`
+ *  env override, then the `fleet-zone` default google-ingress.sh itself uses. */
+function resolveDnsZone(cfg = {}, env = {}) {
+  const fromCfg = cfg.dnsZone;
+  if (fromCfg != null && String(fromCfg).trim() !== '') return String(fromCfg).trim();
+  const fromEnv = env.DNS_ZONE;
+  if (fromEnv != null && String(fromEnv).trim() !== '') return String(fromEnv).trim();
+  return DEFAULT_DNS_ZONE;
+}
+
+/** Derive the GCE region from a zone (`us-central1-a` → `us-central1`), or null
+ *  when the zone is missing/unparseable so the caller omits REGION and lets
+ *  google-ingress.sh apply its own default. A GCE zone is a region plus a
+ *  trailing `-<letter>`. */
+function regionFromZone(zone) {
+  if (zone == null) return null;
+  const z = String(zone).trim();
+  if (z === '') return null;
+  const m = /^(.+)-[a-z]$/.exec(z);
+  return m ? m[1] : null;
+}
+
+/** Build the local invocation for `scripts/gcp/vm-ingress.sh <N> <up|down>` —
+ *  one VM's per-surface NEG → IAP backend → URL-map host rule. Returns
+ *  `{ command, args, env }`; `env` carries only the keys the script reads beyond
+ *  its own defaults (GCP_PROJECT, ZONE, FLEET_DOMAIN, APP_PORT, INSTANCE_PREFIX,
+ *  VM_SLUG). `appPort` is the resolved per-project dev-server port the preview
+ *  NEG targets; omit it (down, or unknown) to let the script's bootstrap default
+ *  stand. An optional `slug` becomes `VM_SLUG`, which makes the script route the
+ *  VM's custom `<slug>.<surface>.<domain>` host (the same slug must be passed on
+ *  `down` so the matching host rule is removed); omit it to keep the numeric
+ *  `vm-<N>` host. */
+function vmIngressInvocation({
+  scriptPath,
+  vmNumber,
+  action,
+  domain,
+  project,
+  zone,
+  appPort,
+  instancePrefix,
+  slug,
+} = {}) {
+  if (!scriptPath) throw new Error('vmIngressInvocation requires scriptPath');
+  if (action !== 'up' && action !== 'down') {
+    throw new Error(`vm-ingress action must be 'up' or 'down', got: ${action}`);
+  }
+  const n = Number(vmNumber);
+  if (!Number.isInteger(n) || n <= 0) {
+    throw new Error(`vm-ingress requires a positive VM number, got: ${vmNumber}`);
+  }
+  if (!domain) throw new Error('vmIngressInvocation requires a fleet domain');
+  const env = { FLEET_DOMAIN: domain };
+  if (project) env.GCP_PROJECT = project;
+  if (zone) env.ZONE = zone;
+  if (appPort != null && String(appPort).trim() !== '') env.APP_PORT = String(appPort);
+  if (instancePrefix) env.INSTANCE_PREFIX = instancePrefix;
+  if (slug != null && String(slug).trim() !== '') env.VM_SLUG = String(slug).trim();
+  return { command: 'bash', args: [scriptPath, String(n), action], env };
+}
+
+/** Build the local invocation for `scripts/gcp/vm-ingress.sh <N> status` — the
+ *  read-only probe that reports which of {NEG, backend service, host rule} exist
+ *  per surface plus an overall `verdict=wired|partial|unwired`. Returns
+ *  `{ command, args, env }`. Like `vmIngressInvocation` but the action is fixed
+ *  to `status` and APP_PORT is omitted — status describes resources by NAME, and
+ *  names key off the VM number, never the port. An optional `slug` rides through
+ *  as VM_SLUG so the host-rule presence check matches the VM's custom
+ *  `<slug>.<surface>.<domain>` host rather than the numeric `vm-<N>` one. */
+function vmIngressStatusInvocation({
+  scriptPath,
+  vmNumber,
+  domain,
+  project,
+  zone,
+  slug,
+} = {}) {
+  if (!scriptPath) throw new Error('vmIngressStatusInvocation requires scriptPath');
+  const n = Number(vmNumber);
+  if (!Number.isInteger(n) || n <= 0) {
+    throw new Error(`vm-ingress status requires a positive VM number, got: ${vmNumber}`);
+  }
+  if (!domain) throw new Error('vmIngressStatusInvocation requires a fleet domain');
+  const env = { FLEET_DOMAIN: domain };
+  if (project) env.GCP_PROJECT = project;
+  if (zone) env.ZONE = zone;
+  if (slug != null && String(slug).trim() !== '') env.VM_SLUG = String(slug).trim();
+  return { command: 'bash', args: [scriptPath, String(n), 'status'], env };
+}
+
+/** Build the local invocation for `scripts/gcp/vm-ingress.sh <N> grant-access` —
+ *  the grant-only pass that (re-)applies one VM's default IAP access grant
+ *  (persisted collaborators + standing operator + automation SA) WITHOUT touching
+ *  any NEG/BES/host-rule resource. Returns `{ command, args, env }`. Like
+ *  `vmIngressStatusInvocation` but the action is fixed to `grant-access` and
+ *  APP_PORT is omitted — the grant targets backend services by NAME (keyed off the
+ *  VM number), never the port. An optional `slug` rides through as VM_SLUG for
+ *  symmetry with the other actions, though the grant itself is slug-independent.
+ *  Reconcile spawns this for already-`wired` VMs its re-wire loop skips, so a
+ *  healed VM whose backends already exist still (re-)grants the SA accessor. */
+function vmIngressGrantAccessInvocation({
+  scriptPath,
+  vmNumber,
+  domain,
+  project,
+  zone,
+  slug,
+} = {}) {
+  if (!scriptPath) throw new Error('vmIngressGrantAccessInvocation requires scriptPath');
+  const n = Number(vmNumber);
+  if (!Number.isInteger(n) || n <= 0) {
+    throw new Error(`vm-ingress grant-access requires a positive VM number, got: ${vmNumber}`);
+  }
+  if (!domain) throw new Error('vmIngressGrantAccessInvocation requires a fleet domain');
+  const env = { FLEET_DOMAIN: domain };
+  if (project) env.GCP_PROJECT = project;
+  if (zone) env.ZONE = zone;
+  if (slug != null && String(slug).trim() !== '') env.VM_SLUG = String(slug).trim();
+  return { command: 'bash', args: [scriptPath, String(n), 'grant-access'], env };
+}
+
+/** Parse the `verdict=<wired|partial|unwired>` line vm-ingress.sh `status`
+ *  prints, returning that verdict or null when the output carries none (e.g. the
+ *  script errored before emitting it). Pure string parsing so the reconcile
+ *  decision below stays unit-testable without spawning the script. */
+function parseIngressStatusVerdict(stdout) {
+  if (typeof stdout !== 'string') return null;
+  for (const line of stdout.split('\n')) {
+    const m = /^verdict=(wired|partial|unwired)\s*$/.exec(line.trim());
+    if (m) return m[1];
+  }
+  return null;
+}
+
+/** Parse the `verdict=<torn-down|orphaned-ingress>` line vm-ingress.sh `down`
+ *  prints, returning that verdict or null when the output carries none (e.g. the
+ *  script died before emitting it, or was spawned with an older script). Pure
+ *  string parsing so the dashboard's destroy-teardown handling — clean vs record
+ *  an orphaned-ingress attention item — stays unit-testable without spawning the
+ *  script. Mirrors parseIngressStatusVerdict for the `down` verdict vocabulary. */
+function parseIngressTeardownVerdict(stdout) {
+  if (typeof stdout !== 'string') return null;
+  for (const line of stdout.split('\n')) {
+    const m = /^verdict=(torn-down|orphaned-ingress)\s*$/.exec(line.trim());
+    if (m) return m[1];
+  }
+  return null;
+}
+
+/** Pick which rostered VMs reconcile must re-wire from their status verdicts.
+ *  `statuses` is `[{ vmNumber, verdict, ... }]`; a VM is selected iff its verdict
+ *  is `unwired` or `partial` (it has missing LB resources). `wired` VMs are left
+ *  untouched — no redundant gcloud churn — and an unknown/null verdict (the probe
+ *  failed) is conservatively SKIPPED rather than blindly re-wired. Returns the
+ *  selected entries in their input order, preserving any extra fields (slug) the
+ *  caller threads through. Pure. */
+function selectIngressReconcileTargets(statuses) {
+  if (!Array.isArray(statuses)) return [];
+  return statuses.filter((s) => s && (s.verdict === 'unwired' || s.verdict === 'partial'));
+}
+
+/** Pick which rostered VMs reconcile must apply the grant-only IAP pass to — the
+ *  complement of the re-wire loop that closes the SA-accessor heal-path gap. A VM
+ *  is selected iff its verdict is `wired`: its backend services already exist (so
+ *  they are the grant's target) but the re-wire loop SKIPS it, meaning `up` — the
+ *  only path that (re-)grants the automation SA the IAP accessor — never runs for
+ *  it. Re-wire targets (`unwired`/`partial`) are excluded here because their `up`
+ *  already grants; an unknown/null verdict (the probe failed) is excluded too —
+ *  the BES may not exist, so a grant would only warn. The result is disjoint from
+ *  `selectIngressReconcileTargets`, so no VM is both re-wired and grant-passed.
+ *  Returns the selected entries in input order, preserving extra fields (slug).
+ *  Pure. */
+function selectIngressGrantOnlyTargets(statuses) {
+  if (!Array.isArray(statuses)) return [];
+  return statuses.filter((s) => s && s.verdict === 'wired');
+}
+
+/** Compute the `process.kill` target that reaps a timed-out reconcile's ENTIRE
+ *  process group. The dashboard spawns `cloud:reconcile-ingress` detached, so the
+ *  child shell leads its own process group (pgid === child.pid) and a negative-pid
+ *  signal reaps every descendant — `node cloud.js` → `bash vm-ingress.sh` →
+ *  `gcloud` — instead of only the immediate `sh -c` child (what an exec/SIGTERM
+ *  timeout does, orphaning the descendants to init where they pile up and wedge the
+ *  dashboard). Returns `-pid` for a real detached child, or `null` when the pid is
+ *  missing / already reaped or would be catastrophic to negate. The `> 1` guard is
+ *  load-bearing: `process.kill(-1, …)` signals EVERY process the user owns and
+ *  `process.kill(0, …)`/`-0` targets the caller's OWN group — a missing pid must
+ *  never be turned into a fleet-wide or self-directed SIGKILL. Pure: pid in, target
+ *  out, so the timeout-kill decision is unit-testable without spawning a process. */
+function reconcileGroupKillTarget(childPid) {
+  if (!Number.isInteger(childPid) || childPid <= 1) return null;
+  return -childPid;
+}
+
+/** Pick the genuine orphans from the VM numbers that currently HAVE LB ingress
+ *  resources (`ingressVmNumbers`, derived from the live backend-service AND NEG
+ *  names). A number is an orphan iff it has ingress but is NOT in the roster,
+ *  has NO in-flight add, AND is NOT a live GCP instance — the false-orphan
+ *  guards (vm--auto-clean-orphaned-lb-ingress) that stop the auto-clean from
+ *  ripping ingress out from under a VM that is mid-provision (its `add-<n>.json`
+ *  sidecar and, moments later, its roster entry land before ingress finishes
+ *  wiring), transiently unlisted, or genuinely running but not yet re-rostered
+ *  (a roster drift — tearing down a *live* VM's ingress would knock it offline).
+ *  Numbers not yet rostered but with an in-flight add, or with a live instance,
+ *  are treated as live, not orphans. `liveVmNumbers` is optional; omit it (or
+ *  pass a non-array) to disable the live-instance guard. Returns the orphan
+ *  numbers deduped and sorted ascending. Pure. */
+function selectOrphanIngressTargets({ ingressVmNumbers, roster, inFlightAdds, liveVmNumbers } = {}) {
+  const rosterSet = new Set((Array.isArray(roster) ? roster : []).map(Number));
+  const inFlightSet = new Set((Array.isArray(inFlightAdds) ? inFlightAdds : []).map(Number));
+  const liveSet = new Set((Array.isArray(liveVmNumbers) ? liveVmNumbers : []).map(Number));
+  const orphans = new Set();
+  for (const raw of Array.isArray(ingressVmNumbers) ? ingressVmNumbers : []) {
+    const n = Number(raw);
+    if (!Number.isInteger(n) || n <= 0) continue;
+    if (rosterSet.has(n) || inFlightSet.has(n) || liveSet.has(n)) continue;
+    orphans.add(n);
+  }
+  return [...orphans].sort((a, b) => a - b);
+}
+
+/** Render a `vm-ingress.sh down` child's termination honestly for an operator log.
+ *  A child killed by a SIGNAL has a NULL exit code, so the old
+ *  `code != null ? code : 0` rendering printed the most misleading string
+ *  available — `exit=0` — for a teardown the supervising timeout had just
+ *  SIGTERMed (the destroy-time bug: `verdict=none, exit=0` on a script that never
+ *  got to finish). Reports the signal when there is one, `spawn-failed` when
+ *  neither a code nor a signal is available, and the real code otherwise. Pure so
+ *  the reporting is unit-testable without spawning anything. */
+function describeTeardownExit({ status, signal } = {}) {
+  if (signal) return `killed=${signal}`;
+  if (status == null) return 'exit=spawn-failed';
+  return `exit=${status}`;
+}
+
+/** Classify one `vm-ingress.sh <N> down` result into cleaned vs failed. A clean
+ *  teardown requires BOTH the verified `verdict === 'torn-down'` AND a zero exit
+ *  — anything else (the `orphaned-ingress` verdict, a null/missing verdict from
+ *  an older script or a crash, a non-zero exit, or a signal kill) is a failure
+ *  the caller surfaces with its cause. `signal` is optional: pass it and a
+ *  timeout-killed child is reported as `killed=SIGTERM` instead of masquerading
+ *  as a clean `exit=0`. Shared by the orphan sweep (npm/cloud.js
+ *  sweepOrphanIngress, spawnSync) and the dashboard's destroy-time teardown
+ *  (server.js tearDownVmIngress, detached spawn) so the two can never drift on
+ *  what "cleaned" means. Returns `{ cleaned: true }` or
+ *  `{ cleaned: false, cause }`. Pure so the cleaned-vs-failed decision is
+ *  unit-testable without spawning the script. */
+function classifyOrphanDownResult({ verdict, status, signal } = {}) {
+  if (verdict === 'torn-down' && status === 0 && !signal) return { cleaned: true };
+  return { cleaned: false, cause: `down verdict=${verdict || 'none'}, ${describeTeardownExit({ status, signal })}` };
+}
+
+/** Parse the machine-readable `CY_ORPHAN_SWEEP <json>` summary line the orphan
+ *  sweep prints into `{ cleaned:[n], failed:[{n, cause}], skipped:[n] }` — plus
+ *  `stillPresent:[n]` (VM numbers that still own ≥1 LB resource) ONLY when the
+ *  summary carries that field. Its ABSENCE (an older reconcile binary that never
+ *  emitted it) is preserved as an absent key, NOT normalized to `[]`, so the
+ *  dashboard's reconcile can tell "sweep reported no residual" (`[]`) apart from
+ *  "sweep can't say" (undefined) and leave stale banners untouched in the latter.
+ *  Returns null when no `CY_ORPHAN_SWEEP` line is present (no sweep ran, or one
+ *  died before emitting) so the dashboard can distinguish "no sweep ran" from
+ *  "sweep ran, found nothing". The LAST such line wins (a reconcile emits one).
+ *  Malformed JSON is skipped, not thrown. Pure so the dashboard's orphan-state
+ *  reconciliation stays unit-testable without spawning the reconcile. */
+function parseOrphanSweepSummary(stdout) {
+  if (typeof stdout !== 'string') return null;
+  let found = null;
+  for (const line of stdout.split('\n')) {
+    const m = /^CY_ORPHAN_SWEEP\s+(\{.*\})\s*$/.exec(line.trim());
+    if (!m) continue;
+    let o;
+    try { o = JSON.parse(m[1]); } catch { continue; }
+    const nums = (arr) => (Array.isArray(arr) ? arr.map(Number).filter((n) => Number.isInteger(n) && n > 0) : []);
+    found = {
+      cleaned: nums(o.cleaned),
+      failed: (Array.isArray(o.failed) ? o.failed : [])
+        .filter((f) => f && Number.isInteger(Number(f.n)) && Number(f.n) > 0)
+        .map((f) => ({ n: Number(f.n), cause: String(f.cause || 'auto-clean did not verify clean') })),
+      skipped: nums(o.skipped),
+    };
+    if (Array.isArray(o.stillPresent)) found.stillPresent = nums(o.stillPresent);
+  }
+  return found;
+}
+
+/** The reconcile-direction mirror of {@link selectOrphanIngressTargets}: given
+ *  the VM numbers that currently carry an orphan-attention banner (`orphanKeys`,
+ *  the keys of the dashboard's orphanedIngress map), the live `roster`, and the
+ *  sweep's `stillPresent` set (VM numbers that still own ≥1 LB resource), decide
+ *  which banners are STALE and must be cleared. A banner is stale iff its VM is
+ *  (a) back in the roster — a destroy→re-add churn re-rostered it, freezing an
+ *  old `failed` banner the sweep never revisits (the VM-3 case) — or (b) reported
+ *  as owning no residual LB resource (absent from `stillPresent`), e.g. a VM
+ *  cleaned outside a recorded sweep. When `stillPresent` is NOT an array (an older
+ *  reconcile binary that never emitted the field) this returns `[]` so the caller
+ *  leaves every banner untouched rather than clearing all of them — the absence of
+ *  the field means "the sweep can't say," not "nothing is present." Junk keys
+ *  (non-positive / non-integer) are ignored. Returns the VM numbers to clear,
+ *  deduped ascending. Pure so the reconcile is unit-testable without the server. */
+function staleOrphanEntries({ orphanKeys, roster, stillPresent } = {}) {
+  if (!Array.isArray(stillPresent)) return [];
+  const present = new Set(stillPresent.map(Number));
+  const rosterSet = new Set((Array.isArray(roster) ? roster : []).map(Number));
+  const stale = new Set();
+  for (const raw of Array.isArray(orphanKeys) ? orphanKeys : []) {
+    const n = Number(raw);
+    if (!Number.isInteger(n) || n <= 0) continue;
+    if (rosterSet.has(n) || !present.has(n)) stale.add(n);
+  }
+  return [...stale].sort((a, b) => a - b);
+}
+
+/** Decide the next step of the add-time ingress CONVERGE loop
+ *  (vm--add-time-url-wiring-converges-reliably). The detached add-tail no longer
+ *  fires `vm-ingress.sh up` ONCE and trusts the slow periodic reconcile to catch
+ *  up — it re-runs `up`+`status` on its own tight cadence until the VM's verdict
+ *  is `wired` or a bounded deadline elapses. This pure decision keeps that loop
+ *  unit-testable without spawning the script or sleeping:
+ *
+ *    - `verdict === 'wired'`  → `{ done: true, wired: true }`. Converged; stop.
+ *    - elapsed ≥ `deadlineMs` → `{ done: true, wired: false }`. Give up; the
+ *      caller surfaces `wiring-failed` (a real attention reason) rather than a
+ *      silent `partial` left for the sweep.
+ *    - otherwise → `{ done: false, wired: false, delayMs }` where `delayMs` is an
+ *      exponential backoff `baseDelayMs * 2**attempt` capped at `maxDelayMs`, so
+ *      the first retries are quick (seconds) and later ones ease off.
+ *
+ *  `attempt` is 0-based (the delay AFTER the Nth attempt). Pure. */
+function ingressConvergeDecision({
+  verdict,
+  elapsedMs,
+  attempt,
+  deadlineMs,
+  baseDelayMs,
+  maxDelayMs,
+} = {}) {
+  if (verdict === 'wired') return { done: true, wired: true, delayMs: 0 };
+  const elapsed = Number(elapsedMs) || 0;
+  const deadline = Number(deadlineMs);
+  if (Number.isFinite(deadline) && elapsed >= deadline) {
+    return { done: true, wired: false, delayMs: 0 };
+  }
+  const base = Number(baseDelayMs) || 0;
+  const max = Number.isFinite(Number(maxDelayMs)) ? Number(maxDelayMs) : base;
+  const a = Math.max(0, Number(attempt) || 0);
+  const delayMs = Math.min(max, base * 2 ** a);
+  return { done: false, wired: false, delayMs };
+}
+
+// =============================================================================
+// Async add-time wiring (vm--async-add-time-lb-ingress-wiring)
+// =============================================================================
+// The Add used to BLOCK ~10 min on synchronous LB ingress wiring even after the
+// container was healthy and serving. The fix returns the Add at container-healthy
+// and converges the wiring in a DETACHED background tail (wire → access, the same
+// ordering cmdUp ran inline) that outlives `cloud:up` — so the dashboard's reap of
+// `cloud:up` can never SIGTERM an in-flight `backend-services create` (the add-time
+// 503 strand the old 900s synchronous await was protecting against). The
+// already-existing per-VM auto-rewire gather is the convergence backstop; it must
+// NOT race the in-flight detached tail, so the tail drops an in-flight marker the
+// dashboard reads via the pure `isWiringInflight` below.
+
+/** Build the local invocation that re-execs THIS node entry (`scriptPath`, i.e.
+ *  cloud.js) as a DETACHED `wire-add-tail` child: it runs the slow wiring + access
+ *  grant in the background so the Add returns at container-healthy. Returns
+ *  `{ command, args, env }`; the child reads `cfg`/`appPort` back out of the env
+ *  (`CY_WIRE_ADD_CFG` JSON, `CY_WIRE_ADD_PORT`) rather than re-deriving them, so the
+ *  background tail wires EXACTLY what the foreground Add would have. `nodePath` is
+ *  the node executable to re-exec (caller passes `process.execPath`); kept a
+ *  parameter so this stays pure/testable without reading the ambient env. */
+function detachedAddTailInvocation({ nodePath, scriptPath, cfg, appPort } = {}) {
+  if (!nodePath) throw new Error('detachedAddTailInvocation requires nodePath');
+  if (!scriptPath) throw new Error('detachedAddTailInvocation requires scriptPath');
+  const env = { CY_WIRE_ADD_CFG: JSON.stringify(cfg || {}) };
+  if (appPort != null && String(appPort).trim() !== '') {
+    env.CY_WIRE_ADD_PORT = String(appPort);
+  }
+  return { command: nodePath, args: [scriptPath, 'wire-add-tail'], env };
+}
+
+/** The per-VM marker file the detached add-tail holds while its wiring is still
+ *  running, so the dashboard's auto-rewire gather skips a VM whose FIRST wiring
+ *  attempt is in flight (never firing a competing `vm-ingress.sh up`). Pure path
+ *  derivation; the create/unlink/stat I/O lives at the call sites (cloud.js writes
+ *  it, server.js reads it), mirroring how the status/verdict split keeps the
+ *  decision testable without touching the filesystem. */
+function ingressInflightMarkerPath(dir, vmNumber) {
+  const n = Number(vmNumber);
+  if (!Number.isInteger(n) || n <= 0) {
+    throw new Error(`ingressInflightMarkerPath requires a positive VM number, got: ${vmNumber}`);
+  }
+  return `${dir.replace(/\/+$/, '')}/cy-ingress-inflight-vm-${n}`;
+}
+
+/** The per-VM marker file the detached add-tail drops when its converge loop
+ *  exhausts the deadline WITHOUT reaching `wired`
+ *  (vm--add-time-url-wiring-converges-reliably). It is the durable signal that
+ *  the URL is genuinely `wiring-failed` — a real attention reason the card shows
+ *  with the re-run command — as opposed to still-`wiring` (the inflight marker
+ *  above) or a transient probe miss. The tail clears any prior failed marker when
+ *  it re-enters the loop and unlinks it on success, so a later self-heal doesn't
+ *  leave a stale failure behind. Pure path derivation; the create/unlink/stat I/O
+ *  lives at the call sites, mirroring the inflight-marker split. */
+function ingressFailedMarkerPath(dir, vmNumber) {
+  const n = Number(vmNumber);
+  if (!Number.isInteger(n) || n <= 0) {
+    throw new Error(`ingressFailedMarkerPath requires a positive VM number, got: ${vmNumber}`);
+  }
+  return `${dir.replace(/\/+$/, '')}/cy-ingress-failed-vm-${n}`;
+}
+
+/** Decide whether a freshly-added VM's detached wiring is still in flight, given
+ *  the marker file's CONTENTS (the unix-ms start timestamp the tail wrote) and the
+ *  current time. Returns false when the marker is missing/empty (`content` null) —
+ *  no tail running — or STALE (older than `maxAgeMs`): a tail that crashed without
+ *  unlinking its marker must not pin auto-rewire off forever, so once the staleness
+ *  bound (comfortably past the ~10-min worst-case wiring) elapses the backstop takes
+ *  over. Pure: the caller does the `readFileSync`/`existsSync`. */
+function isWiringInflight(content, now, maxAgeMs) {
+  if (content == null) return false;
+  const started = Number(String(content).trim());
+  if (!Number.isFinite(started) || started <= 0) return false;
+  return (Number(now) - started) < Number(maxAgeMs);
+}
+
+/** Build the local invocation for `scripts/gcp/iap-access.sh <N> <grant|revoke>`
+ *  — grant or revoke `roles/iap.httpsResourceAccessor` on one VM's per-surface
+ *  backend services for a set of principals (the per-VM access whitelist).
+ *  Returns `{ command, args, env }`; `env.VM_ACCESS` is the comma-joined,
+ *  trimmed principal list the script reads (omitted when empty, so a revoke with
+ *  nothing to remove is a clean no-op), plus GCP_PROJECT / ZONE when supplied.
+ *  Mirrors `vmIngressInvocation` so the spawn shape stays unit-testable without
+ *  ever invoking gcloud. The principal STRINGS are passed through verbatim —
+ *  validation/normalization is the sidecar's job (npm/fleet-vm-access.js); this
+ *  builder only shapes the argv/env. */
+function iapAccessInvocation({
+  scriptPath,
+  vmNumber,
+  action,
+  principals,
+  project,
+  zone,
+} = {}) {
+  if (!scriptPath) throw new Error('iapAccessInvocation requires scriptPath');
+  if (action !== 'grant' && action !== 'revoke') {
+    throw new Error(`iap-access action must be 'grant' or 'revoke', got: ${action}`);
+  }
+  const n = Number(vmNumber);
+  if (!Number.isInteger(n) || n <= 0) {
+    throw new Error(`iap-access requires a positive VM number, got: ${vmNumber}`);
+  }
+  const list = Array.isArray(principals)
+    ? principals
+    : String(principals == null ? '' : principals).split(',');
+  const joined = list.map((p) => String(p).trim()).filter(Boolean).join(',');
+  const env = {};
+  if (joined) env.VM_ACCESS = joined;
+  if (project) env.GCP_PROJECT = project;
+  if (zone) env.ZONE = zone;
+  return { command: 'bash', args: [scriptPath, String(n), action], env };
+}
+
+/** Extract the `BLOCKED:` / `Next valid action:` recovery pair out of a failed
+ *  script's combined stdout+stderr. PURE.
+ *
+ *  The CLI error conventions (CLAUDE.md) require those two markers to be emitted
+ *  together, and the actionable content is the `Next valid action:` body — an
+ *  org-policy remedy, a re-run command. Callers previously surfaced
+ *  `out.slice(-400)`, a blind tail that just as easily lands mid-stack-trace and
+ *  drops the remedy entirely; that opacity is most of why the 2026-07-30 VM-10
+ *  lockout took hours to diagnose.
+ *
+ *  Returns the text from the FIRST `BLOCKED:` marker to the end of the output,
+ *  whitespace-collapsed for a single-line toast, capped at `maxLen`. Returns ''
+ *  when the output carries no recovery pair, so callers can fall back to their
+ *  existing tail rather than showing an empty message. */
+function extractRecoveryLines(output, maxLen = 600) {
+  const text = String(output == null ? '' : output);
+  const start = text.indexOf('BLOCKED:');
+  if (start === -1) return '';
+  if (!/Next valid action:/.test(text.slice(start))) return '';
+  const block = text.slice(start).replace(/\s+/g, ' ').trim();
+  return block.length > maxLen ? `${block.slice(0, maxLen - 1)}…` : block;
+}
+
+/** Build the local invocation for `scripts/gcp/google-ingress.sh` — the shared,
+ *  VM-independent LB infra (static IP, wildcard cert, Cloud DNS, URL map, IAP
+ *  brand). Returns `{ command, args, env }`; `env` carries only the keys the
+ *  script reads beyond its defaults (GCP_PROJECT, REGION, ZONE, FLEET_DOMAIN,
+ *  DNS_ZONE). The script is idempotent, so re-invoking against existing infra is
+ *  a safe no-op. */
+function sharedIngressInvocation({
+  scriptPath,
+  domain,
+  project,
+  zone,
+  dnsZone,
+  region,
+} = {}) {
+  if (!scriptPath) throw new Error('sharedIngressInvocation requires scriptPath');
+  if (!domain) throw new Error('sharedIngressInvocation requires a fleet domain');
+  const env = { FLEET_DOMAIN: domain };
+  if (project) env.GCP_PROJECT = project;
+  if (region) env.REGION = region;
+  if (zone) env.ZONE = zone;
+  if (dnsZone) env.DNS_ZONE = dnsZone;
+  return { command: 'bash', args: [scriptPath], env };
+}
+
+/** Build the `gcloud compute backend-services update <bes> --global
+ *  --timeout=<sec>` invocation — the periodic ingress reconcile's per-BES
+ *  WebSocket-timeout converge. On a GCP External HTTPS LB the backend-service
+ *  timeout is the MAX WebSocket lifetime (not a request/idle timeout), so a BES
+ *  left at GCP's 30s default guillotines the editor's /ws/terminal socket every
+ *  30s (the Build-tab "Reconnecting…" loop). Returns `{ command, args }`;
+ *  `project` adds `--project <id>` when supplied (omitted so gcloud's active
+ *  config applies). `timeoutSec` defaults to {@link BES_WEBSOCKET_TIMEOUT_SEC}
+ *  (86400 = 24h). Pure argv shaping, unit-tested without invoking gcloud — the
+ *  same single-source posture as the bash always-converge in vm-ingress.sh. See
+ *  plan vm--lb-backend-service-websocket-timeout. */
+function backendServiceTimeoutInvocation({ bes, project, timeoutSec = BES_WEBSOCKET_TIMEOUT_SEC } = {}) {
+  if (bes == null || String(bes).trim() === '') {
+    throw new Error('backendServiceTimeoutInvocation requires a backend-service name');
+  }
+  const sec = Number(timeoutSec);
+  if (!Number.isInteger(sec) || sec <= 0) {
+    throw new Error(`backendServiceTimeoutInvocation requires a positive timeoutSec, got: ${timeoutSec}`);
+  }
+  const args = [
+    'compute', 'backend-services', 'update', String(bes).trim(),
+    '--global', `--timeout=${sec}`,
+  ];
+  if (project != null && String(project).trim() !== '') {
+    args.push('--project', String(project).trim());
+  }
+  return { command: 'gcloud', args };
+}
+
+// The shared VPC firewall rule admitting Google's LB + health-check ranges to the
+// fleet. FLEET_FIREWALL_RULE_NAME is the DEPLOYED name — hand-created during the
+// 2026-06-15 LB pilot and adopted as canonical so the converge reaches the live
+// rule with no create/delete against a running fleet. FLEET_FIREWALL_RULE_NAMES
+// additionally carries the drifted name google-ingress.sh managed before the
+// unification (the words reversed), because a converge that describes only one
+// name silently no-ops on a fleet wired under the other. Converging every
+// candidate that exists is safe: `update --rules=tcp` only rewrites allowed
+// ports, never a rule's source ranges or target scope.
+// See plan vm--self-healing-app-port-agnostic-fleet-lb-firewall.
+const FLEET_FIREWALL_RULE_NAME = 'fleet-ingress-allow-lb';
+const FLEET_FIREWALL_RULE_NAMES = [FLEET_FIREWALL_RULE_NAME, 'fleet-ingress-lb-allow'];
+
+/** Build the `gcloud compute firewall-rules describe <rule>` invocation that reads
+ *  the fleet firewall's currently-allowed protocols/ports. The
+ *  `value(allowed[].map().firewall_rule().list())` projection is the same one
+ *  google-ingress.sh:350-351 reads, so the bash converge and the reconcile-cadence
+ *  converge judge the rule from a byte-identical view of it. `project` adds
+ *  `--project <id>` when supplied (omitted so gcloud's active config applies).
+ *  Pure argv shaping, unit-tested without invoking gcloud.
+ *  See plan vm--self-healing-app-port-agnostic-fleet-lb-firewall. */
+function fleetFirewallDescribeInvocation({ ruleName, project } = {}) {
+  if (ruleName == null || String(ruleName).trim() === '') {
+    throw new Error('fleetFirewallDescribeInvocation requires a firewall rule name');
+  }
+  const args = [
+    'compute', 'firewall-rules', 'describe', String(ruleName).trim(),
+    "--format=value(allowed[].map().firewall_rule().list())",
+  ];
+  if (project != null && String(project).trim() !== '') {
+    args.push('--project', String(project).trim());
+  }
+  return { command: 'gcloud', args };
+}
+
+/** Build the `gcloud compute firewall-rules update <rule> --rules=tcp` invocation
+ *  that converges the fleet firewall to all-tcp. `update --rules=…` rewrites ONLY
+ *  the allowed protocols/ports — it leaves the rule's source ranges and target
+ *  scope untouched — so converging a live rule can never widen or narrow who the
+ *  rule applies to. That is load-bearing: the deployed rule's scope is the one
+ *  currently admitting LB traffic to the fleet, and this converge preserves it.
+ *  `project` adds `--project <id>` when supplied. Pure argv shaping, unit-tested
+ *  without invoking gcloud.
+ *  See plan vm--self-healing-app-port-agnostic-fleet-lb-firewall. */
+function fleetFirewallConvergeInvocation({ ruleName, project } = {}) {
+  if (ruleName == null || String(ruleName).trim() === '') {
+    throw new Error('fleetFirewallConvergeInvocation requires a firewall rule name');
+  }
+  const args = [
+    'compute', 'firewall-rules', 'update', String(ruleName).trim(),
+    '--rules=tcp',
+  ];
+  if (project != null && String(project).trim() !== '') {
+    args.push('--project', String(project).trim());
+  }
+  return { command: 'gcloud', args };
+}
+
+/** Decide whether a described firewall rule still needs the all-tcp converge —
+ *  the JS twin of the bash guard at google-ingress.sh:352-358. Returns false ONLY
+ *  for the exact all-tcp value `"tcp"`; every port-pinned list
+ *  (`"tcp:14199,tcp:5173,tcp:3000"` — the VM-6-on-8080 outage), every
+ *  protocol-narrowed value, and an empty/absent describe read as "needs converge".
+ *  Erring toward converging is safe: `update --rules=tcp` is idempotent. */
+function firewallNeedsConverge(currentRulesString) {
+  if (currentRulesString == null) return true;
+  return String(currentRulesString).trim() !== 'tcp';
+}
+
+/** Decide whether a described firewall rule admits TCP traffic to `port` — the
+ *  tripwire for a masked "HEALTHY but unreachable" preview. The preview BES health
+ *  check runs on the fixed editor port (14199), so a rule that admits 14199 but not
+ *  the app's real port reads HEALTHY while every real request is dropped: exactly
+ *  the VM-6-on-8080 outage. Parses the same `allowed` projection
+ *  {@link fleetFirewallDescribeInvocation} reads: bare `tcp` admits every port,
+ *  `tcp:5173,tcp:3000` admits only those, and `tcp:1000-2000` admits an inclusive
+ *  range. Returns false for an unparseable/empty rule string or a non-numeric port
+ *  — a gap we cannot disprove is reported, never silently passed. */
+function firewallAdmitsPort(currentRulesString, port) {
+  const p = Number(port);
+  if (!Number.isInteger(p) || p <= 0) return false;
+  if (currentRulesString == null) return false;
+  const entries = String(currentRulesString)
+    .split(',')
+    .map((e) => e.trim())
+    .filter(Boolean);
+  return entries.some((entry) => {
+    const [proto, ports] = entry.split(':');
+    if (proto.trim().toLowerCase() !== 'tcp') return false;
+    // Bare `tcp` (no port suffix) is the all-tcp converge target — admits everything.
+    if (ports == null || ports.trim() === '') return true;
+    const [lo, hi] = ports.trim().split('-');
+    const low = Number(lo);
+    if (!Number.isInteger(low)) return false;
+    if (hi == null) return low === p;
+    const high = Number(hi);
+    if (!Number.isInteger(high)) return false;
+    return p >= low && p <= high;
+  });
+}
+
+/** Build the `gcloud compute backend-services list --global --format=value(name)`
+ *  invocation the orphan sweep uses to enumerate every LB-ingress backend by
+ *  name (the sweep maps those names back to VM numbers via
+ *  google-ingress.vmNumberFromResourceName). `project` adds `--project <id>` when
+ *  supplied (omitted so gcloud's active config applies). Pure argv shaping,
+ *  unit-tested without invoking gcloud. */
+function backendServicesListInvocation({ project } = {}) {
+  const args = ['compute', 'backend-services', 'list', '--global', '--format=value(name)'];
+  if (project != null && String(project).trim() !== '') {
+    args.push('--project', String(project).trim());
+  }
+  return { command: 'gcloud', args };
+}
+
+/** Build the `gcloud compute network-endpoint-groups list --format=value(name)`
+ *  invocation the orphan sweep uses to enumerate every zonal NEG by name. A
+ *  partial teardown can leave a `vm-<N>-<surface>-neg` behind after its backend
+ *  service is already gone (the VM-5/6 case), so keying orphan candidates off
+ *  backend services alone misses that residue — the sweep unions these NEG owners
+ *  in. `project` adds `--project <id>` when supplied (omitted so gcloud's active
+ *  config applies). Pure argv shaping, unit-tested without invoking gcloud. */
+function negListInvocation({ project } = {}) {
+  const args = ['compute', 'network-endpoint-groups', 'list', '--format=value(name)'];
+  if (project != null && String(project).trim() !== '') {
+    args.push('--project', String(project).trim());
+  }
+  return { command: 'gcloud', args };
+}
+
+/** Build the `gcloud compute instances list --format=value(name)` invocation the
+ *  orphan sweep uses for its defensive live-instance guard: a VM whose GCP
+ *  instance is actually running must never have its ingress torn down, even if a
+ *  roster drift left it unrostered (a successful teardown would knock a live VM
+ *  offline). The names map back to VM numbers via fleet-reconcile.parseRunningFleetVms.
+ *  `project` adds `--project <id>` when supplied. Pure argv shaping, unit-tested
+ *  without invoking gcloud. */
+function instancesListInvocation({ project } = {}) {
+  const args = ['compute', 'instances', 'list', '--format=value(name)'];
+  if (project != null && String(project).trim() !== '') {
+    args.push('--project', String(project).trim());
+  }
+  return { command: 'gcloud', args };
+}
+
+module.exports = {
+  BES_WEBSOCKET_TIMEOUT_SEC,
+  backendServiceTimeoutInvocation,
+  DEFAULT_DNS_ZONE,
+  resolveIngressDomain,
+  resolveDnsZone,
+  regionFromZone,
+  vmIngressInvocation,
+  vmIngressStatusInvocation,
+  vmIngressGrantAccessInvocation,
+  parseIngressStatusVerdict,
+  parseIngressTeardownVerdict,
+  selectIngressReconcileTargets,
+  selectIngressGrantOnlyTargets,
+  reconcileGroupKillTarget,
+  selectOrphanIngressTargets,
+  describeTeardownExit,
+  classifyOrphanDownResult,
+  parseOrphanSweepSummary,
+  staleOrphanEntries,
+  backendServicesListInvocation,
+  negListInvocation,
+  instancesListInvocation,
+  FLEET_FIREWALL_RULE_NAME,
+  FLEET_FIREWALL_RULE_NAMES,
+  fleetFirewallDescribeInvocation,
+  fleetFirewallConvergeInvocation,
+  firewallNeedsConverge,
+  firewallAdmitsPort,
+  ingressConvergeDecision,
+  detachedAddTailInvocation,
+  ingressInflightMarkerPath,
+  ingressFailedMarkerPath,
+  isWiringInflight,
+  iapAccessInvocation,
+  extractRecoveryLines,
+  sharedIngressInvocation,
+};