@codeyam-editor/codeyam-editor 0.1.12 → 0.1.13-staging.g3c6e0c9

package/npm/app-port-drift.js

@@ -0,0 +1,169 @@
+'use strict';
+
+// Detect a fleet VM whose app port drifted away from the port its container
+// publishes on the host.
+//
+// A VM's compose render publishes the app port resolved ONCE at bootstrap
+// (`apps[0].port ?? port ?? 5173`). When the project later moves its app port
+// in-repo, the editor's preview proxy follows the new port while the host keeps
+// publishing the old one: docker-proxy accepts the load balancer's connection
+// and immediately resets it, and every in-container probe reads green because
+// the editor probes its own proxy on the new port. The only comparison that can
+// see it is DECLARED (editor.json + editor.local.json) against PUBLISHED (what
+// the running container actually maps on the host) — that is what this module
+// does. It never repairs: changing the publication means recreating the
+// container, which drops every live agent session.
+//
+// Pure helpers plus an injectable collector, so the reconcile and `vm-ingress.sh
+// status` share one rule and the unit tests never reach for SSH.
+
+/** The editor's control port. Pinned by contract, so on a VM's editor container
+ *  the app surface is the OTHER published mapping — the same disambiguation
+ *  `rcc_recover_ports` (scripts/lib/render-cloud-compose.sh) makes. */
+const EDITOR_CONTROL_PORT = 14199;
+
+const PORT_DRIFT_RUNBOOK = "docs/preview-port-drift.md";
+
+function isValidPort(n) {
+  return Number.isInteger(n) && n > 0 && n < 65536;
+}
+
+/** Remote command that prints the editor container's published port list
+ *  (`docker ps` `{{.Ports}}`), selecting the container by the control port it
+ *  publishes rather than by a name, so a client VM whose compose project is not
+ *  named `codeyam-editor` resolves the same way. Reads the LIVE bindings — no
+ *  compose-file path to guess. Pure string builder. */
+function buildPublishedAppPortReadCommand() {
+  return `docker ps --filter publish=${EDITOR_CONTROL_PORT} --format '{{.Ports}}' 2>/dev/null`;
+}
+
+/** Parse that command's output into the published app port, or null.
+ *
+ *  `{{.Ports}}` reads like `0.0.0.0:3000->3000/tcp, [::]:3000->3000/tcp,
+ *  10.0.0.5:14199->14199/tcp`. The app port is the single host port that is not
+ *  the control port. Anything else — no container, more than one container, no
+ *  control-port mapping, zero or several candidate app ports — returns null:
+ *  an unreadable publication must never be reported as a mismatch. */
+function parsePublishedAppPort(stdout) {
+  if (typeof stdout !== "string") return null;
+  const lines = stdout
+    .split("\n")
+    .map((l) => l.trim())
+    .filter(Boolean);
+  if (lines.length !== 1) return null;
+  const hostPorts = new Set();
+  for (const m of lines[0].matchAll(/:(\d+)->\d+\/tcp/g)) hostPorts.add(Number(m[1]));
+  if (!hostPorts.delete(EDITOR_CONTROL_PORT)) return null;
+  if (hostPorts.size !== 1) return null;
+  const [port] = hostPorts;
+  return isValidPort(port) ? port : null;
+}
+
+/** Parse `cloud-session-idle.sh` stdout into `idle` | `active` | null. Null
+ *  (unknown) is deliberately NOT idle — the same asymmetry that script and
+ *  `converge-vm.sh --converge-env` honour. */
+function parseIdleVerdict(stdout) {
+  if (typeof stdout !== "string") return null;
+  const v = stdout.trim().split("\n").pop();
+  return v === "idle" || v === "active" ? v : null;
+}
+
+/** The remediations that apply to one drifted VM, cheapest first.
+ *
+ *  The session-preserving pin always applies. The re-publish (recreate the
+ *  container so the host publishes the declared port) is named ONLY when the
+ *  VM is known idle, because it drops the PTY broker and every live session. */
+function appPortDriftRemediations({ vmNumber, declared, published, idle }) {
+  const out = [
+    {
+      kind: "pin-to-published",
+      preservesSessions: true,
+      text:
+        `on VM-${vmNumber}, run \`codeyam-editor editor config-override apps.0.port ${published}\` ` +
+        `in the project — pins the app back to the published port with an in-place proxy rebind, no restart`,
+    },
+  ];
+  if (idle === "idle") {
+    out.push({
+      kind: "republish-declared",
+      preservesSessions: false,
+      text:
+        `VM-${vmNumber} is idle, so the container can instead be recreated to publish ${declared}: ` +
+        `change the app-port mapping in its docker-compose.cloud.yml from ${published} to ${declared} ` +
+        `and recreate the editor container (see ${PORT_DRIFT_RUNBOOK})`,
+    });
+  }
+  return out;
+}
+
+/** Compare one VM's declared app port against its published host port.
+ *
+ *  `deps.readDeclaredPort` / `deps.readPublishedPort` return a port or null;
+ *  `deps.readIdleVerdict` (optional) returns `idle` | `active` | null and is
+ *  only consulted once a drift is found. Returns `[]` when the two agree or
+ *  when EITHER cannot be read — no 5173 fallback is ever used for the
+ *  comparison — else a one-element findings array. The published read is
+ *  skipped when the declared port is unreadable, so a VM with no scaffolded
+ *  project costs one probe, not two. */
+async function collectAppPortDrift(vm, deps) {
+  const { vmNumber, slug = null } = vm || {};
+  const declared = await deps.readDeclaredPort();
+  if (!isValidPort(declared)) return [];
+  const published = await deps.readPublishedPort();
+  if (!isValidPort(published) || published === declared) return [];
+  const idle = deps.readIdleVerdict ? await deps.readIdleVerdict() : null;
+  return [
+    {
+      vmNumber,
+      slug,
+      declared,
+      published,
+      idle,
+      remediations: appPortDriftRemediations({ vmNumber, declared, published, idle }),
+    },
+  ];
+}
+
+/** Render one finding as the operator-facing warning line(s). */
+function formatAppPortDriftWarning(finding) {
+  const label = finding.slug || `vm-${finding.vmNumber}`;
+  const lines = [
+    `  VM-${finding.vmNumber} ${label}: PORT DRIFT — the project declares app port ` +
+      `${finding.declared} but the container publishes ${finding.published}, so the ` +
+      `load balancer cannot reach the Live Preview (in-container probes still read healthy).`,
+  ];
+  for (const r of finding.remediations) lines.push(`    fix: ${r.text}`);
+  if (finding.idle !== "idle") {
+    lines.push(
+      `    (recreating the container to publish ${finding.declared} is not offered: ` +
+        `the VM is ${finding.idle || "not confirmed idle"} and a recreate drops live sessions)`,
+    );
+  }
+  return lines.join("\n");
+}
+
+/** Render the single machine-readable field `vm-ingress.sh status` prints.
+ *  Kept separate from the `verdict=` line on purpose: the verdict drives the
+ *  reconcile's re-`up` loop, and re-`up`ping cannot fix a port-publication
+ *  drift. */
+function formatPortDriftStatusLine(findings, { declared = null, published = null } = {}) {
+  if (findings.length) {
+    const f = findings[0];
+    return `port-drift=yes declared=${f.declared} published=${f.published}`;
+  }
+  if (isValidPort(declared) && isValidPort(published)) {
+    return `port-drift=no declared=${declared} published=${published}`;
+  }
+  return "port-drift=unknown";
+}
+
+module.exports = {
+  EDITOR_CONTROL_PORT,
+  buildPublishedAppPortReadCommand,
+  parsePublishedAppPort,
+  parseIdleVerdict,
+  appPortDriftRemediations,
+  collectAppPortDrift,
+  formatAppPortDriftWarning,
+  formatPortDriftStatusLine,
+};

package/npm/cloud.js

@@ -105,6 +105,17 @@ const {
   firewallNeedsConverge,
   firewallAdmitsPort,
 } = require("./ingress-invoke");
+// Declared-vs-published app-port comparison (pure; unit-tested in
+// app-port-drift.test.js). The SSH readers that feed it live below, beside
+// readVmAppPort.
+const {
+  buildPublishedAppPortReadCommand,
+  parsePublishedAppPort,
+  parseIdleVerdict,
+  collectAppPortDrift,
+  formatAppPortDriftWarning,
+  formatPortDriftStatusLine,
+} = require("./app-port-drift");
 // Shared fleet-roster writer/reader. `cloud:up` rosters the VM it just
 // provisioned so the dashboard, the cloudflared-monitor tunnel publisher, and
 // the watchdog see it within one poll cycle — without a manual fleet-vms.json
@@ -114,7 +125,7 @@ const { addToRoster, readRoster } = require("./fleet-roster");
 // Tells a docker container-name conflict (a concurrent run already started this
 // container) apart from a real `docker compose up -d` failure, so the abort path
 // can converge instead of escalating to instance deletion.
-const { classifyComposeUpFailure } = require("./compose-conflict");
+const { classifyComposeUpFailure, isGcloudTransportFailure } = require("./compose-conflict");
 // The two streamed bootstrap scripts, named once for this file's STREAMED_SCRIPTS
 // and for the advance's provisioning pre-flight, which checks the same set.
 const { BOOTSTRAP_SCRIPT, PROJECT_BOOTSTRAP_SCRIPT } = require("./provisioning-contract");
@@ -881,8 +892,8 @@ function parseEditorComposeState(stdout) {
 }
 
 /** Decide whether `cloud:up`'s abort handler may delete the instance it is
- *  unwinding. Pure `{ createdThisRun, containerUp, rostered, probeInconclusive }
- *  → 'delete' | 'keep'`, matching the `reapVerdict` / `reattachVerdict` /
+ *  unwinding. Pure `{ createdThisRun, containerUp, rostered, probeInconclusive,
+ *  remoteCommandNeverRan } → 'delete' | 'keep'`, matching the `reapVerdict` / `reattachVerdict` /
  *  `reconcileRosteredVm` pattern in npm/fleet-cleanup.js, so the whole matrix is
  *  exhaustively testable without a VM.
  *
@@ -904,9 +915,20 @@ function parseEditorComposeState(stdout) {
  *    SAFE: a failing probe is how we reach the abort handler in the first place,
  *    so it can never count as evidence the VM is dead. An orphan costs one
  *    `npm run cloud:destroy`; a wrong delete costs a 15-40 minute rebuild plus
- *    any unpushed work on the VM. */
-function abortTeardownDecision({ createdThisRun, containerUp, rostered, probeInconclusive } = {}) {
+ *    any unpushed work on the VM.
+ *  - `remoteCommandNeverRan` — the error that brought us here was a transport
+ *    failure: the gcloud client never reached the VM (an OS Login quota 429 on
+ *    2026-10-01), so the step it was running never executed. Same class as
+ *    `probeInconclusive` — absence of evidence — and it must fail SAFE the same
+ *    way. On that date the probes truthfully reported "container down, not
+ *    rostered", but only BECAUSE compose-up never ran; deleting on that reading
+ *    destroyed VM-8 after ~19 minutes of finished build. A run that never managed
+ *    to execute its step has learned nothing about the VM, so it has not earned
+ *    the right to delete one. Defense in depth: the compose-up retry should keep
+ *    this from being reached at all. */
+function abortTeardownDecision({ createdThisRun, containerUp, rostered, probeInconclusive, remoteCommandNeverRan } = {}) {
   if (!createdThisRun) return "keep";
+  if (remoteCommandNeverRan) return "keep";
   if (probeInconclusive) return "keep";
   if (containerUp) return "keep";
   if (rostered) return "keep";
@@ -1268,13 +1290,24 @@ function provisionFailureMessage(verb, instanceName, result) {
   );
 }
 
-// A bounded, pure classifier that keys on status === 255 (gcloud-ssh's exit
-// for an ssh failure) and a signature match against known transient strings.
+// A bounded, pure classifier with two independent branches:
+//  - status === 255 (gcloud-ssh's exit for an ssh failure) plus a signature
+//    match against known transient SSH-transport strings;
+//  - a gcloud CLIENT-SIDE API failure (an OS Login quota 429), at any status.
+//    gcloud does not exit 255 for these, so the status code is evidence for the
+//    first branch, never a precondition for reading the text — gating on it is
+//    what kept the 2026-10-01 VM-8 quota 429 out of the retry path.
 function isTransientSshFailure(result) {
-  if (!result || result.status !== 255) {
+  if (!result || result.status === 0 || result.status == null) {
     return false;
   }
   const output = String(result.stderr || "") + String(result.stdout || "");
+  if (isGcloudTransportFailure(output)) {
+    return true;
+  }
+  if (result.status !== 255) {
+    return false;
+  }
   const transientRegex = /kex_exchange_identification|Connection closed|failed to connect to backend|Failed to connect to port 22|Connection refused|Connection reset|Connection timed out|4003|4047/i;
   return transientRegex.test(output);
 }
@@ -2471,16 +2504,34 @@ async function ensureEditorContainerUp(cfg) {
   // stream and capture, so the operator's view is preserved by echoing the
   // captured streams immediately below instead of live. Do NOT revert this to
   // "inherit"; classification depends on it.
-  const up = runGcloud(
-    buildGcloudArgs("ssh-probe", cfg, {
-      command: `cd ~/projects/${repoName} && ${composeCmd} up -d`,
-    }),
-    { stdio: ["ignore", "pipe", "pipe"] },
+  //
+  // Through the retry wrapper: a transport failure (an OS Login quota 429 on
+  // 2026-10-01) means `up -d` never ran, so the honest response is to try again.
+  // Retrying is safe without a did-it-run probe — `up -d` is idempotent here, and
+  // a retry that races a container into existence lands on the name-conflict
+  // verdict below, which converges.
+  const up = await runGcloudSshWithRetry(
+    "ssh-probe",
+    cfg,
+    { command: `cd ~/projects/${repoName} && ${composeCmd} up -d` },
+    { stdio: ["ignore", "pipe", "pipe"], label: "docker compose up -d" },
   );
   if (up.stdout) process.stderr.write(up.stdout);
   if (up.stderr) process.stderr.write(up.stderr);
   if (up.status !== 0) {
     const verdict = classifyComposeUpFailure({ status: up.status, stderr: `${up.stderr || ""}\n${up.stdout || ""}` });
+    if (verdict === "transport-error") {
+      // Retries exhausted and gcloud still never reached the VM. Say so, and mark
+      // the error so the abort handler leaves the VM alone: nothing about the VM
+      // has been learned, so it cannot be judged broken.
+      const err = new Error(
+        "could not reach the VM to run docker compose up -d: gcloud failed before connecting " +
+          "(an OS Login API quota 429 — too many concurrent fleet ssh calls). The remote command " +
+          "never ran; the VM itself was not shown to be broken.",
+      );
+      err.remoteCommandNeverRan = true;
+      throw err;
+    }
     if (verdict !== "name-conflict") {
       throw new Error("docker compose up -d failed on the VM");
     }
@@ -2991,32 +3042,66 @@ function buildVmAppPortReadCommand(repoName) {
  *  `VITE_PORT` is genuinely correct there (it is what `resolved_port()` lands
  *  on) and stays quiet. */
 async function readVmAppPort(cfg) {
-  const repoName = resolveProject(cfg).name;
-  const read = runGcloud(
-    buildGcloudArgs("ssh-probe", cfg, {
-      command: buildVmAppPortReadCommand(repoName),
-    }),
-    { stdio: ["ignore", "pipe", "ignore"] },
-  );
-  if (read.status !== 0 || typeof read.stdout !== "string") {
+  const { port, failure } = await probeVmAppPort(cfg);
+  if (failure && failure.kind === "ssh") {
     console.error(
-      `[cloud] WARNING: could not read the VM's app port over SSH — ` +
+      `[cloud] WARNING: ${failure.what} — ` +
         `falling back to ${VITE_PORT}. If the app does not serve on ` +
         `${VITE_PORT}, the laptop port-forward targets the wrong port and the ` +
         `Live Preview will show "localhost refused to connect".`,
     );
-    return VITE_PORT;
-  }
-  if (read.stdout.includes(VM_APP_PORT_NO_CONFIG)) {
+  } else if (failure) {
     console.error(
-      `[cloud] WARNING: no .codeyam/editor.json found under ~/projects on the ` +
-        `VM (looked for '${repoName}', then any project dir) — falling back to ` +
+      `[cloud] WARNING: ${failure.what} — falling back to ` +
         `${VITE_PORT}. If the project is scaffolded under a different name and ` +
         `serves on another port, the port-forward targets the wrong port and ` +
         `the Live Preview will show "localhost refused to connect".`,
     );
-    return VITE_PORT;
   }
+  return port == null ? VITE_PORT : port;
+}
+
+/** The probe behind `readVmAppPort`, without the fallback: resolves to
+ *  `{ port, failure: null }` when the VM's config was read, or
+ *  `{ port: null, failure: { kind: "ssh" | "no-config", what } }` when it was
+ *  not. Callers that must not act on a guess — the ingress reconcile, where a
+ *  5173 fallback would wire a preview NEG endpoint to a port nothing publishes —
+ *  use this to tell a resolved port from a fallback. Prints nothing; `what` is
+ *  the human description of the miss for the caller's own warning. */
+async function probeVmAppPort(cfg) {
+  const repoName = resolveProject(cfg).name;
+  const read = runGcloud(
+    buildGcloudArgs("ssh-probe", cfg, {
+      command: buildVmAppPortReadCommand(repoName),
+    }),
+    { stdio: ["ignore", "pipe", "ignore"] },
+  );
+  if (read.status !== 0 || typeof read.stdout !== "string") {
+    return {
+      port: null,
+      failure: { kind: "ssh", what: "could not read the VM's app port over SSH" },
+    };
+  }
+  const { port, miss } = parseVmDeclaredAppPort(read.stdout);
+  if (miss === "no-config") {
+    return {
+      port: null,
+      failure: {
+        kind: "no-config",
+        what:
+          `no .codeyam/editor.json found under ~/projects on the ` +
+          `VM (looked for '${repoName}', then any project dir)`,
+      },
+    };
+  }
+  return { port, failure: null };
+}
+
+/** Resolve `buildVmAppPortReadCommand`'s output into `{ port, miss }` — shared
+ *  by `probeVmAppPort` and the port-drift probe, so both resolve the declared
+ *  port identically. A no-config VM is a miss with a null port, never 5173. */
+function parseVmDeclaredAppPort(stdout) {
+  if (stdout.includes(VM_APP_PORT_NO_CONFIG)) return { port: null, miss: "no-config" };
   const parse = (s) => {
     try {
       const v = JSON.parse(s);
@@ -3025,8 +3110,116 @@ async function readVmAppPort(cfg) {
       return {};
     }
   };
-  const [baseJson = "", localJson = ""] = read.stdout.split("<<<LOCAL>>>");
-  return resolveConfiguredAppPort(parse(baseJson), parse(localJson));
+  const [baseJson = "", localJson = ""] = stdout.split("<<<LOCAL>>>");
+  return { port: resolveConfiguredAppPort(parse(baseJson), parse(localJson)), miss: null };
+}
+
+/** Separates the declared-port read from the `docker ps` read in the combined
+ *  probe below. */
+const VM_PUBLISHED_PORT_MARKER = "<<<PUBLISHED>>>";
+
+/** Read the declared app port AND the host port the VM's editor container
+ *  publishes (live `docker ps` bindings) in ONE SSH round-trip, so the roster
+ *  pass costs one extra probe per VM. Either side is null when unreadable. */
+function readVmAppPortsForDrift(cfg) {
+  const read = runGcloud(
+    buildGcloudArgs("ssh-probe", cfg, {
+      command:
+        `${buildVmAppPortReadCommand(resolveProject(cfg).name)}; ` +
+        `echo '${VM_PUBLISHED_PORT_MARKER}'; ${buildPublishedAppPortReadCommand()}`,
+    }),
+    { stdio: ["ignore", "pipe", "ignore"] },
+  );
+  if (read.status !== 0) return { declared: null, published: null };
+  return parseVmAppPortsForDrift(read.stdout);
+}
+
+/** Split the combined probe's output into `{ declared, published }`. Output
+ *  without the marker — an SSH session that exited 0 having printed nothing, a
+ *  truncated read — is unreadable on BOTH sides. Resolving its empty config
+ *  half would land on the 5173 default and could report a drift that does not
+ *  exist. */
+function parseVmAppPortsForDrift(stdout) {
+  if (typeof stdout !== "string" || !stdout.includes(VM_PUBLISHED_PORT_MARKER)) {
+    return { declared: null, published: null };
+  }
+  const [declaredOut, publishedOut] = stdout.split(VM_PUBLISHED_PORT_MARKER);
+  return {
+    declared: declaredOut.trim() ? parseVmDeclaredAppPort(declaredOut).port : null,
+    published: parsePublishedAppPort(publishedOut),
+  };
+}
+
+/** Read the VM's `cloud-session-idle.sh` verdict (`idle` | `active` | null).
+ *  Unknown — including a VM whose checkout predates the script — is null, and
+ *  null is never treated as idle. */
+function readVmIdleVerdict(cfg) {
+  const read = runGcloud(
+    buildGcloudArgs("ssh-probe", cfg, {
+      command:
+        `s="$HOME/codeyam-editor/scripts/cloud-session-idle.sh"; ` +
+        `if [ -r "$s" ]; then bash "$s" 2>/dev/null; fi; true`,
+    }),
+    { stdio: ["ignore", "pipe", "ignore"] },
+  );
+  return read.status === 0 ? parseIdleVerdict(read.stdout) : null;
+}
+
+/** Compare one VM's declared app port against its published host port over the
+ *  SSH probe. Returns the findings plus both raw reads (for the status line).
+ *  See npm/app-port-drift.js for the rule. */
+async function collectVmAppPortDrift(cfg, vmNumber, slug) {
+  const { declared, published } = readVmAppPortsForDrift(cfg);
+  const findings = await collectAppPortDrift(
+    { vmNumber, slug },
+    {
+      readDeclaredPort: async () => declared,
+      readPublishedPort: async () => published,
+      readIdleVerdict: async () => readVmIdleVerdict(cfg),
+    },
+  );
+  return { findings, declared, published };
+}
+
+/** Port-drift pass over the whole roster, on the periodic reconcile that already
+ *  walks it. Reports, never repairs (the fix recreates the container or changes
+ *  the project's port — both the operator's call). Per-VM failures are logged
+ *  and swallowed, like the preview-origin heal beside it. */
+async function reportAppPortDriftAcrossRoster(cfg, roster, deps = {}) {
+  const collect = deps.collectVmAppPortDrift || collectVmAppPortDrift;
+  const drifted = [];
+  for (const n of roster) {
+    const perVmCfg = { ...cfg, instanceName: `${fleetInstancePrefix()}${n}` };
+    try {
+      const { findings } = await collect(perVmCfg, n, slugFor(n));
+      for (const f of findings) {
+        console.error(formatAppPortDriftWarning(f));
+        drifted.push(f);
+      }
+    } catch (e) {
+      console.error(`  VM-${n} ${slugFor(n) || `vm-${n}`}: port-drift check skipped (${e && e.message})`);
+    }
+  }
+  return drifted;
+}
+
+/** `npm run cloud:app-port-drift -- --instance-name <name>` — one VM's
+ *  declared-vs-published comparison as the single `port-drift=` line
+ *  `vm-ingress.sh status` appends, plus the human warning on stderr. */
+async function cmdAppPortDrift(args, projectDir = rootDir) {
+  const flags = parseFlags(args);
+  const cfg = { ...(loadCloudConfig(projectDir) || {}), ...flags };
+  const vmNumber = resolveFleetVmNumber(cfg.instanceName || "");
+  if (vmNumber == null) {
+    throw new Error("cloud:app-port-drift requires --instance-name <numbered fleet VM>.");
+  }
+  const { findings, declared, published } = await collectVmAppPortDrift(
+    cfg,
+    vmNumber,
+    slugFor(vmNumber),
+  );
+  for (const f of findings) console.error(formatAppPortDriftWarning(f));
+  console.log(formatPortDriftStatusLine(findings, { declared, published }));
 }
 
 /** Build the shell command that seeds a JSON patch into a host-side
@@ -3947,7 +4140,7 @@ function fleetInstancePrefix() {
  *  (the VM is provisioned; ingress can be re-wired), Destroy likewise warns so a
  *  teardown still removes the instance. `vmAccess` (up only) is the Add's intended
  *  principal set, which the script's grant step uses in place of its sidecar read. */
-function runVmIngress(cfg, action, vmNumber, domain, appPort, slug, vmAccess) {
+function runVmIngress(cfg, action, vmNumber, domain, appPort, slug, vmAccess, { appPortUnresolved = false } = {}) {
   const scriptPath = path.join(rootDir, "scripts", "gcp", "vm-ingress.sh");
   const inv = vmIngressInvocation({
     scriptPath,
@@ -3960,6 +4153,7 @@ function runVmIngress(cfg, action, vmNumber, domain, appPort, slug, vmAccess) {
     instancePrefix: fleetInstancePrefix(),
     slug,
     vmAccess,
+    appPortUnresolved,
   });
   return spawnSync(inv.command, inv.args, {
     stdio: "inherit",
@@ -4773,12 +4967,18 @@ async function cmdUp(args, projectDir = rootDir) {
     // VM is in right now, then let the pure decision rule — see
     // abortTeardownDecision's doc comment for why each input is load-bearing.
     const state = probeAbortTeardownState(cfg, createdThisRun);
-    const verdict = abortTeardownDecision({ createdThisRun, ...state });
+    // An error that never reached the VM proves nothing about it — see the
+    // `remoteCommandNeverRan` paragraph of abortTeardownDecision.
+    const remoteCommandNeverRan =
+      Boolean(err && err.remoteCommandNeverRan) || isGcloudTransportFailure(err && err.message);
+    const verdict = abortTeardownDecision({ createdThisRun, ...state, remoteCommandNeverRan });
     if (createdThisRun && verdict === "keep") {
       // Deliberately loud and specific. The old blanket "tearing down the VM
       // cloud:up created this run (no orphan left behind)" was actively
       // misleading here — it asserted no orphan while deleting a working VM.
-      const why = state.probeInconclusive
+      const why = remoteCommandNeverRan
+        ? "the failing step never reached the VM (a gcloud transport/quota error), so nothing shows the VM is broken"
+        : state.probeInconclusive
         ? "its liveness could not be confirmed (probe failed), and an unconfirmed VM is never deleted"
         : state.containerUp
           ? "its editor container is RUNNING — another run brought it up"
@@ -5071,6 +5271,25 @@ async function healPreviewOriginsAcrossRoster(cfg, roster) {
  *  resources already present and creates only the missing backend / host rule.
  *  A no-op in legacy quick-tunnel mode (no fleet domain). Invoked on demand and
  *  automatically at dashboard boot so a fleet that lost ingress self-heals. */
+/** Resolve one reconcile target's app port for `vm-ingress.sh up`, refusing to
+ *  guess. Resolves `{ appPort, appPortUnresolved: false }` on a real read, or
+ *  `{ appPort: null, appPortUnresolved: true }` — with a warning naming the VM —
+ *  when the probe failed, so the script leaves the preview NEG's endpoints alone
+ *  instead of wiring (and pruning to) a 5173 fallback over a working endpoint.
+ *  `deps.probeVmAppPort` is injectable so tests never reach for SSH. */
+async function resolveReconcileAppPort(perVmCfg, vmNumber, deps = {}) {
+  const probe = deps.probeVmAppPort || probeVmAppPort;
+  const { port, failure } = await probe(perVmCfg);
+  if (!failure) return { appPort: port, appPortUnresolved: false };
+  console.error(
+    `[cloud] WARNING: VM-${vmNumber}: ${failure.what} — skipping the preview ` +
+      `endpoint converge rather than wiring ${VITE_PORT} over a working endpoint. ` +
+      `If the preview NEG targets the wrong port, Live Preview requests fail ` +
+      `with "upstream connect error" until a later reconcile resolves it.`,
+  );
+  return { appPort: null, appPortUnresolved: true };
+}
+
 async function cmdReconcileIngress(args, projectDir = rootDir) {
   ensureGcloud();
   parseFlags(args);
@@ -5143,17 +5362,23 @@ async function cmdReconcileIngress(args, projectDir = rootDir) {
   for (const t of targets) {
     const label = t.slug || `vm-${t.vmNumber}`;
     // Resolve the VM's real dev-server port so a re-created preview NEG targets
-    // the right port (not the 5173 bootstrap fallback). Best-effort — readVmAppPort
-    // itself falls back to VITE_PORT on any SSH/parse failure.
+    // the right port. NOT readVmAppPort: its VITE_PORT fallback on an SSH/parse
+    // failure is indistinguishable from a VM genuinely serving 5173, and `up`
+    // prunes the preview NEG to the port it is given — a fallback would detach the
+    // working endpoint and wire one nothing publishes. Unresolved → the script
+    // leaves the preview endpoints alone; every other resource still converges.
     const perVmCfg = { ...cfg, instanceName: `${fleetInstancePrefix()}${t.vmNumber}` };
-    const appPort = await readVmAppPort(perVmCfg);
-    const res = runVmIngress(perVmCfg, "up", t.vmNumber, domain, appPort, t.slug);
+    const { appPort, appPortUnresolved } = await resolveReconcileAppPort(perVmCfg, t.vmNumber);
+    const res = runVmIngress(perVmCfg, "up", t.vmNumber, domain, appPort, t.slug, undefined, {
+      appPortUnresolved,
+    });
     console.error(
       `  VM-${t.vmNumber} ${label}: ${t.verdict} → ${res.status === 0 ? "re-wired" : "re-wire FAILED"}`,
     );
   }
 
   await healPreviewOriginsAcrossRoster(cfg, roster);
+  await reportAppPortDriftAcrossRoster(cfg, roster);
 
   // Converge each rostered backend-service's timeout to 86400 (24h). A fully-wired
   // BES still at GCP's 30s default reports `wired` above and is skipped by the
@@ -5722,6 +5947,7 @@ const HANDLERS = {
   status: cmdStatus,
   "ingress-up": cmdIngressUp,
   "reconcile-ingress": cmdReconcileIngress,
+  "app-port-drift": cmdAppPortDrift,
   "wire-add-tail": cmdWireAddTail,
   "reconcile-credentials": cmdReconcileCredentials,
   "reconcile-admin": cmdReconcileAdmin,
@@ -5816,6 +6042,8 @@ module.exports = {
   seedPreviewOrigin,
   healPreviewOriginAtReady,
   healPreviewOriginsAcrossRoster,
+  reportAppPortDriftAcrossRoster,
+  cmdAppPortDrift,
   buildGcloudArgs,
   sshScriptCommand,
   bootstrapRunnerScript,
@@ -5905,6 +6133,10 @@ module.exports = {
   classifyGitRemoteError,
   decideCredentialHeal,
   readVmAppPort,
+  probeVmAppPort,
+  resolveReconcileAppPort,
+  parseVmDeclaredAppPort,
+  parseVmAppPortsForDrift,
   buildVmAppPortReadCommand,
   resolveLaptopGitIdentity,
   gitBranchRefExists,

package/npm/compose-conflict.js

@@ -31,12 +31,44 @@
 // whole captured stream rather than line- or anchor-based.
 const NAME_CONFLICT_FRAGMENTS = [/\bConflict\./i, /container name/i, /is already in use/i];
 
+// The stable fragments of a gcloud CLIENT-SIDE API failure — gcloud's own HTTP
+// client failed before any ssh was attempted, so the remote command never ran.
+// Observed verbatim in the 2026-10-01 VM-8 add log, when an OS Login
+// loginProfile read hit the `ReadRequestsPerMinutePerUser` quota (limit 60):
+//
+//   ERROR: (gcloud.compute.ssh) HttpError accessing <https://oslogin.googleapis.com/…
+//   /loginProfile?…>: response: <{'status': 429}>, content <{"error": {"code": 429,
+//   "message": "Quota exceeded for quota metric …", "status": "RESOURCE_EXHAUSTED",
+//   "details": [{"reason": "RATE_LIMIT_EXCEEDED"}]}}
+//
+// Same discipline as the conflict fragments, for the same reason — Google
+// rephrases these: require gcloud's `HttpError accessing` wrapper AND at least
+// one quota token, never a single token, so an app-level message that merely
+// contains `429` is not swept in.
+const GCLOUD_HTTP_ERROR_FRAGMENT = /HttpError accessing/;
+const GCLOUD_QUOTA_FRAGMENTS = [/'status':\s*429\b/, /RESOURCE_EXHAUSTED/, /RATE_LIMIT_EXCEEDED/];
+
+/** True when `text` carries a gcloud client-side API quota failure, i.e. the
+ *  gcloud client never reached the VM. Shared by {@link classifyComposeUpFailure}
+ *  and `cloud.js`'s transient-ssh predicate and abort handler, so the three
+ *  cannot drift apart on what counts as "never reached the VM". */
+function isGcloudTransportFailure(text) {
+  const s = String(text == null ? '' : text);
+  return GCLOUD_HTTP_ERROR_FRAGMENT.test(s) && GCLOUD_QUOTA_FRAGMENTS.some((re) => re.test(s));
+}
+
 /** Classify the outcome of `docker compose up -d`.
  *
  *  Returns one of:
  *  - `'ok'` — the command succeeded; there is no failure to classify. Guards
  *    against a caller that classifies unconditionally, so a zero status can
  *    never be read as a conflict.
+ *  - `'transport-error'` — the gcloud ssh client never reached the VM (an OS
+ *    Login quota 429 on 2026-10-01), so `up -d` did not run and nothing has been
+ *    learned about the box. Reading this as `'failed'` is what drove the abort
+ *    handler into deleting VM-8 after it had already built its editor image.
+ *    The caller retries; it must never treat this as a provisioning failure.
+ *    Checked first so a transport failure can never be read as a daemon message.
  *  - `'name-conflict'` — the daemon refused because a container of that name
  *    already exists, i.e. a concurrent run already started it. The caller
  *    re-probes for a running container and converges instead of throwing.
@@ -50,10 +82,11 @@ const NAME_CONFLICT_FRAGMENTS = [/\bConflict\./i, /container name/i, /is already
 function classifyComposeUpFailure({ status, stderr } = {}) {
   if (status === 0) return 'ok';
   const text = String(stderr == null ? '' : stderr);
+  if (isGcloudTransportFailure(text)) return 'transport-error';
   if (NAME_CONFLICT_FRAGMENTS.every((re) => re.test(text))) {
     return 'name-conflict';
   }
   return 'failed';
 }
 
-module.exports = { classifyComposeUpFailure };
+module.exports = { classifyComposeUpFailure, isGcloudTransportFailure };