@codeyam-editor/codeyam-editor 0.1.13 → 0.1.14-staging.g3274a6f

package/npm/cloud.js

@@ -935,6 +935,62 @@ function abortTeardownDecision({ createdThisRun, containerUp, rostered, probeInc
   return "delete";
 }
 
+/** Fold what the abort-causing error proves into the probed VM state, giving
+ *  the full {@link abortTeardownDecision} input (minus `createdThisRun`).
+ *
+ *  - `remoteCommandNeverRan` — set by a fatal ssh step that classified a
+ *    transport failure, or sniffed from a message that carries the quota text.
+ *  - `sshCauseUnknown` — a fatal ssh step could not establish whether its
+ *    command ran. That is absence of evidence too, so it also raises
+ *    `probeInconclusive` (fail SAFE) rather than adding a parallel verdict input.
+ *  Pure: `(err, state) → state`, so the err → verdict path is testable without
+ *  running cloud:up. */
+function abortEvidence(err, state = {}) {
+  const remoteCommandNeverRan =
+    Boolean(err && err.remoteCommandNeverRan) || isGcloudTransportFailure(err && err.message);
+  const sshCauseUnknown = Boolean(err && err.sshCauseUnknown);
+  return {
+    ...state,
+    probeInconclusive: Boolean(state.probeInconclusive) || sshCauseUnknown,
+    remoteCommandNeverRan,
+    sshCauseUnknown,
+  };
+}
+
+/** The operator-facing line for a VM this run created but the abort handler
+ *  KEPT. Deliberately loud and specific: the old blanket "tearing down the VM
+ *  cloud:up created this run (no orphan left behind)" was actively misleading —
+ *  it asserted no orphan while deleting a working VM. When nothing was learned
+ *  about the VM (the step never ran, or its cause could not be established), the
+ *  likely-right next move is to finish provisioning it in place, so the line
+ *  names that recovery (`converge-vm.sh <n>` for a fleet VM). `state` is
+ *  {@link abortEvidence}'s output. Pure. */
+function abortKeptMessage({ instanceName, vmNumber, state = {} } = {}) {
+  const why = state.remoteCommandNeverRan
+    ? "the failing step never reached the VM (a gcloud transport/quota error), so nothing shows the VM is broken"
+    : state.sshCauseUnknown
+      ? "the failing ssh step could not establish its cause (the command may never have run), so nothing shows the VM is broken"
+      : state.probeInconclusive
+        ? "its liveness could not be confirmed (probe failed), and an unconfirmed VM is never deleted"
+        : state.containerUp
+          ? "its editor container is RUNNING — another run brought it up"
+          : "it is in the fleet roster and visible to the dashboard";
+  const nothingLearned = state.remoteCommandNeverRan || state.sshCauseUnknown;
+  const recover = nothingLearned
+    ? `  To finish provisioning it in place: ${
+        vmNumber != null
+          ? `scripts/operator/converge-vm.sh ${vmNumber}`
+          : `npm run cloud:up -- --instance-name ${instanceName}`
+      }\n`
+    : "";
+  return (
+    `✗ provisioning aborted — the VM ${instanceName} was LEFT IN PLACE because ${why}.\n` +
+    `  This run created the instance, but that alone does not authorize deleting a VM that is in service.\n` +
+    recover +
+    `  If it really should go, remove it deliberately: npm run cloud:destroy`
+  );
+}
+
 /** Gather the CURRENT-STATE inputs {@link abortTeardownDecision} needs, so the
  *  abort handler reads gather → decide → act instead of inlining probes among
  *  the delete calls.
@@ -1340,6 +1396,80 @@ async function runGcloudSshWithRetry(verb, cfg, extras, { stdio, retries = 5, la
   return lastResult;
 }
 
+/** What a failed ssh step proves about the VM, from its captured result:
+ *  - `'transport'` — the gcloud client or the ssh connection failed before the
+ *    remote command ran (an OS Login quota 429, a refused/reset connection), so
+ *    nothing about the VM has been learned.
+ *  - `'unknown'` — ssh itself exited 255 with no recognized signature, the call
+ *    was killed/never spawned (no status), or gcloud reported an error of its
+ *    own (`ERROR: (gcloud.…)` — an auth or IAP refusal exits 1 like a remote
+ *    command would). Whether the remote command ran cannot be established.
+ *  - `'remote'` — the remote command ran and exited non-zero: real evidence.
+ *  Pure, so the matrix is testable without a VM. Only meaningful for a non-zero
+ *  result; the caller has already branched on success. */
+function classifySshStepFailure(result) {
+  if (isTransientSshFailure(result)) return "transport";
+  if (!result || result.status == null || result.status === 255) return "unknown";
+  if (/ERROR: \(gcloud\./.test(String(result.stderr || ""))) return "unknown";
+  return "remote";
+}
+
+/** Run a provisioning ssh step whose failure is FATAL — it throws, and cloud:up's
+ *  abort handler then decides whether to delete the VM. Two guards keep that
+ *  decision honest, and every fatal step needs both (VM-8 on 2026-10-01 and VM-37
+ *  on 2026-10-08 were the same OS Login 429 at two different steps, a week apart,
+ *  each at a call site that had neither):
+ *
+ *  1. The retry wrapper, so a transient quota 429 is retried instead of being
+ *     fatal on first occurrence.
+ *  2. CAPTURED stderr, never `"inherit"` — a step that cannot see why it failed
+ *     cannot classify the failure. The streams are echoed immediately so the
+ *     operator's view is kept (`echoStdout` for steps whose stdout used to go to
+ *     the terminal; steps that parse stdout leave it off).
+ *
+ *  On failure the thrown error carries the classification for
+ *  abortTeardownDecision: `remoteCommandNeverRan` for a transport failure,
+ *  `sshCauseUnknown` when it cannot be established, and nothing for a genuine
+ *  remote failure — so a really broken VM still takes the destructive path. A
+ *  remote failure keeps `failureMessage` verbatim. */
+async function runFatalSshStep(cfg, command, { label, failureMessage, echoStdout = false, sleepFn } = {}) {
+  const result = await runGcloudSshWithRetry(
+    "ssh-probe",
+    cfg,
+    { command },
+    { stdio: ["ignore", "pipe", "pipe"], label, sleepFn },
+  );
+  // Both echoes go to OUR stderr, as at the compose-up site: this process's
+  // stdout carries the machine-read `CY_DEGRADED` lines the dashboard scans.
+  if (echoStdout && result.stdout) process.stderr.write(result.stdout);
+  if (result.stderr) process.stderr.write(result.stderr);
+  if (result.status === 0) return result;
+  throw fatalSshStepError(result, failureMessage);
+}
+
+/** The error a failed fatal ssh step throws, carrying its
+ *  {@link classifySshStepFailure} verdict as the flag abortTeardownDecision
+ *  reads. A transport failure names its cause in the message (the quota text
+ *  itself was echoed to stderr, not carried here); a remote failure keeps
+ *  `failureMessage` verbatim. Pure: result in, Error out. */
+function fatalSshStepError(result, failureMessage) {
+  const cause = classifySshStepFailure(result);
+  if (cause === "transport") {
+    const why = isGcloudTransportFailure(`${result.stderr || ""}\n${result.stdout || ""}`)
+      ? "an OS Login API quota 429 — too many concurrent fleet ssh calls"
+      : "the ssh connection to the VM could not be established";
+    const err = new Error(
+      `${failureMessage}: gcloud failed before reaching the VM (${why}), after retrying. ` +
+        "The remote command never ran; the VM itself was not shown to be broken.",
+    );
+    err.remoteCommandNeverRan = true;
+    return err;
+  }
+  const err = new Error(failureMessage);
+  if (cause === "unknown") err.sshCauseUnknown = true;
+  return err;
+}
+
 function ensureGcloud() {
   const result = spawnSync("gcloud", ["version"], { stdio: "ignore" });
   if (result.error || result.status !== 0) {
@@ -1579,6 +1709,7 @@ async function waitForIapReady(cfg) {
   const probeStderrs = [];
   while (Date.now() < deadline) {
     attempts++;
+    // Tolerant ssh step: this loop is its own retry, so no wrapper.
     const probe = runGcloud(
       buildGcloudArgs("ssh-probe", cfg, { command: "true" }),
       { stdio: ["ignore", "pipe", "pipe"] },
@@ -1836,6 +1967,7 @@ function sshScriptCommand(scriptBody, scriptArgs = [], envVars = {}) {
 
 async function ensureBootstrapped(cfg, { sleepFn } = {}) {
   console.error("Checking VM bootstrap state...");
+  // Tolerant ssh step: a failed probe falls through to the idempotent bootstrap.
   const probe = runGcloud(
     buildGcloudArgs("ssh-probe", cfg, {
       command: "command -v docker >/dev/null && docker image inspect codeyam-editor:local >/dev/null 2>&1",
@@ -1946,6 +2078,7 @@ async function ensureBootstrapped(cfg, { sleepFn } = {}) {
 // single outcome string ("success" | "failed" | "running" | "absent"). Used
 // both for the idempotent re-attach decision and after each follow session.
 function probeBootstrapOutcome(cfg) {
+  // Tolerant ssh step: unparseable output reads as "absent", which re-attaches.
   const r = runGcloud(
     buildGcloudArgs("ssh-probe", cfg, { command: bootstrapStatusCommand() }),
     { stdio: ["ignore", "pipe", "pipe"] },
@@ -1956,6 +2089,7 @@ function probeBootstrapOutcome(cfg) {
 // Fetch the tail of the VM-side bootstrap log for an actionable failure
 // message — the operator sees WHY the build failed, not a bare exit code.
 function bootstrapLogTail(cfg, lines = 40) {
+  // Tolerant ssh step: diagnostic only, falls back to "(log unavailable)".
   const r = runGcloud(
     buildGcloudArgs("ssh-probe", cfg, {
       command: `tail -n ${lines} ${BOOTSTRAP_LOG_PATH} 2>/dev/null`,
@@ -2179,6 +2313,7 @@ function decideProjectBootstrapFollowAction({ outcome, now, deadline } = {}) {
 // to a single outcome string ("success" | "failed" | "running" | "absent").
 // Used both for the idempotent re-attach decision and after each follow session.
 function probeProjectBootstrapOutcome(cfg, repoName) {
+  // Tolerant ssh step: unparseable output reads as "absent", which re-attaches.
   const r = runGcloud(
     buildGcloudArgs("ssh-probe", cfg, { command: projectBootstrapStatusCommand(repoName) }),
     { stdio: ["ignore", "pipe", "pipe"] },
@@ -2232,6 +2367,7 @@ function projectBootstrapEnv(cfg = {}, { env = process.env, githubToken, gitIden
 // Fetch the tail of the VM-side project-bootstrap log for an actionable failure
 // message — the operator sees WHY the install failed, not a bare exit code.
 function projectBootstrapLogTail(cfg, repoName, lines = 40) {
+  // Tolerant ssh step: diagnostic only, falls back to "(log unavailable)".
   const r = runGcloud(
     buildGcloudArgs("ssh-probe", cfg, {
       command: `tail -n ${lines} ${projectBootstrapLogPath(repoName)} 2>/dev/null`,
@@ -2308,6 +2444,7 @@ async function ensureProjectBootstrapped(cfg, { sleepFn } = {}) {
   // correctly re-run (the re-run is idempotent — skip-on-exists per step). The
   // two structural checks stay as belt-and-suspenders for a legacy VM whose
   // marker predates this change but is otherwise wired.
+  // Tolerant ssh step: a failed probe falls through to the idempotent bootstrap.
   const probe = runGcloud(
     buildGcloudArgs("ssh-probe", cfg, {
       command: projectBootstrapDoneProbeCommand(repoName),
@@ -2446,6 +2583,8 @@ async function ensureProjectBootstrapped(cfg, { sleepFn } = {}) {
 function probeEditorComposeState(cfg) {
   const repoName = resolveProject(cfg).name;
   const composeCmd = `docker compose -f docker-compose.cloud.yml`;
+  // Tolerant ssh step: a failure reports `inconclusive`, which every caller
+  // treats as fail-safe.
   const probe = runGcloud(
     buildGcloudArgs("ssh-probe", cfg, {
       command: `cd ~/projects/${repoName} 2>/dev/null && ${composeCmd} ps --format json 2>/dev/null || true`,
@@ -2989,13 +3128,13 @@ async function seedGithubCredentials(cfg, opts = {}) {
     `echo ${b64} | base64 -d > "$t" && chmod 600 "$t" && ` +
     `if [ -s "$t" ]; then cat "$t" > "$f" && chmod 600 "$f" && rm -f "$t" && echo OK; ` +
     `else rm -f "$t"; echo EMPTY-WRITE-REFUSED >&2; exit 1; fi`;
-  const result = runGcloud(
-    buildGcloudArgs("ssh-probe", cfg, { command: remoteCmd }),
-    { stdio: ["ignore", "pipe", "inherit"] },
-  );
-  if (result.status !== 0) {
-    throw new Error("Failed to write ~/.codeyam-cloud-git-credentials on the VM");
-  }
+  // Fatal ssh step — see runFatalSshStep. stdout stays unechoed, as it always
+  // was: it carries only the `OK` marker.
+  await runFatalSshStep(cfg, remoteCmd, {
+    label: "write git credentials",
+    failureMessage: "Failed to write ~/.codeyam-cloud-git-credentials on the VM",
+    sleepFn: opts.sleepFn,
+  });
 }
 
 /** Sentinel the remote read prints when it cannot find an `editor.json` under
@@ -3070,6 +3209,8 @@ async function readVmAppPort(cfg) {
  *  the human description of the miss for the caller's own warning. */
 async function probeVmAppPort(cfg) {
   const repoName = resolveProject(cfg).name;
+  // Tolerant ssh step: a failure is returned as `{ kind: "ssh" }` and the
+  // caller falls back to VITE_PORT with a loud warning.
   const read = runGcloud(
     buildGcloudArgs("ssh-probe", cfg, {
       command: buildVmAppPortReadCommand(repoName),
@@ -3122,6 +3263,7 @@ const VM_PUBLISHED_PORT_MARKER = "<<<PUBLISHED>>>";
  *  publishes (live `docker ps` bindings) in ONE SSH round-trip, so the roster
  *  pass costs one extra probe per VM. Either side is null when unreadable. */
 function readVmAppPortsForDrift(cfg) {
+  // Tolerant ssh step: a failure reads as "unknown" (nulls), never as drift.
   const read = runGcloud(
     buildGcloudArgs("ssh-probe", cfg, {
       command:
@@ -3154,6 +3296,7 @@ function parseVmAppPortsForDrift(stdout) {
  *  Unknown — including a VM whose checkout predates the script — is null, and
  *  null is never treated as idle. */
 function readVmIdleVerdict(cfg) {
+  // Tolerant ssh step: a failure returns null (no verdict).
   const read = runGcloud(
     buildGcloudArgs("ssh-probe", cfg, {
       command:
@@ -3300,7 +3443,7 @@ async function seedPreviewOrigin(
   // One round-trip reads both the current editor.local.json (for the
   // skip-on-already-set guard) and stack.json (for the sameOriginSafe flag that
   // decides same-origin subpath vs dedicated origin). Mirrors readVmAppPort's
-  // separator-delimited cat.
+  // separator-delimited cat. Tolerant ssh step: a failed read parses as `{}`.
   const read = runGcloud(
     buildGcloudArgs("ssh-probe", cfg, {
       command:
@@ -3406,15 +3549,12 @@ async function seedPreviewOrigin(
   const merged = deepMergeJson(current, overlay);
   const mergedJson = JSON.stringify(merged, null, 2);
   const b64 = Buffer.from(mergedJson, "utf8").toString("base64");
-  const write = runGcloud(
-    buildGcloudArgs("ssh-probe", cfg, {
-      command: buildEditorLocalSeedWriteCommand({ repoName, remotePath, b64 }),
-    }),
-    { stdio: ["ignore", "inherit", "inherit"] },
-  );
-  if (write.status !== 0) {
-    throw new Error(`Failed to write ${remotePath} on VM`);
-  }
+  // Fatal ssh step — see runFatalSshStep.
+  await runFatalSshStep(cfg, buildEditorLocalSeedWriteCommand({ repoName, remotePath, b64 }), {
+    label: "seed editor.local.json (previewOrigin)",
+    failureMessage: `Failed to write ${remotePath} on VM`,
+    echoStdout: true,
+  });
 }
 
 /** READY-probe auto-heal: after the editor container is verifiably Up, ask the
@@ -3448,6 +3588,8 @@ async function seedPreviewOrigin(
  *  `cmdReconcileIngress`, the ~15-minute roster sweep that genuinely does run,
  *  which is what makes a mid-life flip self-heal without an Add. */
 async function healPreviewOriginAtReady(cfg, appPort = VITE_PORT, { resolveAppPort = null } = {}) {
+  // Tolerant ssh steps (this probe and the reload below): the heal is
+  // best-effort and never fails the add.
   const probe = runGcloud(
     buildGcloudArgs("ssh-probe", cfg, {
       command: `curl -fsS --max-time 5 http://localhost:${EDITOR_PORT}/api/config 2>/dev/null || echo '{}'`,
@@ -3516,8 +3658,9 @@ async function healPreviewOriginAtReady(cfg, appPort = VITE_PORT, { resolveAppPo
  *  the provider seed MUST run before ensureEditorContainerUp, because the editor
  *  caches config.provider at startup and reports it as currentProvider; a post-start
  *  seed leaves the running editor (and therefore the dashboard badge) on the default
- *  "claude" until a config reload. */
-async function seedAgentApiKey(cfg) {
+ *  "claude" until a config reload. `sleepFn` is the retry backoff seam, injectable
+ *  as on ensureBootstrapped. */
+async function seedAgentApiKey(cfg, { sleepFn } = {}) {
   // The provider seed is unconditional — `buildEditorLocalEnvPatch` always
   // returns a patch carrying `provider`, so the chosen agent always lands on
   // the VM. The API-key seed is opportunistic: when the matching key isn't set
@@ -3544,6 +3687,8 @@ async function seedAgentApiKey(cfg) {
     : `Seeding provider '${cfg.agent}' into ${remotePath}...`;
   console.error(seedLabel);
 
+  // Tolerant ssh step: a failed read falls back to `{}` below, so it needs no
+  // retry wrapper. The write it feeds is the fatal one.
   const read = runGcloud(
     buildGcloudArgs("ssh-probe", cfg, {
       command: `cat ${remotePath} 2>/dev/null || echo '{}'`,
@@ -3566,15 +3711,15 @@ async function seedAgentApiKey(cfg) {
   const merged = deepMergeJson(current, patch);
   const mergedJson = JSON.stringify(merged, null, 2);
   const b64 = Buffer.from(mergedJson, "utf8").toString("base64");
-  const write = runGcloud(
-    buildGcloudArgs("ssh-probe", cfg, {
-      command: buildEditorLocalSeedWriteCommand({ repoName, remotePath, b64 }),
-    }),
-    { stdio: ["ignore", "inherit", "inherit"] },
-  );
-  if (write.status !== 0) {
-    throw new Error(`Failed to write ${remotePath} on VM`);
-  }
+  // Fatal ssh step — see runFatalSshStep. This exact write, then a bare
+  // `stdio: "inherit"` runGcloud, is the step whose oslogin 429 deleted VM-37 on
+  // 2026-10-08: no retry, and the quota text never reached the thrown message.
+  await runFatalSshStep(cfg, buildEditorLocalSeedWriteCommand({ repoName, remotePath, b64 }), {
+    label: "seed editor.local.json (provider)",
+    failureMessage: `Failed to write ${remotePath} on VM`,
+    echoStdout: true,
+    sleepFn,
+  });
   // NOTE: seedAgentApiKey deliberately does NOT install the build-agent CLI.
   // The CLI install is a POST-container-start step owned solely by
   // ensureEditorContainerUp (which runs `docker compose up -d` first). Seeding
@@ -3627,6 +3772,7 @@ async function seedGreenfieldProjectTitle(cfg) {
   const remotePath = `~/projects/${repoName}/.codeyam/editor.local.json`;
   console.error(`Seeding projectTitle='${patch.projectTitle}' into ${remotePath}...`);
 
+  // Tolerant ssh step: mergeEditorLocalPatchB64 treats a failed read as `{}`.
   const read = runGcloud(
     buildGcloudArgs("ssh-probe", cfg, {
       command: `cat ${remotePath} 2>/dev/null || echo '{}'`,
@@ -3634,15 +3780,12 @@ async function seedGreenfieldProjectTitle(cfg) {
     { stdio: ["ignore", "pipe", "ignore"] },
   );
   const b64 = mergeEditorLocalPatchB64(read.stdout, patch);
-  const write = runGcloud(
-    buildGcloudArgs("ssh-probe", cfg, {
-      command: buildEditorLocalSeedWriteCommand({ repoName, remotePath, b64 }),
-    }),
-    { stdio: ["ignore", "inherit", "inherit"] },
-  );
-  if (write.status !== 0) {
-    throw new Error(`Failed to write ${remotePath} on VM`);
-  }
+  // Fatal ssh step — see runFatalSshStep.
+  await runFatalSshStep(cfg, buildEditorLocalSeedWriteCommand({ repoName, remotePath, b64 }), {
+    label: "seed editor.local.json (projectTitle)",
+    failureMessage: `Failed to write ${remotePath} on VM`,
+    echoStdout: true,
+  });
 }
 
 /** Tri-state probe of the editor container's running state on the VM host.
@@ -3663,6 +3806,7 @@ async function seedGreenfieldProjectTitle(cfg) {
  *  which — pre-fix — made every probe report a non-running container regardless
  *  of its real state and took the whole fleet to zero on 2026-06-30. */
 function probeContainerState(cfg, container) {
+  // Tolerant ssh step: a failure is `"unknown"`, which callers do not count as down.
   const checkProbe = runGcloud(
     buildGcloudArgs("ssh-probe", cfg, {
       command: `docker ps --filter name=^/${container}$ --filter status=running --format '{{.Names}}'`,
@@ -3685,6 +3829,8 @@ function probeContainerRunning(cfg, container) {
 
 /** Probe whether `binary` resolves on PATH inside the running container. */
 function probeAgentBinaryInstalled(cfg, container, binary) {
+  // Tolerant ssh step: a failure reads as "not installed", which leads to the
+  // idempotent install (a fatal step) rather than a throw of its own.
   const probe = runGcloud(
     buildGcloudArgs("ssh-probe", cfg, {
       command: `docker exec ${container} bash -lc 'command -v ${binary}'`,
@@ -3835,16 +3981,13 @@ async function ensureAgentCliInstalled(cfg, waitOpts = {}) {
 
   const hostCommand = `docker exec -i ${container} bash -s <<'EOF'\n${containerScript}\nEOF`;
 
-  const installCmd = runGcloud(
-    buildGcloudArgs("ssh-probe", cfg, {
-      command: hostCommand,
-    }),
-    { stdio: ["ignore", "inherit", "inherit"] }
-  );
-
-  if (installCmd.status !== 0) {
-    throw new Error(`Failed to install build-agent CLI in container for agent: ${agent}`);
-  }
+  // Fatal ssh step — see runFatalSshStep. Safe to retry: the script short-circuits
+  // when the binary is already on PATH.
+  await runFatalSshStep(cfg, hostCommand, {
+    label: "install build-agent CLI",
+    failureMessage: `Failed to install build-agent CLI in container for agent: ${agent}`,
+    echoStdout: true,
+  });
 }
 
 async function openTunnel(cfg, appPort = VITE_PORT) {
@@ -4966,27 +5109,15 @@ async function cmdUp(args, projectDir = rootDir) {
     // `createdThisRun` alone MUST NOT authorize deletion. Gather what state the
     // VM is in right now, then let the pure decision rule — see
     // abortTeardownDecision's doc comment for why each input is load-bearing.
-    const state = probeAbortTeardownState(cfg, createdThisRun);
-    // An error that never reached the VM proves nothing about it — see the
-    // `remoteCommandNeverRan` paragraph of abortTeardownDecision.
-    const remoteCommandNeverRan =
-      Boolean(err && err.remoteCommandNeverRan) || isGcloudTransportFailure(err && err.message);
-    const verdict = abortTeardownDecision({ createdThisRun, ...state, remoteCommandNeverRan });
+    const state = abortEvidence(err, probeAbortTeardownState(cfg, createdThisRun));
+    const verdict = abortTeardownDecision({ createdThisRun, ...state });
     if (createdThisRun && verdict === "keep") {
-      // Deliberately loud and specific. The old blanket "tearing down the VM
-      // cloud:up created this run (no orphan left behind)" was actively
-      // misleading here — it asserted no orphan while deleting a working VM.
-      const why = remoteCommandNeverRan
-        ? "the failing step never reached the VM (a gcloud transport/quota error), so nothing shows the VM is broken"
-        : state.probeInconclusive
-        ? "its liveness could not be confirmed (probe failed), and an unconfirmed VM is never deleted"
-        : state.containerUp
-          ? "its editor container is RUNNING — another run brought it up"
-          : "it is in the fleet roster and visible to the dashboard";
       console.error(
-        `✗ provisioning aborted — the VM ${cfg.instanceName} was LEFT IN PLACE because ${why}.\n` +
-          `  This run created the instance, but that alone does not authorize deleting a VM that is in service.\n` +
-          `  If it really should go, remove it deliberately: npm run cloud:destroy`,
+        abortKeptMessage({
+          instanceName: cfg.instanceName,
+          vmNumber: resolveFleetVmNumber(cfg.instanceName),
+          state,
+        }),
       );
     }
     if (verdict === "delete") {
@@ -5161,6 +5292,7 @@ async function cmdStatus(args, projectDir = rootDir) {
     return;
   }
 
+  // Tolerant ssh step: status display only; a failure prints "?".
   const imgProbe = runGcloud(
     buildGcloudArgs("ssh-probe", cfg, {
       command:
@@ -5698,12 +5830,12 @@ async function probeGitCredentials(cfg) {
     `${projectCd} && if ${composeExec} git -C /workspace ls-remote origin -h HEAD ` +
     `>/dev/null 2>/tmp/cy-lsr.err; then echo LSREMOTE=ok; ` +
     `else echo LSREMOTE=fail; sed 's/^/LSRERR:/' /tmp/cy-lsr.err; fi`;
-  const probe = runGcloud(
-    buildGcloudArgs("ssh-probe", cfg, { command: probeCmd }),
-    { stdio: ["ignore", "pipe", "ignore"] },
-  );
-  if (probe.status !== 0 || typeof probe.stdout !== "string") {
-    throw new Error(`Failed to probe credentials on ${cfg.instanceName} (SSH error).`);
+  // Fatal ssh step — see runFatalSshStep. It runs on the add path too, via
+  // assertGitPushCredentials. stdout is parsed below, so it is not echoed.
+  const failureMessage = `Failed to probe credentials on ${cfg.instanceName} (SSH error).`;
+  const probe = await runFatalSshStep(cfg, probeCmd, { label: "probe git credentials", failureMessage });
+  if (typeof probe.stdout !== "string") {
+    throw new Error(failureMessage);
   }
   const out = probe.stdout;
   return {
@@ -5747,14 +5879,12 @@ async function seedLiveCredentialCache(cfg, token) {
     `echo ${storeB64} | base64 -d | ${composeExec} ` +
     `bash -c 'cat > ${store} && chmod 600 ${store}' && ` +
     `echo APPROVED`;
-  const seed = runGcloud(
-    buildGcloudArgs("ssh-probe", cfg, { command: seedCmd }),
-    { stdio: ["ignore", "pipe", "inherit"] },
-  );
-  if (seed.status !== 0 || !/APPROVED/.test(String(seed.stdout || ""))) {
-    throw new Error(
-      `Failed to seed the live credential cache on ${cfg.instanceName}.`,
-    );
+  // Fatal ssh step — see runFatalSshStep. Safe to retry: `credential approve`
+  // and the in-place store truncate are both idempotent.
+  const failureMessage = `Failed to seed the live credential cache on ${cfg.instanceName}.`;
+  const seed = await runFatalSshStep(cfg, seedCmd, { label: "seed live credential cache", failureMessage });
+  if (!/APPROVED/.test(String(seed.stdout || ""))) {
+    throw new Error(failureMessage);
   }
 }
 
@@ -5804,15 +5934,14 @@ async function cmdReconcileAdmin(args, projectDir = rootDir) {
   console.error(
     `Reconciling admin mode to ${desired === "1" ? "ON" : "off"} on ${cfg.instanceName}...`,
   );
-  const result = runGcloud(
-    buildGcloudArgs("ssh-probe", cfg, {
-      command: buildReconcileAdminCommand(repoName, desired),
-    }),
-    { stdio: "inherit" },
-  );
-  if (result.status !== 0) {
-    throw new Error(`Failed to reconcile admin mode on ${cfg.instanceName}.`);
-  }
+  // Fatal ssh step — see runFatalSshStep. No teardown follows here, but a 429 on
+  // an operator command is still a retry, not a failure. Safe to retry: the
+  // compose line rewrite and `up -d editor` are both idempotent.
+  await runFatalSshStep(cfg, buildReconcileAdminCommand(repoName, desired), {
+    label: "reconcile admin mode",
+    failureMessage: `Failed to reconcile admin mode on ${cfg.instanceName}.`,
+    echoStdout: true,
+  });
   console.error(
     `  ✓ editor service recreated; /api/config now reports admin=${desired === "1" ? "true" : "false"}.`,
   );
@@ -5990,6 +6119,8 @@ module.exports = {
   probeContainerState,
   waitForEditorComposeRunning,
   abortTeardownDecision,
+  abortEvidence,
+  abortKeptMessage,
   probeAbortTeardownState,
   probeAgentBinaryInstalled,
   waitForContainerRunning,
@@ -6080,6 +6211,9 @@ module.exports = {
   formatGcloudFailure,
   isTransientSshFailure,
   runGcloudSshWithRetry,
+  classifySshStepFailure,
+  runFatalSshStep,
+  fatalSshStepError,
   ensureInstance,
   ensureBootstrapped,
   assertEditorContainerRunning,

package/npm/fleet-api-budget.js

@@ -25,15 +25,28 @@
  *    dashboard's own monitoring gathers (bounded in concurrency by hostOpLimiter,
  *    not in rate) and the tunnel watchdog.
  *  - `perAddCallsPerMin` — what one in-flight add is charged. Deliberately its
- *    PEAK, not its average: the fast first minute of the container gate (6 reads)
- *    overlapping cloud:up's own ssh steps or the post-roster burst. Charging the
- *    peak is wrong in the safe direction — it queues a little early rather than
- *    429ing a provision.
+ *    PEAK, not its average. Charging the peak is wrong in the safe direction — it
+ *    queues a little early rather than 429ing a provision.
  *
- *  At these defaults (60 - 20) / 10 = 4 concurrent adds, the same as the job cap,
- *  so an operator who raises MAX_JOB_CONCURRENCY meets this budget instead of the
- *  quota. */
-const API_BUDGET_DEFAULTS = Object.freeze({ budgetPerMin: 60, reservedPerMin: 20, perAddCallsPerMin: 10 });
+ *    Provenance of 15 (2026-10-09, re-derived after VM-37 429'd on 2026-10-08
+ *    under the old 10): the peak is cloud:up's seed → container-gate window,
+ *    where each `gcloud compute ssh` (≥1 loginProfile read) runs back to back.
+ *    Counted from npm/cloud.js on the clone path: app-port probe 1, previewOrigin
+ *    read+write 2, git-credentials write 1, provider seed read+write 2, compose
+ *    state probe + `up -d` 2, container probe + CLI install 2, git-auth probe 1,
+ *    final container assert 1, CLI probe 1 = 13; a greenfield add adds the
+ *    projectTitle read+write = 15. At ~4s per IAP ssh round trip that is ~15
+ *    reads inside one minute. The old 10 was the container gate's first minute
+ *    (6 reads) plus overlap — it did not count the seed burst VM-37 died in.
+ *    This is a STATIC count, not a measured counter; replace it if a measured
+ *    per-add rate is ever taken. The batch `dash_add` path is NOT the gap:
+ *    server.js admits each reserved VM separately through tryStartOrQueue, and
+ *    addVm reaches startJob synchronously, so the job cap holds per VM.
+ *
+ *  At these defaults an add is admitted while 20 + 15·inFlight + 15 ≤ 60, i.e.
+ *  2 concurrent adds — below the job cap of 4, so this budget, not the cap, is
+ *  what paces a batch. API_CALLS_PER_ADD_PER_MIN overrides it on the dashboard. */
+const API_BUDGET_DEFAULTS = Object.freeze({ budgetPerMin: 60, reservedPerMin: 20, perAddCallsPerMin: 15 });
 
 // A positive finite number, else 0 — so garbage degrades toward admitting work,
 // never toward a phantom load that would freeze admission forever.

package/npm/fleet-build-data.js

@@ -55,6 +55,24 @@ function buildVmEnvelope(v, ctx) {
     // a definite `false` that no probe had ever established.
     ...fr.normalizeReachability(v),
     hasSession: !!v.hasSession,
+    // The three session-age signals the codeyam-fleet-latest Phase 1b idle
+    // triage reads by name. server.js computes all three per poll onto `v`, but
+    // they were never copied here, so /data carried none of them and the
+    // procedure's ACTIVE/ENDED table fell through to UNCLEAR on every HOLD VM.
+    // `sessionIdleSeconds`: seconds since the session last did anything, or null
+    // when the editor is too old to report it — never coerced to 0, which would
+    // read as "just active".
+    sessionIdleSeconds:
+      typeof v.sessionIdleSeconds === 'number' && Number.isFinite(v.sessionIdleSeconds)
+        ? v.sessionIdleSeconds
+        : null,
+    // 'none' | 'active' | 'stale' | 'unknown' (npm/fleet-session-activity.js).
+    // An unpolled VM has no verdict at all; it reads 'unknown', which blocks
+    // like 'active', so an absent signal can never clear a VM as stale.
+    sessionActivity: v.sessionActivity || 'unknown',
+    // Boolean twin of the `session-wedged:` prose in degradedReasons, so a
+    // program can read it without parsing English.
+    sessionWedged: v.sessionWedged === true,
     // Three-state: true = broker confirmed up, false = confirmed down, null =
     // editor unreachable so broker state is unknown. Never conflate "down" with
     // "unreachable" — that mismatch is what made the card show "broker down"
@@ -183,14 +201,16 @@ function buildVmEnvelope(v, ctx) {
   };
 }
 
-// The fields the card UI depends on being present in every /data VM entry. This
+// The fields the card UI — and the codeyam-fleet-latest idle triage, which reads
+// /data directly — depend on being present in every /data VM entry. This
 // is the contract `buildVmEnvelope` must satisfy: the regression that prompted
 // this module was a field the card read but the envelope didn't emit, so the
 // test asserts the envelope keys are a superset of this list. Adding a field the
 // card renders means adding it here AND in buildVmEnvelope — the test fails loud
 // if you do one without the other.
 const REQUIRED_CARD_FIELDS = [
-  'n', 'url', 'previewUrl', 'appPort', 'previewServing', 'previewIngress', 'reachable', 'reachableSource', 'hasSession', 'brokerLive', 'brokerStatus',
+  'n', 'url', 'previewUrl', 'appPort', 'previewServing', 'previewIngress', 'reachable', 'reachableSource', 'hasSession',
+  'sessionIdleSeconds', 'sessionActivity', 'sessionWedged', 'brokerLive', 'brokerStatus',
   'awaitingInput', 'needsCleanup', 'feature', 'selectedPlan', 'step', 'mode', 'label', 'cycleClosed', 'status', 'orphan',
   'degradedReasons', 'setupSteps', 'attentionReasons',
   'isHead', 'headHolderUnreachable', 'queuePos', 'queueLen',