@codeyam-editor/codeyam-editor 0.1.11-staging.gf7d5b1a → 0.1.12-staging.g83e4a0c

package/npm/cloud.js

@@ -115,6 +115,9 @@ const { addToRoster, readRoster } = require("./fleet-roster");
 // container) apart from a real `docker compose up -d` failure, so the abort path
 // can converge instead of escalating to instance deletion.
 const { classifyComposeUpFailure } = require("./compose-conflict");
+// The two streamed bootstrap scripts, named once for this file's STREAMED_SCRIPTS
+// and for the advance's provisioning pre-flight, which checks the same set.
+const { BOOTSTRAP_SCRIPT, PROJECT_BOOTSTRAP_SCRIPT } = require("./provisioning-contract");
 
 // =============================================================================
 // Pure helpers — every export here is unit-tested in cloud.test.js without
@@ -1540,6 +1543,7 @@ async function waitForIapReady(cfg) {
   const deadline = Date.now() + 180_000;
   let attempts = 0;
   let lastErr = "";
+  const probeStderrs = [];
   while (Date.now() < deadline) {
     attempts++;
     const probe = runGcloud(
@@ -1550,17 +1554,42 @@ async function waitForIapReady(cfg) {
       console.error(`  ✓ IAP ready (${attempts} ${attempts === 1 ? "probe" : "probes"})`);
       return;
     }
-    lastErr = String(probe.stderr || "").trim().split("\n").slice(-1)[0] || "";
+    const stderr = String(probe.stderr || "").trim();
+    probeStderrs.push(stderr);
+    lastErr = stderr.split("\n").slice(-1)[0] || "";
     // Exponential-ish backoff: 2s, 3s, 5s, 8s, then cap at 8s.
     const delay = Math.min(2000 * Math.pow(1.5, attempts - 1), 8000);
     await new Promise((r) => setTimeout(r, delay));
   }
+  const cause = iapProbeCause(probeStderrs);
   throw new Error(
     `IAP did not become ready for ${cfg.instanceName} within 180s ` +
-      `(${attempts} probes; last error: ${lastErr || "unknown"})`,
+      `(${attempts} probes; ${cause ? `likely cause: ${cause}; ` : ""}` +
+      `last error: ${lastErr || "unknown"})`,
   );
 }
 
+/** Name the likely cause behind a run of failed IAP SSH probes. The last stderr
+ *  line alone is almost always `ssh exited with return code [255]`, which
+ *  collapses three different waits — each with its own expected duration — into
+ *  one unactionable string. The full stderr of each probe still distinguishes
+ *  them. Scans newest-first so the label describes where the wait ENDED (IAP lag
+ *  often gives way to key propagation on a fresh box). Returns null when no
+ *  signature matches; the raw last line is kept beside it either way. */
+function iapProbeCause(probeStderrs) {
+  const signatures = [
+    [/4047|Failed to lookup instance/i, "IAP has not registered the instance yet"],
+    [/Permission denied \(publickey/i, "OS Login SSH key not propagated yet"],
+    [/Connection (refused|closed|reset)/i, "sshd on the instance not accepting connections yet"],
+  ];
+  for (const text of [...(probeStderrs || [])].reverse()) {
+    for (const [re, label] of signatures) {
+      if (re.test(text)) return label;
+    }
+  }
+  return null;
+}
+
 /** Read `process.env[FORWARD_ENV_KEYS_ENV]` (comma-separated key list set by
  *  the fleet dashboard) and return the subset of `process.env` that should
  *  ride along into the VM's project-bootstrap shell. Whitelisted to the
@@ -2216,6 +2245,20 @@ function buildReconcileAdminCommand(repoName, value) {
   );
 }
 
+// Every library `scripts/gcp-project-bootstrap.sh` sources, keyed by the path it
+// sources it AS — i.e. relative to the script's own directory, which on the VM is
+// `$HOME`. A streamed script is a SINGLE FILE, so each of these must be shipped
+// beside it or the script dies at its source line and cloud:up tears the
+// freshly-created VM back down. Adding a `source` line to that script without
+// adding it here has now broken every add on the fleet twice (`lib/npm-pin.sh`
+// 2026-09-04, `lib/render-cloud-compose.sh` 2026-09-18), so this list is the one
+// place to register it: STREAMED_SCRIPTS derives its library entries from this
+// constant rather than repeating them, and a unit test reads the shipped script
+// and fails on any source line missing from it. Keep it a ONE-LINE array literal:
+// the advance's provisioning pre-flight reads it out of a branch's cloud.js TEXT
+// (provisioning-contract.js → parseProjectBootstrapLibs).
+const PROJECT_BOOTSTRAP_LIBS = ["lib/npm-pin.sh", "lib/render-cloud-compose.sh"];
+
 async function ensureProjectBootstrapped(cfg, { sleepFn } = {}) {
   const repoName = resolveProject(cfg).name;
   console.error(`Checking project bootstrap state for ${repoName}...`);
@@ -2245,13 +2288,16 @@ async function ensureProjectBootstrapped(cfg, { sleepFn } = {}) {
   console.error(`  streaming scripts/gcp-project-bootstrap.sh to VM for ${cfg.repo}...`);
   const scriptPath = path.join(rootDir, "scripts", "gcp-project-bootstrap.sh");
   const scriptBody = readStreamedScript(scriptPath, cfg);
-  // gcp-project-bootstrap.sh sources lib/npm-pin.sh relative to its own
-  // directory. Read it through readStreamedScript too, so the library is subject
-  // to the same drift guard as the script that sources it — streaming a reviewed
-  // remote script beside a stale on-disk library would defeat the guard.
-  const bootstrapLibs = {
-    "lib/npm-pin.sh": readStreamedScript(path.join(rootDir, "scripts", "lib", "npm-pin.sh"), cfg),
-  };
+  // gcp-project-bootstrap.sh sources its libraries relative to its own
+  // directory. Read them through readStreamedScript too, so each library is
+  // subject to the same drift guard as the script that sources it — streaming a
+  // reviewed remote script beside a stale on-disk library would defeat the guard.
+  const bootstrapLibs = Object.fromEntries(
+    PROJECT_BOOTSTRAP_LIBS.map((rel) => [
+      rel,
+      readStreamedScript(path.join(rootDir, "scripts", ...rel.split("/")), cfg),
+    ]),
+  );
   const scriptArgs = clientProjectBootstrapArgs(cfg);
   // Git identity, forwarded launch-form keys, fleet-domain ingress and the admin
   // toggle — see projectBootstrapEnv for what each key is for and when it is
@@ -3197,10 +3243,18 @@ async function seedPreviewOrigin(
  *  Best-effort and non-fatal by contract: a probe failure, a still-absent
  *  stack.json (verdict stays `true`), or a heal that can only reach a loopback
  *  bootstrap origin (the operator's `<N>-preview` tunnel not published yet) all
- *  leave the add green. The cloudflared-monitor's periodic tick (section 2) is
- *  the durable owner that heals post-add flips — onboarding finishing minutes
- *  after READY. */
-async function healPreviewOriginAtReady(cfg, appPort = VITE_PORT) {
+ *  leave the add green.
+ *
+ *  TWO TRIGGERS, and the second is the point. This used to be called only from
+ *  the add/READY path, and its own comment named "the cloudflared-monitor's
+ *  periodic tick" as the durable owner of post-add flips. That owner does not
+ *  exist on this fleet: the fleet runs on the GCP HTTPS LB, there is no
+ *  cloudflared tick, and so a VM that developed the failure MID-LIFE — the
+ *  normal case, since the failure needs an app to exist before it can fail to
+ *  hydrate — was never healed by anything. It is now also called per-VM from
+ *  `cmdReconcileIngress`, the ~15-minute roster sweep that genuinely does run,
+ *  which is what makes a mid-life flip self-heal without an Add. */
+async function healPreviewOriginAtReady(cfg, appPort = VITE_PORT, { resolveAppPort = null } = {}) {
   const probe = runGcloud(
     buildGcloudArgs("ssh-probe", cfg, {
       command: `curl -fsS --max-time 5 http://localhost:${EDITOR_PORT}/api/config 2>/dev/null || echo '{}'`,
@@ -3236,7 +3290,14 @@ async function healPreviewOriginAtReady(cfg, appPort = VITE_PORT) {
   // inside the seed answers `true` and re-writes the same-origin sentinel that
   // just failed to hydrate. We got here precisely BECAUSE the editor said
   // `false`, so `false` is the answer the seed must use.
-  await seedPreviewOrigin(cfg, appPort, false);
+  // Resolve the real app port only NOW, past the two guards above, because
+  // reading it costs an SSH round-trip per VM. The add path already has the port
+  // in hand and passes it directly; the periodic sweep passes a resolver instead,
+  // so a fleet of healthy VMs pays nothing and only a VM actually being healed
+  // pays the read. Falls back to whatever `appPort` already is if the resolver
+  // fails, which is the same VITE_PORT bootstrap default it had before.
+  const seedPort = resolveAppPort ? ((await resolveAppPort()) ?? appPort) : appPort;
+  await seedPreviewOrigin(cfg, seedPort, false);
   // Tell the running server to re-read editor.local.json — no restart (the
   // footgun that killed VM7's live Claude session).
   const reload = runGcloud(
@@ -3635,14 +3696,18 @@ function promptForInstanceName(expected) {
 }
 
 // Every file streamed to a VM, and therefore every file the drift guard must
-// compare against the reviewed remote tip. `scripts/lib/npm-pin.sh` is here
-// because gcp-project-bootstrap.sh SOURCES it: a library that reaches the VM is
-// as much streamed content as the script that sources it, and leaving it out
-// meant the drift guard silently vouched for only half of what was shipped.
+// compare against the reviewed remote tip. The libraries are here because
+// gcp-project-bootstrap.sh SOURCES them: a library that reaches the VM is as
+// much streamed content as the script that sources it, and leaving one out meant
+// the drift guard silently vouched for only half of what was shipped. They are
+// DERIVED from PROJECT_BOOTSTRAP_LIBS rather than listed again — when the two
+// were maintained separately, a library added to neither took the whole fleet's
+// provisioning down, and a library added to only one would have been shipped
+// unvouched-for.
 const STREAMED_SCRIPTS = [
-  "scripts/gcp-bootstrap.sh",
-  "scripts/gcp-project-bootstrap.sh",
-  "scripts/lib/npm-pin.sh",
+  BOOTSTRAP_SCRIPT,
+  PROJECT_BOOTSTRAP_SCRIPT,
+  ...PROJECT_BOOTSTRAP_LIBS.map((rel) => `scripts/${rel}`),
 ];
 
 /** The configured editor branch to compare/stream streamed scripts against.
@@ -4965,6 +5030,40 @@ async function cmdIngressUp(args, projectDir = rootDir) {
   );
 }
 
+/** Re-seed the preview origin on any rostered VM whose editor reports the
+ *  subpath-hydration escalation but whose configured origin never followed.
+ *
+ *  The second trigger for `healPreviewOriginAtReady`, whose only other caller is
+ *  the add/READY path — so a VM that developed the failure MID-LIFE was never
+ *  reached by it. That is the normal way this failure appears (it needs an app to
+ *  exist before the app can fail to hydrate), and it is why the same bug recurred
+ *  on four VMs that had each been added long before they broke.
+ *
+ *  Runs over the WHOLE roster, not just the sweep's re-wire targets: a VM can be
+ *  perfectly wired at the load balancer and still be serving its preview from the
+ *  wrong origin — the two conditions are unrelated, and narrowing to the re-wire
+ *  set would miss exactly the healthy-but-wrong VM this exists for. Safe to do so
+ *  because the heal's own two guards return early unless the editor reports a
+ *  not-same-origin-safe stack AND a wrong-mode origin, so a healthy VM costs one
+ *  bounded probe and writes nothing; the app-port read is deferred behind those
+ *  guards, so only a VM actually being healed pays for it.
+ *
+ *  Per-VM failures are logged and swallowed: a preview-origin heal must never
+ *  abort an ingress sweep that has other VMs left to reconcile. */
+async function healPreviewOriginsAcrossRoster(cfg, roster) {
+  for (const n of roster) {
+    const label = slugFor(n) || `vm-${n}`;
+    const perVmCfg = { ...cfg, instanceName: `${fleetInstancePrefix()}${n}` };
+    try {
+      await healPreviewOriginAtReady(perVmCfg, VITE_PORT, {
+        resolveAppPort: () => readVmAppPort(perVmCfg),
+      });
+    } catch (e) {
+      console.error(`  VM-${n} ${label}: preview-origin heal skipped (${e && e.message})`);
+    }
+  }
+}
+
 /** Detect unwired / half-wired rostered VMs and re-wire them idempotently.
  *  Enumerates the fleet roster (the same source the dashboard reads), runs the
  *  read-only `vm-ingress.sh <N> status` probe per VM, and for any reporting
@@ -5054,6 +5153,8 @@ async function cmdReconcileIngress(args, projectDir = rootDir) {
     );
   }
 
+  await healPreviewOriginsAcrossRoster(cfg, roster);
+
   // Converge each rostered backend-service's timeout to 86400 (24h). A fully-wired
   // BES still at GCP's 30s default reports `wired` above and is skipped by the
   // re-wire loop, yet on an External HTTPS LB that 30s is the MAX WebSocket lifetime
@@ -5647,6 +5748,7 @@ async function cloudMain() {
 module.exports = {
   // pure helpers
   isSpawnTimeout,
+  iapProbeCause,
   gcloudCallTimeoutMs,
   provisionFailureMessage,
   GCLOUD_PROVISION_TIMEOUT_MS,
@@ -5713,6 +5815,7 @@ module.exports = {
   buildEditorLocalSeedWriteCommand,
   seedPreviewOrigin,
   healPreviewOriginAtReady,
+  healPreviewOriginsAcrossRoster,
   buildGcloudArgs,
   sshScriptCommand,
   bootstrapRunnerScript,
@@ -5760,6 +5863,7 @@ module.exports = {
   gatherScriptDriftFacts,
   readStreamedScript,
   STREAMED_SCRIPTS,
+  PROJECT_BOOTSTRAP_LIBS,
   // ingress wiring (pure decision + spawn wrappers)
   envPosInt,
   ingressWiringPlan,

package/npm/fleet-add-failure.js

@@ -47,6 +47,13 @@ const { parseRosterMarkers } = require('./fleet-early-roster');
 //      `ensureBootstrapped` skips on the present image, and
 //      `ensureProjectBootstrapped` re-attaches to the in-flight install rather
 //      than starting a second `npm install` over the first.
+//      `iap-not-ready` is the same ground in its strongest form: the throw comes
+//      from waitForIapReady, which ensureInstance calls BEFORE ensureBootstrapped,
+//      ensureProjectBootstrapped or ensureEditorContainerUp — so the provable state
+//      is "an instance exists and nothing else has run". The retry reuses that
+//      live instance and every later step runs for the first time. (VM-4,
+//      2026-09-17: destroy + an identical re-add succeeded in 10 minutes; the
+//      failure was a transient IAP/SSH race, not a broken box.)
 //
 // Ground 2 is deliberately narrow — it is NOT "the VM probably still works".
 // `container-stuck` and `incomplete-workspace` also leave a live VM, but their
@@ -56,6 +63,7 @@ const ADD_FAILURE_RETRYABLE = new Set([
   'instance-vanished',
   'capacity-stockout',
   'project-bootstrap-timeout',
+  'iap-not-ready',
 ]);
 
 // Collect the `    - <path>` bullets the drift guard prints AFTER the "following
@@ -350,6 +358,92 @@ function wedgedInsertRecovery({ code, instance } = {}) {
   );
 }
 
+// Facts for the iap-not-ready arm: waitForIapReady (npm/cloud.js) threw its
+// readiness timeout. Returns { timedOut, instance, budgetSecs, cause } — `cause` is
+// the label waitForIapReady derived from its probe stderr (null when none of its
+// signatures matched, or on a transcript from before the label existed), so the
+// recovery copy can name WHICH wait was blown instead of only `return code [255]`.
+//
+// The signature is emitted from exactly one place, and that place runs before any
+// bootstrap work, which is what lets this arm assert the instance is bare.
+const IAP_NOT_READY = /IAP did not become ready for (\S+) within (\d+)s/;
+
+function iapNotReadyFacts(logTail) {
+  const m = String(logTail || '').match(IAP_NOT_READY);
+  if (!m) return { timedOut: false, instance: null, budgetSecs: null, cause: null };
+  const cause = (String(logTail).match(/likely cause: ([^;)]+)/) || [])[1] || null;
+  return { timedOut: true, instance: m[1], budgetSecs: Number(m[2]), cause: cause && cause.trim() };
+}
+
+// Operator-facing recovery copy for the iap-not-ready arm. It states what the
+// generic arm cannot: the instance EXISTS and is BARE. "A partial instance may
+// exist" is what steered VM-4's operator toward destroy + re-add (15-25 minutes)
+// when Retry resumes on the same box in seconds — and it left nothing on the card
+// saying a RUNNING, billing instance was sitting there.
+function iapNotReadyRecovery({ instance, budgetSecs, cause } = {}) {
+  const named = instance || 'The instance';
+  const within = budgetSecs ? ` within ${budgetSecs}s` : '';
+  const why = cause ? ` (likely cause: ${cause})` : '';
+  return (
+    `${named} was created and is RUNNING, but SSH over IAP did not answer${within}${why}. This is usually a transient ` +
+    'control-plane race. Nothing was installed on it — the wait runs before any bootstrap step — so it is a bare box and ' +
+    'there is nothing partial to reconcile. Retry add resumes on this same instance rather than creating a second one. ' +
+    'Until you retry or destroy it, the instance keeps running and billing.'
+  );
+}
+
+// Facts for the streamed-library-missing arm: a streamed bootstrap script sourced
+// a `$HOME`-rooted `lib/…` file that was never shipped to the VM, and bash
+// reported `<script>: line N: <path>: No such file or directory`. This took ALL
+// fleet provisioning down twice (lib/npm-pin.sh 2026-09-04, lib/render-cloud-
+// compose.sh 2026-09-18). Returns { missing, script, library } — `script` is the
+// basename of the aborting script, `library` the `lib/…` path relative to $HOME,
+// which is how PROJECT_BOOTSTRAP_LIBS (npm/cloud.js) names its entries.
+const STREAMED_LIBRARY_MISSING = /(\S+): line \d+: \/(?:home\/[^/\s]+|root)\/(lib\/[^\s:]+): No such file or directory/;
+
+function streamedLibraryMissingFacts(logTail) {
+  const m = String(logTail || '').match(STREAMED_LIBRARY_MISSING);
+  if (!m) return { missing: false, script: null, library: null };
+  return { missing: true, script: m[1].split('/').pop(), library: m[2] };
+}
+
+// Did cloud:up report that it deleted the instance it created this run? The abort
+// path (npm/cloud.js) prints the teardown marker BEFORE it issues the delete, and
+// prints a `during abort teardown` warning when that delete fails — so the marker
+// alone is a claim of intent, and only the marker WITHOUT the warning is evidence
+// that no orphan was left behind.
+function cleanTeardownReported(logTail) {
+  const log = String(logTail || '');
+  return /tearing down the VM cloud:up created this run \(no orphan left behind\)/.test(log)
+    && !/during abort teardown/.test(log);
+}
+
+// Operator-facing recovery copy for the streamed-library-missing arm. It names
+// what to fix — register the library so it is streamed — and, when the transcript
+// proves a clean teardown, says there is no orphan to hunt for. Every add on this
+// branch fails identically until the registry is fixed, so it never offers Retry.
+function streamedLibraryMissingRecovery({ script, library, cleanTeardown } = {}) {
+  const lib = library || 'a lib/ file';
+  const from = script ? ` sourced by ${script}` : '';
+  const orphan = cleanTeardown
+    ? 'cloud:up already deleted the VM it created, so there is nothing to reconcile or destroy.'
+    : 'The transcript does not confirm the VM was deleted — check whether a partial instance exists before re-adding.';
+  return (
+    `${lib}${from} was never shipped to the VM, so the bootstrap aborted. Add it to PROJECT_BOOTSTRAP_LIBS in npm/cloud.js, ` +
+    `push, and re-add — every add from this branch fails the same way until then. ${orphan}`
+  );
+}
+
+// Operator-facing recovery copy for the generic cloud-up-failed arm. "A partial
+// instance may exist" is only true when the transcript does not show cloud:up
+// deleting what it created; saying it regardless sends the operator hunting for
+// an orphan that is not there.
+function cloudUpFailedRecovery({ cleanTeardown } = {}) {
+  return cleanTeardown
+    ? 'cloud:up failed during provisioning and deleted the VM it created, so nothing was left behind. Read the transcript for the cause before re-adding.'
+    : 'cloud:up failed during provisioning, so a partial instance may exist. Reconcile or destroy this VM before re-adding.';
+}
+
 // Build a verdict, stamping `retryable` from the single-source-of-truth Set so it
 // can never drift from the category.
 function buildFailureVerdict(category, headline, recovery, extra) {
@@ -392,7 +486,12 @@ function buildFailureVerdict(category, headline, recovery, extra) {
 // instance. `wedged-insert` (arm 8) is the third: it sits directly above the
 // generic arm so a create GCE accepted-and-abandoned cannot report as an ordinary
 // provisioning failure whose advice is to destroy the box — a destroy that, on a
-// wedged slot, queues behind the wedge instead of clearing it.
+// wedged slot, queues behind the wedge instead of clearing it. `iap-not-ready`
+// (arm 9) is the fourth: it keeps a bare, retry-safe instance from reporting as a
+// possibly-partial one whose only recovery is destroy + re-add.
+// `streamed-library-missing` (arm 10) is the fifth: it names the unshipped library
+// that broke every add, instead of advising a hunt for an orphan cloud:up already
+// deleted.
 function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy, wiringDeadlineBlown, wiringInFlight } = {}) {
   const log = String(logTail || '');
 
@@ -554,28 +653,64 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
       wedgedInsertRecovery(wedgedInsert));
   }
 
-  // 9. cloud-up-failed — a generic cloud:up failure, reached ONLY when the drift /
-  //    vanished / stuck / stockout / missing-ref / wedged-insert signatures above
-  //    did not match. A partial instance may exist, so not retryable.
+  // 9. iap-not-ready — the instance reached RUNNING but IAP-tunneled SSH never
+  //    answered inside waitForIapReady's budget. That wait is called from
+  //    ensureInstance before any bootstrap work, so the box exists and is bare.
+  //    Retryable on the re-entrancy ground (see ADD_FAILURE_RETRYABLE); the budget
+  //    was already raised 60s → 180s for this race, and VM-4 still blew it, so the
+  //    answer is a cheap retry rather than a longer wait for every dead add.
+  //
+  //    Tested BEFORE cloud-up-failed for the reason arm 6 documents: the provision
+  //    shell echoes cloud:up's tail into every transcript, so matching the generic
+  //    line first refuses the safe retry — exactly the VM-4 failure.
+  const iapNotReady = iapNotReadyFacts(log);
+  if (iapNotReady.timedOut) {
+    return buildFailureVerdict('iap-not-ready', 'SSH never became reachable',
+      iapNotReadyRecovery(iapNotReady));
+  }
+
+  // 10. streamed-library-missing — a streamed bootstrap script sourced a `lib/…`
+  //    file that was never shipped to the VM. The failure that has taken ALL fleet
+  //    provisioning down twice (2026-09-04, 2026-09-18); both times it fell through
+  //    to cloud-up-failed, whose recovery told six operators to hunt for a partial
+  //    instance cloud:up had already deleted, and said nothing about the cause.
+  //
+  //    NOT retryable: re-issuing the same add streams the same scripts without the
+  //    library and fails identically — the missing-branch-ref reasoning. The fix is
+  //    a code change (register the library), which is what the recovery names.
+  //
+  //    Tested BEFORE cloud-up-failed for the reason arm 6 documents: the provision
+  //    shell echoes cloud:up's tail into every transcript, so matching the generic
+  //    line first is precisely the bug being fixed.
+  const missingLib = streamedLibraryMissingFacts(log);
+  if (missingLib.missing) {
+    return buildFailureVerdict('streamed-library-missing', `Bootstrap library never shipped: ${missingLib.library}`,
+      streamedLibraryMissingRecovery({ ...missingLib, cleanTeardown: cleanTeardownReported(log) }));
+  }
+
+  // 11. cloud-up-failed — a generic cloud:up failure, reached ONLY when the drift /
+  //    vanished / stuck / stockout / missing-ref / wedged-insert / iap-not-ready /
+  //    streamed-library signatures above did not match. Not retryable. Whether a
+  //    partial instance may exist is read off the transcript's teardown evidence.
   if (/cloud:up FAILED \(exit/i.test(log)) {
     return buildFailureVerdict('cloud-up-failed', 'Provisioning failed',
-      'cloud:up failed during provisioning, so a partial instance may exist. Reconcile or destroy this VM before re-adding.');
+      cloudUpFailedRecovery({ cleanTeardown: cleanTeardownReported(log) }));
   }
 
-  // 10. interrupted — a signal kill (OOM 137, SIGTERM 143, …) or an explicit
+  // 12. interrupted — a signal kill (OOM 137, SIGTERM 143, …) or an explicit
   //    Killed / out-of-memory marker. The real VM state is unknown. Not retryable.
   if (fc.isSignalKillExit(Number(exitCode)) || /\bKilled\b/.test(log) || /Out of memory/i.test(log)) {
     return buildFailureVerdict('interrupted', 'Provisioning interrupted',
       'The provision was interrupted (killed or out of memory), so the VM state is unknown. Reconcile or destroy this VM before re-adding.');
   }
 
-  // 11. The post-add IAP read-back verdicts. Unlike every signature above, this one
+  // 13. The post-add IAP read-back verdicts. Unlike every signature above, this one
   //    is not produced by the provision transcript at all — it is the BLOCKED
   //    reason iapReadbackOutcome hands to annotateAddFailure, which is what
   //    stops a read-back failure from being the one add failure that reaches the
   //    card with no category at all (and so no differentiated headline, no
   //    recovery, no guarded Retry, and no chance for holdConvergingBadge to hold it
-  //    calm). Tested AFTER all eight hard-failure signatures so a proven failure
+  //    calm). Tested AFTER every hard-failure signature so a proven failure
   //    always wins if both texts are ever present, and BEFORE converging/verifying
   //    because it is strictly more specific than either.
   if (/IAP policy authorizes no human/i.test(log)) {
@@ -599,7 +734,7 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
       `The VM is up but its IAP policy authorizes no human, so the public URL returns "Error code 51" for everyone. Its background wiring is no longer running, so this will not resolve on its own — grant access to fix it. Do NOT destroy or re-add this VM.${remedy ? `\n\n${remedy}` : ''}`);
   }
 
-  // 12. converging — the add reached the early-roster marker (`CY_ROSTER`): the
+  // 14. converging — the add reached the early-roster marker (`CY_ROSTER`): the
   //    container gate passed, the instance is real, and the VM is rostered, yet it
   //    exited non-zero WITHOUT matching any specific hard-failure signature above.
   //    By construction of the provision script, every genuine hard failure (drift,
@@ -617,7 +752,7 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
       'The container is up and the VM is rostered; the LB ingress is still wiring in the background. This resolves on its own once the editor is reachable over the LB — no action needed unless it persists past the wiring deadline.');
   }
 
-  // 13. The residue: no failure signature above matched. The polarity here is
+  // 15. The residue: no failure signature above matched. The polarity here is
   //    INVERTED from the original design. It used to read "no recognized success
   //    signature ⇒ failed", which made red the DEFAULT for "we don't know" — so a
   //    VM whose container was Up and whose editor answered /api/health 200 got a
@@ -739,4 +874,14 @@ function wedgedRetryMessage(n) {
   );
 }
 
-module.exports = { classifyAddFailure, extractDriftFiles, lastNonEmptyLines, canRetryAdd, transcriptRostered, holdConvergingBadge, addFailureCorroborated, containerHealthWitness, stockoutCapacityPhrase, missingBranchRefFacts, missingBranchRefRecovery, deadRefRetryMessage, wedgedInsertFacts, wedgedInsertRecovery, wedgedRetryMessage, CALM_ADD_CATEGORIES, ADD_FAILURE_RETRYABLE };
+// Refusal copy for a one-click "Retry add" after a streamed-library abort,
+// extracted for the same reason wedgedRetryMessage was. The retry is refused
+// because every add from this branch fails identically until the library is
+// registered, and the generic refusal would again send the operator after an
+// orphan cloud:up already deleted. Repeats the classified recovery, which names
+// the fix and says whether anything was left behind. Pure; no I/O.
+function streamedLibraryRetryMessage(n, recovery) {
+  return `VM-${n} cannot be retried until the code is fixed: ${recovery || 'a streamed bootstrap library was never shipped to the VM.'}`;
+}
+
+module.exports = { classifyAddFailure, extractDriftFiles, lastNonEmptyLines, canRetryAdd, transcriptRostered, holdConvergingBadge, addFailureCorroborated, containerHealthWitness, stockoutCapacityPhrase, missingBranchRefFacts, missingBranchRefRecovery, deadRefRetryMessage, wedgedInsertFacts, wedgedInsertRecovery, wedgedRetryMessage, iapNotReadyFacts, iapNotReadyRecovery, streamedLibraryMissingFacts, cleanTeardownReported, streamedLibraryMissingRecovery, cloudUpFailedRecovery, streamedLibraryRetryMessage, CALM_ADD_CATEGORIES, ADD_FAILURE_RETRYABLE };

package/npm/fleet-auth.js

@@ -133,18 +133,56 @@ function listAvailableAccounts(vms) {
 // Pick the credential donor for a chosen account: the lowest-numbered VM in
 // `reachableDonors` whose authenticated account is `email`, or null when no
 // reachable VM holds it (or no email was asked for). `vms` is the per-VM poll
-// state keyed by VM number (`state.vms`). Pure.
+// state keyed by VM number (`state.vms`). A donor whose credential failed the
+// integrity probe (`providerAuth.credentialIntact === false`) is skipped as
+// though it were not authenticated — its credential cannot be copied, so the
+// sink would refuse it after an add had already committed to it. An unprobed
+// verdict (null/absent) is usable: a probe failure is not evidence. Pure.
 function selectDonorForAccount(reachableDonors, vms, email) {
   if (!email) return null;
   const want = String(email).toLowerCase();
   for (const n of [...(reachableDonors || [])].sort((a, b) => a - b)) {
     const pa = vms && vms[n] && vms[n].providerAuth;
+    if (pa && pa.credentialIntact === false) continue;
     const got = pa && pa.state === 'authenticated' && pa.account && pa.account.email;
     if (got && String(got).toLowerCase() === want) return n;
   }
   return null;
 }
 
+// Split the reachable donor candidates into those whose credential can be
+// copied and those the integrity probe failed, so the add can drop the failed
+// ones BEFORE resolveCredsSource's first-donor fallback could pick one, and name
+// each dropped donor in the add log. Order is preserved; unprobed is usable.
+// Returns { usable: number[], skipped: number[] }. Pure.
+function partitionDonorsByIntegrity(reachableDonors, vms) {
+  const usable = [];
+  const skipped = [];
+  for (const n of reachableDonors || []) {
+    const pa = vms && vms[n] && vms[n].providerAuth;
+    if (pa && pa.credentialIntact === false) skipped.push(n);
+    else usable.push(n);
+  }
+  return { usable, skipped };
+}
+
+// Fold the dashboard's credential-integrity verdict into a VM's editor-reported
+// providerAuth record. The editor derives `state` from whether the access token
+// works NOW, so a credential with no refresh token still reads `authenticated`
+// (VM-11, 2026-09-17) — a VM that will strand at its next token expiry and that
+// no add can copy from. When the probe says the credential is incomplete, the
+// card must say what the remedy is: `needs-login`. `credentialIntact` rides on
+// the record either way (null = unprobed) so donor selection reads one place.
+// Returns null for a null auth (editor didn't answer). Pure.
+function applyCredentialIntegrity(auth, credentialIntact) {
+  if (!auth) return null;
+  const intact = credentialIntact === true || credentialIntact === false ? credentialIntact : null;
+  if (intact === false && auth.state === 'authenticated') {
+    return { ...auth, state: 'needs-login', credentialIntact: false };
+  }
+  return { ...auth, credentialIntact: intact };
+}
+
 // A plausible Claude account email: one "@", a dotted domain, no whitespace or
 // shell-significant characters. The shared guard for every place an operator
 // can name an account (the Add form's per-launch override, the set-default
@@ -178,6 +216,8 @@ module.exports = {
   deriveDefaultAuthAccount,
   listAvailableAccounts,
   selectDonorForAccount,
+  partitionDonorsByIntegrity,
+  applyCredentialIntegrity,
   isAccountEmail,
   resolveChosenAccount,
   isAuthStartable,