@codeyam-editor/codeyam-editor 0.1.8 → 0.1.9

package/npm/fleet-add-failure.js

@@ -33,7 +33,11 @@ const { parseRosterMarkers } = require('./fleet-early-roster');
 //
 //   1. NOTHING WAS PROVISIONED. The classifier can prove no cloud resource was
 //      created (`preflight-drift` aborts before the first gcloud call;
-//      `instance-vanished` confirms the instance is gone). Retry starts clean.
+//      `instance-vanished` confirms the instance is gone; `capacity-stockout`
+//      pairs a GCE stockout signature with the `Failed to create instance` line
+//      that `ensureInstance` throws from exactly one place — the branch where the
+//      instance was not found AND the create call failed, so no instance, disk,
+//      NEG or backend service can exist). Retry starts clean.
 //
 //   2. THE REMAINING WORK IS RE-ENTRANT. A real VM exists, but every step the
 //      retry would re-run is skip-on-exists or re-attaching, so the retry
@@ -50,6 +54,7 @@ const { parseRosterMarkers } = require('./fleet-early-roster');
 const ADD_FAILURE_RETRYABLE = new Set([
   'preflight-drift',
   'instance-vanished',
+  'capacity-stockout',
   'project-bootstrap-timeout',
 ]);
 
@@ -171,6 +176,180 @@ function nextValidActionRemedy(text) {
   return i === -1 ? '' : lines.slice(i).join('\n').trim();
 }
 
+// Name the capacity GCE was short of, from whatever the stockout transcript
+// happens to expose — for the capacity-stockout recovery line. GCE reports this
+// inconsistently: the 2026-08-20 VM-11 transcript carried `vmType:` / `zone:`
+// bullets under errorInfo.metadatas, while the 2026-08-18 VM-13 spelling
+// (ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS) carried neither. So every part is
+// optional and the phrase degrades to a generic-but-true sentence rather than
+// interpolating `undefined` into operator-facing copy. Extracted alongside
+// nextValidActionRemedy (the same shape: a pure transcript reader that enriches
+// one arm's recovery text) so the degradation is asserted directly.
+//
+// `zone:` is anchored with ^\s* so the neighbouring `zonesAvailable:` bullet —
+// which lists the zones that DID have capacity — cannot be mistaken for the zone
+// that failed. Never throws; returns the generic phrase for empty input.
+function stockoutCapacityPhrase(logTail) {
+  const log = String(logTail || '');
+  const vmType = (log.match(/^\s*vmType:\s*(\S+)/m) || [])[1];
+  const zone = (log.match(/^\s*zone:\s*(\S+)/m) || [])[1];
+  return [vmType, 'capacity', zone ? `in ${zone}` : 'in the target zone']
+    .filter(Boolean)
+    .join(' ');
+}
+
+// The spellings git uses for "that branch is not on the remote", each capturing
+// the ref name. The union is not defensive padding — the two bootstrap scripts
+// reach the remote with DIFFERENT git commands, and git words the same condition
+// differently for each:
+//   `git fetch <url> <branch>`  (clone_editor_repo, scripts/gcp-bootstrap.sh)
+//        → fatal: couldn't find remote ref <ref>
+//   `git clone --branch <b>`    (clone_project, scripts/gcp-project-bootstrap.sh)
+//        → fatal: Remote branch <b> not found in upstream origin
+// Keying on only the first — the one the 2026-08-25 transcripts happened to
+// carry — would miss every client-side clone failure. Same union-of-spellings
+// lesson capacity-stockout learned from GCE.
+//
+// Deliberately NOT included: git's `warning: Could not find remote branch …`,
+// which is a submodule-update warning that accompanies a SUCCEEDING command.
+// Matching it here would let an add that failed for an unrelated reason claim
+// this category — and this arm sits above the generic one, so a false positive
+// would suppress the true cause rather than merely add noise.
+const MISSING_REF_PATTERNS = [
+  /couldn't find remote ref (\S+)/i,
+  /Remote branch (\S+) not found in upstream/i,
+];
+
+// Which branch died, and which config field carried it — read from the transcript.
+//
+// The field matters because clientBranch and editorBranch are configured in
+// different places: an operator who corrects only one re-issues a request that
+// fails identically. It is derived from the log prefix of the nearest preceding
+// marker line, because the two bootstrap scripts brand their output distinctly
+// (`[gcp-bootstrap]` builds the editor image from the editor branch;
+// `[project-bootstrap]` clones the client repo at the client branch). When the
+// transcript carries neither marker the field is left null and the copy names
+// both fields rather than guessing — the same degrade-to-true-but-generic
+// discipline stockoutCapacityPhrase uses. Never throws; returns
+// { ref: null, field: null } for input with no missing-ref signature.
+function missingBranchRefFacts(logTail) {
+  const lines = String(logTail || '').split('\n');
+  for (let i = 0; i < lines.length; i += 1) {
+    let ref = null;
+    for (const re of MISSING_REF_PATTERNS) {
+      const m = lines[i].match(re);
+      if (m) { ref = m[1]; break; }
+    }
+    if (!ref) continue;
+    let field = null;
+    for (let j = i; j >= 0; j -= 1) {
+      if (/\[project-bootstrap\]/.test(lines[j])) { field = 'clientBranch'; break; }
+      if (/\[gcp-bootstrap\]/.test(lines[j])) { field = 'editorBranch'; break; }
+    }
+    return { ref, field };
+  }
+  return { ref: null, field: null };
+}
+
+// Operator-facing recovery copy for the missing-branch-ref arm.
+//
+// Extracted from the arm for the same reason stockoutCapacityPhrase was — it is
+// the one arm whose copy VARIES with what the transcript happened to expose, so
+// pulling it out lets the wording be asserted directly instead of only through
+// classifyAddFailure, and keeps the arm itself a two-line decision.
+//
+// Two properties are load-bearing and are what the tests pin:
+//   - It states the outcome FIRST and never borrows the generic arm's "a partial
+//     instance may exist — reconcile or destroy it" language. At this point git
+//     has aborted before any image build or container work, so that advice sends
+//     the operator hunting for a box that was never created — the inversion that
+//     cost the 2026-08-25 recovery its first hour.
+//   - The remedy is a DIFFERENT action, never "retry". Re-issuing the same
+//     config against a deleted ref fails identically forever, which is exactly
+//     why this category is kept out of ADD_FAILURE_RETRYABLE.
+//
+// `field` is optional: when the transcript carries no bootstrap marker the copy
+// names both fields rather than guessing which one carried the dead ref.
+// Pure; no I/O.
+function missingBranchRefRecovery({ ref, field } = {}) {
+  const carried = field
+    ? `Its ${field} still points at it`
+    : 'Its clientBranch and editorBranch both need checking';
+  return (
+    `Branch '${ref}' is not on origin any more — it was almost certainly pruned when it merged. ` +
+    'Nothing was provisioned: git aborted before any image build or container work, so there is no instance, ' +
+    `disk, NEG or backend service to clean up, and nothing to destroy. ${carried}, so retrying as-is will fail ` +
+    'identically every time. Re-add on a branch that still exists (the current fleet branch is the usual answer).'
+  );
+}
+
+// Facts for the wedged-insert arm: a GCE instance insert that was ACCEPTED and
+// then never finished. Returns { wedged, code, instance } — `code` is the GCE
+// support code when the transcript carried one, so the recovery copy can hand the
+// operator the exact string a Support case needs instead of making them dig it out
+// of the transcript by hand.
+//
+// Two spellings reach here and both are proof, so the match is a UNION:
+//   A. The raw GCE prose, which is what the 2026-09-04 VM-8 transcript carried:
+//      `Internal error. Please try again or contact Google Support. (Code: '…')`
+//      PAIRED with evidence this run actually attempted a create. The pairing is
+//      what keeps the arm narrow — the same prose surfacing from some other
+//      gcloud call is not an insert that wedged, and must not claim to be one.
+//   B. ensureInstance's own fast-fail line (npm/cloud.js), which fires when a
+//      re-add finds the slot still parked in PROVISIONING behind an operation
+//      that never reached DONE. That path exists precisely so the transcript
+//      carries a signature instead of an uninformative 2-minute timeout, so this
+//      arm keys on it directly.
+//
+// Deliberately NOT keyed on elapsed time. A duration heuristic ("PROVISIONING for
+// too long") would misfire on a legitimately slow create; an INTERNAL_ERROR on an
+// accepted insert is provable evidence, which is the same signature-over-duration
+// discipline the capacity-stockout arm uses.
+const WEDGED_INSERT_PROSE = /Internal error\. Please try again or contact Google Support/i;
+const WEDGED_INSERT_FASTFAIL = /stuck in PROVISIONING behind a wedged create operation/i;
+const WEDGED_INSERT_CREATE_ATTEMPTED = /not found\s*—\s*creating|Failed to create instance|did not reach RUNNING/i;
+
+function wedgedInsertFacts(logTail) {
+  const log = String(logTail || '');
+  const wedged = WEDGED_INSERT_FASTFAIL.test(log)
+    || (WEDGED_INSERT_PROSE.test(log) && WEDGED_INSERT_CREATE_ATTEMPTED.test(log));
+  if (!wedged) return { wedged: false, code: null, instance: null };
+  return {
+    wedged: true,
+    code: (log.match(/\(Code:\s*'(-?\d+)'\)/) || [])[1] || null,
+    instance: (log.match(/Checking instance\s+(\S+?)\.\.\./) || [])[1] || null,
+  };
+}
+
+// Operator-facing recovery copy for the wedged-insert arm, extracted for the same
+// reason stockoutCapacityPhrase and missingBranchRefRecovery were: its wording
+// varies with what the transcript exposed, so pulling it out lets the copy be
+// asserted directly rather than only through classifyAddFailure.
+//
+// Three properties are load-bearing and are what the tests pin:
+//   - It never borrows the generic arm's "reconcile or destroy it" line. A delete
+//     against a wedged slot is ACCEPTED and then queues behind the wedged insert
+//     rather than clearing it, so that advice sends the operator to an action that
+//     provably does not work — the same inversion that cost the 2026-08-25
+//     missing-branch-ref recovery its first hour.
+//   - It says plainly that this is not the operator's fault and not their fix.
+//     The only thing that resolves it is GCE-side.
+//   - It names the Support code when one was captured, because that string is
+//     what a Google Cloud Support case is keyed on.
+function wedgedInsertRecovery({ code, instance } = {}) {
+  const named = instance || 'the instance';
+  const cite = code ? ` citing operation error code ${code}` : '';
+  return (
+    `GCE accepted the create for ${named} and then never finished it: the insert stayed at status RUNNING, progress 0 ` +
+    'and eventually attached an INTERNAL_ERROR without ever reaching DONE, leaving the instance in PROVISIONING and its ' +
+    'boot disk in CREATING. Nothing on our side failed and there is nothing here to reconcile. Do NOT retry — the ' +
+    'instance name is held by the wedged operation, so a re-add cannot succeed while it is outstanding. Do NOT try to ' +
+    'destroy it either — a delete is accepted but queues behind the wedged insert instead of clearing it. This needs ' +
+    `GCE-side resolution: open a Google Cloud Support case${cite}. If you need the slot before Google releases it, add ` +
+    'on a different VM number.'
+  );
+}
+
 // Build a verdict, stamping `retryable` from the single-source-of-truth Set so it
 // can never drift from the category.
 function buildFailureVerdict(category, headline, recovery, extra) {
@@ -200,11 +379,20 @@ function buildFailureVerdict(category, headline, recovery, extra) {
 // Returns { category, headline, recovery, retryable, files? }.
 //
 // Detection is ORDER-SENSITIVE. The provision shell echoes cloud:up's tail, so a
-// drift / instance-vanished / container-stuck transcript ALSO contains the
-// generic `cloud:up FAILED (exit N)` line. Each specific (and every retryable)
-// signature is therefore tested BEFORE the generic cloud-up-failed signature, or
-// a retryable drift would misclassify as a non-retryable cloud:up failure and the
-// operator would lose the safe one-click recovery.
+// drift / instance-vanished / container-stuck / capacity-stockout /
+// missing-branch-ref / wedged-insert transcript ALSO contains the generic `cloud:up FAILED
+// (exit N)` line. Each specific (and every retryable) signature is therefore
+// tested BEFORE the generic cloud-up-failed signature, or a retryable drift would
+// misclassify as a non-retryable cloud:up failure and the operator would lose the
+// safe one-click recovery. That is not a hypothetical: it is exactly the bug
+// `capacity-stockout` (arm 6) was added to fix, and its placement immediately
+// above cloud-up-failed is load-bearing. `missing-branch-ref` (arm 7) has the
+// same property for the same reason — it also sits immediately above the generic
+// arm, and its ordering is what stops a dead ref reporting as a half-provisioned
+// instance. `wedged-insert` (arm 8) is the third: it sits directly above the
+// generic arm so a create GCE accepted-and-abandoned cannot report as an ordinary
+// provisioning failure whose advice is to destroy the box — a destroy that, on a
+// wedged slot, queues behind the wedge instead of clearing it.
 function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy, wiringDeadlineBlown, wiringInFlight } = {}) {
   const log = String(logTail || '');
 
@@ -272,33 +460,127 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
       'The container came up but the project was never scaffolded (no .codeyam/editor.json on the VM), so provisioning did not finish onboarding. Reconcile this VM to complete the scaffold, then re-add.');
   }
 
-  // 6. cloud-up-failed — a generic cloud:up failure, reached ONLY when the drift /
-  //    vanished / stuck signatures above did not match. A partial instance may
-  //    exist, so not retryable.
+  // 6. capacity-stockout — GCE had no capacity for the machine type in the target
+  //    zone, so `ensureInstance` (npm/cloud.js) threw from its create branch. This
+  //    fired twice in three days — VM-13 / payper on 2026-08-18 and VM-11 /
+  //    codeyam-cms on 2026-08-20 — and both times classified as the generic
+  //    cloud-up-failed below, whose copy wrongly warned that a partial instance may
+  //    exist and whose non-retryable verdict refused the one failure that needs
+  //    nothing but a retry. Each incident cost manual state surgery on the operator
+  //    host.
+  //
+  //    Retryable on ground 1 (NOTHING WAS PROVISIONED), proven STRUCTURALLY rather
+  //    than by trusting the error prose: the arm fires only on a stockout signature
+  //    AND the `Failed to create instance` line, which ensureInstance throws from
+  //    exactly one place — the branch where the instance was not found and the
+  //    create call then failed. That conjunction is positive proof no cloud
+  //    resource exists, the same standing preflight-drift and instance-vanished
+  //    already have; both incidents were verified by hand to have left no instance,
+  //    disk, NEG or backend service behind. The not-rostered guard is belt-and-
+  //    braces: if a future transcript stockouts on one attempt and then creates a
+  //    box, a roster marker means a real VM exists and this arm must not fire.
+  //
+  //    The signature is a UNION because GCE spells this more than one way: the
+  //    2026-08-18 transcript carried `ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS`
+  //    with `reason: stockout`, while 2026-08-20 carried `reason:
+  //    resource_availability` with `state:STOCKOUT, sub-state:STOCKOUT` and the
+  //    prose below. Keying on only the first would have missed the second. The
+  //    prose alternative uses `\s+` because the message wraps mid-sentence.
+  //
+  //    Tested BEFORE cloud-up-failed because the provision shell echoes cloud:up's
+  //    tail, so a stockout transcript ALSO carries `cloud:up FAILED (exit N)` —
+  //    matching that first is precisely the bug being fixed here.
+  if (/ZONE_RESOURCE_POOL_EXHAUSTED|\bSTOCKOUT\b|reason:\s*resource_availability|does not have\s+enough resources available/i.test(log)
+      && /Failed to create instance/i.test(log)
+      && !transcriptRostered(log)) {
+    return buildFailureVerdict('capacity-stockout', 'No capacity in the zone',
+      `GCE had no ${stockoutCapacityPhrase(log)} at that moment, so the create call failed before anything existed — there is no instance, disk, NEG or backend service to clean up, and nothing to undo. Capacity is transient but its return is not scheduled: wait a few minutes, then Retry add. Do NOT destroy anything.`);
+  }
+
+  // 7. missing-branch-ref — the requested branch no longer exists on origin, so
+  //    git aborted the clone/fetch. Observed live on 2026-08-25: VMs 2, 4, 5 and
+  //    7 each died on `fatal: couldn't find remote ref editor-improvements-75`,
+  //    a branch pruned from origin earlier that day when it merged. This repo
+  //    prunes on merge, so ANY stored feature-branch name expires silently —
+  //    this is a recurring condition, not a one-off.
+  //
+  //    NOT retryable, and deliberately NOT a member of ADD_FAILURE_RETRYABLE
+  //    even though it satisfies ground 1 (nothing was provisioned). Membership
+  //    means "re-issuing the SAME request can succeed"; re-issuing an add
+  //    against a deleted ref fails identically forever. Admitting it would
+  //    automate the guaranteed-fail loop the operator was stuck in by hand. The
+  //    honest remedy is a DIFFERENT request — re-add on a live branch — which is
+  //    what the recovery copy names.
+  //
+  //    Tested BEFORE cloud-up-failed for the reason arm 6 documents: the
+  //    provision shell echoes cloud:up's tail, so this transcript ALSO carries
+  //    the generic `cloud:up FAILED (exit N)` line. Matching that first is the
+  //    bug being fixed — it is what made the real incident report the OPPOSITE
+  //    of the truth, warning that "a partial instance may exist — reconcile or
+  //    destroy it" when nothing had been provisioned at all.
+  const missingRef = missingBranchRefFacts(log);
+  if (missingRef.ref) {
+    return buildFailureVerdict('missing-branch-ref', 'Branch no longer exists',
+      missingBranchRefRecovery(missingRef));
+  }
+
+  // 8. wedged-insert — GCE accepted the instance insert and then never finished
+  //    it. Observed live on 2026-09-04 (VM-8): the insert was submitted at
+  //    14:29:50Z, sat at `status: RUNNING, progress: 0` for hours, and eventually
+  //    attached `INTERNAL_ERROR` WITHOUT ever transitioning to DONE. The instance
+  //    stayed in PROVISIONING and its boot disk in CREATING indefinitely, and a
+  //    subsequent `instances delete` was accepted but queued at PENDING behind the
+  //    wedged insert — so the slot could be neither created nor destroyed.
+  //
+  //    It needs its own category because every neighbour misdescribes it: it is
+  //    NOT capacity-stockout (GCE accepted the request rather than rejecting it),
+  //    NOT cloud-up-failed (nothing in the provisioning script failed), and
+  //    emphatically not the `verifying` fallback it actually landed on — whose
+  //    copy told the operator the add "may still be converging" and to wait, about
+  //    a VM that never would.
+  //
+  //    NOT retryable, and deliberately not a member of ADD_FAILURE_RETRYABLE: a
+  //    retry cannot succeed while the instance name is held by a wedged operation,
+  //    so admitting it would automate a guaranteed-fail loop — the same reasoning
+  //    missing-branch-ref documents. Not in CALM_ADD_CATEGORIES either: it is
+  //    terminal, not calm, and there is nothing left to converge.
+  //
+  //    Tested BEFORE cloud-up-failed for the reason arm 6 documents: the provision
+  //    shell echoes cloud:up's tail into every transcript, so matching the generic
+  //    line first is precisely the bug being fixed.
+  const wedgedInsert = wedgedInsertFacts(log);
+  if (wedgedInsert.wedged) {
+    return buildFailureVerdict('wedged-insert', 'Create wedged in Google Cloud',
+      wedgedInsertRecovery(wedgedInsert));
+  }
+
+  // 9. cloud-up-failed — a generic cloud:up failure, reached ONLY when the drift /
+  //    vanished / stuck / stockout / missing-ref / wedged-insert signatures above
+  //    did not match. A partial instance may exist, so not retryable.
   if (/cloud:up FAILED \(exit/i.test(log)) {
     return buildFailureVerdict('cloud-up-failed', 'Provisioning failed',
       'cloud:up failed during provisioning, so a partial instance may exist. Reconcile or destroy this VM before re-adding.');
   }
 
-  // 7. interrupted — a signal kill (OOM 137, SIGTERM 143, …) or an explicit
+  // 10. interrupted — a signal kill (OOM 137, SIGTERM 143, …) or an explicit
   //    Killed / out-of-memory marker. The real VM state is unknown. Not retryable.
   if (fc.isSignalKillExit(Number(exitCode)) || /\bKilled\b/.test(log) || /Out of memory/i.test(log)) {
     return buildFailureVerdict('interrupted', 'Provisioning interrupted',
       'The provision was interrupted (killed or out of memory), so the VM state is unknown. Reconcile or destroy this VM before re-adding.');
   }
 
-  // 8. The post-add IAP read-back verdicts. Unlike every signature above, this one
+  // 11. The post-add IAP read-back verdicts. Unlike every signature above, this one
   //    is not produced by the provision transcript at all — it is the BLOCKED
   //    reason iapReadbackOutcome hands to annotateAddFailure, which is what
   //    stops a read-back failure from being the one add failure that reaches the
   //    card with no category at all (and so no differentiated headline, no
   //    recovery, no guarded Retry, and no chance for holdConvergingBadge to hold it
-  //    calm). Tested AFTER all seven hard-failure signatures so a proven failure
+  //    calm). Tested AFTER all eight hard-failure signatures so a proven failure
   //    always wins if both texts are ever present, and BEFORE converging/verifying
   //    because it is strictly more specific than either.
   if (/IAP policy authorizes no human/i.test(log)) {
     const remedy = nextValidActionRemedy(log);
-    //  8a. access-pending — no accessors YET. The grant is the LAST step of the
+    //  9a. access-pending — no accessors YET. The grant is the LAST step of the
     //      detached wire-add-tail, so while that tail is alive an empty policy is
     //      the expected mid-flight reading, not a verdict. Calm: this is the state
     //      that used to render a red "Add failed" on healthy VM-4/VM-7 minutes
@@ -307,7 +589,7 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
       return buildFailureVerdict('access-pending', 'Access not granted yet',
         'The VM is up and rostered, but its IAP policy has no human accessors yet — the add\'s background wiring applies the grant as its last step and is still running. This resolves on its own once that tail finishes; no action is needed unless it persists past the wiring deadline.');
     }
-    //  8b. iap-unauthorized — the wiring is done and the policy STILL authorizes
+    //  9b. iap-unauthorized — the wiring is done and the policy STILL authorizes
     //      nobody, so the public URL returns "Error code 51" for every human. This
     //      is the 2026-08-09 case the read-back exists to catch (one step granted
     //      the operator, the next revoked him) and it is a genuine hard failure.
@@ -317,12 +599,12 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
       `The VM is up but its IAP policy authorizes no human, so the public URL returns "Error code 51" for everyone. Its background wiring is no longer running, so this will not resolve on its own — grant access to fix it. Do NOT destroy or re-add this VM.${remedy ? `\n\n${remedy}` : ''}`);
   }
 
-  // 9. converging — the add reached the early-roster marker (`CY_ROSTER`): the
+  // 12. converging — the add reached the early-roster marker (`CY_ROSTER`): the
   //    container gate passed, the instance is real, and the VM is rostered, yet it
   //    exited non-zero WITHOUT matching any specific hard-failure signature above.
   //    By construction of the provision script, every genuine hard failure (drift,
-  //    instance-vanished, container-stuck, incomplete-workspace, cloud-up-failed,
-  //    interrupted) is printed BEFORE `CY_ROSTER`, so a rostered transcript that
+  //    instance-vanished, container-stuck, incomplete-workspace, capacity-stockout,
+  //    cloud-up-failed, interrupted) is printed BEFORE `CY_ROSTER`, so a rostered transcript that
   //    fell through to here is a still-wiring box owned by the detached
   //    wire-add-tail / reconcile-ingress — NOT an add failure. Surface it as an
   //    honest in-flight state instead of the scary red `unknown`. Not retryable:
@@ -335,7 +617,7 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
       'The container is up and the VM is rostered; the LB ingress is still wiring in the background. This resolves on its own once the editor is reachable over the LB — no action needed unless it persists past the wiring deadline.');
   }
 
-  // 10. The residue: no failure signature above matched. The polarity here is
+  // 13. The residue: no failure signature above matched. The polarity here is
   //    INVERTED from the original design. It used to read "no recognized success
   //    signature ⇒ failed", which made red the DEFAULT for "we don't know" — so a
   //    VM whose container was Up and whose editor answered /api/health 200 got a
@@ -345,7 +627,7 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
   //
   //    Red now requires positive proof of death, and everything else is in-flight:
   //
-  //    10a. unknown — a non-zero exit CORROBORATED by a durable fact (see
+  //    11a. unknown — a non-zero exit CORROBORATED by a durable fact (see
   //        addFailureCorroborated). The box is provably dead or provably never
   //        came up, so the loud, terminal escalation is warranted and unchanged.
   //        Still never invites an unproven retry.
@@ -354,7 +636,7 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
       'The add failed for an unclassified reason and the VM is confirmed dead or never came up — inspect the transcript below, then reconcile or destroy this VM before re-adding.');
   }
 
-  //    10b. verifying — the honest default. No failure signature, and nothing
+  //    11b. verifying — the honest default. No failure signature, and nothing
   //        durable proves this VM is dead: it may still be provisioning or
   //        wiring, and the ingress can keep propagating for minutes after the
   //        wrapper is gone. Calm copy, transcript still one click away. NOT
@@ -405,4 +687,56 @@ function holdConvergingBadge({ category, wiringDeadlineBlown } = {}) {
   return CALM_ADD_CATEGORIES.has(category) && !wiringDeadlineBlown;
 }
 
-module.exports = { classifyAddFailure, extractDriftFiles, lastNonEmptyLines, canRetryAdd, transcriptRostered, holdConvergingBadge, addFailureCorroborated, containerHealthWitness, CALM_ADD_CATEGORIES, ADD_FAILURE_RETRYABLE };
+// Refusal copy for a Retry on a card whose branch was deleted from origin.
+//
+// The generic non-retryable refusal says the add "may have left a
+// half-provisioned instance — reconcile or destroy it". For this category that
+// is the OPPOSITE of the truth: git aborts before any image build or container
+// work, so nothing exists. Sending an operator to hunt for a box that was never
+// created is what cost the 2026-08-25 recovery its first hour, so this copy
+// leads with the outcome (nothing to clean up) before naming the mechanism.
+//
+// The remedy is deliberately not "retry". Re-issuing the same config against a
+// deleted ref fails identically forever — which is exactly why the category is
+// kept OUT of ADD_FAILURE_RETRYABLE — so the honest advice is a different
+// action. For a self-host VM the fleet pin names the live branch and a re-add
+// now picks it up, so the copy can be specific; for a client VM the branch is a
+// per-project choice only the operator can make, so it asks instead of guessing.
+//
+// `ref` / `field` come from missingBranchRefFacts and are both optional: the copy
+// degrades to true-but-generic rather than interpolating `undefined`, the same
+// discipline stockoutCapacityPhrase uses. Pure; no I/O.
+function deadRefRetryMessage({ n, ref, field, selfHost = false, pin } = {}) {
+  const which = ref ? `'${ref}'` : 'the requested branch';
+  const carried = field ? ` (${field})` : '';
+  const remedy = selfHost && pin
+    ? `Add VM again — it now picks up the current fleet branch (${pin}) instead of the stored one.`
+    : 'Add VM again with a branch that still exists on origin.';
+  return (
+    `VM-${n}: branch ${which}${carried} no longer exists on origin, so nothing was provisioned — ` +
+    'there is no instance, disk or backend service to clean up. Retrying as-is re-targets ' +
+    `the same dead branch and fails identically every time. ${remedy}`
+  );
+}
+
+// Refusal copy for a one-click "Retry add" against a slot held by a wedged
+// insert, extracted for the same reason deadRefRetryMessage was: the refusal
+// decision belongs to canRetryAdd, the wording belongs here, and only the wording
+// needs asserting.
+//
+// It exists because the generic refusal line is not merely unhelpful here, it is
+// WRONG in a way the operator cannot detect. "may have left a half-provisioned
+// instance — reconcile or destroy it" describes a cleanup that is accepted by GCE
+// and then silently queues behind the wedged insert instead of clearing it, so
+// following it looks like progress and achieves nothing. Say what is true: this is
+// GCE-side, neither retry nor destroy resolves it, and the unblocking move is a
+// Support case (or a different VM number if a box is needed now). Pure; no I/O.
+function wedgedRetryMessage(n) {
+  return (
+    `VM-${n} is stuck inside Google Cloud: GCE accepted the create and never finished it, so the instance name is ` +
+    'held by a wedged operation. A retry cannot succeed while it is outstanding, and a destroy will queue behind it ' +
+    'rather than clear it — this needs a Google Cloud Support case. Add on a different VM number if you need a box now.'
+  );
+}
+
+module.exports = { classifyAddFailure, extractDriftFiles, lastNonEmptyLines, canRetryAdd, transcriptRostered, holdConvergingBadge, addFailureCorroborated, containerHealthWitness, stockoutCapacityPhrase, missingBranchRefFacts, missingBranchRefRecovery, deadRefRetryMessage, wedgedInsertFacts, wedgedInsertRecovery, wedgedRetryMessage, CALM_ADD_CATEGORIES, ADD_FAILURE_RETRYABLE };

package/npm/fleet-branch.js

@@ -46,4 +46,127 @@ function resolveFleetBranch({ envBranch, gitBranch, fallback } = {}) {
   return cleanBranch(fallback) || UNKNOWN_BRANCH;
 }
 
-module.exports = { UNKNOWN_BRANCH, cleanBranch, resolveFleetBranch };
+// ---------- Dead-pin detection ----------
+// `fleet-advance-to.sh` refuses to WRITE a pin for a branch absent from origin,
+// but that guard runs once and nothing re-checks the pin already in place —
+// while the branch it names can be deleted by anyone merging a PR with the
+// delete-branch box ticked. Observed 2026-08-15: PR #104 merged
+// `editor-improvements-72` and deleted it; hours later the operator host still
+// read `FLEET_BRANCH=editor-improvements-72` and nothing said so.
+//
+// The probe itself (`git ls-remote`) is I/O and belongs to server.js, on the
+// SLOW memo cadence rather than the ~3s /data poll. What lives here is the pure
+// classification of its outcome — which is the part a test can actually reach,
+// since a function defined in server.js can never be executed by a test (that
+// file has top-level side effects on require).
+//
+// The tri-state mirrors scripts/operator/lib/branch-on-origin.sh exactly, and
+// the mirroring is the point: `gone` is a DEFINITIVE absence, `indeterminate`
+// means absence was NOT proven (no network, no remote, auth trouble). Widening
+// `indeterminate` into `gone` would turn every offline operator host into a
+// fleet-wide "your pin is dead" alarm.
+
+/** Map a `git ls-remote --exit-code --heads` exit code to the shared tri-state.
+ *  0 -> 'live', 2 -> 'gone', anything else (including a non-numeric or absent
+ *  code) -> 'indeterminate'. Pure. */
+function lsRemoteStatus(exitCode) {
+  if (exitCode === 0) return 'live';
+  if (exitCode === 2) return 'gone';
+  return 'indeterminate';
+}
+
+/** Classify the health of the fleet's branch pin.
+ *
+ *  Returns `{ state, branch, pinned, dead, blocks, recovery }` where `state` is
+ *  'live' | 'gone' | 'indeterminate' | 'unknown'.
+ *
+ *  'unknown' is the un-probeable case — no branch, or the UNKNOWN_BRANCH
+ *  sentinel that `resolveFleetBranch` yields when there is neither a pin nor a
+ *  readable git HEAD. There is no ref to ask origin about, so reporting it as
+ *  `gone` would invent a deletion that never happened.
+ *
+ *  `dead` is true ONLY for 'gone'. It is the single field a caller should gate a
+ *  degradation banner on, so no consumer has to re-derive which states are
+ *  actionable.
+ *
+ *  `blocks` names what a dead pin is actively breaking — both consequences,
+ *  because neither announces itself: the operator-repo refresh cannot fetch a
+ *  deleted ref, and the pin also drives `cloud.json editorBranch`, so newly
+ *  provisioned VMs check out a ref that is not there. Pure; no I/O. */
+function classifyBranchPin({ branch, lsRemoteExitCode, pinned = false } = {}) {
+  const name = cleanBranch(branch);
+  const probeable = name !== null && name !== UNKNOWN_BRANCH;
+  const state = probeable ? lsRemoteStatus(lsRemoteExitCode) : 'unknown';
+  const dead = state === 'gone';
+  return {
+    state,
+    branch: name,
+    pinned: Boolean(pinned),
+    dead,
+    blocks: dead
+      ? [
+        'refresh-operator-repo.sh cannot fetch the pinned branch, so this host keeps serving whatever its clone already had',
+        'cloud.json editorBranch reads the same pin, so newly provisioned VMs check out a ref that does not exist',
+      ]
+      : [],
+    recovery: dead
+      ? `scripts/operator/fleet-advance-to.sh <BRANCH> — move the fleet to a live branch (it refuses a branch that is itself absent from origin)`
+      : null,
+  };
+}
+
+/** Decide whether the fleet pin should override a stored per-VM branch when an
+ *  add is RE-ISSUED, and report every override so the caller can log it.
+ *
+ *  Returns `{ config, overrides }` — a NEW config (the input is never mutated)
+ *  and `[{ field, from, to }]`, empty when nothing changed.
+ *
+ *  The problem this solves: a per-VM config that recorded `editor-improvements-75`
+ *  was not wrong when it was written — it EXPIRED, silently, the moment that
+ *  branch merged and origin pruned it. This repo prunes on merge, so every stored
+ *  feature-branch name has that property. The pin does not expire: it is the
+ *  durable statement of fleet-wide policy, and on 2026-08-25 it was correct
+ *  throughout while four retried cards each failed on their stale snapshot (a
+ *  fresh pool add read the pin and succeeded at the same moment).
+ *
+ *  Three guards, each declining to act rather than acting wrongly:
+ *   - `selfHost` false ⇒ untouched. A client VM's branch is a deliberate
+ *     per-project choice; overwriting it with a fleet pin would silently
+ *     retarget someone else's project. Only the self-host path, where the branch
+ *     IS fleet policy, falls back to the pin.
+ *   - no pin, or the UNKNOWN_BRANCH sentinel ⇒ untouched. There is no branch to
+ *     prefer, and stamping the sentinel would guarantee the failure this exists
+ *     to prevent.
+ *   - `pinState === 'gone'` ⇒ untouched. A pin PROVEN absent from origin is not
+ *     an improvement on a stale-but-possibly-live stored branch; swapping one
+ *     dead ref for another just moves the failure. Note the deliberate
+ *     asymmetry with 'indeterminate', which does NOT decline: absence was not
+ *     proven there, and refusing to converge every time the network blips would
+ *     strand the fleet on stale branches for the sake of a maybe.
+ *
+ *  Pure; no I/O. */
+function pinCorrectedBranches({ cfg, pin, selfHost = false, pinState } = {}) {
+  const config = { ...(cfg || {}) };
+  const overrides = [];
+  const name = cleanBranch(pin);
+  if (!selfHost || !name || name === UNKNOWN_BRANCH || pinState === 'gone') {
+    return { config, overrides };
+  }
+  for (const field of ['clientBranch', 'editorBranch']) {
+    const from = cleanBranch(config[field]);
+    if (from && from !== name) {
+      config[field] = name;
+      overrides.push({ field, from, to: name });
+    }
+  }
+  return { config, overrides };
+}
+
+module.exports = {
+  UNKNOWN_BRANCH,
+  cleanBranch,
+  resolveFleetBranch,
+  lsRemoteStatus,
+  classifyBranchPin,
+  pinCorrectedBranches,
+};

package/npm/fleet-build-data.js

@@ -120,6 +120,17 @@ function buildVmEnvelope(v, ctx) {
     // unreachable VM's last-known config is stale, and an unreachable card has
     // louder problems to report than its preview origin.
     previewMisconfigured: (v.reachable && v.previewMisconfigured) || null,
+    // The project's DECLARED datastore and whether it is answering:
+    // { type, target, local, reachable, lastError }, or null when the project
+    // declares none (a Rust CLI, a static site) — which is the common case and
+    // must render nothing at all. `reachable` is tri-state exactly like
+    // previewServing/previewIngress above: true = probe reached it, false =
+    // confirmed down, null = unknown (editor just booted, or the probe went
+    // stale). Never conflate "down" with "unknown" — only a confirmed false
+    // raises the amber reason. Gated on `reachable` like providerAuth: an
+    // unreachable VM's last-known datastore verdict is stale, and it has louder
+    // problems to report.
+    datastore: (v.reachable && v.datastore) || null,
     effects: v.effects || null,
     // The VM's live-configured build agent (lowercase provider), reported by
     // /api/session-info and merged (sticky) onto the poll state. Drives the
@@ -177,7 +188,7 @@ const REQUIRED_CARD_FIELDS = [
   'degradedReasons', 'setupSteps', 'attentionReasons',
   'isHead', 'headHolderUnreachable', 'queuePos', 'queueLen',
   'job', 'config', 'project', 'variant', 'cardIdentity', 'brokerHistory',
-  'brokerDeath', 'providerAuth', 'previewMisconfigured', 'effects', 'currentProvider',
+  'brokerDeath', 'providerAuth', 'previewMisconfigured', 'datastore', 'effects', 'currentProvider',
   'readiness', 'backendHealth', 'vmAccess',
   'startedAt', 'featureStartedAt', 'attention', 'envOverride',
 ];