@codeyam-editor/codeyam-editor 0.1.7 → 0.1.8

package/npm/cloud.js

@@ -59,6 +59,7 @@ const {
   normalizePrincipal,
   resolveDefaultOperator,
   withDefaultOperator,
+  unauthorizableAddRefusal,
   accessFor,
   setAccess,
   addAccess,
@@ -3608,8 +3609,9 @@ function fleetInstancePrefix() {
  *  scripts/gcp/vm-ingress.sh with the resolved coordinates. Returns the spawnSync
  *  result so callers decide fatality — Add treats a non-zero as a loud warning
  *  (the VM is provisioned; ingress can be re-wired), Destroy likewise warns so a
- *  teardown still removes the instance. */
-function runVmIngress(cfg, action, vmNumber, domain, appPort, slug) {
+ *  teardown still removes the instance. `vmAccess` (up only) is the Add's intended
+ *  principal set, which the script's grant step uses in place of its sidecar read. */
+function runVmIngress(cfg, action, vmNumber, domain, appPort, slug, vmAccess) {
   const scriptPath = path.join(rootDir, "scripts", "gcp", "vm-ingress.sh");
   const inv = vmIngressInvocation({
     scriptPath,
@@ -3621,6 +3623,7 @@ function runVmIngress(cfg, action, vmNumber, domain, appPort, slug) {
     appPort,
     instancePrefix: fleetInstancePrefix(),
     slug,
+    vmAccess,
   });
   return spawnSync(inv.command, inv.args, {
     stdio: "inherit",
@@ -3651,17 +3654,77 @@ function runVmIngressGrantAccess(cfg, vmNumber, domain, slug) {
   });
 }
 
+/** THE principal set that must have IAP access to this VM after the Add: the
+ *  explicit `cfg.vmAccess` collaborators unioned with the resolved default
+ *  operator. Both add-time access paths consume this ONE function —
+ *  `wireVmIngressOnAdd` (which hands it to vm-ingress.sh's grant) and
+ *  `applyVmAccessOnAdd` (which grants it and subtracts it from the stale-revoke
+ *  set). They used to decide independently, and that is precisely how one path
+ *  granted `user:jared@codeyam.com` at 00:09:31 and the other revoked the same
+ *  principal 18 seconds later as a "stale prior tenant" (VMs 4/5/6, 2026-08-09):
+ *  the ingress path read the SIDECAR (still holding the prior tenant) while
+ *  `applyVmAccessOnAdd` resolved an empty operator. One source of truth removes
+ *  the whole class of disagreement. `env` is injectable for tests; the only I/O
+ *  is `resolveDefaultOperator`'s config/gcloud read. */
+function intendedVmPrincipals(cfg = {}, env = process.env) {
+  return withDefaultOperator(
+    normalizeAccessList(cfg.vmAccess || []),
+    resolveDefaultOperator({ env }),
+  );
+}
+
+/** Add preflight: throw — BEFORE any GCE instance exists — when this Add would
+ *  come up on an IAP-gated public URL that authorizes NOBODY. The whole condition
+ *  is env + config (`intendedVmPrincipals` needs no GCP state), which is why it
+ *  belongs beside cmdUp's other cheap boundary validations rather than after a
+ *  machine has been provisioned: checking later means a half-provisioned VM to
+ *  clean up. On 2026-08-09 the un-refused version of this Add reported `done` for
+ *  VMs 4/5/6, left three unreachable URLs, and revoked each slot's prior tenant on
+ *  the way.
+ *
+ *  Only fires in LB mode on a numbered fleet VM with nothing to grant; an explicit
+ *  `--vm-access` satisfies it with no default operator configured, so the
+ *  collaborator flow is unaffected, as is every legacy quick-tunnel Add. The
+ *  refusal text (and its `Next valid action:` remedy naming all three config
+ *  tiers) is the pure `unauthorizableAddRefusal`; this function is the composition
+ *  and the throw. */
+function assertAddCanAuthorizeSomeone(cfg, env = process.env) {
+  let plan;
+  try {
+    plan = ingressWiringPlan(cfg, env);
+  } catch {
+    // A malformed domain is `resolveIngressDomain`'s error to raise on the wiring
+    // path; it must not masquerade here as an authorization refusal.
+    plan = { wire: false };
+  }
+  const refusal = unauthorizableAddRefusal({
+    lbMode: !!plan.wire,
+    vmNumber: plan.wire ? plan.vmNumber : null,
+    intendedCount: plan.wire ? intendedVmPrincipals(cfg, env).length : null,
+  });
+  if (refusal) throw new Error(refusal);
+}
+
 /** Add-time hook: wire VM-N onto the shared LB when a fleet domain is set.
  *  Best-effort + loud — a wiring hiccup must never abort an otherwise-green Add
  *  (the VM is reachable operator-side over SSH regardless), so on failure it
  *  prints the exact re-run command rather than throwing. No-op (and silent) in
- *  legacy quick-tunnel mode or for a non-fleet client project. */
+ *  legacy quick-tunnel mode or for a non-fleet client project.
+ *
+ *  RETURNS the principals this call granted (`intendedVmPrincipals`), or [] when
+ *  it did not wire — the explicit dataflow `applyVmAccessOnAdd` needs to know what
+ *  the Add has already granted before it computes a stale-revoke. Returned on the
+ *  FAILURE path too: a non-zero `vm-ingress.sh up` may still have completed its
+ *  grant step, and a principal that was possibly granted must never be revoked. */
 function wireVmIngressOnAdd(cfg, appPort, env = process.env) {
   const plan = ingressWiringPlan(cfg, env);
-  if (!plan.wire) return;
+  if (!plan.wire) return [];
   const label = plan.slug || `vm-${plan.vmNumber}`;
+  // Hand the intended set to the script so its `apply_default_iap_access` grants
+  // THIS Add's principals instead of reading the sidecar's prior tenant.
+  const vmAccess = intendedVmPrincipals(cfg, env);
   console.error(`  wiring LB ingress for VM-${plan.vmNumber} on ${plan.domain}...`);
-  const res = runVmIngress(cfg, "up", plan.vmNumber, plan.domain, appPort, plan.slug);
+  const res = runVmIngress(cfg, "up", plan.vmNumber, plan.domain, appPort, plan.slug, vmAccess);
   if (res.status === 0) {
     console.error(`  ✓ LB ingress wired: https://${label}.editor.${plan.domain}`);
   } else {
@@ -3683,6 +3746,7 @@ function wireVmIngressOnAdd(cfg, appPort, env = process.env) {
         `(vm-ingress.sh up exited ${res.status == null ? "abnormally" : res.status}). Re-wire: ${rewire}`,
     );
   }
+  return vmAccess;
 }
 
 /** Destroy-time hook: unwire VM-N's LB backends before the instance is deleted,
@@ -3751,8 +3815,15 @@ function runIapAccess(cfg, vmNumber, action, principals) {
  *  a revoke/grant hiccup (e.g. the SA missing IAP-admin) prints the exact re-run
  *  command rather than aborting an otherwise-green Add. `cfg.vmAccess` is the
  *  array (or comma-separated string) cmdUp validated; absent ⇒ an empty desired
- *  list (the clean-slate case). */
-function applyVmAccessOnAdd(cfg, env = process.env) {
+ *  list (the clean-slate case).
+ *
+ *  `grantedThisRun` is what an EARLIER step of this same Add already granted —
+ *  `wireVmIngressOnAdd`'s return value. It is subtracted from the stale-revoke
+ *  set, so a principal this Add just authorized can never be revoked moments later
+ *  as a prior tenant. That is a second, cheaper guard than the shared
+ *  `intendedVmPrincipals` resolver above: it holds even if the two paths drift
+ *  apart again, which is exactly how VMs 4/5/6 broke on 2026-08-09. */
+function applyVmAccessOnAdd(cfg, env = process.env, { grantedThisRun = [] } = {}) {
   const desired = normalizeAccessList(cfg.vmAccess || []);
   let plan;
   try {
@@ -3769,11 +3840,15 @@ function applyVmAccessOnAdd(cfg, env = process.env) {
   // never inaccessible-by-default (the Error-51 footgun). The default operator is
   // NOT persisted to the sidecar (it's a standing grant, not a per-VM
   // collaborator), so `desired` alone drives the sidecar + stale-revoke below.
-  const toGrant = withDefaultOperator(desired, resolveDefaultOperator({ env }));
+  const toGrant = intendedVmPrincipals(cfg, env);
   const stale = accessFor(vmNumber);
   // Never revoke a principal we're about to (re-)grant — a prior tenant who was
-  // also the default operator must not be revoked-then-regranted for nothing.
-  const staleOnly = stale.filter((p) => !toGrant.includes(p));
+  // also the default operator must not be revoked-then-regranted for nothing —
+  // NOR one an earlier step of this same Add already granted. `toGrant` alone
+  // protected nothing when it was empty: the ingress step had granted the operator
+  // seconds earlier and this revoke took it straight back off (VMs 4/5/6).
+  const grantedAlready = new Set([...toGrant, ...normalizeAccessList(grantedThisRun || [])]);
+  const staleOnly = stale.filter((p) => !grantedAlready.has(p));
 
   if (plan.wire) {
     if (staleOnly.length > 0) {
@@ -3890,12 +3965,23 @@ async function convergeVmIngressOnAdd(cfg, appPort, env = process.env, deps = {}
   }
   if (!plan.wire) {
     // Nothing to converge — the no-op wire keeps the clean-slate invariant.
-    wireVmIngressOnAdd(cfg, appPort, env);
-    return { wired: true, reason: "no-wire" };
-  }
-
+    const granted = wireVmIngressOnAdd(cfg, appPort, env);
+    return { wired: true, reason: "no-wire", grantedThisRun: granted };
+  }
+
+  // The intended set, resolved ONCE and handed to every `up` attempt so each
+  // converge iteration grants the same principals — and reported back to the
+  // caller as `grantedThisRun` so the access reconcile that follows cannot revoke
+  // them. Every `up` runs the script's grant step, so these are granted whether
+  // the loop converges or hits its deadline. Injectable via `deps` like the rest
+  // of this loop's slow I/O: the resolve spawns `gcloud config get-value account`
+  // on its last tier, and a unit test must never depend on the gcloud identity
+  // that happens to be active on the machine running it.
+  const grantedThisRun =
+    deps.grantedThisRun != null ? deps.grantedThisRun : intendedVmPrincipals(cfg, env);
   const runUp =
-    deps.runUp || ((p) => runVmIngress(cfg, "up", p.vmNumber, p.domain, appPort, p.slug));
+    deps.runUp
+    || ((p) => runVmIngress(cfg, "up", p.vmNumber, p.domain, appPort, p.slug, grantedThisRun));
   const probeVerdict = deps.probeVerdict || ((p) => probeVmIngressVerdict(cfg, p));
   const sleep = deps.sleep || ((ms) => new Promise((r) => setTimeout(r, ms)));
   const now = deps.now || (() => Date.now());
@@ -3938,7 +4024,7 @@ async function convergeVmIngressOnAdd(cfg, appPort, env = process.env, deps = {}
     if (decision.done) {
       if (decision.wired) {
         console.error(`  ✓ LB ingress wired: https://${label}.editor.${plan.domain}`);
-        return { wired: true, reason: "wired", attempts: attempt + 1 };
+        return { wired: true, reason: "wired", attempts: attempt + 1, grantedThisRun };
       }
       const rewire =
         `FLEET_DOMAIN=${plan.domain}${plan.slug ? ` VM_SLUG=${plan.slug}` : ""} ` +
@@ -3955,7 +4041,7 @@ async function convergeVmIngressOnAdd(cfg, appPort, env = process.env, deps = {}
         `CY_DEGRADED ingress-wiring-failed: VM-${plan.vmNumber} public URL did not wire ` +
           `within the add window (last verdict=${verdict || "unknown"}). Re-wire: ${rewire}`,
       );
-      return { wired: false, reason: "deadline", attempts: attempt + 1 };
+      return { wired: false, reason: "deadline", attempts: attempt + 1, grantedThisRun };
     }
     attempt += 1;
     await sleep(decision.delayMs);
@@ -3984,8 +4070,8 @@ function startDetachedAddWiring(cfg, appPort, env = process.env) {
   if (!plan.wire) {
     // Nothing slow to detach: wiring is a no-op and the access reconcile is a cheap
     // sidecar write. Run them inline so the clean-slate invariant still holds.
-    wireVmIngressOnAdd(cfg, appPort, env);
-    applyVmAccessOnAdd(cfg, env);
+    const grantedThisRun = wireVmIngressOnAdd(cfg, appPort, env);
+    applyVmAccessOnAdd(cfg, env, { grantedThisRun });
     return;
   }
   const marker = ingressInflightMarkerPath(INGRESS_INFLIGHT_DIR, plan.vmNumber);
@@ -4017,8 +4103,8 @@ function startDetachedAddWiring(cfg, appPort, env = process.env) {
         `running it inline instead.`,
     );
     try { fs.unlinkSync(marker); } catch { /* marker may not exist */ }
-    wireVmIngressOnAdd(cfg, appPort, env);
-    applyVmAccessOnAdd(cfg, env);
+    const grantedThisRun = wireVmIngressOnAdd(cfg, appPort, env);
+    applyVmAccessOnAdd(cfg, env, { grantedThisRun });
   }
 }
 
@@ -4050,8 +4136,11 @@ async function cmdWireAddTail() {
     // Converge the URL to `verdict=wired` (retry-until-wired), not a single-shot
     // `up` left to the slow reconcile sweep — then the IAP access grant, in the
     // SAME order cmdUp ran them inline.
-    await convergeVmIngressOnAdd(cfg, appPort);
-    applyVmAccessOnAdd(cfg);
+    // Carry what the converge granted into the access reconcile, so the revoke
+    // step cannot take back a principal this same Add just authorized — the exact
+    // grant-then-revoke-18s-later sequence `/tmp/cy-wire-add-vm-4.log` recorded.
+    const { grantedThisRun } = await convergeVmIngressOnAdd(cfg, appPort);
+    applyVmAccessOnAdd(cfg, process.env, { grantedThisRun });
   } finally {
     if (marker) {
       try { fs.unlinkSync(marker); } catch { /* already gone — fine */ }
@@ -4163,6 +4252,8 @@ async function cmdUp(args, projectDir = rootDir) {
     }
   }
 
+  assertAddCanAuthorizeSomeone(cfg);
+
   checkScriptDrift(cfg);
 
   await ensureFirewallRule(cfg);
@@ -5320,6 +5411,8 @@ module.exports = {
   ingressWiringPlan,
   fleetInstancePrefix,
   runVmIngress,
+  intendedVmPrincipals,
+  assertAddCanAuthorizeSomeone,
   wireVmIngressOnAdd,
   probeVmIngressVerdict,
   convergeVmIngressOnAdd,

package/npm/fleet-add-failure.js

@@ -99,10 +99,13 @@ function transcriptRostered(logTail) {
 // The categories that describe an add whose outcome is NOT yet proven bad — the
 // calm, in-flight side of the classifier. `converging` is a rostered box whose
 // ingress is still wiring; `verifying` is the residue: no failure signature
-// matched and nothing durable proves the VM is dead. Both render a calm card and
-// neither may use the word "failed". Consumed by holdConvergingBadge and by
-// deriveStatus (server.js) to pick the matching calm status.
-const CALM_ADD_CATEGORIES = new Set(['converging', 'verifying']);
+// matched and nothing durable proves the VM is dead; `access-pending` is a VM that
+// is up and rostered but whose IAP policy has no accessors YET, because its
+// detached wiring tail — which applies the grant last — is still running. All
+// three render a calm card and none may use the word "failed". Consumed by
+// holdConvergingBadge and by deriveStatus (server.js) to pick the matching calm
+// status.
+const CALM_ADD_CATEGORIES = new Set(['converging', 'verifying', 'access-pending']);
 
 // Is a non-zero add outcome CORROBORATED by a durable fact — i.e. do we have
 // positive proof of death, rather than merely an absence of proof of life?
@@ -157,6 +160,17 @@ function containerHealthWitness({ reachable, inVmHealthy } = {}) {
   return null;
 }
 
+// Lift the `Next valid action:` remedy out of a BLOCKED reason so a classified
+// card's recovery line stays the SAME text the producing check already emitted,
+// instead of a second copy of the same instructions that can silently drift from
+// it. Returns '' when the text carries no remedy, in which case the caller's
+// static copy stands on its own.
+function nextValidActionRemedy(text) {
+  const lines = String(text || '').split('\n');
+  const i = lines.findIndex((l) => /^\s*Next valid action:/.test(l));
+  return i === -1 ? '' : lines.slice(i).join('\n').trim();
+}
+
 // Build a verdict, stamping `retryable` from the single-source-of-truth Set so it
 // can never drift from the category.
 function buildFailureVerdict(category, headline, recovery, extra) {
@@ -179,6 +193,9 @@ function buildFailureVerdict(category, headline, recovery, extra) {
 //   instanceAlive        does GCP still list the instance?
 //   containerHealthy     did the in-VM editor answer /api/health?
 //   wiringDeadlineBlown  did the add-time wire tail drop its failed marker?
+//   wiringInFlight       is that tail still RUNNING? Only the IAP read-back arm
+//                        reads it, to tell "no accessors yet" from "no accessors,
+//                        ever" — the grant is the tail's last step.
 //
 // Returns { category, headline, recovery, retryable, files? }.
 //
@@ -188,7 +205,7 @@ function buildFailureVerdict(category, headline, recovery, extra) {
 // signature is therefore tested BEFORE the generic cloud-up-failed signature, or
 // a retryable drift would misclassify as a non-retryable cloud:up failure and the
 // operator would lose the safe one-click recovery.
-function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy, wiringDeadlineBlown } = {}) {
+function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy, wiringDeadlineBlown, wiringInFlight } = {}) {
   const log = String(logTail || '');
 
   // 1. preflight-drift — the streamed-script drift guard aborted BEFORE any
@@ -270,7 +287,37 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
       'The provision was interrupted (killed or out of memory), so the VM state is unknown. Reconcile or destroy this VM before re-adding.');
   }
 
-  // 8. converging — the add reached the early-roster marker (`CY_ROSTER`): the
+  // 8. The post-add IAP read-back verdicts. Unlike every signature above, this one
+  //    is not produced by the provision transcript at all — it is the BLOCKED
+  //    reason iapReadbackOutcome hands to annotateAddFailure, which is what
+  //    stops a read-back failure from being the one add failure that reaches the
+  //    card with no category at all (and so no differentiated headline, no
+  //    recovery, no guarded Retry, and no chance for holdConvergingBadge to hold it
+  //    calm). Tested AFTER all seven hard-failure signatures so a proven failure
+  //    always wins if both texts are ever present, and BEFORE converging/verifying
+  //    because it is strictly more specific than either.
+  if (/IAP policy authorizes no human/i.test(log)) {
+    const remedy = nextValidActionRemedy(log);
+    //  8a. access-pending — no accessors YET. The grant is the LAST step of the
+    //      detached wire-add-tail, so while that tail is alive an empty policy is
+    //      the expected mid-flight reading, not a verdict. Calm: this is the state
+    //      that used to render a red "Add failed" on healthy VM-4/VM-7 minutes
+    //      before their grant step had even run (2026-08-11).
+    if (wiringInFlight) {
+      return buildFailureVerdict('access-pending', 'Access not granted yet',
+        'The VM is up and rostered, but its IAP policy has no human accessors yet — the add\'s background wiring applies the grant as its last step and is still running. This resolves on its own once that tail finishes; no action is needed unless it persists past the wiring deadline.');
+    }
+    //  8b. iap-unauthorized — the wiring is done and the policy STILL authorizes
+    //      nobody, so the public URL returns "Error code 51" for every human. This
+    //      is the 2026-08-09 case the read-back exists to catch (one step granted
+    //      the operator, the next revoked him) and it is a genuine hard failure.
+    //      Not retryable: the VM exists and is fine — re-adding it would duplicate
+    //      a live box when what is missing is one grant.
+    return buildFailureVerdict('iap-unauthorized', 'No one can reach this VM',
+      `The VM is up but its IAP policy authorizes no human, so the public URL returns "Error code 51" for everyone. Its background wiring is no longer running, so this will not resolve on its own — grant access to fix it. Do NOT destroy or re-add this VM.${remedy ? `\n\n${remedy}` : ''}`);
+  }
+
+  // 9. converging — the add reached the early-roster marker (`CY_ROSTER`): the
   //    container gate passed, the instance is real, and the VM is rostered, yet it
   //    exited non-zero WITHOUT matching any specific hard-failure signature above.
   //    By construction of the provision script, every genuine hard failure (drift,
@@ -280,15 +327,15 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
   //    wire-add-tail / reconcile-ingress — NOT an add failure. Surface it as an
   //    honest in-flight state instead of the scary red `unknown`. Not retryable:
   //    the VM exists and its detached wiring owns finishing it, so a one-click
-  //    retry could duplicate/orphan a live box. Tested LAST so a hard-failure
-  //    signature (all printed pre-roster) always wins — `converging` can never
-  //    shadow a proven failure.
+  //    retry could duplicate/orphan a live box. Tested after every hard-failure
+  //    signature (all printed pre-roster) so a proven failure always wins —
+  //    `converging` can never shadow one.
   if (transcriptRostered(log)) {
     return buildFailureVerdict('converging', 'Add still converging',
       'The container is up and the VM is rostered; the LB ingress is still wiring in the background. This resolves on its own once the editor is reachable over the LB — no action needed unless it persists past the wiring deadline.');
   }
 
-  // 9. The residue: no failure signature above matched. The polarity here is
+  // 10. The residue: no failure signature above matched. The polarity here is
   //    INVERTED from the original design. It used to read "no recognized success
   //    signature ⇒ failed", which made red the DEFAULT for "we don't know" — so a
   //    VM whose container was Up and whose editor answered /api/health 200 got a
@@ -298,7 +345,7 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
   //
   //    Red now requires positive proof of death, and everything else is in-flight:
   //
-  //    9a. unknown — a non-zero exit CORROBORATED by a durable fact (see
+  //    10a. unknown — a non-zero exit CORROBORATED by a durable fact (see
   //        addFailureCorroborated). The box is provably dead or provably never
   //        came up, so the loud, terminal escalation is warranted and unchanged.
   //        Still never invites an unproven retry.
@@ -307,7 +354,7 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
       'The add failed for an unclassified reason and the VM is confirmed dead or never came up — inspect the transcript below, then reconcile or destroy this VM before re-adding.');
   }
 
-  //    9b. verifying — the honest default. No failure signature, and nothing
+  //    10b. verifying — the honest default. No failure signature, and nothing
   //        durable proves this VM is dead: it may still be provisioning or
   //        wiring, and the ingress can keep propagating for minutes after the
   //        wrapper is gone. Calm copy, transcript still one click away. NOT

package/npm/fleet-memory.js

@@ -290,8 +290,16 @@ function restoreFromSnapshot({ snapshot, maxAgeMs, nowIso, sidecarKeys } = {}) {
  *  Degrades to 'cgroup=unavailable' rather than printing zeros off-cgroup, so a
  *  laptop log cannot be misread as a cgroup at 0MB.
  *
+ *  `detachedWiringCount` — when supplied — appends `tails=N`, which is what makes
+ *  the RSS/cgroup gap ATTRIBUTABLE rather than merely visible. On 2026-08-11 the
+ *  cgroup read 2154MB against a 96MB RSS and the whole diagnosis was "those ~2GB
+ *  are this process's own transient add children, not a leak". `RSS=96MB
+ *  cgroup=2154MB tails=4` reads as a batch draining on its own; `RSS=1300MB
+ *  cgroup=1400MB tails=0` reads as a genuine leak. Omitted (the pre-existing
+ *  callers, and any host with no tail accounting) prints exactly as before.
+ *
  *  Pure; no I/O. */
-function formatMemoryText({ rssBytes, memoryCurrent, memoryHigh } = {}) {
+function formatMemoryText({ rssBytes, memoryCurrent, memoryHigh, detachedWiringCount } = {}) {
   const mb = (b) => (Number(b) / 1024 / 1024).toFixed(0);
   const current = Number(memoryCurrent);
   const high = Number(memoryHigh);
@@ -308,6 +316,8 @@ function formatMemoryText({ rssBytes, memoryCurrent, memoryHigh } = {}) {
   } else {
     parts.push('cgroup=unavailable');
   }
+  const tails = Number(detachedWiringCount);
+  if (Number.isFinite(tails) && tails >= 0) parts.push(`tails=${tails}`);
   return parts.join(' ');
 }
 
@@ -323,11 +333,26 @@ function formatMemoryText({ rssBytes, memoryCurrent, memoryHigh } = {}) {
  *  protecting. The dashboard's own self-update counts, being a heavy job by any
  *  other name.
  *
- *  Pure; no I/O — the caller passes state.jobs / state.dashboardUpdate. */
-function hasJobInFlight({ jobs, dashboardUpdate } = {}) {
+ *  `detachedWiringCount` is the THIRD source of busy, and the one this guard was
+ *  blind to: an Add settles at container-healthy while its detached wire-add-tail
+ *  keeps spawning gcloud children for another 5-10 min in the SAME cgroup. Judging
+ *  "safe to restart" off `state.jobs` alone reported no job in flight while that
+ *  work was live — on 2026-08-11 the escalation crossed the restart threshold at
+ *  86% of memory.high and the only thing that held it back was an unrelated job
+ *  happening to still be `running`. Had it settled thirty seconds earlier the
+ *  dashboard would have restarted itself mid-batch and orphaned four adds, which
+ *  is precisely the failure class the detached-provision work exists to prevent.
+ *  Omitting the key behaves exactly as before, so every current caller is
+ *  unaffected.
+ *
+ *  Pure; no I/O — the caller passes state.jobs / state.dashboardUpdate and the
+ *  live-tail count. */
+function hasJobInFlight({ jobs, dashboardUpdate, detachedWiringCount } = {}) {
   const busy = (j) => !!j && (j.state === 'running' || j.state === 'recovering');
   const anyJob = Object.values((jobs && typeof jobs === 'object') ? jobs : {}).some(busy);
-  return anyJob || !!(dashboardUpdate && dashboardUpdate.state === 'running');
+  const tails = Number(detachedWiringCount);
+  const anyTail = Number.isFinite(tails) && tails > 0;
+  return anyJob || anyTail || !!(dashboardUpdate && dashboardUpdate.state === 'running');
 }
 
 /** Should this escalation level be announced to the journal?

package/npm/fleet-queue.js

@@ -126,9 +126,150 @@ function reconcileQueueDraft(draftOrder, liveOrder) {
   return out;
 }
 
+// Consecutive failed reads before the panel stops trusting its mirror. At the
+// default QUEUE_POLL_MS of 5000 this is ~15s of sustained failure — long enough
+// that the ordinary 20s-timeout blip the next tick clears only ever reaches
+// 'degraded', short enough that a real credential outage is flagged in seconds.
+// Named rather than inlined so the verdict and the tooltip copy read the same
+// number from one place.
+const QUEUE_STALE_AFTER_FAILURES = 3;
+
+/** Merge one commit-queue poll result over the previous memo.
+ *
+ *  THE RULE: a failed read RETAINS the previous `entries` AND the previous
+ *  `readAt`, and records `error`. Retaining the entries is deliberate — a stale
+ *  order beats a blank panel for an operator mid-triage. Leaving `readAt`
+ *  untouched is what makes the staleness legible: it is the ONLY signal that
+ *  distinguishes a frozen view from a live one.
+ *
+ *  Advancing `readAt` on a failure is precisely the bug this exists to prevent.
+ *  On 2026-08-14 /etc/codeyam/git-credentials on the operator host went to 0
+ *  bytes and every origin git operation began failing; the queue panel went on
+ *  naming VM-5 as HEAD for 3.5 hours after VM-5 had released, while the real
+ *  queue was VM-2 -> VM-8 -> VM-4. Nothing on the page distinguished it from
+ *  live. This is the same lesson mergeDashboardRevision records from
+ *  2026-07-31, applied to the second poller.
+ *
+ *  A previous `readAt` of null (no successful read has EVER happened) stays
+ *  null and is not coerced to a timestamp — "never read" and "read a moment
+ *  ago" must stay distinguishable, the same reason mergeDashboardRevision
+ *  refuses to push a null `behind` through Number().
+ *
+ *  Success is the absence of `error`. Pure; no I/O. */
+function mergeQueuePoll({ prev, entries, readAt, error } = {}) {
+  const base = (prev && typeof prev === 'object') ? prev : {};
+  const str = (v) => ((typeof v === 'string' && v) ? v : null);
+  const prevEntries = Array.isArray(base.entries) ? base.entries : [];
+  const prevReadAt = str(base.readAt);
+  const prevFailures = (typeof base.consecutiveFailures === 'number'
+    && Number.isFinite(base.consecutiveFailures) && base.consecutiveFailures >= 0)
+    ? base.consecutiveFailures : 0;
+  const err = str(error);
+  if (err) {
+    return {
+      entries: prevEntries,
+      readAt: prevReadAt,
+      error: err,
+      consecutiveFailures: prevFailures + 1,
+    };
+  }
+  return {
+    entries: Array.isArray(entries) ? entries : [],
+    readAt: str(readAt) || prevReadAt,
+    error: null,
+    consecutiveFailures: 0,
+  };
+}
+
+/** What does the current queue memo mean for the operator?
+ *
+ *  Levels:
+ *    'fresh'    — the last read succeeded. Render exactly as before; every
+ *                 control stays live.
+ *    'degraded' — erroring, but under QUEUE_STALE_AFTER_FAILURES consecutive
+ *                 failures. A blip. Noted quietly, controls stay live — a
+ *                 single 20s-timeout hiccup that the next tick clears must not
+ *                 cry wolf or lock the operator out of their own queue.
+ *    'stale'    — erroring at or past the threshold. The mirror is not to be
+ *                 trusted and writes computed against it are blocked.
+ *    'unknown'  — erroring and NO read has ever succeeded (the boot window with
+ *                 broken credentials). Worth saying out loud: an empty-looking
+ *                 queue here means "we cannot see the queue", not "go".
+ *
+ *  AGE IS DELIBERATELY NOT AN INPUT. A genuinely quiet queue is not a broken
+ *  one, so escalation keys on consecutive read FAILURES only; age is carried by
+ *  projectQueueFreshness purely to phrase the message ("last read 3h 12m ago").
+ *  An hours-old `readAt` with no error is 'fresh' and must stay that way. Taking
+ *  a clock here would imply otherwise, so the signature omits one — the same
+ *  take-only-what-you-use shape as classifyDashboardStaleness.
+ *
+ *  Pure; no I/O. */
+function classifyQueueFreshness({ readAt, error, consecutiveFailures } = {}) {
+  if (!(typeof error === 'string' && error)) return 'fresh';
+  if (!(typeof readAt === 'string' && readAt)) return 'unknown';
+  const failures = (typeof consecutiveFailures === 'number' && Number.isFinite(consecutiveFailures))
+    ? consecutiveFailures : 0;
+  return failures >= QUEUE_STALE_AFTER_FAILURES ? 'stale' : 'degraded';
+}
+
+/** The `/data` view of the queue memo, with the freshness VERDICT resolved
+ *  server-side.
+ *
+ *  The panel is then a pure renderer. Two consumers re-deriving "is this stale"
+ *  from a raw timestamp would drift, and `public/index.html` is an inline
+ *  browser script with no module boundary — any decision left in it could only
+ *  ever be source-scan tested, which is how it ships bugs green. Same split, and
+ *  same reason, as projectDashboardRevision.
+ *
+ *  `now` is injected (epoch ms) rather than read from the clock so the
+ *  projection stays pure and its tests deterministic. `ageSec` is null when no
+ *  successful read has happened or no clock was supplied.
+ *
+ *  Pure; no I/O. */
+function projectQueueFreshness(memo = {}, now = null) {
+  const at = (typeof memo.readAt === 'string' && memo.readAt) ? Date.parse(memo.readAt) : NaN;
+  const clock = (typeof now === 'number' && Number.isFinite(now)) ? now : null;
+  const ageSec = (Number.isFinite(at) && clock !== null)
+    ? Math.max(0, Math.floor((clock - at) / 1000))
+    : null;
+  return { ...memo, level: classifyQueueFreshness(memo), ageSec };
+}
+
+/** The operator-facing reason a queue WRITE must be refused, or null when writes
+ *  are allowed.
+ *
+ *  A write computed against a mirror known to be frozen is never the right
+ *  action, and the silent-no-op outcome is worse than a refusal: a Release
+ *  fires at a uuid that no longer holds the head, reports "released head
+ *  bb65addf", changes nothing, and leaves the real head held. A reorder is
+ *  worse still — computeReorderTargets validates the posted order against the
+ *  very mirror that is wrong, so a doomed reorder looks perfectly valid.
+ *
+ *  The verbatim git error is interpolated rather than summarised to "poll
+ *  failed". On 2026-08-14 the entire recovery was legible from one line —
+ *  `could not read Username for 'https://github.com'` — and truncating it would
+ *  have cost the same 3.5 hours.
+ *
+ *  Lives here rather than in server.js so the refusal COPY is directly
+ *  assertable: server.js has top-level side effects on require, so a message
+ *  built there could only ever be source-scan tested. Pure; no I/O. */
+function queueWriteBlockReason(memo = {}) {
+  const level = classifyQueueFreshness(memo);
+  if (level !== 'stale' && level !== 'unknown') return null;
+  const why = (typeof memo.error === 'string' && memo.error) ? ` (${memo.error})` : '';
+  return level === 'unknown'
+    ? `commit queue has never been read successfully${why} — cannot act on an unknown queue`
+    : `commit queue read is failing${why} — the displayed order cannot be trusted; fix the read before changing the queue`;
+}
+
 module.exports = {
   decideAutoReleaseGoneHead,
   computeReorderTargets,
   nudgeQueueOrder,
   reconcileQueueDraft,
+  mergeQueuePoll,
+  classifyQueueFreshness,
+  projectQueueFreshness,
+  queueWriteBlockReason,
+  QUEUE_STALE_AFTER_FAILURES,
 };