@codeyam-editor/codeyam-editor 0.1.6 → 0.1.8

package/npm/fleet-add-failure.js

@@ -96,6 +96,81 @@ function transcriptRostered(logTail) {
   return parseRosterMarkers(String(logTail || '')).length > 0;
 }
 
+// The categories that describe an add whose outcome is NOT yet proven bad — the
+// calm, in-flight side of the classifier. `converging` is a rostered box whose
+// ingress is still wiring; `verifying` is the residue: no failure signature
+// matched and nothing durable proves the VM is dead; `access-pending` is a VM that
+// is up and rostered but whose IAP policy has no accessors YET, because its
+// detached wiring tail — which applies the grant last — is still running. All
+// three render a calm card and none may use the word "failed". Consumed by
+// holdConvergingBadge and by deriveStatus (server.js) to pick the matching calm
+// status.
+const CALM_ADD_CATEGORIES = new Set(['converging', 'verifying', 'access-pending']);
+
+// Is a non-zero add outcome CORROBORATED by a durable fact — i.e. do we have
+// positive proof of death, rather than merely an absence of proof of life?
+//
+// This is the conjunction that gates the red `unknown` verdict. Before it, the
+// classifier's fallthrough treated "no recognized success signature" as failure,
+// so "we don't know" rendered as a terminal red "Add failed" telling the operator
+// to destroy a VM whose container was Up and whose editor answered /api/health
+// 200 (VM-4, 2026-07-31). Red now requires BOTH halves:
+//
+//   1. A non-zero exit code. A *missing* code is not proof of death — today it
+//      usually means the wrapper was signal-killed while its provision ran on,
+//      which is exactly the case that was misreading as failure. `null`,
+//      `undefined`, a non-integer, and 0 all fail this half.
+//   2. At least one durable corroborating fact:
+//        - `instanceAlive === false`   GCP says the instance is gone.
+//        - `containerHealthy === false` the in-VM editor was probed and is not
+//          answering — the box exists but never came up.
+//        - `wiringDeadlineBlown`       the wire-add-tail gave up (the failed
+//          marker is the tail's own honest deadline signal).
+//
+// Every fact is TRI-STATE on purpose: `undefined`/`null` means "not probed" and
+// corroborates nothing. Only an explicit `false` counts, so an unprobed VM can
+// never be escalated to red by absence of evidence.
+function addFailureCorroborated({ exitCode, instanceAlive, containerHealthy, wiringDeadlineBlown } = {}) {
+  const code = exitCode == null ? NaN : Number(exitCode);
+  if (!Number.isInteger(code) || code === 0) return false;
+  return instanceAlive === false || containerHealthy === false || !!wiringDeadlineBlown;
+}
+
+// Resolve the two independent health signals into the single tri-state
+// `containerHealthy` fact addFailureCorroborated consumes. Pure so the precedence
+// — which decides whether a VM can be escalated to red — is unit-tested; the
+// server (addFailureFacts) supplies the live signals.
+//
+//   reachable    the editor answered over the LB.
+//   inVmHealthy  the STRAND_PROBE /api/health read from INSIDE the container.
+//
+// LB reachability wins when present: it is the stronger witness (it proves the
+// whole URL→ingress→backend path, not just the process). Otherwise the in-VM
+// witness answers — this is the still-wiring case, where the LB necessarily
+// cannot see a VM whose ingress is what we are waiting on, yet the editor itself
+// is demonstrably up.
+//
+// Returns `null` when NEITHER signal was gathered. That is the load-bearing case:
+// an unprobed VM must corroborate nothing, so it can never be escalated to red by
+// absence of evidence. `false` is reserved for a VM that was genuinely probed and
+// did not answer.
+function containerHealthWitness({ reachable, inVmHealthy } = {}) {
+  if (reachable) return true;
+  if (inVmHealthy != null) return !!inVmHealthy;
+  return null;
+}
+
+// Lift the `Next valid action:` remedy out of a BLOCKED reason so a classified
+// card's recovery line stays the SAME text the producing check already emitted,
+// instead of a second copy of the same instructions that can silently drift from
+// it. Returns '' when the text carries no remedy, in which case the caller's
+// static copy stands on its own.
+function nextValidActionRemedy(text) {
+  const lines = String(text || '').split('\n');
+  const i = lines.findIndex((l) => /^\s*Next valid action:/.test(l));
+  return i === -1 ? '' : lines.slice(i).join('\n').trim();
+}
+
 // Build a verdict, stamping `retryable` from the single-source-of-truth Set so it
 // can never drift from the category.
 function buildFailureVerdict(category, headline, recovery, extra) {
@@ -108,7 +183,19 @@ function buildFailureVerdict(category, headline, recovery, extra) {
 //
 //   logTail   the last lines of the provision transcript (string; '' when the
 //             log was unreadable — must never throw)
-//   exitCode  the provision's exit code (number; used for signal-kill detection)
+//   exitCode  the provision's exit code (number; used for signal-kill detection
+//             and as the first half of the proof-of-death conjunction)
+//
+// The remaining inputs are the durable facts the server gathers (the module
+// stays pure — it never probes). All are tri-state; omit any that was not
+// probed and it corroborates nothing. See addFailureCorroborated.
+//
+//   instanceAlive        does GCP still list the instance?
+//   containerHealthy     did the in-VM editor answer /api/health?
+//   wiringDeadlineBlown  did the add-time wire tail drop its failed marker?
+//   wiringInFlight       is that tail still RUNNING? Only the IAP read-back arm
+//                        reads it, to tell "no accessors yet" from "no accessors,
+//                        ever" — the grant is the tail's last step.
 //
 // Returns { category, headline, recovery, retryable, files? }.
 //
@@ -118,7 +205,7 @@ function buildFailureVerdict(category, headline, recovery, extra) {
 // signature is therefore tested BEFORE the generic cloud-up-failed signature, or
 // a retryable drift would misclassify as a non-retryable cloud:up failure and the
 // operator would lose the safe one-click recovery.
-function classifyAddFailure({ logTail, exitCode } = {}) {
+function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy, wiringDeadlineBlown, wiringInFlight } = {}) {
   const log = String(logTail || '');
 
   // 1. preflight-drift — the streamed-script drift guard aborted BEFORE any
@@ -200,7 +287,37 @@ function classifyAddFailure({ logTail, exitCode } = {}) {
       'The provision was interrupted (killed or out of memory), so the VM state is unknown. Reconcile or destroy this VM before re-adding.');
   }
 
-  // 8. converging — the add reached the early-roster marker (`CY_ROSTER`): the
+  // 8. The post-add IAP read-back verdicts. Unlike every signature above, this one
+  //    is not produced by the provision transcript at all — it is the BLOCKED
+  //    reason iapReadbackOutcome hands to annotateAddFailure, which is what
+  //    stops a read-back failure from being the one add failure that reaches the
+  //    card with no category at all (and so no differentiated headline, no
+  //    recovery, no guarded Retry, and no chance for holdConvergingBadge to hold it
+  //    calm). Tested AFTER all seven hard-failure signatures so a proven failure
+  //    always wins if both texts are ever present, and BEFORE converging/verifying
+  //    because it is strictly more specific than either.
+  if (/IAP policy authorizes no human/i.test(log)) {
+    const remedy = nextValidActionRemedy(log);
+    //  8a. access-pending — no accessors YET. The grant is the LAST step of the
+    //      detached wire-add-tail, so while that tail is alive an empty policy is
+    //      the expected mid-flight reading, not a verdict. Calm: this is the state
+    //      that used to render a red "Add failed" on healthy VM-4/VM-7 minutes
+    //      before their grant step had even run (2026-08-11).
+    if (wiringInFlight) {
+      return buildFailureVerdict('access-pending', 'Access not granted yet',
+        'The VM is up and rostered, but its IAP policy has no human accessors yet — the add\'s background wiring applies the grant as its last step and is still running. This resolves on its own once that tail finishes; no action is needed unless it persists past the wiring deadline.');
+    }
+    //  8b. iap-unauthorized — the wiring is done and the policy STILL authorizes
+    //      nobody, so the public URL returns "Error code 51" for every human. This
+    //      is the 2026-08-09 case the read-back exists to catch (one step granted
+    //      the operator, the next revoked him) and it is a genuine hard failure.
+    //      Not retryable: the VM exists and is fine — re-adding it would duplicate
+    //      a live box when what is missing is one grant.
+    return buildFailureVerdict('iap-unauthorized', 'No one can reach this VM',
+      `The VM is up but its IAP policy authorizes no human, so the public URL returns "Error code 51" for everyone. Its background wiring is no longer running, so this will not resolve on its own — grant access to fix it. Do NOT destroy or re-add this VM.${remedy ? `\n\n${remedy}` : ''}`);
+  }
+
+  // 9. converging — the add reached the early-roster marker (`CY_ROSTER`): the
   //    container gate passed, the instance is real, and the VM is rostered, yet it
   //    exited non-zero WITHOUT matching any specific hard-failure signature above.
   //    By construction of the provision script, every genuine hard failure (drift,
@@ -210,17 +327,42 @@ function classifyAddFailure({ logTail, exitCode } = {}) {
   //    wire-add-tail / reconcile-ingress — NOT an add failure. Surface it as an
   //    honest in-flight state instead of the scary red `unknown`. Not retryable:
   //    the VM exists and its detached wiring owns finishing it, so a one-click
-  //    retry could duplicate/orphan a live box. Tested LAST so a hard-failure
-  //    signature (all printed pre-roster) always wins — `converging` can never
-  //    shadow a proven failure.
+  //    retry could duplicate/orphan a live box. Tested after every hard-failure
+  //    signature (all printed pre-roster) so a proven failure always wins —
+  //    `converging` can never shadow one.
   if (transcriptRostered(log)) {
     return buildFailureVerdict('converging', 'Add still converging',
       'The container is up and the VM is rostered; the LB ingress is still wiring in the background. This resolves on its own once the editor is reachable over the LB — no action needed unless it persists past the wiring deadline.');
   }
 
-  // 9. unknown — fallback. Never invite an unproven retry.
-  return buildFailureVerdict('unknown', 'Add failed',
-    'The add failed for an unclassified reason — inspect the transcript below, then reconcile or destroy this VM before re-adding.');
+  // 10. The residue: no failure signature above matched. The polarity here is
+  //    INVERTED from the original design. It used to read "no recognized success
+  //    signature ⇒ failed", which made red the DEFAULT for "we don't know" — so a
+  //    VM whose container was Up and whose editor answered /api/health 200 got a
+  //    terminal red "Add failed" card advising the operator to destroy it (VM-4,
+  //    2026-07-31; the wrapper had been killed mid-flight, so its transcript was
+  //    truncated before CY_ROSTER and it never reached the `converging` arm above).
+  //
+  //    Red now requires positive proof of death, and everything else is in-flight:
+  //
+  //    10a. unknown — a non-zero exit CORROBORATED by a durable fact (see
+  //        addFailureCorroborated). The box is provably dead or provably never
+  //        came up, so the loud, terminal escalation is warranted and unchanged.
+  //        Still never invites an unproven retry.
+  if (addFailureCorroborated({ exitCode, instanceAlive, containerHealthy, wiringDeadlineBlown })) {
+    return buildFailureVerdict('unknown', 'Add failed',
+      'The add failed for an unclassified reason and the VM is confirmed dead or never came up — inspect the transcript below, then reconcile or destroy this VM before re-adding.');
+  }
+
+  //    10b. verifying — the honest default. No failure signature, and nothing
+  //        durable proves this VM is dead: it may still be provisioning or
+  //        wiring, and the ingress can keep propagating for minutes after the
+  //        wrapper is gone. Calm copy, transcript still one click away. NOT
+  //        retryable — nothing is proven about the box either way, so a one-click
+  //        retry could still duplicate a live GCP instance (the ADD_FAILURE_RETRYABLE
+  //        grounds are unchanged by this inversion).
+  return buildFailureVerdict('verifying', 'Add outcome unconfirmed',
+    'No failure signature was found in the transcript and nothing proves this VM is dead, so the add may still be converging — the ingress keeps propagating for minutes after the provision wrapper exits. Inspect the transcript below; reconcile this VM only if it is still unresolved past the wiring deadline.');
 }
 
 // Pure guard for the operator "Retry add" action: given a VM number and its
@@ -238,15 +380,29 @@ function canRetryAdd({ n, job } = {}) {
 }
 
 // Pure badge-hold predicate for deriveStatus (server.js): hold the calm in-flight
-// `add-converging` card — instead of the red add-failed — ONLY when the add was
-// classified `converging` AND its detached first-wiring is still in flight. Once
-// the wire-add-tail fails/deadlines (`addTailFailed`) or is gone (`addTailInflight`
-// false), or the category is anything else, return false so the caller falls
-// through to the red add-failed status — the operator-chosen escalation. Kept pure
-// (like canRetryAdd / canClearBadge) so the badge decision is unit-tested without a
+// card — instead of the red add-failed — while the add is still converging.
+//
+// The hold used to require `addTailInflight`, i.e. that the detached wire-add-tail
+// PROCESS was still alive. That premise was wrong: the tail's exit says nothing
+// about LB convergence, because the EXTERNAL_MANAGED backend services keep
+// propagating for minutes afterwards. VM-4's backends were still being created at
+// 18:17-18:20Z, long after its job had been declared failed at 18:15Z — the tail
+// had exited, so the calm hold released and the card went red on a healthy VM.
+//
+// The hold is now driven by the question that actually matters: is this add still
+// converging, and has its wiring deadline genuinely blown? `wiringDeadlineBlown`
+// (the failed marker the tail drops when it gives up) stays the release signal —
+// tail liveness is no longer a requirement for calm, so the calm state survives
+// the multi-minute propagation window instead of ending at process exit.
+//
+// Both calm categories qualify (see CALM_ADD_CATEGORIES): a rostered box that is
+// still wiring, and the unproven residue. Any proven-failure category returns
+// false and falls through to the red add-failed status — the operator-chosen
+// escalation, which this predicate narrows but never softens. Kept pure (like
+// canRetryAdd / canClearBadge) so the badge decision is unit-tested without a
 // live dashboard.
-function holdConvergingBadge({ category, addTailInflight, addTailFailed } = {}) {
-  return category === 'converging' && !!addTailInflight && !addTailFailed;
+function holdConvergingBadge({ category, wiringDeadlineBlown } = {}) {
+  return CALM_ADD_CATEGORIES.has(category) && !wiringDeadlineBlown;
 }
 
-module.exports = { classifyAddFailure, extractDriftFiles, lastNonEmptyLines, canRetryAdd, transcriptRostered, holdConvergingBadge, ADD_FAILURE_RETRYABLE };
+module.exports = { classifyAddFailure, extractDriftFiles, lastNonEmptyLines, canRetryAdd, transcriptRostered, holdConvergingBadge, addFailureCorroborated, containerHealthWitness, CALM_ADD_CATEGORIES, ADD_FAILURE_RETRYABLE };

package/npm/fleet-cleanup.js

@@ -337,6 +337,132 @@ function reattachNextStep(verdict) {
   return 'probe'; // 'done' and 'reconcile' both confirm via the gcloud probe
 }
 
+// May the PERIODIC stranded-reconcile pass drive a recover/roster probe for this
+// VM, or is a provision already running that a second cloud:up would collide
+// with? This is the guard the boot re-attach path has always had and the
+// stranded path never did — the 2026-08-07 asymmetry that destroyed VM-4/VM-5.
+//
+// On that day an `add` job badge read `error` while its detached `cloud:up`
+// child was alive and healthy (a source-build bootstrap runs ~40 minutes;
+// `npm install` alone took 25m). `pollStrandedInstances` builds `activeAddNs`
+// from BADGE state and deliberately excludes an `error` add so the bounded
+// bootstrap-recovery budget can retry on a timer — so the VM read
+// running-but-unrostered, crossed STRANDED_ADOPT_THRESHOLD in ~3 minutes, and
+// the pass launched a SECOND cloud:up on top of the live one. The two raced on
+// `docker compose up -d`, the loser hit `Conflict. The container name
+// "/codeyam-editor-editor-1" is already in use`, and its abort handler deleted
+// the instance. The badge is a derived, lossy signal that was wrong for 30
+// minutes straight; LIVENESS is the ground truth, so this gate reads exactly
+// what `reattachAdd` reads — the exit marker plus a `kill -0` of the recorded
+// pid — and composes `reattachVerdict`/`reattachNextStep` rather than forking a
+// parallel matrix.
+//
+//   'watch'  no marker + a live child   → the provision is genuinely in flight.
+//                                         Re-attach and watch (mirroring
+//                                         runStartupReap); NEVER provision.
+//   'defer'  no marker, no live child, and no pid was EVER recorded, inside the
+//            ambiguity window            → we cannot tell whether a child is
+//                                         running. Do nothing this tick.
+//   'probe'  everything else             → existing behavior: drive the gcloud
+//                                         probe (reconcileViaGcloud →
+//                                         reapVerdict).
+//
+// The 'defer' rung exists because `childAlive` is FALSE-NEGATIVE-PRONE on one
+// specific path: when `strandedReconcileMeta` finds no sidecar it reconstructs
+// cfg from launch history and persists `{ cfg }` with NO pid, so `isPidAlive`
+// reads false for a VM whose child may well be alive — and, worse, that pid-less
+// sidecar is what every later tick reads back, blinding the guard permanently.
+// `pidRecorded` separates "we knew the pid and the process is gone" (real
+// evidence → probe) from "we never had a pid" (absence of evidence → defer).
+// Erring toward NOT provisioning is the asymmetric-cost choice: waiting one more
+// 60s tick costs nothing, being wrong costs a destroyed VM.
+//
+// The window is what keeps 'defer' from becoming a permanent refusal that would
+// break stranded adoption outright: once more than `ambiguityWindowMs` has
+// elapsed since we first saw the pid-less sidecar, no plausible provision child
+// is still running, so the probe proceeds and the self-heal path is preserved.
+// An unknown/unstamped elapsed is treated as "just noticed" (defer), again the
+// safe direction; a caller that supplies no window at all opts out entirely and
+// keeps the old probe-always behavior.
+//
+// Pure, so the whole matrix is testable without a VM: server.js maps the verdict
+// to its side effect.
+function strandedRecoverGate({
+  markerPresent,
+  exitCode,
+  childAlive,
+  pidRecorded,
+  sinceMs,
+  ambiguityWindowMs,
+} = {}) {
+  // A live child is a live child — reuse the existing classifier so 'running'
+  // keeps meaning exactly what the boot path means by it.
+  if (reattachNextStep(reattachVerdict({ markerPresent, exitCode, childAlive })) === 'watch') {
+    return 'watch';
+  }
+  // A marker is durable proof the child reached its EXIT trap: nothing is in
+  // flight, so the probe is safe. (Which probe outcome follows — roster,
+  // recover-bootstrap, surface-error — stays reapVerdict's call, not ours.)
+  if (markerPresent) return 'probe';
+  // We knew the pid and `kill -0` says it is gone: real evidence, so probe.
+  if (pidRecorded) return 'probe';
+  const window = Number(ambiguityWindowMs);
+  if (!Number.isFinite(window) || window <= 0) return 'probe'; // caller opted out
+  const elapsed = Number(sinceMs);
+  if (!Number.isFinite(elapsed)) return 'defer'; // unstamped — treat as just-noticed
+  return elapsed < window ? 'defer' : 'probe';
+}
+
+// Is an `add` genuinely IN FLIGHT for this VM right now?
+//
+// Extracted from pollVanishedInstances, where the inline expression was
+// `addPending.has(n) || !!(state.jobs[n] && state.jobs[n].action === 'add')` —
+// whose second clause is also true for a TERMINAL (`done` / `error`) add job.
+// That permanently marks the VM busy, so `reconcileRosteredVm` can never return
+// 'cleanup' and a rostered VM whose instance is gone can never be auto-reaped.
+// Live evidence from the same 2026-08-07 incident: VM-5's instance was deleted
+// at ~12:46 and its card logged `vanish-reconcile: VM-5 unreachable …` once a
+// minute for 50 consecutive polls, only clearing after a manual `clear-badge`
+// deleted the terminal job record.
+//
+// The on-disk sidecar clause is already correct and is preserved: a sidecar is
+// removed on the add's success path, so its presence really does mean in-flight.
+// The job clause now filters on job STATE the same way `activeAddNs` and
+// `pollStartupProbes` do. Pure so the caller's computation is directly
+// assertable — the bug was never in `reconcileRosteredVm`'s contract, it was in
+// what this expression fed it.
+const IN_FLIGHT_ADD_STATES = ['running', 'queued', 'recovering'];
+
+function hasInFlightAdd({ sidecarPending, job } = {}) {
+  if (sidecarPending) return true;
+  if (!job || job.action !== 'add') return false;
+  return IN_FLIGHT_ADD_STATES.includes(job.state);
+}
+
+// WHICH of reconcileRosteredVm's two distinct 'leave' causes fired, as a tag the
+// caller renders. Pure so the distinction is directly assertable.
+//
+// 'leave' is returned both when the GCP probe was INCONCLUSIVE (exists null /
+// undefined — absence of evidence, never act) and when the instance is
+// CONFIRMED gone but the VM is busy. The old single log line conflated them into
+// a self-contradiction — it announced "instance-gone unconfirmed (exists=false)"
+// when exists=false IS the confirmation — which cost hours of code-reading
+// during the 2026-08-07 incident to work out which branch was actually firing.
+//
+//   'probe-inconclusive'  exists is null/undefined — retry next tick
+//   'live-session'        instance confirmed gone but a session is live
+//   'in-flight-add'       instance confirmed gone but an add is in flight
+//   null                  not a 'leave' case at all
+//
+// Mirrors reconcileRosteredVm's own precedence (exists-uncertainty first, then
+// hasSession, then hasPendingAdd) so the tag can never disagree with the verdict.
+function vanishLeaveReason({ exists, hasSession, hasPendingAdd } = {}) {
+  if (exists === null || exists === undefined) return 'probe-inconclusive';
+  if (hasSession) return 'live-session';
+  if (hasPendingAdd) return 'in-flight-add';
+  return null;
+}
+
 // Anti-loop budget for the `recover-bootstrap` path. Pure: given the count of
 // recoveries ALREADY persisted on the sidecar, return the next attempt number,
 // whether the reap should proceed (re-run cloud:up) or treat the VM as
@@ -537,4 +663,7 @@ module.exports = {
   isSignalKillExit,
   reattachVerdict,
   reattachNextStep,
+  strandedRecoverGate,
+  hasInFlightAdd,
+  vanishLeaveReason,
 };

package/npm/fleet-job-badge.js

@@ -15,21 +15,37 @@
 // Should the VM's last terminal-error job badge be reconciled against live
 // health — i.e. presented as "recovered" rather than a live red error?
 //
-//   job        the carried job record ({ state, action, ... }) or null
-//   reachable  the VM's editor answered this poll (true = healthy / responding)
-//   drifted    a project-dir-drift signal is present (reachable-but-unwired /
-//              half-provisioned add — genuinely needs-recovery, owned by the
-//              companion reconcile-inflight-jobs plan; never falsely cleared here)
-//   offBranch  the VM's live client branch differs from its target branch
-//   degraded   the job completed but a wiring sub-step is flagged degraded
+//   job          the carried job record ({ state, action, ... }) or null
+//   reachable    the VM's editor answered this poll over the LB (true = healthy)
+//   inVmHealthy  the in-VM editor answered /api/health from INSIDE the container
+//                (the STRAND_PROBE `HEALTH=1` witness gathered over docker-exec)
+//   drifted      a project-dir-drift signal is present (reachable-but-unwired /
+//                half-provisioned add — genuinely needs-recovery, owned by the
+//                companion reconcile-inflight-jobs plan; never falsely cleared here)
+//   offBranch    the VM's live client branch differs from its target branch
+//   degraded     the job completed but a wiring sub-step is flagged degraded
 //
 // Returns true only when the job is a terminal error AND the VM is demonstrably
-// fine (reachable, not drifted, on-branch, not degraded). An unreachable /
-// drifted / off-branch VM keeps its error surfaced — a real problem must stay
-// visible.
-function shouldReconcileFailedBadge({ job, reachable, drifted, offBranch, degraded } = {}) {
+// fine (healthy, not drifted, on-branch, not degraded). A drifted / off-branch /
+// unwitnessed VM keeps its error surfaced — a real problem must stay visible.
+//
+// HEALTH IS WITNESSED TWO WAYS, and either suffices. `reachable` alone made the
+// self-heal unable to fire in exactly the case that needs it most: on the GCE
+// operator host `reachable` is derived from the LB probe, which needs the very
+// ingress the add is still wiring. That is a circular dependency — the badge
+// needs reachable, reachable needs ingress, and ingress is what we are waiting
+// for — so a still-wiring VM sat at unreachableStreak 9 with a red add-error it
+// could not clear by construction (VM-4, 2026-07-31). The dashboard already held
+// the disconfirming evidence: gatherLauncherStrand runs STRAND_PROBE over
+// docker-exec for exactly these unreachable-with-streak VMs and reads HEALTH=1
+// from the in-VM /api/health.
+//
+// This widens ONE input; it does not relax the verdict. Every other guard is
+// untouched, and the "absence of evidence never auto-acts" posture holds — an
+// unprobed VM has neither witness and stays unreconciled.
+function shouldReconcileFailedBadge({ job, reachable, inVmHealthy, drifted, offBranch, degraded } = {}) {
   if (!job || job.state !== 'error') return false;
-  if (!reachable) return false;
+  if (!reachable && !inVmHealthy) return false;
   if (drifted) return false;
   if (offBranch) return false;
   if (degraded) return false;

package/npm/fleet-memory.js

@@ -290,8 +290,16 @@ function restoreFromSnapshot({ snapshot, maxAgeMs, nowIso, sidecarKeys } = {}) {
  *  Degrades to 'cgroup=unavailable' rather than printing zeros off-cgroup, so a
  *  laptop log cannot be misread as a cgroup at 0MB.
  *
+ *  `detachedWiringCount` — when supplied — appends `tails=N`, which is what makes
+ *  the RSS/cgroup gap ATTRIBUTABLE rather than merely visible. On 2026-08-11 the
+ *  cgroup read 2154MB against a 96MB RSS and the whole diagnosis was "those ~2GB
+ *  are this process's own transient add children, not a leak". `RSS=96MB
+ *  cgroup=2154MB tails=4` reads as a batch draining on its own; `RSS=1300MB
+ *  cgroup=1400MB tails=0` reads as a genuine leak. Omitted (the pre-existing
+ *  callers, and any host with no tail accounting) prints exactly as before.
+ *
  *  Pure; no I/O. */
-function formatMemoryText({ rssBytes, memoryCurrent, memoryHigh } = {}) {
+function formatMemoryText({ rssBytes, memoryCurrent, memoryHigh, detachedWiringCount } = {}) {
   const mb = (b) => (Number(b) / 1024 / 1024).toFixed(0);
   const current = Number(memoryCurrent);
   const high = Number(memoryHigh);
@@ -308,6 +316,8 @@ function formatMemoryText({ rssBytes, memoryCurrent, memoryHigh } = {}) {
   } else {
     parts.push('cgroup=unavailable');
   }
+  const tails = Number(detachedWiringCount);
+  if (Number.isFinite(tails) && tails >= 0) parts.push(`tails=${tails}`);
   return parts.join(' ');
 }
 
@@ -323,11 +333,26 @@ function formatMemoryText({ rssBytes, memoryCurrent, memoryHigh } = {}) {
  *  protecting. The dashboard's own self-update counts, being a heavy job by any
  *  other name.
  *
- *  Pure; no I/O — the caller passes state.jobs / state.dashboardUpdate. */
-function hasJobInFlight({ jobs, dashboardUpdate } = {}) {
+ *  `detachedWiringCount` is the THIRD source of busy, and the one this guard was
+ *  blind to: an Add settles at container-healthy while its detached wire-add-tail
+ *  keeps spawning gcloud children for another 5-10 min in the SAME cgroup. Judging
+ *  "safe to restart" off `state.jobs` alone reported no job in flight while that
+ *  work was live — on 2026-08-11 the escalation crossed the restart threshold at
+ *  86% of memory.high and the only thing that held it back was an unrelated job
+ *  happening to still be `running`. Had it settled thirty seconds earlier the
+ *  dashboard would have restarted itself mid-batch and orphaned four adds, which
+ *  is precisely the failure class the detached-provision work exists to prevent.
+ *  Omitting the key behaves exactly as before, so every current caller is
+ *  unaffected.
+ *
+ *  Pure; no I/O — the caller passes state.jobs / state.dashboardUpdate and the
+ *  live-tail count. */
+function hasJobInFlight({ jobs, dashboardUpdate, detachedWiringCount } = {}) {
   const busy = (j) => !!j && (j.state === 'running' || j.state === 'recovering');
   const anyJob = Object.values((jobs && typeof jobs === 'object') ? jobs : {}).some(busy);
-  return anyJob || !!(dashboardUpdate && dashboardUpdate.state === 'running');
+  const tails = Number(detachedWiringCount);
+  const anyTail = Number.isFinite(tails) && tails > 0;
+  return anyJob || anyTail || !!(dashboardUpdate && dashboardUpdate.state === 'running');
 }
 
 /** Should this escalation level be announced to the journal?
@@ -356,6 +381,49 @@ function shouldAnnounceMemoryLevel({ level, lastLevel } = {}) {
   return !QUIET_MEMORY_LEVELS.includes(level);
 }
 
+/** Resolve the directory holding THIS process's memory.current / memory.high,
+ *  or null when no cgroup-derived reading is available.
+ *
+ *  The cgroup-v2 hierarchy ROOT exposes no memory.current and no memory.high —
+ *  only a leaf cgroup does. Reading `/sys/fs/cgroup/memory.current` directly is
+ *  therefore ENOENT on every systemd host, and that is exactly what made the
+ *  2026-07-30 memory remedy inert: both reads failed, every cgroup number
+ *  degraded to null, decideMemoryRestart answered 'no-cgroup-limit' on every
+ *  tick forever, and the drain silently fell back to the RSS threshold the
+ *  incident had already proven cannot fire before the event loop stalls.
+ *
+ *  Takes the CONTENTS of /proc/self/cgroup rather than reading it, so the path
+ *  derivation — the part that was wrong — is unit-testable on a macOS dev box
+ *  with no cgroup filesystem at all. That untestability is why a root-vs-leaf
+ *  path error shipped green.
+ *
+ *  Returns null for every shape that has no leaf memory file to read:
+ *    • the unified line `0::/` — the process IS in the root cgroup, which
+ *      genuinely has no memory files; this must NOT look like a valid path;
+ *    • cgroup v1 (`N:controller:/path` lines, no `0::` entry);
+ *    • empty, unreadable, or malformed input (a non-Linux host has no
+ *      /proc/self/cgroup at all, and the caller passes the null through).
+ *
+ *  Pure; no I/O. */
+function resolveCgroupMemoryDir({ procSelfCgroup, root } = {}) {
+  if (typeof procSelfCgroup !== 'string') return null;
+  const base = (typeof root === 'string' && root)
+    ? root.replace(/\/+$/, '')
+    : '/sys/fs/cgroup';
+  for (const line of procSelfCgroup.split('\n')) {
+    const trimmed = line.trim();
+    // cgroup v2 unified hierarchy is the single `0::<path>` entry. v1 lines
+    // (`N:controller:/path`) never match, which is the intended null path.
+    if (!trimmed.startsWith('0::')) continue;
+    const rel = trimmed.slice(3);
+    if (!rel.startsWith('/')) return null;
+    const clean = rel.replace(/\/+$/, '');
+    if (!clean) return null;
+    return base + clean;
+  }
+  return null;
+}
+
 /** Keep only the restart timestamps still inside the rolling window.
  *
  *  Split out from decideMemoryRestart so the "how many restarts have I already
@@ -386,6 +454,7 @@ module.exports = {
   formatMemoryText,
   hasJobInFlight,
   shouldAnnounceMemoryLevel,
+  resolveCgroupMemoryDir,
   pruneRestartWindow,
   buildStateSnapshot,
   decodeStateSnapshot,