@codeyam-editor/codeyam-editor 0.1.5 → 0.1.7

package/npm/fleet-ghcr-secret.js

@@ -0,0 +1,129 @@
+'use strict';
+
+// npm/fleet-ghcr-secret.js — pure decisions for sourcing the write:packages GHCR
+// push token from GCP Secret Manager instead of a laptop shell env or a baked
+// unit env.
+//
+// Why this exists: the fleet dashboard needs CODEYAM_GHCR_PUSH_TOKEN
+// (write:packages) for self-feeding — after one VM compiles the editor it
+// publishes the binary so the next VM pulls it (~2 min) instead of recompiling
+// (~10 min). On the GCE operator host the token was never wired in at all, so
+// self-feeding was structurally off and the "Self-feeding disabled" banner
+// (npm/fleet-prewarm.js::pushTokenDiagnostic) was permanent — and every
+// reprovision would drop any hand-set value. Reading it from Secret Manager at
+// boot via the attached service account (metadata creds) means the token never
+// lives in the unit env, a git checkout, or a laptop shell, survives reprovision
+// + rotation (a dashboard restart picks up the latest version), and both the
+// banner and the actual `cloud:up` publish get the token from one source.
+//
+// This module holds only the *decisions* (fetch-or-not, which secret, argv,
+// payload parse) so each case is unit-tested without a live gcloud. The I/O — the
+// `${GCLOUD} secrets versions access` subprocess and assigning the resolved value
+// onto process.env — lives in the dashboard server shell that calls these.
+
+// The default Secret Manager secret id the operator host reads when no explicit
+// name is configured. Kept in sync with the name provision-operator-host.sh
+// grants the SA accessor on and prints in its create-command hint.
+const DEFAULT_PUSH_TOKEN_SECRET = 'codeyam-ghcr-push-token';
+
+// Decide whether to hydrate the push token from Secret Manager, and from which
+// secret. The rule, in precedence order:
+//
+//   token present            → never fetch. An explicit env var always wins, so
+//                              the laptop path and any operator override are
+//                              untouched (strictly back-compatible).
+//   explicit secretNameEnv   → fetch that named secret (works anywhere gcloud
+//                              can reach it, laptop included, if the operator
+//                              opts in by setting CODEYAM_GHCR_PUSH_TOKEN_SECRET).
+//   onGce, no env            → fetch the DEFAULT secret. The operator host runs
+//                              with the SA attached (metadata creds), so this is
+//                              the zero-config self-feeding path.
+//   otherwise                → don't fetch. A plain laptop with neither the token
+//                              nor a secret name configured must NOT fire a
+//                              gcloud call that would just fail — no noise.
+//
+// Returns { fetch:false, reason } or { fetch:true, secretName, reason }.
+function secretFetchPlan({ token, onGce, secretNameEnv } = {}) {
+  if (token && String(token).trim()) {
+    return { fetch: false, reason: 'env token present — Secret Manager not consulted' };
+  }
+  const explicit = secretNameEnv && String(secretNameEnv).trim();
+  const secretName = explicit ? String(secretNameEnv).trim() : (onGce ? DEFAULT_PUSH_TOKEN_SECRET : '');
+  if (!secretName) {
+    return { fetch: false, reason: 'no push token, no secret configured, and not on GCE' };
+  }
+  return {
+    fetch: true,
+    secretName,
+    reason: explicit ? 'explicit CODEYAM_GHCR_PUSH_TOKEN_SECRET' : 'default secret on GCE (SA creds)',
+  };
+}
+
+// The argv suffix (after the ${GCLOUD} chokepoint prefix) that reads the latest
+// version of a secret. `latest` follows rotation: storing a new secret version
+// and restarting the dashboard picks up the new token with no reprovision. A
+// truthy `project` appends `--project <id>` so the read is pinned to the fleet
+// project rather than gcloud's ambient default; omitted when blank. Keeping the
+// project branch here (not at the call site) leaves the dashboard's I/O edge a
+// branch-free passthrough.
+function secretAccessArgv(secretName, project) {
+  const argv = ['secrets', 'versions', 'access', 'latest', '--secret', String(secretName || '')];
+  const proj = project && String(project).trim();
+  if (proj) argv.push('--project', proj);
+  return argv;
+}
+
+// Interpret one `gcloud secrets versions access` result into the outcome the
+// dashboard applies: the resolved token (empty on any failure), the console
+// method to log through ('log' | 'error'), and the operator-facing message. This
+// is where the fail-soft branching lives — a gcloud error, an empty payload, and
+// a real token each map to a distinct outcome — so the dashboard's I/O edge stays
+// thin (exec → interpret → assign) and every branch is unit-tested here.
+//
+//   hadError    truthy when the exec callback received an Error
+//   stdout      gcloud's stdout (the raw secret payload on success)
+//   stderr      gcloud's stderr (or the Error message) — tail is surfaced on error
+//   secretName  the secret that was read, for the message
+//   reason      the secretFetchPlan reason, for the message
+function interpretSecretAccess({ hadError, stdout, stderr, secretName, reason } = {}) {
+  const name = String(secretName || '');
+  const why = reason ? ` (${reason})` : '';
+  if (hadError) {
+    return {
+      token: '',
+      logLevel: 'error',
+      message: `push-token hydrate: could not read Secret Manager secret '${name}'${why}: ${String(stderr || '').slice(-200)}`,
+    };
+  }
+  const token = parseSecretPayload(stdout);
+  if (!token) {
+    return {
+      token: '',
+      logLevel: 'error',
+      message: `push-token hydrate: secret '${name}' resolved empty — self-feeding stays off`,
+    };
+  }
+  return {
+    token,
+    logLevel: 'log',
+    message: `push-token hydrate: loaded write:packages token from Secret Manager secret '${name}'${why} — self-feeding enabled`,
+  };
+}
+
+// Parse gcloud's stdout into the token. gcloud prints the raw secret payload with
+// no added framing, so this is just a trim — which also defends against a secret
+// stored with a trailing newline (the common `echo ... | gcloud secrets create`
+// footgun). A GitHub PAT never contains surrounding whitespace, so trimming is
+// safe. Non-string / null input → '' (fail-soft; the banner simply stays).
+function parseSecretPayload(stdout) {
+  if (typeof stdout !== 'string') return '';
+  return stdout.trim();
+}
+
+module.exports = {
+  DEFAULT_PUSH_TOKEN_SECRET,
+  secretFetchPlan,
+  secretAccessArgv,
+  parseSecretPayload,
+  interpretSecretAccess,
+};

package/npm/fleet-git-maintenance.js

@@ -0,0 +1,142 @@
+'use strict';
+
+/**
+ * Pure helpers for keeping the operator host's git clone healthy across the
+ * dashboard's high-frequency `git fetch origin <branch>` polling.
+ *
+ * The failure mode this exists to break: `pollPlans` fetches every ~12s in
+ * /opt/codeyam-editor. On a full or slow disk an interrupted fetch abandons a
+ * multi-hundred-MB `.git/objects/pack/tmp_pack_*` file. Those accumulate (~3GB
+ * observed), fill the disk, which makes the NEXT fetch fail sooner and leave
+ * another temp pack — a runaway that ends with a 100%-full disk and a wedged
+ * dashboard. Two mechanisms break the loop: prune the abandoned temp packs, and
+ * back off the fetch cadence while the remote keeps failing so a broken remote
+ * costs one attempt a minute instead of five a minute.
+ *
+ * Everything here is pure so it is unit-testable without a git repo or a clock;
+ * server.js owns the exec + the wall clock and passes `nowMs` in.
+ */
+
+// Backoff bounds. The floor is one normal poll interval (no penalty for a
+// single blip) and the ceiling keeps a long outage at a ~5 min retry rather
+// than backing off to effectively never.
+const DEFAULT_BASE_BACKOFF_MS = 30_000;
+const DEFAULT_MAX_BACKOFF_MS = 300_000;
+// Temp packs younger than this may belong to a fetch that is still running —
+// deleting those would sabotage a healthy fetch, so only reap the abandoned.
+const DEFAULT_STALE_PACK_MIN_AGE_MINUTES = 10;
+// `git gc --auto` is cheap when there is nothing to do, but not free; hourly is
+// often enough to retire loose objects the poll churn leaves behind.
+const DEFAULT_GC_INTERVAL_MS = 3_600_000;
+
+/** The zero state for a clone that has not yet been fetched from. */
+function initialFetchState() {
+  return { consecutiveFailures: 0, suppressedUntilMs: 0, lastGcAtMs: 0 };
+}
+
+/**
+ * Exponential backoff for consecutive fetch failures: 0 while healthy, then
+ * base, 2x base, 4x base ... capped at maxMs.
+ * @param {number} consecutiveFailures How many fetches have failed in a row.
+ * @param {Object} [opts] `{ baseMs, maxMs }` overrides.
+ * @returns {number} Milliseconds to suppress fetching for. 0 means "fetch now".
+ */
+function nextBackoffMs(consecutiveFailures, opts) {
+  const baseMs = (opts && opts.baseMs) || DEFAULT_BASE_BACKOFF_MS;
+  const maxMs = (opts && opts.maxMs) || DEFAULT_MAX_BACKOFF_MS;
+  if (!consecutiveFailures || consecutiveFailures < 1) return 0;
+  // 2^30 ms already exceeds any sane maxMs; clamp the exponent so a very long
+  // outage cannot overflow into Infinity before the cap is applied.
+  const exponent = Math.min(consecutiveFailures - 1, 30);
+  return Math.min(baseMs * Math.pow(2, exponent), maxMs);
+}
+
+/**
+ * Whether a fetch should be attempted now, or skipped because the clone is
+ * inside its post-failure backoff window.
+ * @param {Object} state A fetch state from initialFetchState().
+ * @param {number} nowMs Current epoch ms.
+ * @returns {boolean} True when the caller should attempt the fetch.
+ */
+function shouldAttemptFetch(state, nowMs) {
+  if (!state) return true;
+  return nowMs >= (state.suppressedUntilMs || 0);
+}
+
+/**
+ * Advances the state after a failed fetch: one more consecutive failure, and a
+ * suppression window sized by the backoff curve. Returns a NEW state — callers
+ * reassign rather than mutate, so a stale reference can never half-apply.
+ * @param {Object} state The prior fetch state.
+ * @param {number} nowMs Current epoch ms.
+ * @param {Object} [opts] `{ baseMs, maxMs }` backoff overrides.
+ * @returns {Object} The next fetch state.
+ */
+function recordFetchFailure(state, nowMs, opts) {
+  const prior = state || initialFetchState();
+  const consecutiveFailures = (prior.consecutiveFailures || 0) + 1;
+  return {
+    ...prior,
+    consecutiveFailures,
+    suppressedUntilMs: nowMs + nextBackoffMs(consecutiveFailures, opts)
+  };
+}
+
+/**
+ * Clears the failure streak after a successful fetch, so recovery is immediate
+ * — one good fetch drops the clone straight back to the normal poll cadence
+ * rather than walking the backoff curve back down.
+ * @param {Object} state The prior fetch state.
+ * @returns {Object} The next fetch state.
+ */
+function recordFetchSuccess(state) {
+  const prior = state || initialFetchState();
+  return { ...prior, consecutiveFailures: 0, suppressedUntilMs: 0 };
+}
+
+/**
+ * Whether `git gc --auto` is due on this clone.
+ * @param {Object} state The fetch state (carries lastGcAtMs).
+ * @param {number} nowMs Current epoch ms.
+ * @param {number} [intervalMs] How often gc should run.
+ * @returns {boolean} True when gc is due.
+ */
+function shouldRunGc(state, nowMs, intervalMs) {
+  const every = intervalMs || DEFAULT_GC_INTERVAL_MS;
+  const last = (state && state.lastGcAtMs) || 0;
+  return nowMs - last >= every;
+}
+
+/** Stamps the gc clock. Returns a NEW state. */
+function recordGcRun(state, nowMs) {
+  return { ...(state || initialFetchState()), lastGcAtMs: nowMs };
+}
+
+/**
+ * The shell command that reaps abandoned `tmp_pack_*` files from a clone's pack
+ * directory. Age-gated so a temp pack belonging to an in-flight fetch is never
+ * deleted out from under it, and `-maxdepth 1` keeps it from wandering into
+ * whatever else lives under .git.
+ * @param {string} repoRoot Absolute path to the clone's working tree.
+ * @param {number} [minAgeMinutes] Only delete temp packs older than this.
+ * @returns {string} A `find ... -delete` command safe to exec.
+ */
+function stalePackPruneCommand(repoRoot, minAgeMinutes) {
+  const age = minAgeMinutes === undefined ? DEFAULT_STALE_PACK_MIN_AGE_MINUTES : minAgeMinutes;
+  return `find "${repoRoot}/.git/objects/pack" -maxdepth 1 -type f -name 'tmp_pack_*' -mmin +${age} -delete`;
+}
+
+module.exports = {
+  initialFetchState,
+  nextBackoffMs,
+  shouldAttemptFetch,
+  recordFetchFailure,
+  recordFetchSuccess,
+  shouldRunGc,
+  recordGcRun,
+  stalePackPruneCommand,
+  DEFAULT_BASE_BACKOFF_MS,
+  DEFAULT_MAX_BACKOFF_MS,
+  DEFAULT_STALE_PACK_MIN_AGE_MINUTES,
+  DEFAULT_GC_INTERVAL_MS
+};

package/npm/fleet-host-op-limiter.js

@@ -0,0 +1,68 @@
+'use strict';
+// Global host-CPU concurrency limiter for the Fleet Dashboard's gcloud-ssh work
+// (scripts/operator/fleet-dashboard/server.js requires this). No gcloud/SSH I/O
+// itself — this module owns ONE guarantee: at most `concurrency` wrapped tasks
+// run at any instant, the rest queue FIFO. The server funnels its per-VM
+// gcloud-ssh gathers through a single shared instance so the operator host runs
+// a bounded number of `gcloud compute ssh` (Python, ~100 MB RSS + real CPU
+// apiece) processes REGARDLESS of fleet size.
+//
+// The bug this fixes: monitoring fanned out one `gcloud compute ssh … docker
+// exec` per VM per gather with no host-wide cap. `pollEffects` sized its
+// reachable-sweep and launcher-strand-sweep worker pools INDEPENDENTLY
+// (EFFECT_GATHER_CONCURRENCY each), and neither shared a budget with the heavy
+// mutation spawns — so 3 gather SSHs + 3 strand SSHs + N concurrent build SSHs
+// could all run at once. On an 8-vCPU operator host that meant 40–60 load, the
+// gcloud calls hanging in D-state and piling up faster than they completed, and
+// the dashboard itself CPU-starved (local_http=000). A single shared limiter
+// caps the instantaneous gcloud process count so the fan-out can never outrun
+// the host again.
+//
+// Deliberately a general async semaphore (not gcloud-aware): the caller passes a
+// thunk that does the I/O, so this stays pure and unit-testable with no network.
+
+// Build a concurrency limiter. `run(fn)` enqueues the zero-arg thunk `fn` and
+// returns a promise that settles with fn's resolved value (or rejects with its
+// error). At most `concurrency` thunks are in flight at once; the rest wait in a
+// FIFO queue and start as slots free. A thunk's slot is released whether it
+// resolves OR rejects, so one failing gcloud call can never leak a permanent
+// slot and starve the queue. `concurrency` is floored to 1 (a 0 / negative /
+// NaN value would otherwise wedge every task in the queue forever).
+function createHostOpLimiter(opts) {
+  const o = opts || {};
+  const raw = Number(o.concurrency);
+  const concurrency = Number.isFinite(raw) && raw >= 1 ? Math.floor(raw) : 1;
+  let active = 0;
+  const queue = [];
+
+  const pump = () => {
+    while (active < concurrency && queue.length > 0) {
+      const { fn, resolve, reject } = queue.shift();
+      active += 1;
+      // `Promise.resolve().then(fn)` normalizes a thunk that returns a value, a
+      // promise, OR throws synchronously into one promise chain — so a synchronous
+      // throw releases the slot via the reject path instead of escaping `pump`.
+      Promise.resolve().then(fn).then(
+        (val) => { active -= 1; pump(); resolve(val); },
+        (err) => { active -= 1; pump(); reject(err); },
+      );
+    }
+  };
+
+  const run = (fn) => new Promise((resolve, reject) => {
+    queue.push({ fn, resolve, reject });
+    pump();
+  });
+
+  return {
+    run,
+    // Introspection for tests + operator diagnostics; never load-bearing.
+    active: () => active,
+    queued: () => queue.length,
+    concurrency,
+  };
+}
+
+module.exports = {
+  createHostOpLimiter,
+};

package/npm/fleet-iap-token.js

@@ -0,0 +1,165 @@
+'use strict';
+// IAP identity-token caching for the fleet dashboard's per-VM probes.
+//
+// On the GCE operator host the dashboard reaches each VM through its
+// IAP-fronted LB hostname, which requires an OIDC ID token whose audience is
+// the fleet's IAP OAuth client id.
+//
+// There are two mint paths, and they are NOT interchangeable:
+//
+//   - `buildGceIdentityTokenRequest` — the metadata server's identity endpoint
+//     with `format=full`. The only one IAP actually accepts, because IAP
+//     authorizes on the token's `email` claim and only `format=full` carries it.
+//     This is the dashboard's path on the operator host.
+//   - `gcloudMintArgs` — `gcloud auth print-identity-token`, which mirrors
+//     `scripts/operator/fleet-redeploy.sh`. Retained for the off-GCE (laptop)
+//     path, where a human account's gcloud session is the credential. It does
+//     NOT emit an email claim, so it cannot authenticate the on-GCE probe.
+//
+// The dashboard polls every VM every few seconds, so minting per probe would
+// fork a gcloud process per VM per poll. Instead we mint once and cache until
+// shortly before the token's own expiry.
+//
+// Pure logic only — the token cache, the JWT expiry read, and the gcloud argv
+// are all pure; the caller injects `mintToken` (the actual subprocess) and
+// `now`. Keeps it unit-testable in isolation like vm-urls.js.
+
+// Refresh this long before the token's stated expiry. An IAP token lives ~1h;
+// renewing a minute early means an in-flight poll can never race the boundary
+// and get a 401 that would render as "auth misconfigured" on every card.
+const EXPIRY_SKEW_MS = 60_000;
+
+// Fallback lifetime when a minted token carries no readable `exp`. Deliberately
+// short: re-minting slightly too often is cheap, serving a dead token is not.
+const FALLBACK_TTL_MS = 5 * 60_000;
+
+/** The gcloud argv that mints an IAP identity token for `clientId`.
+ *  Mirrors fleet-redeploy.sh's `gcloud_sa auth print-identity-token
+ *  --audiences=<IAP_CLIENT_ID>` so both sides request the same audience. */
+function gcloudMintArgs({ clientId } = {}) {
+  const id = clientId == null ? '' : String(clientId).trim();
+  if (id === '') throw new Error('gcloudMintArgs requires an IAP client id');
+  return ['auth', 'print-identity-token', `--audiences=${id}`];
+}
+
+// The GCE metadata server's OIDC identity endpoint. Reachable only from a GCE
+// instance, and only with the `Metadata-Flavor: Google` header.
+const GCE_IDENTITY_URL =
+  'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity';
+
+/** The metadata-server request that mints an IAP identity token for `audience`.
+ *
+ *  Two details are load-bearing and were validated live against the fleet IAP
+ *  on 2026-07-19:
+ *
+ *  - `format=full` is what makes the token carry `email` + `email_verified`.
+ *    IAP authorizes on the email, so the default (`format=standard`) token —
+ *    and equally `gcloud auth print-identity-token`, which also omits email —
+ *    is rejected with a 401 no matter how correct the audience is.
+ *  - `audience` must be the MANUAL OAuth client id the backends are configured
+ *    with. Google-managed IAP exposes no usable programmatic audience: the
+ *    client id recovered from IAP's browser 302 is rejected as `Invalid JWT
+ *    audience`. See scripts/gcp/google-ingress.sh step 8.
+ */
+function buildGceIdentityTokenRequest({ audience } = {}) {
+  const aud = audience == null ? '' : String(audience).trim();
+  if (aud === '') throw new Error('buildGceIdentityTokenRequest requires an IAP client id');
+  return {
+    url: `${GCE_IDENTITY_URL}?audience=${encodeURIComponent(aud)}&format=full`,
+    headers: { 'Metadata-Flavor': 'Google' },
+  };
+}
+
+/** Which mint path applies, given the host and the configured audience.
+ *
+ *  Three outcomes, and the distinction between the last two is the fix:
+ *    'unconfigured' — no client id. There is no audience to mint for, so the
+ *                     probe must report 'auth-misconfigured' rather than mint
+ *                     a token IAP will reject.
+ *    'metadata'     — on the GCE operator host. Only the metadata endpoint's
+ *                     format=full token carries the `email` claim IAP
+ *                     authorizes on.
+ *    'gcloud'       — off GCE (laptop), where there is no metadata server and
+ *                     a human gcloud session is the credential.
+ */
+function selectMintStrategy({ onGce, clientId } = {}) {
+  const id = clientId == null ? '' : String(clientId).trim();
+  if (id === '') return 'unconfigured';
+  return onGce ? 'metadata' : 'gcloud';
+}
+
+/** Read the `exp` claim (ms since epoch) out of a JWT without verifying it.
+ *  We are not authenticating the token — the LB does that — we only need to
+ *  know when to stop reusing it. Returns null for anything unparseable, which
+ *  sends the caller to FALLBACK_TTL_MS. */
+function decodeJwtExpiryMs(token) {
+  const raw = token == null ? '' : String(token).trim();
+  const parts = raw.split('.');
+  if (parts.length !== 3) return null;
+  try {
+    const payload = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8'));
+    const exp = Number(payload && payload.exp);
+    if (!Number.isFinite(exp) || exp <= 0) return null;
+    return exp * 1000;
+  } catch {
+    return null;
+  }
+}
+
+/** Build a caching IAP-token provider.
+ *
+ *  `mintToken()` must resolve to a raw token string (or null/'' on failure);
+ *  `now()` returns epoch ms. Returns `{ getToken }`, where getToken() resolves
+ *  to a usable token or null. Null is a first-class outcome — the probe layer
+ *  turns it into 'auth-misconfigured' rather than a false "unreachable", so a
+ *  broken credential is legible on the card instead of looking like a dead VM.
+ *
+ *  Concurrent callers share one in-flight mint: the dashboard probes N VMs in
+ *  parallel each poll, and without this a cold cache would fork N gcloud
+ *  processes simultaneously.
+ */
+function createIapTokenProvider({ mintToken, now, skewMs = EXPIRY_SKEW_MS } = {}) {
+  if (typeof mintToken !== 'function') {
+    throw new Error('createIapTokenProvider requires a mintToken function');
+  }
+  const clock = typeof now === 'function' ? now : Date.now;
+
+  let cached = null;
+  let expiresAtMs = 0;
+  let inFlight = null;
+
+  async function mintAndCache() {
+    const token = await mintToken();
+    const value = token == null ? '' : String(token).trim();
+    if (value === '') {
+      cached = null;
+      expiresAtMs = 0;
+      return null;
+    }
+    const exp = decodeJwtExpiryMs(value);
+    cached = value;
+    expiresAtMs = exp == null ? clock() + FALLBACK_TTL_MS : exp;
+    return cached;
+  }
+
+  async function getToken() {
+    if (cached !== null && clock() + skewMs < expiresAtMs) return cached;
+    if (inFlight === null) {
+      inFlight = mintAndCache().finally(() => { inFlight = null; });
+    }
+    return inFlight;
+  }
+
+  return { getToken };
+}
+
+module.exports = {
+  EXPIRY_SKEW_MS,
+  FALLBACK_TTL_MS,
+  GCE_IDENTITY_URL,
+  gcloudMintArgs,
+  buildGceIdentityTokenRequest,
+  selectMintStrategy,
+  decodeJwtExpiryMs,
+  createIapTokenProvider,
+};

package/npm/fleet-job-badge.js

@@ -15,21 +15,37 @@
 // Should the VM's last terminal-error job badge be reconciled against live
 // health — i.e. presented as "recovered" rather than a live red error?
 //
-//   job        the carried job record ({ state, action, ... }) or null
-//   reachable  the VM's editor answered this poll (true = healthy / responding)
-//   drifted    a project-dir-drift signal is present (reachable-but-unwired /
-//              half-provisioned add — genuinely needs-recovery, owned by the
-//              companion reconcile-inflight-jobs plan; never falsely cleared here)
-//   offBranch  the VM's live client branch differs from its target branch
-//   degraded   the job completed but a wiring sub-step is flagged degraded
+//   job          the carried job record ({ state, action, ... }) or null
+//   reachable    the VM's editor answered this poll over the LB (true = healthy)
+//   inVmHealthy  the in-VM editor answered /api/health from INSIDE the container
+//                (the STRAND_PROBE `HEALTH=1` witness gathered over docker-exec)
+//   drifted      a project-dir-drift signal is present (reachable-but-unwired /
+//                half-provisioned add — genuinely needs-recovery, owned by the
+//                companion reconcile-inflight-jobs plan; never falsely cleared here)
+//   offBranch    the VM's live client branch differs from its target branch
+//   degraded     the job completed but a wiring sub-step is flagged degraded
 //
 // Returns true only when the job is a terminal error AND the VM is demonstrably
-// fine (reachable, not drifted, on-branch, not degraded). An unreachable /
-// drifted / off-branch VM keeps its error surfaced — a real problem must stay
-// visible.
-function shouldReconcileFailedBadge({ job, reachable, drifted, offBranch, degraded } = {}) {
+// fine (healthy, not drifted, on-branch, not degraded). A drifted / off-branch /
+// unwitnessed VM keeps its error surfaced — a real problem must stay visible.
+//
+// HEALTH IS WITNESSED TWO WAYS, and either suffices. `reachable` alone made the
+// self-heal unable to fire in exactly the case that needs it most: on the GCE
+// operator host `reachable` is derived from the LB probe, which needs the very
+// ingress the add is still wiring. That is a circular dependency — the badge
+// needs reachable, reachable needs ingress, and ingress is what we are waiting
+// for — so a still-wiring VM sat at unreachableStreak 9 with a red add-error it
+// could not clear by construction (VM-4, 2026-07-31). The dashboard already held
+// the disconfirming evidence: gatherLauncherStrand runs STRAND_PROBE over
+// docker-exec for exactly these unreachable-with-streak VMs and reads HEALTH=1
+// from the in-VM /api/health.
+//
+// This widens ONE input; it does not relax the verdict. Every other guard is
+// untouched, and the "absence of evidence never auto-acts" posture holds — an
+// unprobed VM has neither witness and stays unreconciled.
+function shouldReconcileFailedBadge({ job, reachable, inVmHealthy, drifted, offBranch, degraded } = {}) {
   if (!job || job.state !== 'error') return false;
-  if (!reachable) return false;
+  if (!reachable && !inVmHealthy) return false;
   if (drifted) return false;
   if (offBranch) return false;
   if (degraded) return false;

package/npm/fleet-live-status.js

@@ -42,13 +42,13 @@ function liveSessionStatus(v, brokerStatus) {
   // Unreachable: we polled nothing reliable. Defer to the caller's
   // asleep/unreachable mapping — a stale broker signal must not mask it.
   if (!v.reachable) return null;
-  if (v.hasSession && v.awaitingUserInput) return 'waiting';
+  if (v.hasSession && v.awaitingUserInput) return awaitingVerdict(v);
   if (v.hasSession) return 'building';
   // No editor-owned session, but a live agent may still be working in the PTY
   // broker (decoupled lifetime — see docs/process-model.md). The broker
   // tri-state is already debounced upstream; agent-live ⇒ working, not idle.
   if (brokerStatus === 'agent-live') {
-    if (v.awaitingUserInput) return 'waiting';
+    if (v.awaitingUserInput) return awaitingVerdict(v);
     // The agent is parked alive in the broker's build slot, but its workflow
     // reached the terminal step (editor reports needsCleanup) — finished and
     // idle, not building. A mid-feature post-restart agent is NOT terminal and
@@ -59,6 +59,26 @@ function liveSessionStatus(v, brokerStatus) {
   return 'idle';
 }
 
+// Does the VM have a task/feature genuinely running? `feature`/`featureStartedAt`
+// (from /api/session-info) are both None at a fresh prompt and Some once a build
+// feature is selected, so either being present means a task is in flight.
+function hasActiveFeature(v) {
+  return !!(v.feature || v.featureStartedAt);
+}
+
+// Discriminate an `awaitingUserInput` agent between two very different shapes
+// (observed live 2026-06-30, VM-1): an agent parked at its prompt with NO task
+// running is IDLE — ready and waiting for YOU to start something — not the same
+// as a running task BLOCKED on a human answer. Only the latter is 'waiting'
+// (which the needs-you overlay later escalates). A no-feature, non-terminal
+// awaiting agent reads 'idle' so a freshly-launched healthy VM never false-flags
+// "Waiting"/"Needs You". `needsCleanup` (the terminal-done signal the module
+// already trusts) keeps a finished session out of the idle bucket.
+function awaitingVerdict(v) {
+  if (!v.needsCleanup && !hasActiveFeature(v)) return 'idle';
+  return 'waiting';
+}
+
 // Distinguish a freshly-rostered VM whose editor is still STARTING UP — a normal
 // cold boot that hasn't bound :14199 yet, or (the VM-2 case) the ~10–15-min
 // self-host `rebuild-self --force-in-container` compile that recompiles the
@@ -109,15 +129,27 @@ function classifyStartupProbe({ stdout, err } = {}) {
 
 // Pure gate deciding whether pollStartupProbes should fire the (costly) IAP-SSH
 // build probe for a given VM this tick. We only probe a VM that is PLAUSIBLY
-// still starting — present, unreachable, GCP still lists it RUNNING, NOT already
-// owned by an active heavy job, and either recently launched OR already confirmed
+// still starting — present, unreachable, GCP still lists it RUNNING, not blocked
+// by an active heavy job, and either recently launched OR already confirmed
 // building on a prior tick (so a ~15-min self-host compile keeps being probed
 // past the grace window with no fixed cap). A healthy/reachable VM, an
 // asleep/vanished one, or a long-idle one that simply went unreachable is never
-// probed. Pure so pollStartupProbes stays a thin SSH loop and this decision
-// matrix is exhaustively unit-testable without booting the server.
-function shouldProbeStartup({ vmPresent = false, reachable = false, gcpRunning = false, jobActive = false, recentlyLaunched = false, wasBuilding = false } = {}) {
-  return vmPresent && !reachable && gcpRunning && !jobActive && (recentlyLaunched || wasBuilding);
+// probed.
+//
+// The add-job exception (observed live 2026-06-30, VM-1): an active ADD job does
+// NOT block the probe. The long self-host container build runs *under* the add,
+// so we must probe DURING it to prime `wasBuilding` early — otherwise, by the
+// time the add completes the recentlyLaunched grace has already lapsed, no prior
+// probe ever ran (the old `!jobActive` guard suppressed them all), and the
+// still-booting editor falls straight to red 'unreachable' instead of calm
+// 'editor-starting'. Priming `wasBuilding` during the add lets the probe chain
+// carry the card calm past the grace with no fixed cap. Any OTHER active heavy
+// job (rebuild/destroy/reconfigure/…) still gates the probe out — its own status
+// already owns the card. Pure so pollStartupProbes stays a thin SSH loop and this
+// decision matrix is exhaustively unit-testable without booting the server.
+function shouldProbeStartup({ vmPresent = false, reachable = false, gcpRunning = false, jobActive = false, jobIsAdd = false, recentlyLaunched = false, wasBuilding = false } = {}) {
+  const blockedByJob = jobActive && !jobIsAdd;
+  return vmPresent && !reachable && gcpRunning && !blockedByJob && (recentlyLaunched || wasBuilding);
 }
 
 module.exports = { liveSessionStatus, classifyStartingVsUnreachable, classifyStartupProbe, shouldProbeStartup };