@codeyam-editor/codeyam-editor 0.1.2 → 0.1.3

package/npm/fleet-cleanup.js

@@ -76,31 +76,136 @@ function classifyDescribeError(err, stderr) {
 }
 
 // Decide what the startup reap should do for one sidecar. Pure: takes the
-// decoded sidecar + GCP-existence verdict + (for adds) container-up verdict,
-// returns a tag the caller dispatches on.
-//   verdict             when                                          dispatch
-//   'cleanup'           destroy gone, OR add gone                     runDestroyCleanup / runAddCleanup
-//   'reconcile-add'     add up AND container up                       ensure roster + recordLaunch, then clear
-//   'recover-bootstrap' add instance up but container missing/down    re-run cloud:up to finish the half-done bootstrap
-//   'clear-only'        destroy sidecar but instance still exists     just clear the sidecar
-//   'leave'             gcloud uncertain (exists === null)             leave for the next boot
-// The (exists, containerUp) split is what distinguishes a finished add from a
-// half-provisioned one: a dashboard SIGKILL'd mid-`cloud:up` leaves a GCP
+// decoded sidecar + GCP-existence verdict + (for adds) container-up verdict +
+// (for up adds) project-wiring verdict, returns a tag the caller dispatches on.
+//   verdict             when                                              dispatch
+//   'cleanup'           destroy gone, OR add gone                         runDestroyCleanup / runAddCleanup
+//   'reconcile-add'     add up, container up, project wired (or unknown)  ensure roster + recordLaunch, then clear
+//   'half-provisioned'  add up, container up, but project NOT wired       surface needs-recovery card (finish-wire from cfg)
+//   'recover-bootstrap' add instance up but container missing/down        re-run cloud:up to finish the half-done bootstrap
+//   'clear-only'        destroy sidecar but instance still exists         just clear the sidecar
+//   'leave'             gcloud uncertain (exists === null)                 leave for the next boot
+// The (exists, containerUp) split distinguishes a finished add from a container-
+// down half-provisioned one: a dashboard SIGKILL'd mid-`cloud:up` leaves a GCP
 // instance with no docker/container, which the old (exists-only) matrix wrongly
-// classified 'reconcile-add' — adding an unreachable VM to the roster.
-// `containerUp` is only consulted for an add whose instance exists; destroy and
-// the instance-gone add path ignore it. Splitting the matrix out lets the table
-// be exhaustively tested without any of the side-effects in server.js.
-function reapVerdict({ action, exists, containerUp }) {
+// classified 'reconcile-add' — adding an unreachable VM to the roster. The
+// `projectWired` axis closes the remaining gap (improve39 VM-7): an instance +
+// container that ARE up but whose `/workspace` is unwired (no git remote, empty
+// manifest) is NOT a finished add — it's half-provisioned and must surface a
+// recoverable card, not be silently reconciled "done". Only a POSITIVE unwired
+// signal (`projectWired === false`) downgrades to 'half-provisioned'; an
+// unknown/unprobed wiring (`null`/`undefined`) stays 'reconcile-add' so a flaky
+// probe can't strand a healthy VM. `containerUp`/`projectWired` are only consulted
+// for an add whose instance exists; destroy and the instance-gone add path ignore
+// them. Splitting the matrix out lets the table be exhaustively tested without any
+// of the side-effects in server.js.
+function reapVerdict({ action, exists, containerUp, projectWired }) {
   if (exists === null) return 'leave';
   if (action === 'destroy') return exists ? 'clear-only' : 'cleanup';
   if (action === 'add') {
     if (!exists) return 'cleanup';
-    return containerUp ? 'reconcile-add' : 'recover-bootstrap';
+    if (!containerUp) return 'recover-bootstrap';
+    return projectWired === false ? 'half-provisioned' : 'reconcile-add';
   }
   return 'leave';
 }
 
+// Classify whether a VM's editor `/workspace` is actually wired to a client
+// project, from a best-effort probe of two signals: the git remote URL and
+// whether `package.json` carries a real `name`. Pure: server.js runs the SSH +
+// `docker exec` probe and hands the raw stdout/err here so the decision is
+// unit-testable without a live VM.
+//   - err present (ssh/docker blip, timeout)        → null  (unknown — do NOT
+//     strand a healthy VM on a flaky probe; reapVerdict treats null as wired)
+//   - a git remote OR a non-empty package.json name → true  (wired)
+//   - neither (empty / no-git workspace)            → false (half-provisioned —
+//     the VM-7 signature: `/workspace` with no remote and an empty manifest)
+// The probe emits the two signals separated by the `---CY---` sentinel so this
+// parser is independent of locale/line-ending noise in the ssh transport.
+function classifyProjectWiring({ stdout, err } = {}) {
+  if (err) return null;
+  const parts = String(stdout == null ? '' : stdout).split('---CY---').map((s) => s.trim());
+  const remote = parts[0] || '';
+  const nameCount = Number((parts[1] || '0').trim()) || 0;
+  if (remote || nameCount > 0) return true;
+  return false;
+}
+
+// Durable exit marker for a detached `add` provision (improve40). The add's
+// `cloud:up` shell is spawned in its OWN session (detached) so a dashboard
+// SIGKILL — the 2GB launchd hard cap, exactly the kill that aborted the VM-7
+// add mid-flight — no longer reaches it; the provision keeps running. When that
+// detached shell finally exits it writes `add-<n>.exit` (its trailing exit code
+// + an ISO timestamp) into PENDING_JOBS_DIR, alongside the `add-<n>.json`
+// sidecar. The marker is the GROUND-TRUTH completion signal the next boot reads
+// instead of inferring `in-flight → error` from the lost child: a marker means
+// "the provision finished while the dashboard was away," its code says whether
+// it finished OK. The name is derived here so the shell `trap` (server.js) and
+// the JS reader build the exact same path.
+function exitMarkerName(n) {
+  return `add-${n}.exit`;
+}
+
+// Parse an exit-marker file body into { exitCode, finishedAt }, or null when the
+// body is missing/blank/malformed. Format is two lines — the numeric exit code
+// then an ISO timestamp — written by the shell trap. A non-integer first line is
+// treated as no-marker (null) so a half-written / corrupt marker falls back to
+// the gcloud reconcile path rather than being read as a bogus exit code. Pure:
+// takes the text so it's unit-testable without touching disk.
+function parseExitMarker(text) {
+  if (text == null) return null;
+  const lines = String(text).split('\n').map((s) => s.trim()).filter(Boolean);
+  if (!lines.length) return null;
+  const code = Number(lines[0]);
+  if (!Number.isInteger(code)) return null;
+  return { exitCode: code, finishedAt: lines[1] || null };
+}
+
+// Read VM-n's exit marker from `dir`, or null when it's absent/unreadable/
+// malformed (the still-running and never-written cases both surface as null).
+function readExitMarker(dir, n) {
+  try { return parseExitMarker(fs.readFileSync(path.join(dir, exitMarkerName(n)), 'utf8')); }
+  catch { return null; }
+}
+
+// Remove a consumed exit marker; a missing file is the already-cleared path, not
+// an error (mirrors clearPendingJob).
+function clearExitMarker(dir, n) {
+  try { fs.unlinkSync(path.join(dir, exitMarkerName(n))); }
+  catch (_) { /* missing is fine */ }
+}
+
+// Decide how a snapshot-restored `add` should be re-attached after a dashboard
+// restart, BEFORE any gcloud probe. Pure: takes the durable signals the boot
+// path gathers — whether the exit marker exists, its recorded code, and whether
+// the detached provision child is still alive (recorded pid, `kill -0`).
+//   verdict       when                                          dispatch
+//   'done'        marker present, exit code 0                   roster + recordLaunch, clear sidecar/marker
+//   'error'       marker present, non-zero exit code            surface the real failure, clear sidecar/marker
+//   'running'     no marker yet, child still alive              RE-ATTACH: keep polling, do NOT mark error
+//   'reconcile'   child gone, no usable marker                  fall through to fc.reapVerdict (gcloud probe)
+// This is the durable fix's brain: a provision that survived the kill (still
+// running, or finished while we were away) is classified from GROUND TRUTH, so
+// it is never blanket-converted `in-flight → error` the way the snapshot-only
+// path did (the improve39 VM-7 false `add-failed`). Only when the child is gone
+// AND left no parseable marker do we fall back to inferring state from gcloud —
+// `reconcile` hands that exact (exists, containerUp, projectWired) decision to
+// fc.reapVerdict. A marker whose first line isn't an integer is treated as
+// absent (parseExitMarker → null) so a corrupt marker reconciles rather than
+// asserting a bogus code. The matrix is exhaustively tested without a live VM.
+function reattachVerdict({ markerPresent, exitCode, childAlive } = {}) {
+  if (markerPresent) {
+    // Guard the null/undefined coercion BEFORE Number() — `Number(null)` is 0,
+    // which would masquerade as a clean exit. A marker with no parseable code
+    // (half-written / corrupt) must reconcile, not assert success.
+    const code = exitCode == null ? NaN : Number(exitCode);
+    if (Number.isInteger(code)) return code === 0 ? 'done' : 'error';
+    return 'reconcile'; // marker present but no parseable code — let gcloud decide
+  }
+  if (childAlive) return 'running';
+  return 'reconcile';
+}
+
 // Anti-loop budget for the `recover-bootstrap` path. Pure: given the count of
 // recoveries ALREADY persisted on the sidecar, return the next attempt number,
 // whether the reap should proceed (re-run cloud:up) or treat the VM as
@@ -113,6 +218,66 @@ function recoverBootstrapDecision({ recoverAttempts, max = 3 }) {
   return { attempt, proceed: attempt <= max, max };
 }
 
+// improve40 — select the orphaned `add` jobs that must be reconciled from the
+// VM's REAL provision state even though their per-job sidecar is GONE. The
+// durable re-attach path (sidecar + exit marker + recorded pid) cannot fire for
+// these — a restart left only the snapshot record — so the snapshot job's
+// stashed `config` is the sole cfg source and a live gcloud probe is the only
+// ground truth. Pure: takes the restored jobs map + the set of `<action>-<n>`
+// sidecar keys the sidecar reap already owns; returns the [{ n, cfg }] the boot
+// path hands to reconcileViaGcloud. A job is selected when ALL hold:
+//   - it is an `add`,
+//   - it is flagged `restartOrphaned` (restoreFromSnapshot downgraded a
+//     running add → error because the dashboard restarted before it finished —
+//     this is the ONLY error state we re-probe; an add that failed IN-PROCESS
+//     carries a plain terminal `error` with no flag and stays surfaced), AND
+//   - NO `add-<n>` sidecar exists (a sidecar means the sidecar reap already
+//     owns it — never double-reconcile).
+// `cfg` rides from the job's `config`; null when the snapshot never carried one
+// (older snapshot) — the caller still probes, but a roster reconcile then can't
+// recordLaunch, exactly mirroring the sidecar path's `meta.cfg`-absent case.
+function orphanedAddsToReconcile({ jobs, sidecarKeys } = {}) {
+  const sidecars = sidecarKeys instanceof Set ? sidecarKeys : new Set(sidecarKeys || []);
+  const src = (jobs && typeof jobs === 'object') ? jobs : {};
+  const out = [];
+  for (const [k, j] of Object.entries(src)) {
+    if (!j || typeof j !== 'object') continue;
+    if (j.action !== 'add') continue;
+    if (!j.restartOrphaned) continue;
+    if (sidecars.has(`add-${k}`)) continue;
+    out.push({ n: Number(k), cfg: j.config || null });
+  }
+  return out;
+}
+
+// improve41 — the destroy-direction mirror of orphanedAddsToReconcile: decide
+// what to do with a VM that is STILL in the roster but whose GCP instance state
+// we have just re-probed. The bug this closes: a Destroy that races a dashboard
+// restart can leave the instance deleted while the startup reap took the
+// `clear-only` path (instance still present at reap time) and dropped the only
+// sidecar — so nothing ever runs roster cleanup and the VM polls forever
+// "unreachable." A sidecar-independent reconcile, keyed on REAL GCP existence
+// rather than on a pending-job sidecar, catches it. Pure: server.js runs the
+// gcloud describe (→ exists, via classifyDescribeError) and reads session /
+// pending-add state, and hands the three facts here so the matrix is testable
+// without a live VM.
+//   verdict     when                                        dispatch
+//   'cleanup'   instance gone AND the VM is idle            runDestroyCleanup
+//   'leave'     exists uncertain (null), OR the VM is busy  do nothing this pass
+//   'keep'      instance still exists                       do nothing (healthy)
+// Fail-safe by construction: cleanup fires ONLY on a DEFINITIVE absence
+// (exists === false) — a flaky/transient describe (null) is always 'leave', so a
+// gcloud blip can never evict a healthy VM (mirroring reapVerdict's `leave`
+// posture). "Busy" (a live session or an in-flight add to this number) also
+// holds the entry for an operator rather than auto-removing — the shouldn't-
+// happen-but-fail-safe case where a vanished instance still reports a session.
+function reconcileRosteredVm({ exists, hasSession, hasPendingAdd } = {}) {
+  if (exists === null || exists === undefined) return "leave";
+  if (exists) return "keep";
+  if (hasSession || hasPendingAdd) return "leave";
+  return "cleanup";
+}
+
 module.exports = {
   parsePendingJobFilename,
   persistPendingJob,
@@ -120,5 +285,13 @@ module.exports = {
   listPendingJobs,
   classifyDescribeError,
   reapVerdict,
+  reconcileRosteredVm,
+  orphanedAddsToReconcile,
+  classifyProjectWiring,
   recoverBootstrapDecision,
+  exitMarkerName,
+  parseExitMarker,
+  readExitMarker,
+  clearExitMarker,
+  reattachVerdict,
 };

package/npm/fleet-effect-status.js

@@ -0,0 +1,179 @@
+'use strict';
+// Pure predicate module for the Fleet Dashboard's per-button "will clicking
+// this change the VM, or is it already in that state?" indicators
+// (scripts/operator/fleet-dashboard/server.js requires this). Given an
+// already-gathered FACTS object — no I/O, no git/docker exec; server.js does
+// the impure fact-gathering — computeButtonEffects returns a per-button verdict
+// map. Mirrors the pure-module-plus-colocated-vitest-test shape of
+// npm/fleet-reset.js so the whole decision table is unit-testable without
+// spawning a process.
+//
+// Each verdict is { effect, reason }:
+//   - 'changes'     — clicking does something; `reason` says what.
+//   - 'noop'        — already in that state; clicking changes nothing.
+//   - 'recommended' — soft signal (run-setup only): a reset/update landed after
+//                     the last successful setup. NEVER a confident green no-op.
+//   - 'unknown'     — not yet checked, VM unreachable, or a needed fact is
+//                     missing. We never CLAIM a safe no-op we couldn't verify —
+//                     silent failure here would be worse than no indicator.
+//
+// The predicate for each button is derived from what that button actually
+// touches (its tooltip in public/index.html is the source of truth): the sync
+// buttons key off a git commit comparison, the reset buttons off
+// working-tree/session/`.codeyam` state, and rebuild off a build-commit
+// comparison.
+
+// Short sha for compact reason strings. Tolerates null / short input.
+function short(sha) {
+  return sha ? String(sha).slice(0, 7) : '?';
+}
+
+// "origin/<branch>" — or a bare "origin" when the branch is unknown.
+function originRef(branch) {
+  return branch ? `origin/${branch}` : 'origin';
+}
+
+// "N commit(s) behind origin/<branch>" with correct singular/plural.
+function behindReason(behind, branch) {
+  return `${behind} commit${behind === 1 ? '' : 's'} behind ${originRef(branch)}`;
+}
+
+// Compose a short reason listing which signals make a reset non-trivial, so the
+// operator sees WHAT a Reset would discard rather than a bare "changes". The
+// `agent` (a live build agent in the broker) leads the list — it's the most
+// consequential thing a reset's `pty-broker stop` would silently discard.
+function resetReason({ agent, session, dirty, behind, codeyam }) {
+  const parts = [];
+  if (agent) parts.push('running build agent');
+  if (dirty) parts.push('uncommitted changes');
+  if (behind) parts.push('source behind origin');
+  if (session) parts.push('active session');
+  if (codeyam) parts.push('.codeyam state');
+  return parts.length ? `will discard: ${parts.join(', ')}` : 'will reset state';
+}
+
+// The action ids governed by an effect indicator — the same ids server.js /
+// index.html use for the buttons. Reconfigure / Env / Destroy are deliberately
+// absent (one-shots, not converge-to-a-state actions, so a no-op predicate is
+// meaningless for them).
+const ACTION_IDS = [
+  'update-client-source',
+  'update-editor-source',
+  'reset-editor-source',
+  'reset-client',
+  'rebuild-binary',
+  'update-reset-everything',
+  'run-setup',
+];
+
+// facts (a plain object gathered by server.js — no I/O):
+//   {
+//     reachable: boolean,
+//     hasSession: boolean,
+//     client:  { headSha, originSha, behind, dirty } | null,     // /workspace
+//     editor:  { headSha, originSha, behind, dirty, buildCommit } | null,
+//     codeyamDirty: boolean,
+//     setupRanSince: boolean,        // a setup completed after the last reset/update
+//     clientBranch: string | null,   // for reason text only
+//     editorBranch: string | null,   // for reason text only
+//   }
+function computeButtonEffects(facts) {
+  const f = facts || {};
+
+  // Unreachable VM (or no facts at all): every governed button is unknown. We
+  // could not reach the editor to gather facts, so never render a green no-op.
+  if (!f.reachable) {
+    const out = {};
+    for (const id of ACTION_IDS) out[id] = { effect: 'unknown', reason: 'VM unreachable' };
+    return out;
+  }
+
+  const client = f.client || null;
+  const editor = f.editor || null;
+  const hasSession = !!f.hasSession;
+  const codeyamDirty = !!f.codeyamDirty;
+  // A live build agent in the broker. A reset's first step is `pty-broker stop`,
+  // which kills it regardless of whether a workflow session is registered — so
+  // this is discardable work even when hasSession is false. Explicit-true only:
+  // a null/absent value (older or unreachable VM) must NOT flip a genuine noop.
+  const brokerLive = f.brokerLive === true;
+
+  // (1) Update client — ff-advance /workspace to origin tip. Changes iff behind.
+  const updateClient = client == null
+    ? { effect: 'unknown', reason: 'client state not checked' }
+    : client.behind > 0
+      ? { effect: 'changes', reason: behindReason(client.behind, f.clientBranch) }
+      : { effect: 'noop', reason: 'up to date with ' + originRef(f.clientBranch) };
+
+  // (2) Update editor — advance the editor source the binary builds from.
+  const updateEditor = editor == null
+    ? { effect: 'unknown', reason: 'editor state not checked' }
+    : editor.behind > 0
+      ? { effect: 'changes', reason: behindReason(editor.behind, f.editorBranch) }
+      : { effect: 'noop', reason: 'up to date with ' + originRef(f.editorBranch) };
+
+  // (5) Rebuild binary — advances source then compiles. Changes iff the running
+  // binary's build commit differs from the editor origin tip. Unknown when we
+  // don't have both commits to compare.
+  const rebuild = (editor == null || !editor.buildCommit || !editor.originSha)
+    ? { effect: 'unknown', reason: 'binary/source commit not checked' }
+    : editor.buildCommit !== editor.originSha
+      ? { effect: 'changes', reason: `binary at ${short(editor.buildCommit)}, ${originRef(f.editorBranch)} at ${short(editor.originSha)}` }
+      : { effect: 'noop', reason: 'binary built from ' + originRef(f.editorBranch) + ' tip' };
+
+  // (3) Reset editor — discards uncommitted + session, advances source, wipes
+  // .codeyam. No-op ONLY when clean at origin tip with no session.
+  const resetEditor = editor == null
+    ? { effect: 'unknown', reason: 'editor state not checked' }
+    : (brokerLive || hasSession || editor.dirty || editor.behind > 0 || codeyamDirty)
+      ? { effect: 'changes', reason: resetReason({ agent: brokerLive, session: hasSession, dirty: editor.dirty, behind: editor.behind > 0, codeyam: codeyamDirty }) }
+      : { effect: 'noop', reason: 'already clean at origin tip, no session' };
+
+  // (3b) Reset client — hard-clean /workspace + .codeyam, drop the session.
+  // No source-advance, so `behind` is NOT part of this predicate.
+  const resetClient = client == null
+    ? { effect: 'unknown', reason: 'client state not checked' }
+    : (brokerLive || client.dirty || codeyamDirty || hasSession)
+      ? { effect: 'changes', reason: resetReason({ agent: brokerLive, session: hasSession, dirty: client.dirty, behind: false, codeyam: codeyamDirty }) }
+      : { effect: 'noop', reason: 'already clean, no session' };
+
+  // (4b) Update & Reset Everything — composite. Changes iff ANY component would
+  // change: rebuild-binary, reset-editor-source, or the client being
+  // behind/dirty. Reason summarizes which parts.
+  const everything = (() => {
+    if (editor == null || client == null) {
+      return { effect: 'unknown', reason: 'state not checked' };
+    }
+    const parts = [];
+    if (rebuild.effect === 'changes') parts.push('rebuild');
+    if (resetEditor.effect === 'changes') parts.push('reset editor');
+    if (client.behind > 0 || client.dirty) parts.push('reset client');
+    if (parts.length > 0) return { effect: 'changes', reason: parts.join(' + ') };
+    // No part would change. But if a component was itself unverifiable, don't
+    // claim a confident no-op — fall back to unknown.
+    if (rebuild.effect === 'unknown' || resetEditor.effect === 'unknown') {
+      return { effect: 'unknown', reason: 'state not checked' };
+    }
+    return { effect: 'noop', reason: 'everything already at origin tip, no session' };
+  })();
+
+  // (run-setup) Honest soft signal — there is no cheap upstream proof that
+  // `npm run setup` would change nothing (DB/migration state isn't
+  // introspected). `recommended` when a reset/update landed after the last
+  // successful setup; otherwise `unknown`. Never a confident green no-op.
+  const runSetup = f.setupRanSince
+    ? { effect: 'unknown', reason: 'no setup-affecting change since last setup' }
+    : { effect: 'recommended', reason: 'a reset/update landed since the last setup' };
+
+  return {
+    'update-client-source': updateClient,
+    'update-editor-source': updateEditor,
+    'reset-editor-source': resetEditor,
+    'reset-client': resetClient,
+    'rebuild-binary': rebuild,
+    'update-reset-everything': everything,
+    'run-setup': runSetup,
+  };
+}
+
+module.exports = { computeButtonEffects, ACTION_IDS };

package/npm/fleet-grouping.js

@@ -0,0 +1,56 @@
+// Group an already-sorted/filtered VM list by client project for the Fleet
+// Dashboard's grouped layout.
+//
+// Returns an ORDERED array of { project, vms } groups: real project groups
+// first, sorted case-insensitively by name (locale compare); the
+// unknown/starting-up bucket (project === null) pinned LAST. Within each group
+// the incoming order of `vms` is preserved untouched, so the caller's active
+// Sort mode continues to drive within-group order — grouping never re-sorts the
+// cards, it only partitions them.
+//
+// `projectKey(v)` resolves the group key for a VM and defaults to the same
+// identity `card()` renders in the dashboard:
+//   (v.cardIdentity && v.cardIdentity.project) || v.project || null
+// It is injectable so unit tests can drive grouping without constructing full
+// card payloads. A null/empty key routes the VM into the trailing unknown
+// bucket so VMs mid-provision (blank identity) never silently disappear.
+//
+// Pure; no I/O.
+
+function defaultProjectKey(v) {
+  return (v && v.cardIdentity && v.cardIdentity.project) || (v && v.project) || null;
+}
+
+function groupVmsByProject(vms, projectKey = defaultProjectKey) {
+  // First pass: stable bucketing. `Map` preserves insertion order, so the
+  // first VM of each project fixes that group's relative position before we
+  // re-order — but we re-sort named groups below regardless, so insertion
+  // order only matters for tie-stability within equal-name groups (none here).
+  const named = new Map();
+  let unknown = null;
+
+  for (const v of vms) {
+    const raw = projectKey(v);
+    const key = raw == null || raw === "" ? null : raw;
+    if (key === null) {
+      if (!unknown) unknown = { project: null, vms: [] };
+      unknown.vms.push(v);
+      continue;
+    }
+    let group = named.get(key);
+    if (!group) {
+      group = { project: key, vms: [] };
+      named.set(key, group);
+    }
+    group.vms.push(v);
+  }
+
+  const groups = Array.from(named.values());
+  groups.sort((a, b) =>
+    a.project.localeCompare(b.project, undefined, { sensitivity: "base" })
+  );
+  if (unknown) groups.push(unknown);
+  return groups;
+}
+
+module.exports = { groupVmsByProject, defaultProjectKey };

package/npm/fleet-job-badge.js

@@ -0,0 +1,39 @@
+'use strict';
+
+// npm/fleet-job-badge.js — pure verdict for the fleet dashboard's failed-job
+// badge reconciliation (improve39). A VM's last terminal-error heavy job
+// (add/error, reset-all error, reset-editor error, …) used to stick on the
+// dashboard card indefinitely — even after the VM was recovered and is healthy
+// again — because the card renders the last job's error with no check against
+// the VM's live state, so a recovered VM keeps screaming a red error and
+// misleads the operator. This decides whether such a badge should be RECONCILED
+// (presented as "recovered" rather than a live error) because the VM is
+// demonstrably fine now. The live signals are gathered in server.js (the
+// untestable server shell); the verdict lives here so the cases are unit-tested
+// without a live VM.
+
+// Should the VM's last terminal-error job badge be reconciled against live
+// health — i.e. presented as "recovered" rather than a live red error?
+//
+//   job        the carried job record ({ state, action, ... }) or null
+//   reachable  the VM's editor answered this poll (true = healthy / responding)
+//   drifted    a project-dir-drift signal is present (reachable-but-unwired /
+//              half-provisioned add — genuinely needs-recovery, owned by the
+//              companion reconcile-inflight-jobs plan; never falsely cleared here)
+//   offBranch  the VM's live client branch differs from its target branch
+//   degraded   the job completed but a wiring sub-step is flagged degraded
+//
+// Returns true only when the job is a terminal error AND the VM is demonstrably
+// fine (reachable, not drifted, on-branch, not degraded). An unreachable /
+// drifted / off-branch VM keeps its error surfaced — a real problem must stay
+// visible.
+function shouldReconcileFailedBadge({ job, reachable, drifted, offBranch, degraded } = {}) {
+  if (!job || job.state !== 'error') return false;
+  if (!reachable) return false;
+  if (drifted) return false;
+  if (offBranch) return false;
+  if (degraded) return false;
+  return true;
+}
+
+module.exports = { shouldReconcileFailedBadge };

package/npm/fleet-memory.js

@@ -74,17 +74,30 @@ function decodeStateSnapshot(buf) {
  *  Returns null when the snapshot is missing, malformed, or older than
  *  maxAgeMs (stale snapshots are worse than no snapshot — pollVMs will
  *  repopulate from real data). Otherwise returns { jobs, vms, orphans }
- *  with two safety transforms applied to jobs:
+ *  with these safety transforms applied to jobs:
  *
- *    • `running` → `error` with a log marker, because the child process
- *      died with the dashboard; we can't possibly have a live job tracker
- *      after a process restart.
+ *    • `running` WITH a pending-job sidecar → `recovering` with a deferral
+ *      marker, NOT `error`. A heavy job that stashed a recoverable sidecar
+ *      (its cfg) must defer its final state to the startup reap, which
+ *      actually checks gcloud — the blanket `error` was the improve39 VM-7
+ *      bug: a mid-add restart stranded an instance that was really up as a
+ *      dead `add-failed`. `recovering` is an honest "the reap will resolve
+ *      this" surface; the reap then supersedes it (reconcile/half-provisioned
+ *      /cleanup) once it has the gcloud verdict.
+ *    • `running` WITHOUT a sidecar → `error` with a log marker, because the
+ *      child process died with the dashboard and there's nothing to recover
+ *      from — un-recoverable history, the old behavior.
  *    • `queued`  → dropped, because the in-memory pendingJobQueue array
  *      is gone, so promoteFromJobQueue can never start them. The operator
  *      will re-click (the queued click had done no work yet).
  *
+ *  `sidecarKeys` is the set of `${action}-${n}` strings the caller read from
+ *  the pending-jobs dir (server.js passes it; tests pass a plain array/Set).
+ *  Omitted → no job has a sidecar → every running job downgrades to `error`
+ *  exactly as before, so existing callers/tests are unaffected.
+ *
  *  Pure; no I/O. */
-function restoreFromSnapshot({ snapshot, maxAgeMs, nowIso } = {}) {
+function restoreFromSnapshot({ snapshot, maxAgeMs, nowIso, sidecarKeys } = {}) {
   if (!snapshot || typeof snapshot !== 'object') return null;
   if (typeof snapshot.timestamp !== 'string') return null;
   const ts = Date.parse(snapshot.timestamp);
@@ -95,16 +108,33 @@ function restoreFromSnapshot({ snapshot, maxAgeMs, nowIso } = {}) {
   if (!Number.isFinite(cap) || cap <= 0) return null;
   if (ageMs > cap) return null;
 
+  const sidecars = sidecarKeys instanceof Set ? sidecarKeys : new Set(sidecarKeys || []);
   const srcJobs = (snapshot.jobs && typeof snapshot.jobs === 'object') ? snapshot.jobs : {};
   const jobs = {};
   for (const [k, j] of Object.entries(srcJobs)) {
     if (!j || typeof j !== 'object') continue;
     if (j.state === 'queued') continue;
     if (j.state === 'running') {
+      if (sidecars.has(`${j.action}-${k}`)) {
+        jobs[k] = {
+          ...j,
+          state: 'recovering',
+          log: `${j.log ? j.log + '\n' : ''}(snapshot-restored — deferring to startup reap)`,
+        };
+        continue;
+      }
       jobs[k] = {
         ...j,
         state: 'error',
         finishedAt: nowIso,
+        // improve40: mark this as a restart-orphan, NOT a genuine in-process
+        // failure. The job was `running` when the restart downgraded it and has
+        // NO sidecar to reconcile from — so the boot path must probe the VM's
+        // REAL provision state (orphanedAddsToReconcile → gcloud reap) rather
+        // than leave a dead `add-failed` badge on a VM that may have actually
+        // provisioned (the VM-7 hole). An add that failed IN-PROCESS keeps its
+        // own terminal `error` with no flag, so it is never re-probed/cleared.
+        restartOrphaned: true,
         log: `${j.log ? j.log + '\n' : ''}(snapshot-restored — dashboard restarted before this finished)`,
       };
       continue;