@yemi33/minions 0.1.2305 → 0.1.2307

package/engine/shared.js

@@ -1946,6 +1946,86 @@ function mutateControl(mutator) {
   }, { defaultValue: { state: 'stopped', pid: null }, skipWriteIfUnchanged: true });
 }
 
+// ── Crash-loop detection (W-mr2c46590003e3ee) ──────────────────────────────
+// Three independent mechanisms can respawn a dead/frozen engine process:
+// engine/supervisor.js#checkEngine / #checkEngineHung, engine/watchdog.js
+// (OS-scheduled external recovery), and dashboard.js's in-process 30s
+// watchdog. None of them counted or throttled repeated crashes, so a genuine
+// crash-loop (engine dying every few minutes) silently self-healed forever
+// with no human-visible signal — see the 2026-07-01 00:20-00:30 incident
+// (4 respawns in ~5 minutes, notes/inbox/dallas-W-mr2azk6i000y06e0-*.md).
+//
+// recordEngineRespawn() gives all three mechanisms a single rolling counter
+// persisted in control.json so it survives process restarts: every call
+// appends {ts, source}, prunes entries older than CRASH_LOOP_WINDOW_MS, and
+// once the count within the window reaches CRASH_LOOP_THRESHOLD it flips a
+// dashboard-visible `control.crashLoopAlert` flag and writes a deduped
+// (one-per-day) inbox note.
+//
+// Callers MUST invoke this only at the point an actual respawn action is
+// taken — not on every health-check tick. The existing
+// RESPAWN_IN_PROGRESS_WINDOW_MS / post-spawn-grace guards already in each
+// mechanism prevent two watchers from double-spawning for the same crash, so
+// recording only real spawns keeps a single crash from being counted 2-3x
+// across the three watchers.
+const CRASH_LOOP_WINDOW_MS = Number(process.env.MINIONS_CRASH_LOOP_WINDOW_MS) || 10 * 60 * 1000; // 10 min
+const CRASH_LOOP_THRESHOLD = Number(process.env.MINIONS_CRASH_LOOP_THRESHOLD) || 3;
+
+function recordEngineRespawn(source) {
+  const now = Date.now();
+  let crashLoop = false;
+  let count = 1;
+  let windowEntries = [];
+  try {
+    mutateControl((control) => {
+      const prior = Array.isArray(control.engineRespawns) ? control.engineRespawns : [];
+      const entries = prior
+        .filter(e => e && Number.isFinite(e.ts) && (now - e.ts) <= CRASH_LOOP_WINDOW_MS)
+        .concat([{ ts: now, source: String(source || 'unknown') }]);
+      // Defensive cap — the window filter above already bounds this in
+      // practice, but guards against unbounded growth under clock skew or an
+      // operator-widened window.
+      control.engineRespawns = entries.slice(-50);
+      windowEntries = control.engineRespawns;
+      count = windowEntries.length;
+      if (count >= CRASH_LOOP_THRESHOLD) {
+        crashLoop = true;
+        control.crashLoopAlert = {
+          count,
+          windowMs: CRASH_LOOP_WINDOW_MS,
+          firstAt: new Date(windowEntries[0].ts).toISOString(),
+          lastAt: new Date(now).toISOString(),
+          sources: windowEntries.map(e => e.source),
+        };
+      } else if (control.crashLoopAlert) {
+        // Window rolled off past incidents — clear a stale alert so the
+        // dashboard doesn't keep showing a resolved crash-loop forever.
+        delete control.crashLoopAlert;
+      }
+      return control;
+    });
+  } catch (e) {
+    log('warn', `recordEngineRespawn: control.json update failed: ${e.message}`);
+  }
+
+  if (crashLoop) {
+    try {
+      const lines = windowEntries.map(e => `- ${e.source} @ ${new Date(e.ts).toISOString()}`).join('\n');
+      const body = `# Engine crash-loop detected\n\n` +
+        `${count} engine respawns in the last ${Math.round(CRASH_LOOP_WINDOW_MS / 60000)} minutes ` +
+        `(threshold: ${CRASH_LOOP_THRESHOLD}).\n\n` +
+        `Respawns:\n${lines}\n\n` +
+        `The engine keeps self-healing (each respawn succeeded) but is crashing repeatedly — ` +
+        `investigate \`engine/engine-stdio.log\` around the timestamps above before this becomes ` +
+        `a silent outage.\n`;
+      writeToInbox('engine', 'crash-loop-detected', body, null, { severity: 'critical' });
+    } catch (e) {
+      log('warn', `recordEngineRespawn: inbox alert failed: ${e.message}`);
+    }
+  }
+  return { count, crashLoop };
+}
+
 // W-mp60tw0u000j3931: Lock-safe read-modify-write for engine/state.json — the
 // persistent cross-restart engine-level state file (migration markers,
 // one-shot reconciliation versions, etc.). Mirrors mutateControl's shape.
@@ -2904,6 +2984,14 @@ const ENGINE_DEFAULTS = {
   // & Review Loop, or set `engine.autoFixPaused: true` in config.json.
   autoFixPaused: false, // hard-stop kill-switch / master override (see comment above)
   autoConsolidateMemory: false, // opt-in: periodically spawn engine/kb-sweep-runner.js from the tick loop (4h cadence). Inbox→notes consolidation already runs every tick via consolidateInbox; this flag only controls the KB sweep.
+  // W-mqtvnnj1000357fa — fleet-wide fallback for live-checkout auto-stash. When
+  // true, a dirty live-checkout tree is `git stash push --include-untracked`'d
+  // before dispatch instead of failing with FAILURE_CLASS.LIVE_CHECKOUT_DIRTY,
+  // so dispatch proceeds without manual operator intervention. Per-project
+  // `project.liveCheckoutAutoStash` overrides this (resolveLiveCheckoutAutoStash).
+  // The engine never auto-pops the stash — the operator runs `git stash pop`.
+  // Default false so a fresh install behaves identically to before this knob.
+  liveCheckoutAutoStash: false,
   prNoOpFixPauseAttempts: 2, // pause one PR automation cause after repeated no-op fixes for unchanged evidence
   quarantineAutoRecoveryMax: 2, // #2996 follow-up: cap on auto-flipping WORKTREE_DIRTY/WORKTREE_DIVERGENT failures back to pending (the quarantine is self-healing so the next dispatch starts clean; the cap prevents infinite loops if quarantine itself keeps failing).
   // W-mq5n1zx5 — Layer 1a/2b: harden the quarantine rename path against
@@ -3254,11 +3342,12 @@ const ENGINE_DEFAULTS = {
   // When ON, a dirty/broken live-checkout tree is `git fetch origin` +
   // `git reset --hard origin/<branch>`'d before dispatch instead of failing
   // LIVE_CHECKOUT_DIRTY. Per-project `project.liveCheckoutAutoReset` overrides
-  // this. Default OFF — auto-reset is destructive (it discards the operator's
-  // uncommitted changes), so it is strictly opt-in. Resolution precedence lives
-  // in `resolveLiveCheckoutAutoReset`. Fires ONLY on confirmed dirty-tree
-  // detection, never on mid-operation / blob-fetch / tooling failures.
-  liveCheckoutAutoReset: false,
+  // this. Default ON — discarding uncommitted dirt via reset-to-remote keeps
+  // live-checkout branches from diverging (the old `false` default accumulated
+  // `minions: auto-save agent WIP` commits on every dirty exit). Resolution
+  // precedence lives in `resolveLiveCheckoutAutoReset`. Fires ONLY on confirmed
+  // dirty-tree detection, never on mid-operation / blob-fetch / tooling failures.
+  liveCheckoutAutoReset: true,
   orphanHolderScanTimeoutMs: 5000,          // 5s ceiling for the cross-platform holder scan (PowerShell / /proc walk / lsof)
   ccMaxTurns: 50, // max tool-use turns per CC/doc-chat call before CLI stops (per response, not per session)
   ccWorkerIdleTimeoutMs: 30 * 60 * 1000, // W-mr0qs0vw: idle-reaper window for the persistent `copilot --acp` worker pool (engine/cc-worker-pool.js). After this much inactivity with no in-flight turn the warm ACP process is killed; the next message cold-spawns a fresh session with NO memory of prior turns (CC shows a "context cleared after inactivity" notice). Tradeoff: shorter = less idle memory/process footprint, longer = more context durability across gaps between messages. Wired into the pool via ccWorkerPool.setIdleTimeoutMs() on every reloadConfig(); clamped to [60000, 28800000] (1min–8h) in the settings POST handler.
@@ -4555,6 +4644,7 @@ const FAILURE_CLASS = {
   LIVE_CHECKOUT_FAILED: 'live-checkout-failed', // #305 (live-checkout dispatch mode): prepareLiveCheckout THREW before agent spawn (helper guard, ref validation, or a transient `git status`/`rev-parse`/`checkout` failure) — distinct from the confirmed-dirty result (LIVE_CHECKOUT_DIRTY) which the helper returns, not throws. A thrown error is NOT proof the tree is dirty, so it must not be over-classified as dirty. Retryable with bounded backoff (NOT in dispatch.js neverRetry): racy branch-lock handoff, just-finished sibling dispatch, or transient git errors frequently clear on the next attempt; the engine auto-retries up to ENGINE_DEFAULTS.maxRetries before giving up. Genuinely terminal underlying reasons (auth, validation) still short-circuit via the reason-string check in isRetryableFailureReason.
   LIVE_CHECKOUT_MID_OPERATION: 'live-checkout-mid-operation', // P-a7f3c1d9 (live-checkout dispatch mode): spawnAgent could not switch/create the target branch in project.localPath because the operator tree is mid-operation — an in-progress merge/rebase/cherry-pick/bisect or a detached HEAD. Distinct from LIVE_CHECKOUT_DIRTY (uncommitted changes): here the tree may be clean but the branch op cannot proceed. Engine refuses to spawn (it never runs `git reset`/`git clean`/`git rebase --abort` against the operator tree). Non-retryable — operator must finish or abort the in-progress operation, or checkout a branch, before re-dispatch.
   LIVE_CHECKOUT_BLOB_FETCH: 'live-checkout-blob-fetch', // PL-live-checkout-reliability-hardening (live-checkout dispatch mode): `git checkout <existing-branch>` in project.localPath failed because the tree could not be materialized — on a Scalar/GVFS-managed ADO partial (blobless) clone, switching onto a branch whose tree differs from HEAD hydrates the changed paths' blobs through the GVFS cache server (`*.gvfscache.dev.azure.com`), a different endpoint than the main git remote that receives NO auth in the headless engine shell, so the fetch fails DETERMINISTICALLY. Distinct from LIVE_CHECKOUT_FAILED (transient) because retrying reproduces it identically — surfaced once as an operator-actionable refusal instead of retry-storming to the cap. Non-retryable — the operator hydrates the branch once with their own credentials (`git checkout <branch>` interactively, or `scalar prefetch` / `git -C <repo> fetch`), then re-dispatches. The engine NEVER forces, resets, cleans, or stashes the operator tree, and best-effort switches HEAD back to the original ref so the tree is not stranded half-populated.
+  LIVE_CHECKOUT_WORKTREE_CONFLICT: 'live-checkout-worktree-conflict', // W-mr28h2j2000y0de1 (live-checkout dispatch mode): `git checkout <existing-branch>` in project.localPath failed because that exact branch is ALSO checked out in a SECOND worktree elsewhere (a leftover from a prior isolated-worktree dispatch, a manually-created worktree, or a stale worktree left by a checkoutMode change). git refuses deterministically with `fatal: '<branch>' is already used by worktree at '<path>'`. Distinct from LIVE_CHECKOUT_FAILED (transient) because this is a STRUCTURAL conflict — it does NOT clear on retry, ever, until a human or the engine removes/reassigns the other worktree; retrying just reproduces it identically and burns maxRetries. Non-retryable — the operator either `git worktree remove <path>` (freeing the branch) if that worktree is stale, or finishes/commits/pushes from it directly if it holds real WIP. The engine NEVER forces, resets, cleans, or stashes the operator tree, and best-effort switches HEAD back to the original ref so the tree is not stranded.
   INVALID_WORKDIR: 'invalid-workdir',     // P-714ef144: dispatch carried a meta.workdir override that failed validation — non-string, absolute path, drive-letter prefix, null byte, ".." segment, or post-resolve containment escape against project.localPath / worktree root. Engine refuses to spawn (the subpath would either be unreachable on disk or point outside the operator's allowed surface). Non-retryable — operator must fix the WI's meta.workdir before re-dispatch. Inbox alert lists the offending value + the resolved-vs-base mismatch.
   MODEL_UNAVAILABLE: 'model-unavailable', // W-mpg6isvy000xca4d: requested model returned overloaded_error / 503 / service_unavailable. Retriable — engine swaps in the runtime-appropriate fallback model on next spawn (Claude leans on --fallback-model already plumbed; Copilot overrides --model with engine.copilotFallbackModel).
   WORKSPACE_MANIFEST_REPO: 'workspace-manifest-repo-forbidden', // W-mq07avbk000m5543: dispatch routed an agent to a project/repo not present in its workspace_manifest.allowed_repos. Structural — never retryable until the manifest is widened or a different agent is chosen.
@@ -9060,7 +9150,7 @@ module.exports = {
   assertStateFileSize,
   withFileLock,
   mutateJsonFileLocked,
-  mutateControl,
+  mutateControl, recordEngineRespawn,
   mutateEngineState, // W-mp60tw0u000j3931
   readEngineState,   // W-mp60tw0u000j3931
   mutateCooldowns,

package/engine/supervisor.js

@@ -5,7 +5,14 @@
  * Spawned `detached: true` by `minions start`/`restart` after the engine and
  * dashboard are up. Polls every SUPERVISOR_INTERVAL_MS:
  *   - engine PID from `engine/control.json`
- *   - dashboard PID via port-listener probe (port 7331 by default)
+ *   - dashboard via a port-listener probe (port 7331 by default) AND, when the
+ *     port is bound, an HTTP `/api/health` probe. The HTTP layer catches the
+ *     "frozen but still listening" dashboard (wedged event loop) that the
+ *     port-listen probe alone reports as healthy — it respawns only after
+ *     DASH_HEALTH_MAX_FAILS consecutive failed probes so a dashboard merely
+ *     busy with a heavy synchronous getStatus() rebuild isn't killed. What
+ *     counts as "responsive" (and why engine 'degraded'/'stopped' does NOT mean
+ *     the dashboard is frozen) is documented at _probeDashboardHealth.
  *
  * When either is dead AND the stop-intent flag is NOT set, respawns the dead
  * one in the same way the CLI does (detached, stdio routed to the engine-
@@ -38,6 +45,17 @@ const MINIONS_DIR = path.resolve(__dirname, '..');
 function _sharedOrNull() {
   try { return require('./shared'); } catch { return null; }
 }
+// Crash-loop counter (W-mr2c46590003e3ee) — best-effort. shared.js owns the
+// counter/threshold/inbox-alert logic (engine/shared.js#recordEngineRespawn)
+// so supervisor.js, engine/watchdog.js, and dashboard.js's in-process
+// watchdog all feed the same rolling window in control.json instead of each
+// keeping (and silently ignoring) their own. Falls through to a no-op when
+// shared.js can't load, matching this file's existing fail-open posture.
+function _recordCrashLoopRespawn(source) {
+  const shared = _sharedOrNull();
+  if (!shared || typeof shared.recordEngineRespawn !== 'function') return;
+  try { shared.recordEngineRespawn(source); } catch { /* best-effort */ }
+}
 function _engineDir() { return _sharedOrNull()?.ENGINE_DIR || __dirname; }
 const STATIC_ENGINE_DIR = __dirname; // for require-time paths that can't be lazy
 function CONTROL_PATH_FN() { return path.join(_engineDir(), 'control.json'); }
@@ -76,6 +94,16 @@ function _resolveDashPort() {
 const POST_SPAWN_GRACE_MS = Number(process.env.MINIONS_SUPERVISOR_GRACE_MS) || 15000;
 // #421 — heartbeat age threshold before supervisor considers the engine event loop frozen.
 const SUPERVISOR_STALE_ENGINE_HEARTBEAT_MS = Number(process.env.MINIONS_SUPERVISOR_STALE_HEARTBEAT_MS) || 180000; // 3 min
+// Dashboard HTTP liveness (frozen-but-listening detection). A dashboard whose
+// event loop is wedged keeps the socket LISTENING, so the port-listen probe
+// alone reports it healthy while the in-browser SPA shows the "unreachable"
+// banner forever. We layer an HTTP /api/health probe on top: only respawn after
+// DASH_HEALTH_MAX_FAILS *consecutive* failed probes so a dashboard that is
+// merely busy with a heavy synchronous getStatus() rebuild (documented to block
+// the loop 15-25s) isn't mistaken for a hang. With the default 30s interval,
+// 3 strikes ≈ 90s of continuous unresponsiveness before we act.
+const DASH_HEALTH_TIMEOUT_MS = Number(process.env.MINIONS_SUPERVISOR_DASH_HEALTH_TIMEOUT_MS) || 5000;
+const DASH_HEALTH_MAX_FAILS = Number(process.env.MINIONS_SUPERVISOR_DASH_HEALTH_FAILS) || 3;
 const isWin = process.platform === 'win32';
 
 function safeReadJson(p) {
@@ -354,6 +382,39 @@ function spawnDashboard() {
 // the engine.
 let _lastEngineRespawnAt = 0;
 let _lastDashboardRespawnAt = 0;
+// Consecutive failed dashboard HTTP health probes. Reset on any healthy probe,
+// after a respawn, and inside the post-spawn grace window.
+let _dashHealthFailStreak = 0;
+
+// Lazy, defensive HTTP health probe. Reuses restart-health.js's httpGetJson
+// (rule: reuse before re-implementing) but never lets a load failure (e.g. a
+// mid-upgrade broken shared.js that restart-health requires at module top)
+// break the supervisor. Returns null when the probe helper is unavailable so
+// the caller can FALL BACK to port-only liveness instead of false-respawning a
+// healthy dashboard.
+function _probeDashboardHealth(port, timeoutMs = DASH_HEALTH_TIMEOUT_MS) {
+  let getJson = null;
+  try {
+    const rh = require('./restart-health');
+    if (rh && rh._private && typeof rh._private.httpGetJson === 'function') {
+      getJson = rh._private.httpGetJson;
+    }
+  } catch { /* restart-health/shared unavailable — fall through */ }
+  if (!getJson) return Promise.resolve(null);
+  return getJson(`http://127.0.0.1:${port}/api/health`, timeoutMs)
+    // "Responsive" == the dashboard event loop executed handleHealth and returned
+    // a well-formed 2xx envelope, REGARDLESS of the engine-derived status value.
+    // /api/health reports 'degraded' when the engine is paused/stopping and
+    // 'stopped' when the engine is down — those are legitimate ENGINE states
+    // (owned by checkEngine/checkEngineHung), not evidence the DASHBOARD process
+    // is frozen. Gating on status==='healthy' would respawn a perfectly
+    // responsive dashboard every time an operator pauses the engine, tearing
+    // down the socket and tripping the in-browser "unreachable" banner for no
+    // reason. Only a failed/timed-out request (res.ok false or a rejection)
+    // means the dashboard's own event loop is actually wedged.
+    .then(res => !!(res && res.ok && res.json && typeof res.json.status === 'string'))
+    .catch(() => false);
+}
 
 // Window during which a `null` pid + recent `restarted_at` is interpreted as
 // "another watchdog is currently respawning the engine — don't double-spawn."
@@ -364,12 +425,29 @@ let _lastDashboardRespawnAt = 0;
 // to enter cli.js:443 and write its own PID on cold Windows boots.
 const RESPAWN_IN_PROGRESS_WINDOW_MS = Number(process.env.MINIONS_SUPERVISOR_RESPAWN_WINDOW_MS) || 60000;
 
+// #3758 — 'degraded' is a FAILURE state (set by dashboard.js's frozen-tick
+// watchdog, see dashboard.js#_markEngineAsDegradedIfFrozen), not a deliberate
+// user-intent state like 'paused'/'stopping'/'stopped'. When a frozen tick loop
+// finally unblocks, engine.js#tickInner reads control.state !== 'running' and
+// calls process.exit(0) — self-exiting exactly BECAUSE it saw 'degraded'. The
+// dashboard.js comment documents the expected recovery: "the PID-dead watchdog
+// then auto-restarts it." Excluding 'degraded' here left that promise unmet —
+// the engine PID goes away and the control plane silently stays down until a
+// manual `minions restart`. Respawn eligibility must include both 'running'
+// (still up) and 'degraded' (crashed/crashing out of a real failure); only
+// 'paused', 'stopping', and 'stopped' reflect actual user intent to stay down.
+function _isRespawnEligibleEngineState(state) {
+  return state === 'running' || state === 'degraded';
+}
+
 function checkEngine(now) {
   if (now - _lastEngineRespawnAt < POST_SPAWN_GRACE_MS) return;
   const control = safeReadJson(CONTROL_PATH_FN());
-  // Only respawn when control.json says "running" — paused/stopped/stopping
-  // are legitimate states the supervisor must not override.
-  if (!control || control.state !== 'running') return;
+  // Respawn when control.json says "running" (PID died unexpectedly) or
+  // "degraded" (watchdog-flagged failure state — see _isRespawnEligibleEngineState
+  // above). "paused"/"stopped"/"stopping" are legitimate user-intent states the
+  // supervisor must not override.
+  if (!control || !_isRespawnEligibleEngineState(control.state)) return;
   if (control.pid && isPidAlive(control.pid)) return;
 
   // Cross-watchdog race guard: dashboard.js's in-process engine watchdog
@@ -391,6 +469,7 @@ function checkEngine(now) {
   reapStrayProcesses(path.join(MINIONS_DIR, 'engine.js'), 'engine');
   const newPid = spawnEngine();
   _lastEngineRespawnAt = now;
+  _recordCrashLoopRespawn('supervisor:dead-pid');
   console.log(`[supervisor] Engine respawned (new PID: ${newPid})`);
 }
 
@@ -403,7 +482,11 @@ function checkEngine(now) {
 function checkEngineHung(now) {
   if (now - _lastEngineRespawnAt < POST_SPAWN_GRACE_MS) return;
   const control = safeReadJson(CONTROL_PATH_FN());
-  if (!control || control.state !== 'running') return;
+  // Include 'degraded' (see _isRespawnEligibleEngineState) so a permanently
+  // frozen tick loop that never gets a chance to self-exit (tickInner never
+  // re-enters) still gets force-restarted once its own heartbeat goes stale,
+  // instead of being ignored forever the moment the state flips to 'degraded'.
+  if (!control || !_isRespawnEligibleEngineState(control.state)) return;
   // Only acts when the PID is alive — a dead PID is handled by checkEngine().
   if (!control.pid || !isPidAlive(control.pid)) return;
 
@@ -421,34 +504,73 @@ function checkEngineHung(now) {
   reapStrayProcesses(path.join(MINIONS_DIR, 'engine.js'), 'engine');
   const newPid = spawnEngine();
   _lastEngineRespawnAt = now;
+  _recordCrashLoopRespawn('supervisor:stale-heartbeat');
   console.log(`[supervisor] Engine restarted due to stale heartbeat (new PID: ${newPid})`);
 }
 
-function checkDashboard(now) {
-  if (now - _lastDashboardRespawnAt < POST_SPAWN_GRACE_MS) return;
+async function checkDashboard(now) {
+  if (now - _lastDashboardRespawnAt < POST_SPAWN_GRACE_MS) { _dashHealthFailStreak = 0; return; }
   const dashPort = _resolveDashPort();
   const pids = listeningPidsForPort(dashPort);
-  if (pids.length > 0) return;
-
-  console.log(`[supervisor] Dashboard not listening on port ${dashPort} — respawning...`);
-  // Reap stray dashboards first. Besides bounding the count, this collapses the
-  // port-desync case: orphan dashboards holding other ports are killed, freeing
-  // the canonical port for the fresh one whose beacon _resolveDashPort() reads.
-  reapStrayProcesses(path.join(MINIONS_DIR, 'dashboard.js'), 'dashboard');
-  const newPid = spawnDashboard();
-  _lastDashboardRespawnAt = now;
-  console.log(`[supervisor] Dashboard respawned (new PID: ${newPid})`);
+
+  const respawn = (why) => {
+    console.log(`[supervisor] Dashboard ${why} on port ${dashPort} — respawning...`);
+    // Reap stray dashboards first. Besides bounding the count, this collapses the
+    // port-desync case: orphan dashboards holding other ports are killed, freeing
+    // the canonical port for the fresh one whose beacon _resolveDashPort() reads.
+    // Critically, for the frozen-but-listening case the reap also KILLS the wedged
+    // process still holding the port so spawnDashboard() can bind it.
+    reapStrayProcesses(path.join(MINIONS_DIR, 'dashboard.js'), 'dashboard');
+    const newPid = spawnDashboard();
+    _lastDashboardRespawnAt = now;
+    _dashHealthFailStreak = 0;
+    console.log(`[supervisor] Dashboard respawned (new PID: ${newPid})`);
+  };
+
+  // Hard-down: nothing bound to the port → respawn immediately.
+  if (pids.length === 0) {
+    _dashHealthFailStreak = 0;
+    respawn('not listening');
+    return;
+  }
+
+  // Port is bound. Verify the event loop is actually responsive — a frozen-but-
+  // listening dashboard keeps the socket LISTENING yet serves the in-browser
+  // "unreachable" banner. Probe /api/health; only respawn after a streak of
+  // failures so a legitimately busy getStatus() rebuild isn't mistaken for a hang.
+  const healthy = await _probeDashboardHealth(dashPort);
+  if (healthy === null) {
+    // Probe helper unavailable — fall back to port-only liveness (the pre-existing
+    // behavior). Don't count this as a failure; we won't respawn a bound dashboard
+    // we can't actually prove is frozen.
+    _dashHealthFailStreak = 0;
+    return;
+  }
+  if (healthy) { _dashHealthFailStreak = 0; return; }
+
+  _dashHealthFailStreak++;
+  console.log(`[supervisor] Dashboard bound but unresponsive on port ${dashPort} (${_dashHealthFailStreak}/${DASH_HEALTH_MAX_FAILS} strikes)`);
+  if (_dashHealthFailStreak < DASH_HEALTH_MAX_FAILS) return;
+  respawn('frozen (health probe failed)');
 }
 
-function tick() {
+let _tickInFlight = false;
+async function tick() {
+  // The dashboard health probe is async; guard against a slow probe overlapping
+  // the next interval fire. setInterval invoking an async fn is fire-and-forget,
+  // so this flag is the only thing preventing concurrent ticks.
+  if (_tickInFlight) return;
+  _tickInFlight = true;
   try {
     if (isStopIntentSet()) return;
     const now = Date.now();
     checkEngine(now);
     checkEngineHung(now);
-    checkDashboard(now);
+    await checkDashboard(now);
   } catch (e) {
     console.error(`[supervisor] tick error: ${e && e.message}`);
+  } finally {
+    _tickInFlight = false;
   }
 }
 
@@ -517,8 +639,11 @@ module.exports = {
     openAppendFd,
     checkEngine,
     checkEngineHung,
+    _isRespawnEligibleEngineState,
     checkDashboard,
+    _probeDashboardHealth,
     SUPERVISOR_STALE_ENGINE_HEARTBEAT_MS,
+    DASH_HEALTH_MAX_FAILS,
     tick,
     // Path getters honor MINIONS_TEST_DIR via shared.ENGINE_DIR, so test
     // isolation correctly redirects writes/reads under createTestMinionsDir.

package/engine/watchdog.js

@@ -249,6 +249,16 @@ async function tick(opts) {
     });
     if (child && typeof child.unref === 'function') child.unref();
     logLine(minionsHome, `spawned minions ${action} pid=${child && child.pid} (detached)`);
+    // Crash-loop counter (W-mr2c46590003e3ee) — best-effort, injected so this
+    // module stays dependency-free of shared.js (matches isStopIntentSet /
+    // confirmPortUp above). Caller (bin/minions.js) wires opts.recordRespawn
+    // to shared.recordEngineRespawn so all three engine-respawn mechanisms
+    // (supervisor.js, this watchdog, dashboard.js's in-process watchdog)
+    // share one rolling window in control.json instead of each silently
+    // ignoring repeated crashes.
+    if (typeof opts.recordRespawn === 'function') {
+      try { opts.recordRespawn(`watchdog:${action}`); } catch { /* best-effort */ }
+    }
     return { healthy: false, action, spawnedPid: child && child.pid };
   } catch (err) {
     logLine(minionsHome, `FAILED to spawn minions ${action}: ${err && err.message || err}`);