@yemi33/minions 0.1.2304 → 0.1.2306

package/engine/supervisor.js

@@ -5,7 +5,14 @@
  * Spawned `detached: true` by `minions start`/`restart` after the engine and
  * dashboard are up. Polls every SUPERVISOR_INTERVAL_MS:
  *   - engine PID from `engine/control.json`
- *   - dashboard PID via port-listener probe (port 7331 by default)
+ *   - dashboard via a port-listener probe (port 7331 by default) AND, when the
+ *     port is bound, an HTTP `/api/health` probe. The HTTP layer catches the
+ *     "frozen but still listening" dashboard (wedged event loop) that the
+ *     port-listen probe alone reports as healthy — it respawns only after
+ *     DASH_HEALTH_MAX_FAILS consecutive failed probes so a dashboard merely
+ *     busy with a heavy synchronous getStatus() rebuild isn't killed. What
+ *     counts as "responsive" (and why engine 'degraded'/'stopped' does NOT mean
+ *     the dashboard is frozen) is documented at _probeDashboardHealth.
  *
  * When either is dead AND the stop-intent flag is NOT set, respawns the dead
  * one in the same way the CLI does (detached, stdio routed to the engine-
@@ -76,6 +83,16 @@ function _resolveDashPort() {
 const POST_SPAWN_GRACE_MS = Number(process.env.MINIONS_SUPERVISOR_GRACE_MS) || 15000;
 // #421 — heartbeat age threshold before supervisor considers the engine event loop frozen.
 const SUPERVISOR_STALE_ENGINE_HEARTBEAT_MS = Number(process.env.MINIONS_SUPERVISOR_STALE_HEARTBEAT_MS) || 180000; // 3 min
+// Dashboard HTTP liveness (frozen-but-listening detection). A dashboard whose
+// event loop is wedged keeps the socket LISTENING, so the port-listen probe
+// alone reports it healthy while the in-browser SPA shows the "unreachable"
+// banner forever. We layer an HTTP /api/health probe on top: only respawn after
+// DASH_HEALTH_MAX_FAILS *consecutive* failed probes so a dashboard that is
+// merely busy with a heavy synchronous getStatus() rebuild (documented to block
+// the loop 15-25s) isn't mistaken for a hang. With the default 30s interval,
+// 3 strikes ≈ 90s of continuous unresponsiveness before we act.
+const DASH_HEALTH_TIMEOUT_MS = Number(process.env.MINIONS_SUPERVISOR_DASH_HEALTH_TIMEOUT_MS) || 5000;
+const DASH_HEALTH_MAX_FAILS = Number(process.env.MINIONS_SUPERVISOR_DASH_HEALTH_FAILS) || 3;
 const isWin = process.platform === 'win32';
 
 function safeReadJson(p) {
@@ -354,6 +371,39 @@ function spawnDashboard() {
 // the engine.
 let _lastEngineRespawnAt = 0;
 let _lastDashboardRespawnAt = 0;
+// Consecutive failed dashboard HTTP health probes. Reset on any healthy probe,
+// after a respawn, and inside the post-spawn grace window.
+let _dashHealthFailStreak = 0;
+
+// Lazy, defensive HTTP health probe. Reuses restart-health.js's httpGetJson
+// (rule: reuse before re-implementing) but never lets a load failure (e.g. a
+// mid-upgrade broken shared.js that restart-health requires at module top)
+// break the supervisor. Returns null when the probe helper is unavailable so
+// the caller can FALL BACK to port-only liveness instead of false-respawning a
+// healthy dashboard.
+function _probeDashboardHealth(port, timeoutMs = DASH_HEALTH_TIMEOUT_MS) {
+  let getJson = null;
+  try {
+    const rh = require('./restart-health');
+    if (rh && rh._private && typeof rh._private.httpGetJson === 'function') {
+      getJson = rh._private.httpGetJson;
+    }
+  } catch { /* restart-health/shared unavailable — fall through */ }
+  if (!getJson) return Promise.resolve(null);
+  return getJson(`http://127.0.0.1:${port}/api/health`, timeoutMs)
+    // "Responsive" == the dashboard event loop executed handleHealth and returned
+    // a well-formed 2xx envelope, REGARDLESS of the engine-derived status value.
+    // /api/health reports 'degraded' when the engine is paused/stopping and
+    // 'stopped' when the engine is down — those are legitimate ENGINE states
+    // (owned by checkEngine/checkEngineHung), not evidence the DASHBOARD process
+    // is frozen. Gating on status==='healthy' would respawn a perfectly
+    // responsive dashboard every time an operator pauses the engine, tearing
+    // down the socket and tripping the in-browser "unreachable" banner for no
+    // reason. Only a failed/timed-out request (res.ok false or a rejection)
+    // means the dashboard's own event loop is actually wedged.
+    .then(res => !!(res && res.ok && res.json && typeof res.json.status === 'string'))
+    .catch(() => false);
+}
 
 // Window during which a `null` pid + recent `restarted_at` is interpreted as
 // "another watchdog is currently respawning the engine — don't double-spawn."
@@ -364,12 +414,29 @@ let _lastDashboardRespawnAt = 0;
 // to enter cli.js:443 and write its own PID on cold Windows boots.
 const RESPAWN_IN_PROGRESS_WINDOW_MS = Number(process.env.MINIONS_SUPERVISOR_RESPAWN_WINDOW_MS) || 60000;
 
+// #3758 — 'degraded' is a FAILURE state (set by dashboard.js's frozen-tick
+// watchdog, see dashboard.js#_markEngineAsDegradedIfFrozen), not a deliberate
+// user-intent state like 'paused'/'stopping'/'stopped'. When a frozen tick loop
+// finally unblocks, engine.js#tickInner reads control.state !== 'running' and
+// calls process.exit(0) — self-exiting exactly BECAUSE it saw 'degraded'. The
+// dashboard.js comment documents the expected recovery: "the PID-dead watchdog
+// then auto-restarts it." Excluding 'degraded' here left that promise unmet —
+// the engine PID goes away and the control plane silently stays down until a
+// manual `minions restart`. Respawn eligibility must include both 'running'
+// (still up) and 'degraded' (crashed/crashing out of a real failure); only
+// 'paused', 'stopping', and 'stopped' reflect actual user intent to stay down.
+function _isRespawnEligibleEngineState(state) {
+  return state === 'running' || state === 'degraded';
+}
+
 function checkEngine(now) {
   if (now - _lastEngineRespawnAt < POST_SPAWN_GRACE_MS) return;
   const control = safeReadJson(CONTROL_PATH_FN());
-  // Only respawn when control.json says "running" — paused/stopped/stopping
-  // are legitimate states the supervisor must not override.
-  if (!control || control.state !== 'running') return;
+  // Respawn when control.json says "running" (PID died unexpectedly) or
+  // "degraded" (watchdog-flagged failure state — see _isRespawnEligibleEngineState
+  // above). "paused"/"stopped"/"stopping" are legitimate user-intent states the
+  // supervisor must not override.
+  if (!control || !_isRespawnEligibleEngineState(control.state)) return;
   if (control.pid && isPidAlive(control.pid)) return;
 
   // Cross-watchdog race guard: dashboard.js's in-process engine watchdog
@@ -403,7 +470,11 @@ function checkEngine(now) {
 function checkEngineHung(now) {
   if (now - _lastEngineRespawnAt < POST_SPAWN_GRACE_MS) return;
   const control = safeReadJson(CONTROL_PATH_FN());
-  if (!control || control.state !== 'running') return;
+  // Include 'degraded' (see _isRespawnEligibleEngineState) so a permanently
+  // frozen tick loop that never gets a chance to self-exit (tickInner never
+  // re-enters) still gets force-restarted once its own heartbeat goes stale,
+  // instead of being ignored forever the moment the state flips to 'degraded'.
+  if (!control || !_isRespawnEligibleEngineState(control.state)) return;
   // Only acts when the PID is alive — a dead PID is handled by checkEngine().
   if (!control.pid || !isPidAlive(control.pid)) return;
 
@@ -424,31 +495,69 @@ function checkEngineHung(now) {
   console.log(`[supervisor] Engine restarted due to stale heartbeat (new PID: ${newPid})`);
 }
 
-function checkDashboard(now) {
-  if (now - _lastDashboardRespawnAt < POST_SPAWN_GRACE_MS) return;
+async function checkDashboard(now) {
+  if (now - _lastDashboardRespawnAt < POST_SPAWN_GRACE_MS) { _dashHealthFailStreak = 0; return; }
   const dashPort = _resolveDashPort();
   const pids = listeningPidsForPort(dashPort);
-  if (pids.length > 0) return;
-
-  console.log(`[supervisor] Dashboard not listening on port ${dashPort} — respawning...`);
-  // Reap stray dashboards first. Besides bounding the count, this collapses the
-  // port-desync case: orphan dashboards holding other ports are killed, freeing
-  // the canonical port for the fresh one whose beacon _resolveDashPort() reads.
-  reapStrayProcesses(path.join(MINIONS_DIR, 'dashboard.js'), 'dashboard');
-  const newPid = spawnDashboard();
-  _lastDashboardRespawnAt = now;
-  console.log(`[supervisor] Dashboard respawned (new PID: ${newPid})`);
+
+  const respawn = (why) => {
+    console.log(`[supervisor] Dashboard ${why} on port ${dashPort} — respawning...`);
+    // Reap stray dashboards first. Besides bounding the count, this collapses the
+    // port-desync case: orphan dashboards holding other ports are killed, freeing
+    // the canonical port for the fresh one whose beacon _resolveDashPort() reads.
+    // Critically, for the frozen-but-listening case the reap also KILLS the wedged
+    // process still holding the port so spawnDashboard() can bind it.
+    reapStrayProcesses(path.join(MINIONS_DIR, 'dashboard.js'), 'dashboard');
+    const newPid = spawnDashboard();
+    _lastDashboardRespawnAt = now;
+    _dashHealthFailStreak = 0;
+    console.log(`[supervisor] Dashboard respawned (new PID: ${newPid})`);
+  };
+
+  // Hard-down: nothing bound to the port → respawn immediately.
+  if (pids.length === 0) {
+    _dashHealthFailStreak = 0;
+    respawn('not listening');
+    return;
+  }
+
+  // Port is bound. Verify the event loop is actually responsive — a frozen-but-
+  // listening dashboard keeps the socket LISTENING yet serves the in-browser
+  // "unreachable" banner. Probe /api/health; only respawn after a streak of
+  // failures so a legitimately busy getStatus() rebuild isn't mistaken for a hang.
+  const healthy = await _probeDashboardHealth(dashPort);
+  if (healthy === null) {
+    // Probe helper unavailable — fall back to port-only liveness (the pre-existing
+    // behavior). Don't count this as a failure; we won't respawn a bound dashboard
+    // we can't actually prove is frozen.
+    _dashHealthFailStreak = 0;
+    return;
+  }
+  if (healthy) { _dashHealthFailStreak = 0; return; }
+
+  _dashHealthFailStreak++;
+  console.log(`[supervisor] Dashboard bound but unresponsive on port ${dashPort} (${_dashHealthFailStreak}/${DASH_HEALTH_MAX_FAILS} strikes)`);
+  if (_dashHealthFailStreak < DASH_HEALTH_MAX_FAILS) return;
+  respawn('frozen (health probe failed)');
 }
 
-function tick() {
+let _tickInFlight = false;
+async function tick() {
+  // The dashboard health probe is async; guard against a slow probe overlapping
+  // the next interval fire. setInterval invoking an async fn is fire-and-forget,
+  // so this flag is the only thing preventing concurrent ticks.
+  if (_tickInFlight) return;
+  _tickInFlight = true;
   try {
     if (isStopIntentSet()) return;
     const now = Date.now();
     checkEngine(now);
     checkEngineHung(now);
-    checkDashboard(now);
+    await checkDashboard(now);
   } catch (e) {
     console.error(`[supervisor] tick error: ${e && e.message}`);
+  } finally {
+    _tickInFlight = false;
   }
 }
 
@@ -517,8 +626,11 @@ module.exports = {
     openAppendFd,
     checkEngine,
     checkEngineHung,
+    _isRespawnEligibleEngineState,
     checkDashboard,
+    _probeDashboardHealth,
     SUPERVISOR_STALE_ENGINE_HEARTBEAT_MS,
+    DASH_HEALTH_MAX_FAILS,
     tick,
     // Path getters honor MINIONS_TEST_DIR via shared.ENGINE_DIR, so test
     // isolation correctly redirects writes/reads under createTestMinionsDir.

package/engine.js

@@ -138,7 +138,7 @@ const worktreePool = require('./engine/worktree-pool');
 // ─── State Readers (delegated to engine/queries.js) ─────────────────────────
 
 const { getConfig, getControl, getDispatch, getNotes,
-  getAgentStatus, getAgentCharter, getInboxFiles,
+  getAgentStatus, getInboxFiles,
   collectSkillFiles, getSkillIndex, getKnowledgeBaseIndex,
   getPrs, SKILLS_DIR } = queries;
 
@@ -10564,7 +10564,7 @@ module.exports = {
 
   // State readers/writers
   getConfig, getControl, getDispatch, getRouting, getNotes,
-  getAgentStatus, getAgentCharter, getInboxFiles, getPrs,
+  getAgentStatus, getInboxFiles, getPrs,
   validateConfig,
 
   // Dispatch management (re-exported from engine/dispatch.js)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2304",
+  "version": "0.1.2306",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"