@yemi33/minions 0.1.2448 → 0.1.2449

package/engine/{supervisor.js → recovery/supervisor.js}

@@ -1,18 +1,16 @@
 #!/usr/bin/env node
 /**
- * engine/supervisor.js — External watchdog for engine + dashboard.
+ * engine/recovery/supervisor.js — External watchdog for engine + dashboard.
  *
  * Spawned `detached: true` by `minions start`/`restart` after the engine and
  * dashboard are up. Polls every SUPERVISOR_INTERVAL_MS:
  *   - engine PID from `engine/control.json`
- *   - dashboard via a port-listener probe (port 7331 by default) AND, when the
- *     port is bound, an HTTP `/api/health` probe. The HTTP layer catches the
- *     "frozen but still listening" dashboard (wedged event loop) that the
- *     port-listen probe alone reports as healthy — it respawns only after
- *     DASH_HEALTH_MAX_FAILS consecutive failed probes so a dashboard merely
- *     busy with a heavy synchronous getStatus() rebuild isn't killed. What
- *     counts as "responsive" (and why engine 'degraded'/'stopped' does NOT mean
- *     the dashboard is frozen) is documented at _probeDashboardHealth.
+ *   - dashboard via both a port-listener probe (port 7331 by default) and HTTP
+ *     `/api/health`. Either can false-negative on a loaded Windows host, so a
+ *     responsive HTTP result overrides a missed port probe and replacement
+ *     requires DASH_HEALTH_MAX_FAILS consecutive ticks where neither signal
+ *     proves health. What counts as responsive is documented at
+ *     _probeDashboardHealth.
  *
  * When either is dead AND the stop-intent flag is NOT set, respawns the dead
  * one in the same way the CLI does (detached, stdio routed to the engine-
@@ -29,25 +27,30 @@
  * Windows Service via nssm — see docs/supervisor.md.
  */
 
-require('./stdio-timestamps').installIfNotInstalled();
+require('../core/stdio-timestamps').installIfNotInstalled();
 
 const fs = require('fs');
 const path = require('path');
 const os = require('os');
 const { spawn, spawnSync, execSync } = require('child_process');
+const {
+  terminateProcess,
+  commandLineRunsScript,
+  listNodePidsMatchingScript,
+} = require('../processes/process-utils');
 
-// Lazy path getters route through engine/shared.js when available so the
+// Lazy path getters route through engine/core/shared.js when available so the
 // supervisor's notion of "the engine dir" honors MINIONS_TEST_DIR (used by
 // the unit tests). Fall back to __dirname-based resolution when shared.js
 // isn't loadable (e.g. early load / bad upgrade) so the supervisor still
 // boots in production.
-const MINIONS_DIR = path.resolve(__dirname, '..');
+const MINIONS_DIR = path.resolve(__dirname, '..', '..');
 function _sharedOrNull() {
-  try { return require('./shared'); } catch { return null; }
+  try { return require('../core/shared'); } catch { return null; }
 }
 // Crash-loop counter (W-mr2c46590003e3ee) — best-effort. shared.js owns the
-// counter/threshold/inbox-alert logic (engine/shared.js#recordEngineRespawn)
-// so supervisor.js, engine/watchdog.js, and dashboard.js's in-process
+// counter/threshold/inbox-alert logic (engine/core/shared.js#recordEngineRespawn)
+// so supervisor.js, engine/recovery/watchdog.js, and dashboard.js's in-process
 // watchdog all feed the same rolling window in control.json instead of each
 // keeping (and silently ignoring) their own. Falls through to a no-op when
 // shared.js can't load, matching this file's existing fail-open posture.
@@ -56,8 +59,8 @@ function _recordCrashLoopRespawn(source) {
   if (!shared || typeof shared.recordEngineRespawn !== 'function') return;
   try { shared.recordEngineRespawn(source); } catch { /* best-effort */ }
 }
-function _engineDir() { return _sharedOrNull()?.ENGINE_DIR || __dirname; }
-const STATIC_ENGINE_DIR = __dirname; // for require-time paths that can't be lazy
+function _engineDir() { return _sharedOrNull()?.ENGINE_DIR || path.resolve(__dirname, '..'); }
+const STATIC_ENGINE_DIR = path.resolve(__dirname, '..'); // for require-time paths that can't be lazy
 function CONTROL_PATH_FN() { return path.join(_engineDir(), 'control.json'); }
 function STOP_INTENT_PATH_FN() { return _sharedOrNull()?.STOP_INTENT_PATH || path.join(_engineDir(), 'stop-intent.json'); }
 function SUPERVISOR_PID_PATH_FN() { return path.join(_engineDir(), 'supervisor.pid'); }
@@ -94,14 +97,15 @@ function _resolveDashPort() {
 const POST_SPAWN_GRACE_MS = Number(process.env.MINIONS_SUPERVISOR_GRACE_MS) || 15000;
 // #421 — heartbeat age threshold before supervisor considers the engine event loop frozen.
 const SUPERVISOR_STALE_ENGINE_HEARTBEAT_MS = Number(process.env.MINIONS_SUPERVISOR_STALE_HEARTBEAT_MS) || 180000; // 3 min
-// Dashboard HTTP liveness (frozen-but-listening detection). A dashboard whose
-// event loop is wedged keeps the socket LISTENING, so the port-listen probe
-// alone reports it healthy while the in-browser SPA shows the "unreachable"
-// banner forever. We layer an HTTP /api/health probe on top: only respawn after
-// DASH_HEALTH_MAX_FAILS *consecutive* failed probes so a dashboard that is
-// merely busy with a heavy synchronous getStatus() rebuild (documented to block
-// the loop 15-25s) isn't mistaken for a hang. With the default 30s interval,
-// 3 strikes ≈ 90s of continuous unresponsiveness before we act.
+// A recent shell-free heartbeat vetoes a failed PID shell-out. This is shorter
+// than the stale-heartbeat watchdog but long enough to absorb several missed
+// 15s heartbeat writes under host load.
+const SUPERVISOR_FRESH_HEARTBEAT_MS = Number(process.env.MINIONS_SUPERVISOR_FRESH_HEARTBEAT_MS) || 60000;
+// Dashboard liveness needs two independent signals. A frozen event loop can
+// retain its listening socket, while netstat can time out or return no rows on
+// a loaded Windows host even though HTTP still works. Only respawn after both
+// fail for DASH_HEALTH_MAX_FAILS consecutive ticks. With the default 30s
+// interval, 3 strikes is about 90s of continuous unresponsiveness.
 const DASH_HEALTH_TIMEOUT_MS = Number(process.env.MINIONS_SUPERVISOR_DASH_HEALTH_TIMEOUT_MS) || 5000;
 const DASH_HEALTH_MAX_FAILS = Number(process.env.MINIONS_SUPERVISOR_DASH_HEALTH_FAILS) || 3;
 const REAP_WAIT_TIMEOUT_MS = Number(process.env.MINIONS_SUPERVISOR_REAP_TIMEOUT_MS) || 10000;
@@ -112,15 +116,15 @@ function safeReadJson(p) {
   try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch { return null; }
 }
 
-// Single source of truth lives in engine/shared.js (writeStopIntent /
+// Single source of truth lives in engine/core/shared.js (writeStopIntent /
 // clearStopIntent / isStopIntentSet). The supervisor used to re-implement
 // this locally; centralizing it ensures dashboard.js, bin/minions.js,
-// engine/cli.js, and the supervisor all read/write through one contract.
+// engine/operations/cli.js, and the supervisor all read/write through one contract.
 // Fall back to a local file-presence check if shared.js fails to load (e.g.
 // during a bad upgrade) — we'd rather fail-closed than crash the supervisor.
 function isStopIntentSet() {
   try {
-    const shared = require('./shared');
+    const shared = require('../core/shared');
     if (shared && typeof shared.isStopIntentSet === 'function') {
       return shared.isStopIntentSet();
     }
@@ -151,7 +155,7 @@ function isPidAlive(pid, {
       const out = execSync(`tasklist /FI "PID eq ${n}" /NH`, {
         encoding: 'utf8', timeout: 3000, windowsHide: true,
       });
-      // Word-boundary + image-name match (mirrors engine/restart-health.js) so a
+      // Word-boundary + image-name match (mirrors engine/recovery/restart-health.js) so a
       // digit-substring collision in another column can't read a dead pid as
       // alive. Fall back to an inline regex if shared.js isn't loadable.
       const shared = _sharedOrNull();
@@ -206,125 +210,12 @@ function listeningPidsForPort(port) {
 // checkout on the same machine is never touched. Opt out via
 // MINIONS_SUPERVISOR_REAP=0.
 const REAP_ENABLED = process.env.MINIONS_SUPERVISOR_REAP !== '0';
-
-function _normPath(p) { return String(p).replace(/\\/g, '/').toLowerCase(); }
-
-// Split a command line into argv tokens, honoring double-quote grouping
-// (Windows CommandLine quotes any path containing spaces, e.g.
-// `"C:\Program Files\nodejs\node.exe"`). We only need this accurate enough to
-// tell which token is the *script* being executed.
-function _tokenizeCmdline(cmdline) {
-  const toks = [];
-  let cur = '';
-  let inQuote = false;
-  for (const ch of String(cmdline)) {
-    if (ch === '"') { inQuote = !inQuote; continue; }
-    if (!inQuote && (ch === ' ' || ch === '\t')) { if (cur) { toks.push(cur); cur = ''; } continue; }
-    cur += ch;
-  }
-  if (cur) toks.push(cur);
-  return toks;
-}
-
-// True iff `cmdline` RUNS `target` as its script — not merely mentions it as an
-// argument. The script is the first `.js` token after the node executable, so
-// `node .../engine.js start` matches but `node eslint.js .../engine.js` (where
-// engine.js is an argument to a different script) does NOT. Exact, normalized
-// equality also rejects longer-name false positives (`.../engine.js.bak`,
-// `.../engine.json`) that a plain substring `includes` would catch. `target` is
-// already normalized by the caller.
-//
-// This is intentionally stricter than substring matching: every orphan the
-// supervisor needs to reap was spawned by spawnEngine()/spawnDashboard() with a
-// byte-identical `path.join(MINIONS_DIR, '<script>.js')`, so exact script-token
-// match catches all of them while a foreign process that only *names* the path
-// is left alone (fail-safe: never kill on a loose match).
-//
-// ENGINE subcommand gate: for an `engine.js` target we additionally require the
-// token AFTER the script to be `start`, so a short-lived `node engine.js stop`
-// one-shot (spawned by `minions restart`/`down`/`update` and the `stop` verb
-// during the restart window) is NEVER reaped. The long-lived daemon is ALWAYS
-// spawned with `'start'` (supervisor/dashboard spawnEngine + bin/minions.js), so
-// the requirement loses zero real orphans. The `dashboard.js` match stays BARE
-// (no subcommand) — only the engine candidate gets the `start` requirement.
-function _cmdRunsScript(cmdline, target) {
-  const toks = _tokenizeCmdline(cmdline);
-  const isEngineTarget = target === 'engine.js' || target.endsWith('/engine.js');
-  // toks[0] is the node executable; the script is the first later .js token.
-  for (let i = 1; i < toks.length; i++) {
-    const norm = _normPath(toks[i]);
-    if (norm.endsWith('.js')) {
-      if (norm !== target) return false;
-      if (isEngineTarget) {
-        // Only the daemon (`engine.js start`) is reapable; one-shots like
-        // `engine.js stop` are left alone.
-        const next = toks[i + 1] ? _normPath(toks[i + 1]) : '';
-        return next === 'start';
-      }
-      return true;
-    }
-  }
-  return false;
-}
-
-// PIDs of `node` processes that RUN `needle` (a script path) as their script.
-// Cross-platform: PowerShell CIM on Windows, `ps` elsewhere. Best effort —
-// returns [] on any enumeration failure (fail-open: leak rather than risk
-// killing the wrong process).
-//
-// The OS does the coarse filtering (WQL `CommandLine LIKE` / a `grep` on the
-// script basename) so only candidate rows — not the machine's entire node/
-// process table — cross into the JS heap; `_cmdRunsScript` then applies the
-// exact script-position match. On a busy host (CI runners, many node procs)
-// this keeps the buffered stdout to a handful of rows instead of megabytes.
-// `maxBuffer` caps the read defensively in case the coarse filter still matches
-// a lot. The basename is an internal constant (`engine.js`/`dashboard.js`), so
-// it carries no shell/WQL-injection risk.
-const _ENUM_MAX_BUFFER = 8 * 1024 * 1024;
-function listNodePidsMatching(needle) {
-  const target = _normPath(needle);
-  const base = path.basename(needle); // coarse prefilter, e.g. 'engine.js'
-  const pids = [];
-  try {
-    let out = '';
-    if (isWin) {
-      // spawnSync (no shell) + -EncodedCommand sidesteps cmd.exe quoting — an
-      // inline `powershell -Command "... $(...) | ..."` via execSync gets its
-      // `$(...)`/`|` mangled by cmd.exe before PowerShell ever sees them.
-      const ps = `Get-CimInstance Win32_Process -Filter "name='node.exe' AND CommandLine LIKE '%${base}%'" | `
-        + 'ForEach-Object { "$($_.ProcessId)|$($_.CommandLine)" }';
-      const res = spawnSync('powershell',
-        ['-NoProfile', '-NonInteractive', '-EncodedCommand', Buffer.from(ps, 'utf16le').toString('base64')],
-        { encoding: 'utf8', timeout: 8000, windowsHide: true, maxBuffer: _ENUM_MAX_BUFFER });
-      out = (res && res.stdout) || '';
-    } else {
-      // grep prefilters to candidate rows; no match exits 1 -> execSync throws
-      // -> caught below -> [] (correct: nothing to reap).
-      out = execSync(`ps -eo pid=,args= | grep -iF -- '${base}'`,
-        { encoding: 'utf8', timeout: 8000, maxBuffer: _ENUM_MAX_BUFFER });
-    }
-    for (const line of out.split('\n')) {
-      const trimmed = line.trim();
-      if (!trimmed) continue;
-      // Windows rows are "PID|<full command line>"; ps rows are "PID <args>".
-      const m = isWin ? trimmed.match(/^(\d+)\|(.*)$/) : trimmed.match(/^(\d+)\s+(.*)$/);
-      if (!m) continue;
-      const pid = Number(m[1]);
-      if (pid && _cmdRunsScript(m[2], target)) pids.push(pid);
-    }
-  } catch { /* enumeration failed — fail-open with whatever we collected */ }
-  return pids;
-}
-
-function _killHard(pid) {
-  const shared = _sharedOrNull();
-  try {
-    if (shared && typeof shared.killImmediate === 'function') { shared.killImmediate({ pid }); return; }
-  } catch { /* fall through to platform kill */ }
-  try {
-    if (isWin) execSync(`taskkill /F /T /PID ${pid}`, { timeout: 5000, windowsHide: true, stdio: 'ignore' });
-    else process.kill(pid, 'SIGKILL');
-  } catch { /* already gone */ }
+const _cmdRunsScript = commandLineRunsScript;
+const listNodePidsMatching = listNodePidsMatchingScript;
+function _killHard(pid, scriptPath) {
+  terminateProcess('engine.supervisor.daemon-replace', { pid }, {
+    scriptPath,
+  });
 }
 
 function _sleepSync(ms) {
@@ -357,9 +248,17 @@ function reapStrayProcesses(scriptPath, label, options = {}) {
   const listPids = options.listPids || listNodePidsMatching;
   const kill = options.kill || _killHard;
   const pids = listPids(scriptPath).filter(p => p !== process.pid);
+  const expectedPids = (options.expectedPids || [])
+    .map(Number)
+    .filter(pid => Number.isInteger(pid) && pid > 0 && pid !== process.pid);
+  const expectedIsAlive = options.isAlive || isPidAlive;
+  const unproven = expectedPids.filter(pid => expectedIsAlive(pid) && !pids.includes(pid));
+  if (unproven.length > 0) {
+    throw new Error(`Refusing to respawn ${label}: live PID ownership was not proven for ${unproven.join(', ')}`);
+  }
   if (!pids.length) return 0;
   console.log(`[supervisor] Reaping ${pids.length} stray ${label} process(es) before respawn: ${pids.join(', ')}`);
-  for (const pid of pids) kill(pid);
+  for (const pid of pids) kill(pid, scriptPath);
   const waited = waitForPidsToExit(pids, options);
   if (!waited.ok) {
     throw new Error(`Refusing to respawn ${label}: stale PID(s) still alive after ${options.timeoutMs || REAP_WAIT_TIMEOUT_MS}ms: ${waited.remaining.join(', ')}`);
@@ -381,7 +280,7 @@ function openAppendFd(name) {
   // still serves HTTP 200, but you can't see anything it logs).
   const dir = _engineDir();
   try {
-    const shared = require('./shared');
+    const shared = require('../core/shared');
     if (shared && typeof shared.openAppendLogFd === 'function') {
       return shared.openAppendLogFd(name, dir, { fallback: 'ignore' }).fd;
     }
@@ -507,6 +406,12 @@ function _isRespawnEligibleEngineState(state) {
   return state === 'running' || state === 'degraded';
 }
 
+function _engineHeartbeatAgeMs(control, now) {
+  const heartbeat = Number(control?.heartbeat);
+  if (!heartbeat || !Number.isFinite(heartbeat)) return null;
+  return now - heartbeat;
+}
+
 const ENGINE_WATCHDOG_OPERATION_NAMES = [
   'isPidAlive',
   'reapStrayProcesses',
@@ -564,6 +469,15 @@ function checkEngine(now, operations) {
   const ops = _resolveEngineWatchdogOperations(operations);
   if (control.pid && ops.isPidAlive(control.pid)) return;
 
+  const heartbeatAgeMs = _engineHeartbeatAgeMs(control, now);
+  if (control.pid && heartbeatAgeMs !== null && heartbeatAgeMs < SUPERVISOR_FRESH_HEARTBEAT_MS) {
+    console.log(
+      `[supervisor] Engine PID ${control.pid} reported dead by the liveness probe but its heartbeat is ` +
+      `${Math.round(heartbeatAgeMs / 1000)}s old — treating as up (false-negative suppressed)`,
+    );
+    return;
+  }
+
   console.log(`[supervisor] Engine PID ${control.pid || '(none)'} is dead — respawning...`);
   // Reap any orphan engines first so a stale control.json PID can't snowball
   // into an unbounded pile of contending engine processes.
@@ -578,7 +492,7 @@ function checkEngine(now, operations) {
 // event loop has frozen (sync busy-loop, blocked system call). Unlike
 // checkEngine (which only fires when the PID is dead), this fires when the PID
 // is alive but control.heartbeat has not advanced for SUPERVISOR_STALE_ENGINE_HEARTBEAT_MS.
-// The heartbeat is written every ~15s by a dedicated timer in engine/cli.js; if
+// The heartbeat is written every ~15s by a dedicated timer in engine/operations/cli.js; if
 // it goes stale while the PID persists, the event loop is stuck.
 function checkEngineHung(now, operations) {
   if (now - _lastEngineRespawnAt < POST_SPAWN_GRACE_MS) return;
@@ -605,17 +519,26 @@ function checkEngineHung(now, operations) {
     `(${Math.round(heartbeatAge / 1000)}s old, threshold ${SUPERVISOR_STALE_ENGINE_HEARTBEAT_MS / 1000}s) — ` +
     `event loop likely frozen; restarting engine`,
   );
-  ops.reapStrayProcesses(path.join(MINIONS_DIR, 'engine.js'), 'engine');
+  ops.reapStrayProcesses(path.join(MINIONS_DIR, 'engine.js'), 'engine', {
+    expectedPids: [control.pid],
+    isAlive: ops.isPidAlive,
+  });
   const newPid = ops.spawnEngine();
   _lastEngineRespawnAt = now;
   ops.recordCrashLoopRespawn('supervisor:stale-heartbeat');
   console.log(`[supervisor] Engine restarted due to stale heartbeat (new PID: ${newPid})`);
 }
 
-async function checkDashboard(now) {
+async function checkDashboard(now, operations = {}) {
   if (now - _lastDashboardRespawnAt < POST_SPAWN_GRACE_MS) { _dashHealthFailStreak = 0; return; }
-  const dashPort = _resolveDashPort();
-  const pids = listeningPidsForPort(dashPort);
+  const resolveDashPort = operations.resolveDashPort || _resolveDashPort;
+  const findListeningPids = operations.listeningPidsForPort || listeningPidsForPort;
+  const probeDashboardHealth = operations.probeDashboardHealth || _probeDashboardHealth;
+  const reap = operations.reapStrayProcesses || reapStrayProcesses;
+  const spawnDash = operations.spawnDashboard || spawnDashboard;
+  const pidAlive = operations.isPidAlive || isPidAlive;
+  const dashPort = resolveDashPort();
+  const pids = findListeningPids(dashPort);
 
   const respawn = (why) => {
     console.log(`[supervisor] Dashboard ${why} on port ${dashPort} — respawning...`);
@@ -624,38 +547,43 @@ async function checkDashboard(now) {
     // the canonical port for the fresh one whose beacon _resolveDashPort() reads.
     // Critically, for the frozen-but-listening case the reap also KILLS the wedged
     // process still holding the port so spawnDashboard() can bind it.
-    reapStrayProcesses(path.join(MINIONS_DIR, 'dashboard.js'), 'dashboard');
-    const newPid = spawnDashboard(dashPort);
+    reap(path.join(MINIONS_DIR, 'dashboard.js'), 'dashboard', {
+      expectedPids: pids,
+      isAlive: pidAlive,
+    });
+    const newPid = spawnDash(dashPort);
     _lastDashboardRespawnAt = now;
     _dashHealthFailStreak = 0;
     console.log(`[supervisor] Dashboard respawned (new PID: ${newPid})`);
+    return { status: 'respawned', pid: newPid, port: dashPort };
   };
 
-  // Hard-down: nothing bound to the port → respawn immediately.
-  if (pids.length === 0) {
+  const healthy = await probeDashboardHealth(dashPort);
+  if (healthy === true) {
+    if (pids.length === 0) {
+      console.log(
+        `[supervisor] Dashboard port probe missed ${dashPort}, but /api/health responded — ` +
+        'treating as up (false-negative suppressed)',
+      );
+    }
     _dashHealthFailStreak = 0;
-    respawn('not listening');
-    return;
+    return { status: 'healthy', port: dashPort };
   }
 
-  // Port is bound. Verify the event loop is actually responsive — a frozen-but-
-  // listening dashboard keeps the socket LISTENING yet serves the in-browser
-  // "unreachable" banner. Probe /api/health; only respawn after a streak of
-  // failures so a legitimately busy getStatus() rebuild isn't mistaken for a hang.
-  const healthy = await _probeDashboardHealth(dashPort);
-  if (healthy === null) {
-    // Probe helper unavailable — fall back to port-only liveness (the pre-existing
-    // behavior). Don't count this as a failure; we won't respawn a bound dashboard
-    // we can't actually prove is frozen.
+  if (healthy === null && pids.length > 0) {
     _dashHealthFailStreak = 0;
-    return;
+    return { status: 'listening', port: dashPort };
   }
-  if (healthy) { _dashHealthFailStreak = 0; return; }
 
   _dashHealthFailStreak++;
-  console.log(`[supervisor] Dashboard bound but unresponsive on port ${dashPort} (${_dashHealthFailStreak}/${DASH_HEALTH_MAX_FAILS} strikes)`);
-  if (_dashHealthFailStreak < DASH_HEALTH_MAX_FAILS) return;
-  respawn('frozen (health probe failed)');
+  console.log(
+    `[supervisor] Dashboard has no responsive health signal on port ${dashPort} ` +
+    `(${_dashHealthFailStreak}/${DASH_HEALTH_MAX_FAILS} strikes; listeners=${pids.length})`,
+  );
+  if (_dashHealthFailStreak < DASH_HEALTH_MAX_FAILS) {
+    return { status: 'unhealthy', strikes: _dashHealthFailStreak, port: dashPort };
+  }
+  return respawn(pids.length > 0 ? 'frozen (health probe failed)' : 'unreachable');
 }
 
 let _tickInFlight = false;
@@ -747,6 +675,8 @@ module.exports = {
     _isRespawnEligibleEngineState,
     checkDashboard,
     _probeDashboardHealth,
+    _killHard,
+    SUPERVISOR_FRESH_HEARTBEAT_MS,
     SUPERVISOR_STALE_ENGINE_HEARTBEAT_MS,
     DASH_HEALTH_MAX_FAILS,
     tick,