@yemi33/minions 0.1.2225 → 0.1.2226

package/engine/supervisor.js

@@ -27,7 +27,7 @@ require('./stdio-timestamps').installIfNotInstalled();
 const fs = require('fs');
 const path = require('path');
 const os = require('os');
-const { spawn, execSync } = require('child_process');
+const { spawn, spawnSync, execSync } = require('child_process');
 
 // Lazy path getters route through engine/shared.js when available so the
 // supervisor's notion of "the engine dir" honors MINIONS_TEST_DIR (used by
@@ -128,6 +128,139 @@ function listeningPidsForPort(port) {
   } catch { return []; }
 }
 
+// Reap-before-respawn (supervisor respawn-storm fix).
+//
+// checkEngine/checkDashboard only spawn a replacement — they never killed the
+// orphan. When the engine's liveness signal (control.json `pid`) goes stale
+// because a transient EPERM blocked the heartbeat tmp->rename on Windows, the
+// supervisor kept respawning engines WITHOUT reaping the dead-but-still-running
+// ones. Each extra engine contends harder on control.json -> more EPERM -> the
+// PID stays stale -> unbounded accumulation (observed: 36 engines + 41
+// dashboards, dozens of console-window popups per minute). Reaping every stray
+// process that runs THIS install's engine.js/dashboard.js before spawning the
+// replacement bounds the live count to ~1: a persistent EPERM degrades into a
+// slow 1-in-1-out restart cycle instead of a runaway storm.
+//
+// Only ever called from the respawn branch, i.e. AFTER we've already concluded
+// the watched process is dead — so every match is by definition an orphan that
+// is safe to kill. Scoped to MINIONS_DIR's script path so a second Minions
+// checkout on the same machine is never touched. Opt out via
+// MINIONS_SUPERVISOR_REAP=0.
+const REAP_ENABLED = process.env.MINIONS_SUPERVISOR_REAP !== '0';
+
+function _normPath(p) { return String(p).replace(/\\/g, '/').toLowerCase(); }
+
+// Split a command line into argv tokens, honoring double-quote grouping
+// (Windows CommandLine quotes any path containing spaces, e.g.
+// `"C:\Program Files\nodejs\node.exe"`). We only need this accurate enough to
+// tell which token is the *script* being executed.
+function _tokenizeCmdline(cmdline) {
+  const toks = [];
+  let cur = '';
+  let inQuote = false;
+  for (const ch of String(cmdline)) {
+    if (ch === '"') { inQuote = !inQuote; continue; }
+    if (!inQuote && (ch === ' ' || ch === '\t')) { if (cur) { toks.push(cur); cur = ''; } continue; }
+    cur += ch;
+  }
+  if (cur) toks.push(cur);
+  return toks;
+}
+
+// True iff `cmdline` RUNS `target` as its script — not merely mentions it as an
+// argument. The script is the first `.js` token after the node executable, so
+// `node .../engine.js start` matches but `node eslint.js .../engine.js` (where
+// engine.js is an argument to a different script) does NOT. Exact, normalized
+// equality also rejects longer-name false positives (`.../engine.js.bak`,
+// `.../engine.json`) that a plain substring `includes` would catch. `target` is
+// already normalized by the caller.
+//
+// This is intentionally stricter than substring matching: every orphan the
+// supervisor needs to reap was spawned by spawnEngine()/spawnDashboard() with a
+// byte-identical `path.join(MINIONS_DIR, '<script>.js')`, so exact script-token
+// match catches all of them while a foreign process that only *names* the path
+// is left alone (fail-safe: never kill on a loose match).
+function _cmdRunsScript(cmdline, target) {
+  const toks = _tokenizeCmdline(cmdline);
+  // toks[0] is the node executable; the script is the first later .js token.
+  for (let i = 1; i < toks.length; i++) {
+    const norm = _normPath(toks[i]);
+    if (norm.endsWith('.js')) return norm === target;
+  }
+  return false;
+}
+
+// PIDs of `node` processes that RUN `needle` (a script path) as their script.
+// Cross-platform: PowerShell CIM on Windows, `ps` elsewhere. Best effort —
+// returns [] on any enumeration failure (fail-open: leak rather than risk
+// killing the wrong process).
+//
+// The OS does the coarse filtering (WQL `CommandLine LIKE` / a `grep` on the
+// script basename) so only candidate rows — not the machine's entire node/
+// process table — cross into the JS heap; `_cmdRunsScript` then applies the
+// exact script-position match. On a busy host (CI runners, many node procs)
+// this keeps the buffered stdout to a handful of rows instead of megabytes.
+// `maxBuffer` caps the read defensively in case the coarse filter still matches
+// a lot. The basename is an internal constant (`engine.js`/`dashboard.js`), so
+// it carries no shell/WQL-injection risk.
+const _ENUM_MAX_BUFFER = 8 * 1024 * 1024;
+function listNodePidsMatching(needle) {
+  const target = _normPath(needle);
+  const base = path.basename(needle); // coarse prefilter, e.g. 'engine.js'
+  const pids = [];
+  try {
+    let out = '';
+    if (isWin) {
+      // spawnSync (no shell) + -EncodedCommand sidesteps cmd.exe quoting — an
+      // inline `powershell -Command "... $(...) | ..."` via execSync gets its
+      // `$(...)`/`|` mangled by cmd.exe before PowerShell ever sees them.
+      const ps = `Get-CimInstance Win32_Process -Filter "name='node.exe' AND CommandLine LIKE '%${base}%'" | `
+        + 'ForEach-Object { "$($_.ProcessId)|$($_.CommandLine)" }';
+      const res = spawnSync('powershell',
+        ['-NoProfile', '-NonInteractive', '-EncodedCommand', Buffer.from(ps, 'utf16le').toString('base64')],
+        { encoding: 'utf8', timeout: 8000, windowsHide: true, maxBuffer: _ENUM_MAX_BUFFER });
+      out = (res && res.stdout) || '';
+    } else {
+      // grep prefilters to candidate rows; no match exits 1 -> execSync throws
+      // -> caught below -> [] (correct: nothing to reap).
+      out = execSync(`ps -eo pid=,args= | grep -iF -- '${base}'`,
+        { encoding: 'utf8', timeout: 8000, maxBuffer: _ENUM_MAX_BUFFER });
+    }
+    for (const line of out.split('\n')) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      // Windows rows are "PID|<full command line>"; ps rows are "PID <args>".
+      const m = isWin ? trimmed.match(/^(\d+)\|(.*)$/) : trimmed.match(/^(\d+)\s+(.*)$/);
+      if (!m) continue;
+      const pid = Number(m[1]);
+      if (pid && _cmdRunsScript(m[2], target)) pids.push(pid);
+    }
+  } catch { /* enumeration failed — fail-open with whatever we collected */ }
+  return pids;
+}
+
+function _killHard(pid) {
+  const shared = _sharedOrNull();
+  try {
+    if (shared && typeof shared.killImmediate === 'function') { shared.killImmediate({ pid }); return; }
+  } catch { /* fall through to platform kill */ }
+  try {
+    if (isWin) execSync(`taskkill /F /T /PID ${pid}`, { timeout: 5000, windowsHide: true, stdio: 'ignore' });
+    else process.kill(pid, 'SIGKILL');
+  } catch { /* already gone */ }
+}
+
+// Kill every stray process running `scriptPath`, except this supervisor.
+// Returns the number reaped. No-op when MINIONS_SUPERVISOR_REAP=0.
+function reapStrayProcesses(scriptPath, label) {
+  if (!REAP_ENABLED) return 0;
+  const pids = listNodePidsMatching(scriptPath).filter(p => p !== process.pid);
+  if (!pids.length) return 0;
+  console.log(`[supervisor] Reaping ${pids.length} stray ${label} process(es) before respawn: ${pids.join(', ')}`);
+  for (const pid of pids) _killHard(pid);
+  return pids.length;
+}
+
 function openAppendFd(name) {
   // Try to use the same append-fd helper the rest of the codebase uses, fall
   // back to a plain fs.openSync(..., 'a') when it isn't available. We re-route
@@ -226,6 +359,9 @@ function checkEngine(now) {
   }
 
   console.log(`[supervisor] Engine PID ${control.pid || '(none)'} is dead — respawning...`);
+  // Reap any orphan engines first so a stale control.json PID can't snowball
+  // into an unbounded pile of contending engine processes.
+  reapStrayProcesses(path.join(MINIONS_DIR, 'engine.js'), 'engine');
   const newPid = spawnEngine();
   _lastEngineRespawnAt = now;
   console.log(`[supervisor] Engine respawned (new PID: ${newPid})`);
@@ -238,6 +374,10 @@ function checkDashboard(now) {
   if (pids.length > 0) return;
 
   console.log(`[supervisor] Dashboard not listening on port ${dashPort} — respawning...`);
+  // Reap stray dashboards first. Besides bounding the count, this collapses the
+  // port-desync case: orphan dashboards holding other ports are killed, freeing
+  // the canonical port for the fresh one whose beacon _resolveDashPort() reads.
+  reapStrayProcesses(path.join(MINIONS_DIR, 'dashboard.js'), 'dashboard');
   const newPid = spawnDashboard();
   _lastDashboardRespawnAt = now;
   console.log(`[supervisor] Dashboard respawned (new PID: ${newPid})`);
@@ -313,6 +453,9 @@ module.exports = {
     isStopIntentSet,
     isPidAlive,
     listeningPidsForPort,
+    listNodePidsMatching,
+    reapStrayProcesses,
+    _cmdRunsScript,
     openAppendFd,
     checkEngine,
     checkDashboard,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2225",
+  "version": "0.1.2226",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"