@yemi33/minions 0.1.2387 → 0.1.2388

package/engine/supervisor.js

@@ -507,7 +507,39 @@ function _isRespawnEligibleEngineState(state) {
   return state === 'running' || state === 'degraded';
 }
 
-function checkEngine(now) {
+const ENGINE_WATCHDOG_OPERATION_NAMES = [
+  'isPidAlive',
+  'reapStrayProcesses',
+  'spawnEngine',
+  'recordCrashLoopRespawn',
+];
+
+// Isolated state does not isolate this module's source checkout, so tests must
+// provide every OS/process side effect before an engine respawn is possible.
+function _resolveEngineWatchdogOperations(injected) {
+  const provided = injected && typeof injected === 'object' ? injected : {};
+  if (process.env.MINIONS_TEST_DIR) {
+    const missing = ENGINE_WATCHDOG_OPERATION_NAMES
+      .filter(name => typeof provided[name] !== 'function');
+    if (missing.length > 0) {
+      throw new Error(
+        `MINIONS_TEST_DIR is set; inject engine watchdog operations instead of using real process control: ${missing.join(', ')}`,
+      );
+    }
+  }
+  return {
+    isPidAlive: typeof provided.isPidAlive === 'function' ? provided.isPidAlive : isPidAlive,
+    reapStrayProcesses: typeof provided.reapStrayProcesses === 'function'
+      ? provided.reapStrayProcesses
+      : reapStrayProcesses,
+    spawnEngine: typeof provided.spawnEngine === 'function' ? provided.spawnEngine : spawnEngine,
+    recordCrashLoopRespawn: typeof provided.recordCrashLoopRespawn === 'function'
+      ? provided.recordCrashLoopRespawn
+      : _recordCrashLoopRespawn,
+  };
+}
+
+function checkEngine(now, operations) {
   if (now - _lastEngineRespawnAt < POST_SPAWN_GRACE_MS) return;
   const control = safeReadJson(CONTROL_PATH_FN());
   // Respawn when control.json says "running" (PID died unexpectedly) or
@@ -515,7 +547,6 @@ function checkEngine(now) {
   // above). "paused"/"stopped"/"stopping" are legitimate user-intent states the
   // supervisor must not override.
   if (!control || !_isRespawnEligibleEngineState(control.state)) return;
-  if (control.pid && isPidAlive(control.pid)) return;
 
   // Cross-watchdog race guard: dashboard.js's in-process engine watchdog
   // writes `{pid: null, restarted_at}` BEFORE spawning a new engine. During
@@ -530,13 +561,16 @@ function checkEngine(now) {
     }
   }
 
+  const ops = _resolveEngineWatchdogOperations(operations);
+  if (control.pid && ops.isPidAlive(control.pid)) return;
+
   console.log(`[supervisor] Engine PID ${control.pid || '(none)'} is dead — respawning...`);
   // Reap any orphan engines first so a stale control.json PID can't snowball
   // into an unbounded pile of contending engine processes.
-  reapStrayProcesses(path.join(MINIONS_DIR, 'engine.js'), 'engine');
-  const newPid = spawnEngine();
+  ops.reapStrayProcesses(path.join(MINIONS_DIR, 'engine.js'), 'engine');
+  const newPid = ops.spawnEngine();
   _lastEngineRespawnAt = now;
-  _recordCrashLoopRespawn('supervisor:dead-pid');
+  ops.recordCrashLoopRespawn('supervisor:dead-pid');
   console.log(`[supervisor] Engine respawned (new PID: ${newPid})`);
 }
 
@@ -546,7 +580,7 @@ function checkEngine(now) {
 // is alive but control.heartbeat has not advanced for SUPERVISOR_STALE_ENGINE_HEARTBEAT_MS.
 // The heartbeat is written every ~15s by a dedicated timer in engine/cli.js; if
 // it goes stale while the PID persists, the event loop is stuck.
-function checkEngineHung(now) {
+function checkEngineHung(now, operations) {
   if (now - _lastEngineRespawnAt < POST_SPAWN_GRACE_MS) return;
   const control = safeReadJson(CONTROL_PATH_FN());
   // Include 'degraded' (see _isRespawnEligibleEngineState) so a permanently
@@ -554,8 +588,6 @@ function checkEngineHung(now) {
   // re-enters) still gets force-restarted once its own heartbeat goes stale,
   // instead of being ignored forever the moment the state flips to 'degraded'.
   if (!control || !_isRespawnEligibleEngineState(control.state)) return;
-  // Only acts when the PID is alive — a dead PID is handled by checkEngine().
-  if (!control.pid || !isPidAlive(control.pid)) return;
 
   const heartbeat = Number(control.heartbeat);
   if (!heartbeat || !Number.isFinite(heartbeat)) return; // no heartbeat yet (fresh start)
@@ -563,15 +595,20 @@ function checkEngineHung(now) {
   const heartbeatAge = now - heartbeat;
   if (heartbeatAge < SUPERVISOR_STALE_ENGINE_HEARTBEAT_MS) return;
 
+  // Only acts when the PID is alive — a dead PID is handled by checkEngine().
+  if (!control.pid) return;
+  const ops = _resolveEngineWatchdogOperations(operations);
+  if (!ops.isPidAlive(control.pid)) return;
+
   console.log(
     `[supervisor] Engine PID ${control.pid} is alive but heartbeat is stale ` +
     `(${Math.round(heartbeatAge / 1000)}s old, threshold ${SUPERVISOR_STALE_ENGINE_HEARTBEAT_MS / 1000}s) — ` +
     `event loop likely frozen; restarting engine`,
   );
-  reapStrayProcesses(path.join(MINIONS_DIR, 'engine.js'), 'engine');
-  const newPid = spawnEngine();
+  ops.reapStrayProcesses(path.join(MINIONS_DIR, 'engine.js'), 'engine');
+  const newPid = ops.spawnEngine();
   _lastEngineRespawnAt = now;
-  _recordCrashLoopRespawn('supervisor:stale-heartbeat');
+  ops.recordCrashLoopRespawn('supervisor:stale-heartbeat');
   console.log(`[supervisor] Engine restarted due to stale heartbeat (new PID: ${newPid})`);
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2387",
+  "version": "0.1.2388",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"