@yemi33/minions 0.1.2380 → 0.1.2382

package/engine/supervisor.js

@@ -104,6 +104,8 @@ const SUPERVISOR_STALE_ENGINE_HEARTBEAT_MS = Number(process.env.MINIONS_SUPERVIS
 // 3 strikes ≈ 90s of continuous unresponsiveness before we act.
 const DASH_HEALTH_TIMEOUT_MS = Number(process.env.MINIONS_SUPERVISOR_DASH_HEALTH_TIMEOUT_MS) || 5000;
 const DASH_HEALTH_MAX_FAILS = Number(process.env.MINIONS_SUPERVISOR_DASH_HEALTH_FAILS) || 3;
+const REAP_WAIT_TIMEOUT_MS = Number(process.env.MINIONS_SUPERVISOR_REAP_TIMEOUT_MS) || 10000;
+const REAP_WAIT_POLL_MS = Number(process.env.MINIONS_SUPERVISOR_REAP_POLL_MS) || 100;
 const isWin = process.platform === 'win32';
 
 function safeReadJson(p) {
@@ -126,11 +128,27 @@ function isStopIntentSet() {
   return fs.existsSync(STOP_INTENT_PATH_FN());
 }
 
-function isPidAlive(pid) {
-  if (!pid) return false;
+function _readPosixProcessState(pid) {
+  const result = spawnSync('ps', ['-o', 'stat=', '-p', String(pid)], {
+    encoding: 'utf8',
+    timeout: 3000,
+    windowsHide: true,
+    stdio: ['ignore', 'pipe', 'ignore'],
+  });
+  if (result.error || result.status !== 0) return null;
+  return String(result.stdout || '').trim();
+}
+
+function isPidAlive(pid, {
+  platform = process.platform,
+  kill = process.kill,
+  readPosixProcessState = _readPosixProcessState,
+} = {}) {
+  const n = Number(pid);
+  if (!Number.isInteger(n) || n <= 0) return false;
   try {
-    if (isWin) {
-      const out = execSync(`tasklist /FI "PID eq ${pid}" /NH`, {
+    if (platform === 'win32') {
+      const out = execSync(`tasklist /FI "PID eq ${n}" /NH`, {
         encoding: 'utf8', timeout: 3000, windowsHide: true,
       });
       // Word-boundary + image-name match (mirrors engine/restart-health.js) so a
@@ -138,12 +156,16 @@ function isPidAlive(pid) {
       // alive. Fall back to an inline regex if shared.js isn't loadable.
       const shared = _sharedOrNull();
       if (shared && typeof shared.tasklistOutputShowsPid === 'function') {
-        return shared.tasklistOutputShowsPid(out, pid, { imageName: 'node' });
+        return shared.tasklistOutputShowsPid(out, n, { imageName: 'node' });
       }
-      return new RegExp(`\\b${pid}\\b`).test(out) && out.toLowerCase().includes('node');
+      return new RegExp(`\\b${n}\\b`).test(out) && out.toLowerCase().includes('node');
     }
-    process.kill(pid, 0);
-    return true;
+    kill(n, 0);
+    // kill(0) also succeeds for Unix zombies. They cannot execute or retain
+    // runtime resources, and a synchronous reap wait prevents Node from
+    // processing SIGCHLD, so treating them as live deadlocks every respawn.
+    const state = readPosixProcessState(n);
+    return state === null || (state !== '' && !state.startsWith('Z'));
   } catch { return false; }
 }
 
@@ -305,14 +327,43 @@ function _killHard(pid) {
   } catch { /* already gone */ }
 }
 
+function _sleepSync(ms) {
+  if (!(ms > 0)) return;
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+function waitForPidsToExit(pids, {
+  timeoutMs = REAP_WAIT_TIMEOUT_MS,
+  pollMs = REAP_WAIT_POLL_MS,
+  isAlive = isPidAlive,
+  sleep = _sleepSync,
+} = {}) {
+  const unique = [...new Set((pids || []).map(Number).filter(pid => Number.isInteger(pid) && pid > 0))];
+  const deadline = Date.now() + Math.max(0, timeoutMs);
+  let remaining = unique.filter(isAlive);
+  while (remaining.length > 0 && Date.now() < deadline) {
+    sleep(Math.min(pollMs, Math.max(1, deadline - Date.now())));
+    remaining = remaining.filter(isAlive);
+  }
+  return { ok: remaining.length === 0, remaining };
+}
+
 // Kill every stray process running `scriptPath`, except this supervisor.
-// Returns the number reaped. No-op when MINIONS_SUPERVISOR_REAP=0.
-function reapStrayProcesses(scriptPath, label) {
+// Returns the number reaped. Throws instead of spawning a replacement when any
+// old process survives the bounded wait — binding a fallback port or starting a
+// second engine is worse than retrying on the next supervisor tick.
+function reapStrayProcesses(scriptPath, label, options = {}) {
   if (!REAP_ENABLED) return 0;
-  const pids = listNodePidsMatching(scriptPath).filter(p => p !== process.pid);
+  const listPids = options.listPids || listNodePidsMatching;
+  const kill = options.kill || _killHard;
+  const pids = listPids(scriptPath).filter(p => p !== process.pid);
   if (!pids.length) return 0;
   console.log(`[supervisor] Reaping ${pids.length} stray ${label} process(es) before respawn: ${pids.join(', ')}`);
-  for (const pid of pids) _killHard(pid);
+  for (const pid of pids) kill(pid);
+  const waited = waitForPidsToExit(pids, options);
+  if (!waited.ok) {
+    throw new Error(`Refusing to respawn ${label}: stale PID(s) still alive after ${options.timeoutMs || REAP_WAIT_TIMEOUT_MS}ms: ${waited.remaining.join(', ')}`);
+  }
   return pids.length;
 }
 
@@ -372,11 +423,15 @@ function spawnEngine() {
   return proc.pid;
 }
 
-function spawnDashboard() {
+function spawnDashboard(requestedPort = _resolveDashPort()) {
   const out = openAppendFd('dashboard-stdio.log');
   const err = openAppendFd('dashboard-stdio.log');
-  const env = { ...process.env, MINIONS_NO_AUTO_OPEN: '1' };
-  const proc = spawn(process.execPath, [..._sqliteSpawnFlags(), path.join(MINIONS_DIR, 'dashboard.js')], {
+  const env = {
+    ...process.env,
+    MINIONS_NO_AUTO_OPEN: '1',
+    MINIONS_REQUIRE_DASHBOARD_PORT: '1',
+  };
+  const proc = spawn(process.execPath, [..._sqliteSpawnFlags(), path.join(MINIONS_DIR, 'dashboard.js'), String(requestedPort)], {
     cwd: MINIONS_DIR,
     stdio: ['ignore', out, err],
     detached: true,
@@ -533,7 +588,7 @@ async function checkDashboard(now) {
     // Critically, for the frozen-but-listening case the reap also KILLS the wedged
     // process still holding the port so spawnDashboard() can bind it.
     reapStrayProcesses(path.join(MINIONS_DIR, 'dashboard.js'), 'dashboard');
-    const newPid = spawnDashboard();
+    const newPid = spawnDashboard(dashPort);
     _lastDashboardRespawnAt = now;
     _dashHealthFailStreak = 0;
     console.log(`[supervisor] Dashboard respawned (new PID: ${newPid})`);
@@ -647,6 +702,7 @@ module.exports = {
     listeningPidsForPort,
     listNodePidsMatching,
     reapStrayProcesses,
+    waitForPidsToExit,
     _cmdRunsScript,
     openAppendFd,
     checkEngine,

package/engine/timeout.js

@@ -23,6 +23,60 @@ const MINIONS_DIR = shared.MINIONS_DIR;
 // Only needed for engine().activeProcesses and engine().engineRestartGraceUntil — log/ts come from shared.js
 let _engine = null;
 function engine() { if (!_engine) _engine = require('../engine'); return _engine; }
+const _pendingBranchPreservations = new Set();
+
+function resolveDispatchBranchPath(item) {
+  if (item?.worktreePath) return item.worktreePath;
+  if (item?.originalRef && item.meta?.project?.localPath) return item.meta.project.localPath;
+  return null;
+}
+
+function preserveDispatchBranch(item, config) {
+  const branchPath = resolveDispatchBranchPath(item);
+  if (!branchPath || !item.meta?.branch) {
+    return Promise.resolve({ pushed: false, reason: 'invalid-worktree' });
+  }
+  const persistedProject = item.meta?.project || {};
+  const project = getProjects(config).find(p => p.name === persistedProject.name) || persistedProject;
+  if (!project.localPath) return Promise.resolve({ pushed: false, reason: 'invalid-project' });
+  return engine().pushCleanAheadBranch({
+    rootDir: project.localPath,
+    worktreePath: branchPath,
+    branchName: item.meta.branch,
+    mainBranch: shared.resolveMainBranch(project.localPath, project.mainBranch),
+    project,
+    gitOpts: { env: shared.gitEnv() },
+  });
+}
+
+async function _finalizeDeadDispatch(item, reason, failureClass, config, deps = {}) {
+  const preserveBranch = deps.preserveBranch || preserveDispatchBranch;
+  const restoreLiveCheckout = deps.restoreLiveCheckout ||
+    require('./live-checkout').maybeRestoreLiveCheckoutFromRecord;
+  const complete = deps.complete || dispatch().completeDispatch;
+
+  if (resolveDispatchBranchPath(item) && item.meta?.branch) {
+    try {
+      await preserveBranch(item, config);
+    } catch (err) {
+      log('warn', `Orphan branch preservation failed for ${item.id}: ${err.message}`);
+    }
+  }
+  if (item.originalRef && item.meta?.branch && item.meta?.project?.localPath) {
+    try {
+      await restoreLiveCheckout({
+        item,
+        isTerminalFailure: true,
+        resultLabel: 'orphaned',
+        log: (lvl, msg) => log(lvl, msg),
+        writeInboxAlert: dispatch().writeInboxAlert,
+      });
+    } catch (err) {
+      log('warn', 'live-checkout restore (orphan): ' + err.message);
+    }
+  }
+  complete(item.id, DISPATCH_RESULT.ERROR, reason, '', failureClass ? { failureClass } : {});
+}
 
 // Lazy require for dispatch module (also circular via engine)
 let _dispatch = null;
@@ -664,7 +718,27 @@ function checkTimeouts(config) {
   const deadItems = [];
   const legacyAnnotationClears = new Set();
 
-  function completeFromOutput(item, liveLogPath, processExitCode, detectedLogText, hasProcess) {
+  function completeFromOutput(item, liveLogPath, processExitCode, detectedLogText, hasProcess, preservationDone = false, expectedProcInfo = activeProcesses.get(item.id)) {
+    if (!preservationDone && resolveDispatchBranchPath(item) && item.meta?.branch) {
+      if (_pendingBranchPreservations.has(item.id)) return;
+      _pendingBranchPreservations.add(item.id);
+      preserveDispatchBranch(item, config)
+        .catch(err => log('warn', `Output-completion branch preservation failed for ${item.id}: ${err.message}`))
+        .then(() => completeFromOutput(item, liveLogPath, processExitCode, detectedLogText, hasProcess, true, expectedProcInfo))
+        .finally(() => _pendingBranchPreservations.delete(item.id));
+      return;
+    }
+    if (!(getDispatch().active || []).some(active => active.id === item.id)) return;
+    if (hasProcess) {
+      const currentProcInfo = activeProcesses.get(item.id);
+      if (currentProcInfo !== expectedProcInfo ||
+          currentProcInfo?._steeringAt ||
+          currentProcInfo?._steeringMessage ||
+          currentProcInfo?._steeringNoSession) {
+        log('info', `${item.id}: output-detected completion yielded to the close/steering owner after branch preservation`);
+        return;
+      }
+    }
     const isSuccess = processExitCode === 0;
     log('info', `Agent ${item.agent} (${item.id}) completed via output detection (exit code ${processExitCode}, ${isSuccess ? 'success' : 'error'})`);
 
@@ -956,22 +1030,18 @@ function checkTimeouts(config) {
 
   // Clean up dead items
   for (const { item, reason, failureClass } of deadItems) {
-    completeDispatch(item.id, DISPATCH_RESULT.ERROR, reason, '', failureClass ? { failureClass } : {});
-    // PL-live-checkout-reliability-hardening — a live-mode dispatch declared an
-    // orphan (lost process handle, e.g. across an engine restart) leaves the
-    // operator checkout on the agent branch. Restore it from the persisted
-    // record (originalRef present ⇒ live mode). Always a terminal failure here.
-    if (item.originalRef && item.meta?.branch && item.meta?.project?.localPath) {
-      try {
-        require('./live-checkout').maybeRestoreLiveCheckoutFromRecord({
-          item,
-          isTerminalFailure: true,
-          resultLabel: 'orphaned',
-          log: (lvl, msg) => log(lvl, msg),
-          writeInboxAlert: dispatch().writeInboxAlert,
-        }).catch(() => {});
-      } catch (e) { log('warn', 'live-checkout restore (orphan): ' + e.message); }
+    // #853 — stale orphans have no process close event, so preserve their
+    // clean committed branch explicitly before completeDispatch makes the
+    // worktree eligible for teardown/quarantine.
+    if (resolveDispatchBranchPath(item) && item.meta?.branch) {
+      if (_pendingBranchPreservations.has(item.id)) continue;
+      _pendingBranchPreservations.add(item.id);
+      _finalizeDeadDispatch(item, reason, failureClass, config, { complete: completeDispatch })
+        .catch(err => log('warn', `Orphan cleanup failed for ${item.id}: ${err.message}`))
+        .finally(() => _pendingBranchPreservations.delete(item.id));
+      continue;
     }
+    completeDispatch(item.id, DISPATCH_RESULT.ERROR, reason, '', failureClass ? { failureClass } : {});
   }
 
   // Clear legacy blocking-tool annotations; process liveness no longer depends on tool parsing.
@@ -1055,4 +1125,5 @@ module.exports = {
   // exported for testing — steering clamp helpers and deferred checkpoint
   _clampSteeringDeferredMaxMs, _clampSteeringMaxKillRetries, _appendLiveOutputLine, deferSteeringUntilCheckpoint,
   killTrackedProcess, _escalatePlatformKill, // exported for testing — P-6e2a4c15 cancel-vs-kill
+  _finalizeDeadDispatch, // exported for testing
 };

package/engine/untrusted-fence.js

@@ -113,7 +113,8 @@ function buildSource(kind, parts) {
   }
 
   if (k === 'pinned-note' || k === 'team-notes' || k === 'agent-memory'
-   || k === 'wi-reference' || k === 'doc-content' || k === 'doc-selection') {
+   || k === 'wi-reference' || k === 'doc-content' || k === 'doc-selection'
+   || k === 'project-instructions') {
     return parts.path ? `${k}:${get('path')}` : k;
   }
   if (k === 'inbox') {