@yemi33/minions 0.1.1949 → 0.1.1951

package/engine/spawn-agent.js

@@ -38,6 +38,7 @@ const path = require('path');
 const { runFile, cleanChildEnv, killGracefully, killImmediate, killByPidsImmediate, listProcessDescendants, ts, resolveEngineCacheDir } = require('./shared');
 const { resolveRuntime } = require('./runtimes');
 const { acquireAdoTokenSync, isLikelyAdoToken } = require('./ado-token');
+const keepProcessSweep = require('./keep-process-sweep');
 
 // ─── Pure helpers (exported for tests) ──────────────────────────────────────
 
@@ -534,8 +535,49 @@ function main() {
     if (trackedDescendants.size || gotFirstOutput) {
       snapshotDescendants();
       if (trackedDescendants.size) {
-        const reaped = killByPidsImmediate([...trackedDescendants]);
-        try { fs.appendFileSync(debugPath, `DESCENDANTS reaped=${reaped}/${trackedDescendants.size}\n`); } catch {}
+        // W-mp68q6ke0010de68 — opt-in keep_processes flag: agents whose work
+        // item carried `meta.keep_processes: true` may have written
+        // `agents/<id>/keep-pids.json` declaring specific descendant PIDs the
+        // engine MUST NOT reap. Resolve agentId from MINIONS_LIVE_OUTPUT_PATH
+        // (engine.js:1451 sets it to agents/<agentId>/live-output.log) and
+        // subtract validated, alive PIDs from the kill set. Missing or
+        // invalid file → fall through to today's behavior (reap everything).
+        let toKillPids = [...trackedDescendants];
+        let kept = [];
+        let keepRecord = null;
+        let keepReason = null;
+        try {
+          const liveOut = process.env.MINIONS_LIVE_OUTPUT_PATH;
+          const agentId = liveOut ? path.basename(path.dirname(liveOut)) : '';
+          if (agentId) {
+            // W-mp6k7ywi000fa33c — per-WI override (set by engine when
+            // meta.keep_processes_skip_workdir_check is true) bypasses the
+            // requireGitWorkdir check so legitimate non-git keep_processes
+            // use cases (e.g., a daemon under /tmp) still anchor.
+            const reapOpts = process.env.MINIONS_KEEP_PROCESSES_SKIP_WORKDIR_CHECK === '1'
+              ? { requireGitWorkdir: false }
+              : {};
+            const plan = keepProcessSweep.computeReapPlan(toKillPids, agentId, reapOpts);
+            toKillPids = plan.toKill;
+            kept = plan.kept;
+            keepRecord = plan.record;
+            keepReason = plan.reason;
+          }
+        } catch (e) {
+          try { fs.appendFileSync(debugPath, `KEEP-PIDS error: ${e.message}\n`); } catch {}
+        }
+        if (kept.length) {
+          try {
+            fs.appendFileSync(
+              debugPath,
+              `KEPT pids=[${kept.join(',')}] purpose="${(keepRecord?.purpose || '').slice(0, 200)}" wi=${keepRecord?.wi_id || ''}\n`,
+            );
+          } catch {}
+        } else if (keepReason) {
+          try { fs.appendFileSync(debugPath, `KEEP-PIDS skipped: ${keepReason}\n`); } catch {}
+        }
+        const reaped = toKillPids.length ? killByPidsImmediate(toKillPids) : 0;
+        try { fs.appendFileSync(debugPath, `DESCENDANTS reaped=${reaped}/${toKillPids.length} kept=${kept.length}\n`); } catch {}
       }
     }
     // Prefer the 'exit' event's code/signal when present — see note above.

package/engine/timeout.js

@@ -181,19 +181,27 @@ function isTrackedProcessAlive(procInfo) {
   }
 }
 
+// Read the on-disk PID for a dispatch ID, or null if missing/invalid. Shared by
+// `isOsPidAliveForDispatch` and the recycled-PID command-line cross-check below
+// so both paths agree on which integer PID to interrogate.
+function _readDispatchPid(itemId) {
+  const safeId = String(itemId || '').replace(/[:\\/*?"<>|]/g, '-');
+  const pidPath = path.join(ENGINE_DIR, 'tmp', `pid-${safeId}.pid`);
+  let raw;
+  try { raw = fs.readFileSync(pidPath, 'utf8'); }
+  catch { return null; }
+  const pid = parseInt(String(raw).trim(), 10);
+  return Number.isFinite(pid) && pid > 0 ? pid : null;
+}
+
 // Last-resort liveness check via the on-disk PID file (engine/tmp/pid-<safeId>.pid).
 // Used by orphan detection to avoid false-positive kills when the engine has lost the
 // tracked process handle (engine restart, never-tracked spawn, etc.) but the OS-level
 // child process is still alive and healthy. The safeId here mirrors engine.js spawn
 // (id.replace(/[:\\/*?"<>|]/g, '-')) — same pattern engine/cli.js uses to re-attach.
 function isOsPidAliveForDispatch(itemId) {
-  const safeId = String(itemId || '').replace(/[:\\/*?"<>|]/g, '-');
-  const pidPath = path.join(ENGINE_DIR, 'tmp', `pid-${safeId}.pid`);
-  let raw;
-  try { raw = fs.readFileSync(pidPath, 'utf8'); }
-  catch { return false; }
-  const pid = parseInt(String(raw).trim(), 10);
-  if (!Number.isFinite(pid) || pid <= 0) return false;
+  const pid = _readDispatchPid(itemId);
+  if (!pid) return false;
   try { process.kill(pid, 0); return true; }
   catch { return false; }
 }
@@ -324,7 +332,12 @@ function checkTimeouts(config) {
     // detectPhantom downstream lets enforcePrAttachmentContract route phantom
     // hard-fails through the _phantomRetryCount budget instead of bypassing
     // the retry counter entirely (P-d9a3e6f4).
-    runPostCompletionHooks(item, item.agent, processExitCode, fullLogForHooks, config, { detectPhantom: true }).catch(e => log('warn', 'post-completion hooks: ' + e.message));
+    // P-d2a8f6c1: hand the per-spawn nonce to lifecycle for trust-boundary
+    // validation. Read it from activeProcesses before the hasProcess branch
+    // below deletes the entry.
+    const expectedNonce = activeProcesses.get(item.id)?._completionNonce || null;
+    const completionNonceRequired = config?.engine?.completionNonceRequired ?? ENGINE_DEFAULTS.completionNonceRequired;
+    runPostCompletionHooks(item, item.agent, processExitCode, fullLogForHooks, config, { detectPhantom: true, expectedNonce, completionNonceRequired }).catch(e => log('warn', 'post-completion hooks: ' + e.message));
 
     if (hasProcess) {
       shared.killImmediate(activeProcesses.get(item.id)?.proc);
@@ -341,7 +354,7 @@ function checkTimeouts(config) {
 
     const procInfo = activeProcesses.get(item.id);
     const hasProcess = !!procInfo;
-    const processAlive = isTrackedProcessAlive(procInfo);
+    let processAlive = isTrackedProcessAlive(procInfo);
     const liveLogPath = path.join(AGENTS_DIR, item.agent, 'live-output.log');
     let lastActivity = item.started_at ? new Date(item.started_at).getTime() : 0;
 
@@ -475,8 +488,18 @@ function checkTimeouts(config) {
     if (!processAlive && (canReapDeadProcess || silentMs > staleOrphanTimeout) && (Date.now() > engineRestartGraceUntil || canReapDeadProcess)) {
       // Last-resort PID check: lost tracked handle but OS process may still be alive.
       if (isOsPidAliveForDispatch(item.id)) {
-        log('info', `Orphan check: ${item.agent} (${item.id}) silent ${silentSec}s but OS PID is alive — keeping [${_logState}]`);
-        continue;
+        // Cross-check the PID's command line: a recycled OS PID can be alive
+        // under an unrelated process (Windows reuses PIDs aggressively). Without
+        // this cmdline gate, the dispatch is parked indefinitely as 'active'
+        // pinned to a process that has nothing to do with the agent.
+        const livePid = _readDispatchPid(item.id);
+        if (livePid && !shared.isProcessCommandLineMatchingAgent(livePid)) {
+          log('warn', `Orphan check: ${item.agent} (${item.id}) — PID ${livePid} alive but command line doesn't match agent, treating as dead`);
+          processAlive = false;
+        } else {
+          log('info', `Orphan check: ${item.agent} (${item.id}) silent ${silentSec}s but OS PID is alive — keeping [${_logState}]`);
+          continue;
+        }
       }
       // Final safety scan: the normal 64KB tail scan can miss a clean exit if
       // later runtime payloads or diagnostics push the sentinel outside the tail.

package/engine/worktree-pool.js

@@ -0,0 +1,410 @@
+/**
+ * engine/worktree-pool.js — per-project warm pool of git worktrees that can be
+ * recycled across branches (W-mp73ya3e000me6c5).
+ *
+ * Default-off: `ENGINE_DEFAULTS.worktreePoolSize = 0`. Per-project opt-in via
+ * `projects[].worktreePoolSize`. When enabled, completed worktrees are parked
+ * detached at `origin/<mainBranch>` and reused by the next dispatch for the
+ * same project — saving the cold install/build cost on subsequent WIs.
+ *
+ * Concurrency: all state flips go through `mutateJsonFileLocked` so two
+ * spawning ticks cannot double-borrow the same idle entry. Per CLAUDE.md, git
+ * commands NEVER run inside the lock callback — callers must do `tryBorrow`,
+ * then run git in their own scope, then call `evictEntry` on failure.
+ *
+ * Idle worktrees are detached at origin/<main>, NOT checked out on the local
+ * `main` branch. Local-main is typically already checked out in the project
+ * root, and git refuses two checkouts of the same branch. Detached HEAD
+ * sidesteps that entirely while keeping the working tree warm with main's
+ * package.json + lockfile + node_modules.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const shared = require('./shared');
+
+const { log, ENGINE_DEFAULTS, mutateJsonFileLocked, safeJson } = shared;
+
+const WORKTREE_POOL_FILENAME = 'worktree-pool.json';
+const POOL_STATE = Object.freeze({ IDLE: 'idle', BORROWED: 'borrowed' });
+
+function getPoolPath() {
+  return path.join(shared.MINIONS_DIR, 'engine', WORKTREE_POOL_FILENAME);
+}
+
+function _normalizePath(p) {
+  if (!p) return '';
+  let resolved;
+  try { resolved = path.resolve(String(p)); }
+  catch { return ''; }
+  // Strip trailing separators
+  resolved = resolved.replace(/[\\/]+$/, '');
+  // Case-insensitive on Windows
+  if (process.platform === 'win32') resolved = resolved.toLowerCase();
+  // Use forward-slashes for stable comparison across OS path APIs
+  return resolved.replace(/\\/g, '/');
+}
+
+function _nowIso(now) {
+  return new Date(Number.isFinite(now) ? now : Date.now()).toISOString();
+}
+
+function _entriesArray(data) {
+  if (!data || typeof data !== 'object' || !Array.isArray(data.entries)) return [];
+  return data.entries;
+}
+
+function readPool() {
+  const data = safeJson(getPoolPath());
+  if (!data || !Array.isArray(data.entries)) return { entries: [] };
+  return data;
+}
+
+function mutateWorktreePool(mutator) {
+  return mutateJsonFileLocked(getPoolPath(), (data) => {
+    if (!data || typeof data !== 'object' || Array.isArray(data)) data = { entries: [] };
+    if (!Array.isArray(data.entries)) data.entries = [];
+    const next = mutator(data);
+    return next === undefined ? data : next;
+  }, { defaultValue: { entries: [] }, skipWriteIfUnchanged: true });
+}
+
+/**
+ * Resolve effective pool size for a project. Per-project override beats the
+ * fleet-wide default. Returns 0 (disabled) when nothing is configured.
+ */
+function getProjectPoolSize(projectName, config) {
+  config = config || {};
+  if (projectName && Array.isArray(config.projects)) {
+    const proj = config.projects.find(p => p && p.name === projectName);
+    if (proj && Number.isFinite(Number(proj.worktreePoolSize))) {
+      return Math.max(0, Math.floor(Number(proj.worktreePoolSize)));
+    }
+  }
+  const fleet = config.engine?.worktreePoolSize ?? ENGINE_DEFAULTS.worktreePoolSize;
+  if (Number.isFinite(Number(fleet))) return Math.max(0, Math.floor(Number(fleet)));
+  return 0;
+}
+
+function getPoolIdleTtlMs(config) {
+  config = config || {};
+  const v = config.engine?.worktreePoolIdleTtlMs ?? ENGINE_DEFAULTS.worktreePoolIdleTtlMs;
+  if (Number.isFinite(Number(v))) return Math.max(60000, Number(v));
+  return 6 * 3600 * 1000;
+}
+
+function _findEntryByPath(entries, wtPath) {
+  const norm = _normalizePath(wtPath);
+  if (!norm) return -1;
+  return entries.findIndex(e => e && _normalizePath(e.path) === norm);
+}
+
+function isPoolMember(wtPath) {
+  const entries = _entriesArray(readPool());
+  return _findEntryByPath(entries, wtPath) !== -1;
+}
+
+function getEntry(wtPath) {
+  const entries = _entriesArray(readPool());
+  const idx = _findEntryByPath(entries, wtPath);
+  return idx === -1 ? null : entries[idx];
+}
+
+/**
+ * Atomically borrow an idle entry for a project. Returns the borrowed entry
+ * (with state flipped to 'borrowed' on disk) or null when none is available.
+ *
+ * The caller MUST then run the git ops to checkout the new branch, and call
+ * `evictEntry` on failure to release the slot.
+ */
+function tryBorrow(projectName, dispatchId, opts) {
+  opts = opts || {};
+  if (!projectName || !dispatchId) return null;
+  const now = Number.isFinite(opts.now) ? opts.now : Date.now();
+  let borrowed = null;
+  mutateWorktreePool((data) => {
+    const entries = data.entries;
+    // Pick the most-recently-used idle entry for this project (warmer caches).
+    let bestIdx = -1;
+    let bestIdleSince = -Infinity;
+    for (let i = 0; i < entries.length; i++) {
+      const e = entries[i];
+      if (!e || e.project !== projectName || e.state !== POOL_STATE.IDLE) continue;
+      // Skip entries whose path no longer exists on disk.
+      if (!e.path || !fs.existsSync(e.path)) continue;
+      const t = Date.parse(e.idleSince || e.lastUsed || e.createdAt || '') || 0;
+      if (t > bestIdleSince) { bestIdleSince = t; bestIdx = i; }
+    }
+    if (bestIdx === -1) return data;
+    const entry = entries[bestIdx];
+    entry.state = POOL_STATE.BORROWED;
+    entry.borrowedBy = dispatchId;
+    entry.borrowedAt = _nowIso(now);
+    entry.lastUsed = entry.borrowedAt;
+    entry.idleSince = null;
+    borrowed = { ...entry };
+    return data;
+  });
+  return borrowed;
+}
+
+/**
+ * Register a brand-new worktree as a borrowed pool member. Used when a fresh
+ * worktree is created (no idle slot available) and the pool has room — we
+ * track it from creation so the eventual return is a simple state flip.
+ *
+ * Honors capacity inside the locked mutation: if the project is already at or
+ * over `poolSize`, the registration is rejected and the worktree stays
+ * untracked (cleanup will reap it normally).
+ */
+function registerBorrowed(projectName, wtPath, dispatchId, opts) {
+  opts = opts || {};
+  if (!projectName || !wtPath || !dispatchId) return false;
+  const poolSize = Number.isFinite(opts.poolSize) ? Math.max(0, Math.floor(opts.poolSize)) : 0;
+  if (poolSize <= 0) return false;
+  const branch = opts.branch || '';
+  const now = Number.isFinite(opts.now) ? opts.now : Date.now();
+  let registered = false;
+  mutateWorktreePool((data) => {
+    const entries = data.entries;
+    // Already tracked? Update the borrower fields and keep going.
+    const existingIdx = _findEntryByPath(entries, wtPath);
+    if (existingIdx !== -1) {
+      const e = entries[existingIdx];
+      e.state = POOL_STATE.BORROWED;
+      e.borrowedBy = dispatchId;
+      e.borrowedAt = _nowIso(now);
+      e.idleSince = null;
+      if (branch) e.lastBranch = branch;
+      e.lastUsed = e.borrowedAt;
+      registered = true;
+      return data;
+    }
+    // Capacity gate — count entries (idle + borrowed) for this project.
+    const projectCount = entries.filter(e => e && e.project === projectName).length;
+    if (projectCount >= poolSize) return data;
+    entries.push({
+      project: projectName,
+      path: wtPath,
+      state: POOL_STATE.BORROWED,
+      borrowedBy: dispatchId,
+      borrowedAt: _nowIso(now),
+      idleSince: null,
+      lastBranch: branch,
+      createdAt: _nowIso(now),
+      lastUsed: _nowIso(now),
+    });
+    registered = true;
+    return data;
+  });
+  return registered;
+}
+
+/**
+ * Mark a borrowed entry as idle. Caller is responsible for having already run
+ * the git reset/clean/checkout-detach/pull dance — this is purely a state
+ * flip. If the entry is unknown, no-op (caller can decide to register first).
+ */
+function markIdle(wtPath, opts) {
+  opts = opts || {};
+  if (!wtPath) return false;
+  const now = Number.isFinite(opts.now) ? opts.now : Date.now();
+  const branch = opts.branch || '';
+  let flipped = false;
+  mutateWorktreePool((data) => {
+    const idx = _findEntryByPath(data.entries, wtPath);
+    if (idx === -1) return data;
+    const e = data.entries[idx];
+    e.state = POOL_STATE.IDLE;
+    e.borrowedBy = null;
+    e.borrowedAt = null;
+    e.idleSince = _nowIso(now);
+    e.lastUsed = _nowIso(now);
+    if (branch) e.lastBranch = branch;
+    flipped = true;
+    return data;
+  });
+  return flipped;
+}
+
+/**
+ * Return a worktree to the pool — flip an existing borrowed entry to idle, or
+ * insert a fresh entry as idle when the project is under capacity. Both paths
+ * enforce the per-project capacity inside the locked mutation, so two
+ * simultaneous returns cannot blow past `poolSize`.
+ *
+ * Returns one of:
+ *   - 'flipped'   — existing borrowed entry flipped to idle
+ *   - 'inserted'  — new idle entry added (was a fresh worktree)
+ *   - 'rejected'  — pool full or poolSize=0; caller should let normal cleanup
+ *                   reap the worktree
+ *
+ * Caller MUST have already run the git reset/clean/checkout-detach/pull dance
+ * before invoking this function. On any git failure earlier, call `evictEntry`
+ * and skip the return.
+ */
+function returnToPool(projectName, wtPath, opts) {
+  opts = opts || {};
+  if (!projectName || !wtPath) return 'rejected';
+  const poolSize = Number.isFinite(opts.poolSize) ? Math.max(0, Math.floor(opts.poolSize)) : 0;
+  if (poolSize <= 0) return 'rejected';
+  const now = Number.isFinite(opts.now) ? opts.now : Date.now();
+  const branch = opts.branch || '';
+  let outcome = 'rejected';
+  mutateWorktreePool((data) => {
+    const entries = data.entries;
+    const existingIdx = _findEntryByPath(entries, wtPath);
+    if (existingIdx !== -1) {
+      const e = entries[existingIdx];
+      e.state = POOL_STATE.IDLE;
+      e.borrowedBy = null;
+      e.borrowedAt = null;
+      e.idleSince = _nowIso(now);
+      e.lastUsed = _nowIso(now);
+      if (branch) e.lastBranch = branch;
+      outcome = 'flipped';
+      return data;
+    }
+    // Insert path — gate on capacity inside the lock.
+    const projectCount = entries.filter(e => e && e.project === projectName).length;
+    if (projectCount >= poolSize) {
+      outcome = 'rejected';
+      return data;
+    }
+    entries.push({
+      project: projectName,
+      path: wtPath,
+      state: POOL_STATE.IDLE,
+      borrowedBy: null,
+      borrowedAt: null,
+      idleSince: _nowIso(now),
+      lastBranch: branch,
+      createdAt: _nowIso(now),
+      lastUsed: _nowIso(now),
+    });
+    outcome = 'inserted';
+    return data;
+  });
+  return outcome;
+}
+
+/**
+ * Remove an entry from the pool. Used on git failure during borrow/return so
+ * the worktree falls back to normal cleanup ownership.
+ */
+function evictEntry(wtPath, reason) {
+  if (!wtPath) return false;
+  const norm = _normalizePath(wtPath);
+  let removed = false;
+  mutateWorktreePool((data) => {
+    const before = data.entries.length;
+    data.entries = data.entries.filter(e => _normalizePath(e?.path) !== norm);
+    if (data.entries.length !== before) {
+      removed = true;
+      log('info', `worktree-pool: evicted ${wtPath} (${reason || 'unspecified'})`);
+    }
+    return data;
+  });
+  return removed;
+}
+
+/**
+ * Drop entries whose path is gone, whose borrower is no longer active, or
+ * whose idle TTL has expired. `activeDispatchIds` is a Set of dispatch ids
+ * currently in `dispatch.active`. Returns { dropped: [...], evicted: number }.
+ */
+function pruneStale(opts) {
+  opts = opts || {};
+  const now = Number.isFinite(opts.now) ? opts.now : Date.now();
+  const ttlMs = Number.isFinite(opts.idleTtlMs) ? opts.idleTtlMs : getPoolIdleTtlMs(opts.config);
+  const activeIds = opts.activeDispatchIds instanceof Set
+    ? opts.activeDispatchIds
+    : new Set(Array.isArray(opts.activeDispatchIds) ? opts.activeDispatchIds : []);
+  const dropped = [];
+  mutateWorktreePool((data) => {
+    const next = [];
+    for (const e of data.entries) {
+      if (!e || !e.path) { dropped.push({ path: e?.path, reason: 'invalid' }); continue; }
+      if (!fs.existsSync(e.path)) { dropped.push({ path: e.path, reason: 'missing-path' }); continue; }
+      if (e.state === POOL_STATE.BORROWED) {
+        if (!e.borrowedBy || !activeIds.has(e.borrowedBy)) {
+          dropped.push({ path: e.path, reason: 'orphan-borrow' });
+          continue;
+        }
+        next.push(e);
+        continue;
+      }
+      if (e.state === POOL_STATE.IDLE) {
+        const idleAt = Date.parse(e.idleSince || e.lastUsed || e.createdAt || '') || 0;
+        if (idleAt > 0 && (now - idleAt) > ttlMs) {
+          dropped.push({ path: e.path, reason: 'ttl-expired' });
+          continue;
+        }
+        next.push(e);
+        continue;
+      }
+      // Unknown state — drop defensively.
+      dropped.push({ path: e.path, reason: 'unknown-state' });
+    }
+    data.entries = next;
+    return data;
+  });
+  return { dropped, evicted: dropped.length };
+}
+
+/**
+ * Count entries (idle + borrowed) belonging to a project. Used to gate
+ * `markIdle` registrations from fresh-create paths.
+ */
+function countForProject(projectName) {
+  if (!projectName) return 0;
+  return _entriesArray(readPool()).filter(e => e && e.project === projectName).length;
+}
+
+/**
+ * Return the set of borrowed pool paths whose borrower is in `activeIds`.
+ * Used by cleanup.js to protect them from the age/cap sweep.
+ */
+function getActiveBorrowedPaths(activeDispatchIds) {
+  const activeIds = activeDispatchIds instanceof Set
+    ? activeDispatchIds
+    : new Set(Array.isArray(activeDispatchIds) ? activeDispatchIds : []);
+  const out = new Set();
+  for (const e of _entriesArray(readPool())) {
+    if (!e || e.state !== POOL_STATE.BORROWED) continue;
+    if (e.borrowedBy && activeIds.has(e.borrowedBy)) out.add(_normalizePath(e.path));
+  }
+  return out;
+}
+
+function getIdlePaths() {
+  const out = new Set();
+  for (const e of _entriesArray(readPool())) {
+    if (!e || e.state !== POOL_STATE.IDLE) continue;
+    out.add(_normalizePath(e.path));
+  }
+  return out;
+}
+
+module.exports = {
+  WORKTREE_POOL_FILENAME,
+  POOL_STATE,
+  getPoolPath,
+  readPool,
+  mutateWorktreePool,
+  getProjectPoolSize,
+  getPoolIdleTtlMs,
+  isPoolMember,
+  getEntry,
+  tryBorrow,
+  registerBorrowed,
+  markIdle,
+  returnToPool,
+  evictEntry,
+  pruneStale,
+  countForProject,
+  getActiveBorrowedPaths,
+  getIdlePaths,
+  // Exposed for test injection only — not part of public contract.
+  _normalizePath,
+};