@yemi33/minions 0.1.2147 → 0.1.2149

package/engine/shared.js

@@ -2352,6 +2352,30 @@ const ENGINE_DEFAULTS = {
   autoConsolidateMemory: false, // opt-in: periodically spawn engine/kb-sweep-runner.js from the tick loop (4h cadence). Inbox→notes consolidation already runs every tick via consolidateInbox; this flag only controls the KB sweep.
   prNoOpFixPauseAttempts: 2, // pause one PR automation cause after repeated no-op fixes for unchanged evidence
   quarantineAutoRecoveryMax: 2, // #2996 follow-up: cap on auto-flipping WORKTREE_DIRTY/WORKTREE_DIVERGENT failures back to pending (the quarantine is self-healing so the next dispatch starts clean; the cap prevents infinite loops if quarantine itself keeps failing).
+  // W-mq5n1zx5 — Layer 1a/2b: harden the quarantine rename path against
+  // Windows EBUSY/EPERM races where a lingering `git.exe` descendant (leaked
+  // by a status-probe timeout) still holds packfile handles when we try to
+  // rename the worktree dir out of the way. With these on, the engine
+  // retries the rename with jittered backoff (worst case ≈ baseMs * 2^N + N
+  // jitter), then — if everything still fails — asks git to tear down its
+  // own worktree record via `git worktree remove --force` so the next
+  // dispatch isn't blocked by a half-renamed tree. Each knob is overridable
+  // via `config.engine.<name>`.
+  quarantineRenameRetryAttempts: 6,
+  quarantineRenameRetryBaseMs: 250,
+  quarantineForceRemoveFallback: true,
+  // W-mq5n1zx5 — Layer 1b/2a: pre-empt the EBUSY race itself. (1b)
+  // `--no-optional-locks` tells git to skip the .git/index.lock acquire
+  // around the untracked-cache refresh in status; typical probe duration
+  // under AV drops from 6–12s to <500ms, removing the timeout that
+  // creates the leaked child in the first place. (2a) On Windows only,
+  // before attempting the rename we walk live git.exe processes whose
+  // command line points at the worktree path and Stop-Process them, so
+  // a still-alive descendant from a previous status probe doesn't pin
+  // packfile handles. Both gates are belt-and-suspenders — most
+  // incidents are closed by (1b) alone — but both are cheap and idempotent.
+  statusProbeUseNoOptionalLocks: true,
+  statusProbeKillDescendantsWin32: true,
   completionReportRetentionDays: 90, // retain completion report sidecars beyond capped dispatch history
   completionReportMaxFiles: 5000, // hard cap for completion report sidecars during cleanup
   // P-bfa2c-cors-wildcard: extra Origins permitted to receive an
@@ -2538,6 +2562,17 @@ const ENGINE_DEFAULTS = {
   worktreeStuckThreshold: 10,               // consecutive removal failures before escalation
   worktreeStuckSuppressMs: 60 * 60 * 1000,  // 60min — suppress per-tick warn after escalation
   worktreeStuckSlowRetryMs: 30 * 60 * 1000, // 30min — slow-cadence retry window after escalation
+  // W-mq6f2fe0000557fa — orphan-worktree GC: identify the OS process holding
+  // a stuck worktree's cwd and, behind this opt-in flag, kill it. Default OFF
+  // because killing a foreign process is destructive; flipping on rescues the
+  // engine from the W-mq1k8z6o003acd89 / PID 16828 lock-spin pattern automatically.
+  // Auto-reap only fires for processes whose cmdline matches `spawn-agent.js`
+  // AND whose age exceeds `agentTimeout * 2` AND that have no live dispatch row
+  // — i.e. unambiguous orphan agents the engine forgot about. The holder scan
+  // itself ALWAYS runs on every orphan-sweep escalation (the flag only gates
+  // the kill); holder details are appended to the inbox note either way.
+  autoReapOrphanWorktreeHolders: false,
+  orphanHolderScanTimeoutMs: 5000,          // 5s ceiling for the cross-platform holder scan (PowerShell / /proc walk / lsof)
   ccMaxTurns: 50, // max tool-use turns per CC/doc-chat call before CLI stops (per response, not per session)
   ccTurnTimeoutMs: 300000, // W-mpmwxni2000c25c7-b/-d: 5min per-turn no-progress watchdog. The window resets on every liveness signal — token chunk, tool-call notification, tool-update — so an actively-streaming CC/doc-chat turn (long shell command, deep search, sub-agent loop) survives indefinitely up to the outer CC_CALL_TIMEOUT_MS (~1h) ceiling. Only true silence past this window with no progress fires the cancel: the in-flight LLM call is aborted and the handler surfaces `{code:'cc-turn-timeout', retryable:true}` via the typed error envelope so the UI can stop the spinner and offer Retry. Clamped to [10000, 3600000] in the settings POST handler. Independent of CC_CALL_TIMEOUT_MS. Non-streaming doc-chat is the lone wall-clock exception (no progress hooks); see _raceCcDocChatTimeout in dashboard.js for the dual factory/promise shape.
   docSessionMaxEntries: 200, // cap doc-chat session map/disk store by least-recent activity (LRU; sessions are non-expiring otherwise)
@@ -3682,6 +3717,7 @@ const FAILURE_CLASS = {
   WORKTREE_PREFLIGHT: 'worktree-preflight', // Pre-spawn worktree validation rejected (nested-in-project, drive-root collapse) — never retryable
   WORKTREE_DIRTY: 'worktree-dirty',         // #2996: reused worktree had uncommitted edits and the engine could not auto-heal (or already quarantined). Non-retryable for this dispatch — next discovery cycle creates a fresh worktree.
   WORKTREE_DIVERGENT: 'worktree-divergent', // #2996: reused worktree's local branch was N commits ahead of origin (unsafe to reset, may contain unpushed agent work). Engine quarantined the worktree + backed up the local ref; non-retryable for this dispatch.
+  WORKTREE_QUARANTINE_ENV_BLOCKED: 'worktree-quarantine-env-blocked', // W-mq5n1zx5: quarantine rename was attempted but the OS refused to release the worktree dir (Windows EBUSY/EPERM/EACCES from a lingering git.exe descendant). Engine retried with jittered backoff and (when enabled) `git worktree remove --force`; if all paths failed we surface this dedicated class so the WI auto-recovery loop can re-queue WITHOUT bumping the per-agent retry counter (the failure is environmental, not the agent's fault).
   DEPENDENCY_MERGE_SETUP: 'dependency-merge-setup', // Dependency pre-merge plumbing (stash/status/reset) failed before a real file conflict was verified. Retryable so a fresh worktree can recover.
   INVALID_KEEP_PROCESSES_WORKDIR: 'invalid-keep-processes-workdir', // W-mp6k7ywi000fa33c: keep-pids.json declared a cwd that is not a real git worktree (likely a selective copy of the repo) — never retryable; agent must rerun in a real worktree
   INVALID_KEEP_PROCESSES_SCHEMA: 'invalid-keep-processes-schema', // W-mp7i902u000l991f: keep-pids.json failed validation for a reason other than workdir (pids-missing, ttl-too-long, expires_at-missing, pids-too-many, port-invalid, etc.) — agent wrote the wrong shape; never retryable until they fix the file
@@ -6239,6 +6275,189 @@ function listAllProcesses() {
   return process.platform === 'win32' ? _winListProcesses() : _unixListProcesses();
 }
 
+// W-mq6f2fe0000557fa — orphan-worktree GC: identify OS processes whose cwd
+// is at or inside `dir`. Cross-platform with a fail-open contract: on any
+// error or timeout we return [] (the caller continues with its existing
+// escalation path and an unenriched note).
+//
+// Return shape: [{ pid, cmdline, startedAt, ageMs }, ...]
+//   - pid: numeric OS pid
+//   - cmdline: full command line as a single string (best-effort; possibly truncated)
+//   - startedAt: ms-since-epoch process start time (0 when unknown)
+//   - ageMs: Date.now() - startedAt (0 when startedAt is 0)
+//
+// Platform notes:
+//   - Windows: PowerShell `Get-CimInstance Win32_Process` filtered by the
+//     worktree basename appearing in CommandLine. (Win32_Process does not
+//     expose the working directory; the basename match is the most-reliable
+//     heuristic — every spawn-agent invocation embeds the dispatch id, which
+//     is the worktree basename, in its argv.)
+//   - Linux: walk `/proc/*/cwd` and keep PIDs whose readlinkSync resolves
+//     under `dir`. Cmdline read from `/proc/<pid>/cmdline` (NUL-separated).
+//     Start time derived from stat() mtime of the cwd entry as a proxy.
+//   - macOS: `lsof -F p -d cwd` filtered by directory, then `ps -p <pid>` for
+//     cmdline + start time.
+function findProcessesWithCwdInside(dir, opts = {}) {
+  if (!dir || typeof dir !== 'string') return [];
+  let resolved;
+  try { resolved = path.resolve(dir); } catch { return []; }
+  if (!resolved) return [];
+  const timeoutMs = Number(opts.timeoutMs) > 0
+    ? Number(opts.timeoutMs)
+    : (ENGINE_DEFAULTS.orphanHolderScanTimeoutMs || 5000);
+  const now = Date.now();
+
+  try {
+    if (process.platform === 'win32') {
+      return _findWindowsProcessesWithCwdInside(resolved, timeoutMs, now);
+    }
+    if (process.platform === 'linux') {
+      return _findLinuxProcessesWithCwdInside(resolved, timeoutMs, now);
+    }
+    return _findMacProcessesWithCwdInside(resolved, timeoutMs, now);
+  } catch { return []; }
+}
+
+function _findWindowsProcessesWithCwdInside(resolved, timeoutMs, now) {
+  // Win32_Process exposes CommandLine but not the cwd. Match by basename of
+  // the worktree dir (the dispatch id like W-mq1k8z6o003acd89) appearing in
+  // the cmdline — every spawn-agent prompt-file path embeds it.
+  const basename = path.basename(resolved);
+  if (!basename || basename.length < 3) return [];
+  // PowerShell-quote: outer escape ' as ''
+  const psBasename = basename.replace(/'/g, "''");
+  const psResolved = resolved.replace(/'/g, "''");
+  // Use -like with wildcards on BOTH basename and full path so a process
+  // whose cmdline carries the worktree dir (but a different basename
+  // anywhere in argv) also matches.
+  const script = `Get-CimInstance Win32_Process | Where-Object { ($_.CommandLine -like '*${psBasename}*') -or ($_.CommandLine -like '*${psResolved}*') } | ForEach-Object { [PSCustomObject]@{ pid = $_.ProcessId; cmdline = $_.CommandLine; startedAt = if ($_.CreationDate) { [int64](([datetimeoffset]$_.CreationDate).ToUnixTimeMilliseconds()) } else { 0 } } } | ConvertTo-Json -Compress -Depth 2`;
+  let raw;
+  try {
+    raw = _execSync(
+      `powershell -NoProfile -NonInteractive -Command "${script}"`,
+      { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: timeoutMs, windowsHide: true, maxBuffer: 4 * 1024 * 1024 }
+    );
+  } catch { return []; }
+  if (!raw || !String(raw).trim()) return [];
+  let parsed;
+  try { parsed = JSON.parse(raw); }
+  catch { return []; }
+  const arr = Array.isArray(parsed) ? parsed : [parsed];
+  const out = [];
+  for (const r of arr) {
+    if (!r) continue;
+    const pid = Number(r.pid);
+    if (!Number.isInteger(pid) || pid <= 0) continue;
+    const startedAt = Number(r.startedAt) || 0;
+    out.push({
+      pid,
+      cmdline: r.cmdline ? String(r.cmdline) : '',
+      startedAt,
+      ageMs: startedAt > 0 ? Math.max(0, now - startedAt) : 0,
+    });
+  }
+  return out;
+}
+
+function _findLinuxProcessesWithCwdInside(resolved, _timeoutMs, now) {
+  // /proc walk: scan numeric entries, readlink cwd, keep matches.
+  let entries;
+  try { entries = fs.readdirSync('/proc'); }
+  catch { return []; }
+  const prefix = resolved + path.sep;
+  const out = [];
+  for (const e of entries) {
+    if (!/^\d+$/.test(e)) continue;
+    const pid = Number(e);
+    let cwd;
+    try { cwd = fs.readlinkSync(`/proc/${pid}/cwd`); }
+    catch { continue; }
+    if (!cwd) continue;
+    if (cwd !== resolved && !cwd.startsWith(prefix)) continue;
+    let cmdline = '';
+    try {
+      const buf = fs.readFileSync(`/proc/${pid}/cmdline`);
+      cmdline = buf.toString('utf8').replace(/\0/g, ' ').trim();
+    } catch { /* cmdline read can fail on race */ }
+    let startedAt = 0;
+    try {
+      const st = fs.statSync(`/proc/${pid}`);
+      startedAt = st && st.ctimeMs ? Math.floor(st.ctimeMs) : 0;
+    } catch { /* stat can fail on race */ }
+    out.push({
+      pid,
+      cmdline,
+      startedAt,
+      ageMs: startedAt > 0 ? Math.max(0, now - startedAt) : 0,
+    });
+  }
+  return out;
+}
+
+function _findMacProcessesWithCwdInside(resolved, timeoutMs, now) {
+  // `lsof -a -d cwd -F pn` emits PID then cwd path. Filter for dir matches,
+  // then run a single `ps` pass to enrich cmdline + start time.
+  let raw;
+  try {
+    raw = _execSync('lsof -a -d cwd -F pn 2>/dev/null', {
+      encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: timeoutMs, maxBuffer: 4 * 1024 * 1024,
+    });
+  } catch { return []; }
+  if (!raw) return [];
+  const prefix = resolved + path.sep;
+  const pids = new Set();
+  let currentPid = null;
+  for (const line of String(raw).split(/\r?\n/)) {
+    if (!line) continue;
+    const tag = line[0];
+    const val = line.slice(1);
+    if (tag === 'p') {
+      const n = Number(val);
+      currentPid = Number.isInteger(n) && n > 0 ? n : null;
+    } else if (tag === 'n' && currentPid != null) {
+      if (val === resolved || val.startsWith(prefix)) pids.add(currentPid);
+    }
+  }
+  if (pids.size === 0) return [];
+  const out = [];
+  for (const pid of pids) {
+    let line;
+    try {
+      line = String(_execSync(`ps -p ${pid} -o lstart=,command=`, {
+        stdio: ['ignore', 'pipe', 'ignore'], timeout: 2000, encoding: 'utf8',
+      }) || '').trim();
+    } catch { line = ''; }
+    let startedAt = 0;
+    let cmdline = '';
+    if (line) {
+      // ps lstart= emits "Mon Jun  9 07:04:33 2026" (24 chars), then cmdline.
+      const lstart = line.slice(0, 24).trim();
+      cmdline = line.slice(24).trim();
+      const parsed = Date.parse(lstart);
+      if (!Number.isNaN(parsed)) startedAt = parsed;
+    }
+    out.push({
+      pid,
+      cmdline,
+      startedAt,
+      ageMs: startedAt > 0 ? Math.max(0, now - startedAt) : 0,
+    });
+  }
+  return out;
+}
+
+// W-mq6f2fe0000557fa — clear the per-path failure cooldown entry so the
+// post-holder-reap retry can attempt removeWorktree even though the path
+// has already failed >= 3 times. Without this, the cooldown silently
+// suppresses the retry and the orphan dir is left stuck.
+function clearWorktreeFailureCache(wtPath) {
+  if (!wtPath) return false;
+  try {
+    const resolved = path.resolve(wtPath);
+    return _removeWorktreeFailures.delete(resolved);
+  } catch { return false; }
+}
+
 // Cross-check a single PID's command line for a Minions agent invocation
 // (`claude` or `copilot`, including the `node spawn-agent.js --runtime <name>`
 // wrapper and `gh copilot` fallback). Used by orphan/recycled-PID safety:
@@ -6573,6 +6792,61 @@ function _prHeadSha(pr) {
   return String(pr?.headRefOid || pr?.headSha || pr?._adoSourceCommit || pr?._adoHeadCommit || '').trim();
 }
 
+// Target-branch base SHA, normalized across hosts. GitHub PRs carry `baseSha`
+// (engine/github.js:746-749); ADO PRs carry `_adoTargetCommit`
+// (engine/ado.js:1168-1172, inline in pollPrStatus).
+function _prBaseSha(pr) {
+  return String(pr?.baseSha || pr?._adoTargetCommit || '').trim();
+}
+
+// Target ref name, normalized across hosts. GitHub PRs carry `baseRefName`
+// (e.g. "master"); ADO PRs carry `targetRefName` (e.g. "refs/heads/main").
+function _prTargetRefName(pr) {
+  return String(pr?.baseRefName || pr?.targetRefName || '').trim();
+}
+
+// #3079 — Composite key the engine uses to gate merge-conflict re-dispatch.
+// The same-head guard at engine.js:5676 previously compared only source head
+// SHA, missing the case where a PR is retargeted (e.g. parent branch → main)
+// without the source moving. Including the base SHA and target ref name makes
+// the guard target-aware so retargets naturally release the pause.
+function prMergeConflictGuardKey(pr) {
+  return `${_prHeadSha(pr)}|${_prBaseSha(pr)}|${_prTargetRefName(pr)}`;
+}
+
+// #3079 — Helper invoked by the ADO + GitHub poller metadata-apply paths.
+// When the PR's target ref changes (user retargeted via gh pr edit --base or
+// ADO PR edit), clear the stale MERGE_CONFLICT records so the next tick can
+// re-evaluate the conflict against the new target. Other causes
+// (BUILD_FAILURE, REVIEW_FEEDBACK, HUMAN_FEEDBACK) are untouched — they are
+// not target-sensitive. First-poll seeding (no prior target ref recorded)
+// is a no-op so we don't strip state immediately after an upsert.
+//
+// Returns true when state was actually mutated (callers may use this to set
+// the surrounding "updated" flag and trigger a persist).
+function resetMergeConflictStateOnRetarget(pr, newTargetRef) {
+  if (!pr) return false;
+  const next = String(newTargetRef || '').trim();
+  if (!next) return false;
+  const prior = _prTargetRefName(pr);
+  // First-poll seeding: no prior value → record new value via the caller,
+  // do NOT clear merge-conflict state.
+  if (!prior) return false;
+  if (prior === next) return false;
+  let changed = false;
+  if (pr._lastDispatchByCause && pr._lastDispatchByCause[PR_FIX_CAUSE.MERGE_CONFLICT]) {
+    delete pr._lastDispatchByCause[PR_FIX_CAUSE.MERGE_CONFLICT];
+    if (Object.keys(pr._lastDispatchByCause).length === 0) delete pr._lastDispatchByCause;
+    changed = true;
+  }
+  if (pr._noOpFixes && pr._noOpFixes[PR_FIX_CAUSE.MERGE_CONFLICT]) {
+    delete pr._noOpFixes[PR_FIX_CAUSE.MERGE_CONFLICT];
+    if (Object.keys(pr._noOpFixes).length === 0) delete pr._noOpFixes;
+    changed = true;
+  }
+  return changed;
+}
+
 function prFixEvidenceFingerprint(pr, cause = PR_FIX_CAUSE.UNKNOWN) {
   const review = pr?.minionsReview || {};
   const feedback = pr?.humanFeedback || {};
@@ -6596,6 +6870,16 @@ function prFixEvidenceFingerprint(pr, cause = PR_FIX_CAUSE.UNKNOWN) {
     evidence.mergeConflict = !!pr?._mergeConflict;
     evidence.mergeStatus = pr?.mergeStatus || '';
     evidence.mergeConflictDetail = pr?._mergeConflictDetail || '';
+    // #3079 — same rationale as BUILD_FAILURE / REVIEW_FEEDBACK in #2979:
+    // without head/base/target fields the fingerprint was sticky across a
+    // rebase + force-push AND across a PR retarget, so existing
+    // _noOpFixes[MERGE_CONFLICT] pauses never released. Adding head + base
+    // SHA + target ref name gives MERGE_CONFLICT the same natural-unsticking
+    // property the other auto-fix causes already have.
+    evidence.headRefOid = _prHeadSha(pr);
+    evidence.baseSha = _prBaseSha(pr);
+    evidence.targetRefName = _prTargetRefName(pr);
+    evidence.lastPushedAt = pr?.lastPushedAt || '';
   } else {
     evidence.reviewStatus = pr?.reviewStatus || '';
     evidence.lastReviewedAt = pr?.lastReviewedAt || '';
@@ -7221,6 +7505,8 @@ module.exports = {
   PR_FIX_CAUSE,
   getPrFixAutomationCause,
   prFixEvidenceFingerprint,
+  prMergeConflictGuardKey,
+  resetMergeConflictStateOnRetarget,
   getPrNoOpFixRecord,
   isPrNoOpFixCausePaused,
   getPrPausedCauses,
@@ -7233,6 +7519,8 @@ module.exports = {
   killByPidsImmediate,
   isProcessCommandLineMatchingAgent,
   listAllProcesses,
+  findProcessesWithCwdInside,
+  clearWorktreeFailureCache,
   listProcessDescendants,
   listProcessReachable,
   removeWorktree,