@yemi33/minions 0.1.2148 → 0.1.2149

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -292,12 +292,19 @@ async function openSettings() {
292
292
  '<div class="settings-pane-sub">Persistent CC worker pool and git worktree provisioning. Tune these only if you see worktree-create timeouts or slow CC cold-spawns.</div>' +
293
293
  '<div class="settings-stack" style="margin-bottom:12px">' +
294
294
  settingsToggle('CC Worker Pool', 'set-ccUseWorkerPool', (e.ccUseWorkerPool === undefined ? ((e.ccCli || e.defaultCli) === 'copilot') : !!e.ccUseWorkerPool), 'Route Command Center / doc-chat through a persistent copilot --acp worker per tab instead of spawning a fresh CLI per turn. Copilot-only (Agent Client Protocol transport); Claude does not implement ACP, so this toggle has no effect when CC runtime is Claude. Default ON for copilot (cold-spawn ~20s on Windows); forced OFF for non-copilot CC runtimes regardless of this toggle.') +
295
+ // W-mq6f2fe0000557fa — opt-in: auto-kill orphan spawn-agent processes
296
+ // holding a stuck worktree's cwd. Gated by safe-reap criteria (cmdline
297
+ // matches spawn-agent.js + basename + age > agentTimeout * 2). Default
298
+ // OFF because killing a foreign process is destructive.
299
+ settingsToggle('Auto-reap orphan worktree holders', 'set-autoReapOrphanWorktreeHolders', !!e.autoReapOrphanWorktreeHolders,
300
+ 'When ON, the periodic orphan-worktree sweep automatically kills unambiguous orphan `spawn-agent.js` processes holding a stuck worktree\'s cwd (Windows file-lock pattern, e.g. W-mq1k8z6o003acd89 / PID 16828). Safe-reap criteria: cmdline must match spawn-agent.js, reference the worktree basename, AND process age must exceed `engine.agentTimeout * 2`. Default OFF because killing a foreign process is destructive. The holder SCAN runs either way and is appended to the escalation inbox note.') +
295
301
  '</div>' +
296
302
  '<div class="settings-grid-2">' +
297
303
  settingsField('Worktree Create Timeout', 'set-worktreeCreateTimeout', e.worktreeCreateTimeout || 300000, 'ms', 'Timeout for git worktree add (increase for large repos/Windows)') +
298
304
  settingsField('Worktree Create Retries', 'set-worktreeCreateRetries', e.worktreeCreateRetries || 1, '', 'Retry count for transient worktree add failures (0-3)') +
299
305
  settingsField('Worktree Root', 'set-worktreeRoot', e.worktreeRoot || '../worktrees', '', 'Relative or absolute path for git worktrees; on Windows prefer a short path like C:\\wt') +
300
306
  settingsField('Assert-Clean Status Probe Timeout', 'set-assertCleanStatusTimeoutMs', e.assertCleanStatusTimeoutMs || 10000, 'ms', 'Timeout for the `git status --porcelain` preflight probe in assertCleanSharedWorktree. Raise this (e.g. 60000) on GVFS/Plastic-backed monorepos where sparse hydration runs longer than the 10s default. On timeout the engine quarantines the bad worktree and emits a RETRYABLE failure so the next dispatch starts fresh. Clamped 1000–120000ms.') +
307
+ settingsField('Orphan-holder scan timeout', 'set-orphanHolderScanTimeoutMs', e.orphanHolderScanTimeoutMs || 5000, 'ms', 'Cap on the cross-platform scan (PowerShell / /proc walk / lsof) that identifies the OS process holding a stuck worktree\'s cwd. Bumps up only matter on heavily-loaded hosts where the holder scan races the sweep tick. Clamped 1000–30000ms.') +
301
308
  '</div>';
302
309
 
303
310
  const paneCopilot =
@@ -873,6 +880,8 @@ async function saveSettings() {
873
880
  autoFixHumanComments: document.getElementById('set-autoFixHumanComments').checked,
874
881
  autoCompletePrs: document.getElementById('set-autoCompletePrs').checked,
875
882
  ccUseWorkerPool: !!document.getElementById('set-ccUseWorkerPool')?.checked,
883
+ autoReapOrphanWorktreeHolders: !!document.getElementById('set-autoReapOrphanWorktreeHolders')?.checked,
884
+ orphanHolderScanTimeoutMs: document.getElementById('set-orphanHolderScanTimeoutMs')?.value,
876
885
  adoPollEnabled: document.getElementById('set-adoPollEnabled').checked,
877
886
  ghPollEnabled: document.getElementById('set-ghPollEnabled').checked,
878
887
  prPollStatusEvery: document.getElementById('set-prPollStatusEvery').value,
package/dashboard.js CHANGED
@@ -9361,6 +9361,11 @@ What would you like to discuss or change? When you're happy, say "approve" and I
9361
9361
  // 2min ceiling (covers even a GVFS-backed sparse hydration without
9362
9362
  // letting a config typo wedge dispatch for an hour).
9363
9363
  assertCleanStatusTimeoutMs: [1000, 120000],
9364
+ // W-mq6f2fe0000557fa — cap on the cross-platform holder scan for the
9365
+ // orphan-worktree GC. 1s floor (PowerShell startup alone is 0.5–1s);
9366
+ // 30s ceiling (the scan runs once per orphan-sweep tick and a slow
9367
+ // scan must not block the rest of the sweep for minutes).
9368
+ orphanHolderScanTimeoutMs: [1000, 30000],
9364
9369
  idleAlertMinutes: [1], shutdownTimeout: [30000], restartGracePeriod: [60000],
9365
9370
  meetingRoundTimeout: [60000],
9366
9371
  // W-mq066js7000fff1f-c (Gap B/C): steering safety-net knobs.
package/engine/ado.js CHANGED
@@ -395,6 +395,24 @@ function applyAdoPrMetadata(pr, prData) {
395
395
  updated = true;
396
396
  }
397
397
 
398
+ // #3079 — Track target ref + clear stale merge-conflict dispatch state
399
+ // when the PR is retargeted. The same-head guard at engine.js:5676 and
400
+ // the MERGE_CONFLICT fingerprint at shared.js:6595 are now target-aware
401
+ // (via shared.prMergeConflictGuardKey), but pre-existing
402
+ // _lastDispatchByCause[MERGE_CONFLICT] / _noOpFixes[MERGE_CONFLICT]
403
+ // records would still block re-dispatch until something else cleared
404
+ // them. Reset them explicitly on retarget. First-poll seeding
405
+ // (no prior targetRefName) is a no-op so we don't strip state immediately
406
+ // after upsert.
407
+ const nextTargetRef = String(prData.targetRefName || '').trim();
408
+ if (nextTargetRef && pr.targetRefName !== nextTargetRef) {
409
+ if (shared.resetMergeConflictStateOnRetarget(pr, nextTargetRef)) {
410
+ log('info', `ADO: PR ${pr.id || '?'} retargeted ${pr.targetRefName} → ${nextTargetRef}, clearing stale MERGE_CONFLICT dispatch records`);
411
+ }
412
+ pr.targetRefName = nextTargetRef;
413
+ updated = true;
414
+ }
415
+
398
416
  return updated;
399
417
  }
400
418
 
@@ -2600,6 +2618,7 @@ module.exports = {
2600
2618
  fetchAdoPrMetadata,
2601
2619
  fetchSinglePrBuildStatus,
2602
2620
  findOpenPrOnBranch,
2621
+ applyAdoPrMetadata, // #3079 — exported for unit tests of targetRefName + retarget reset
2603
2622
  _resetAdoThrottle, // exported for testing
2604
2623
  _setAdoThrottleForTest, // exported for testing
2605
2624
  _setAdoTokenForTest, // exported for testing
@@ -400,6 +400,7 @@ function isRetryableFailureReason(reason = '', failureClass = '') {
400
400
  FAILURE_CLASS.WORKTREE_PREFLIGHT, // pre-spawn worktree validation — recompute will produce the same failure
401
401
  FAILURE_CLASS.WORKTREE_DIRTY, // #2996: reused worktree was dirty and could not be auto-healed — non-retryable for this dispatch attempt; the engine quarantined the worktree so the next discovery cycle creates a fresh one
402
402
  FAILURE_CLASS.WORKTREE_DIVERGENT, // #2996: reused worktree's local branch had unpushed commits — engine quarantined + backed up the local ref; non-retryable for this dispatch (next discovery creates fresh worktree on origin/<branch>)
403
+ FAILURE_CLASS.WORKTREE_QUARANTINE_ENV_BLOCKED, // W-mq5n1zx5: quarantine rename couldn't release the worktree dir even after retry + force-remove fallback. Non-retryable at the dispatch level so the per-agent retry counter isn't bumped (environmental, not the agent's fault); the WI auto-recovery loop in engine.js#discoverFromWorkItems re-queues without touching _retriesByAgent.
403
404
  FAILURE_CLASS.INVALID_KEEP_PROCESSES_WORKDIR, // W-mp6k7ywi000fa33c — keep-pids cwd is not a real git worktree; re-running won't fix the structural issue
404
405
  FAILURE_CLASS.INVALID_KEEP_PROCESSES_SCHEMA, // W-mp7i902u000l991f — keep-pids.json failed shape validation; re-running with the same wrong file won't fix it
405
406
  FAILURE_CLASS.INVALID_MANAGED_SPAWN, // W-mpbhxg3b000u8411 — managed-spawn.json failed validation; re-running with the same wrong file won't fix it
@@ -770,6 +771,7 @@ function completeDispatch(id, result = DISPATCH_RESULT.SUCCESS, reason = '', res
770
771
  [FAILURE_CLASS.WORKTREE_PREFLIGHT]: 'worktree preflight rejected (nested in project root or rootDir collapsed to drive root)',
771
772
  [FAILURE_CLASS.WORKTREE_DIRTY]: 'reused worktree had uncommitted edits and could not be auto-healed (#2996) — engine quarantined the dir so the next dispatch creates a fresh worktree',
772
773
  [FAILURE_CLASS.WORKTREE_DIVERGENT]: 'reused worktree had unpushed local commits ahead of origin (#2996) — engine backed up the local ref and quarantined the dir so the next dispatch starts from origin/<branch>',
774
+ [FAILURE_CLASS.WORKTREE_QUARANTINE_ENV_BLOCKED]: 'quarantine rename was blocked by the OS (Windows EBUSY/EPERM) even after retry + force-remove fallback — environmental, not the agent\'s fault; auto-recovery loop will re-queue without bumping per-agent retries',
773
775
  [FAILURE_CLASS.INVALID_KEEP_PROCESSES_WORKDIR]: 'keep_processes cwd is not a real git worktree (rerun in a `git worktree add` directory)',
774
776
  [FAILURE_CLASS.INVALID_KEEP_PROCESSES_SCHEMA]: 'keep-pids.json failed shape validation (wrong keys/types/values — see inbox alert for the canonical shape)',
775
777
  [FAILURE_CLASS.INVALID_MANAGED_SPAWN]: 'managed-spawn.json failed validation (bad schema, workdir, or allowlist — see inbox alert)',
package/engine/github.js CHANGED
@@ -748,6 +748,20 @@ async function pollPrStatus(config) {
748
748
  updated = true;
749
749
  }
750
750
 
751
+ // #3079 — Track GitHub target branch name + clear stale merge-conflict
752
+ // dispatch state when the PR is retargeted (e.g. `gh pr edit --base
753
+ // master` after parent PR merges). Mirrors ado.js applyAdoPrMetadata.
754
+ // First-poll seeding (no prior baseRefName) is a no-op so we don't
755
+ // strip state immediately after upsert.
756
+ const nextBaseRefName = String(prData.base?.ref || '').trim();
757
+ if (nextBaseRefName && pr.baseRefName !== nextBaseRefName) {
758
+ if (shared.resetMergeConflictStateOnRetarget(pr, nextBaseRefName)) {
759
+ log('info', `GitHub: PR ${pr.id} retargeted ${pr.baseRefName} → ${nextBaseRefName}, clearing stale MERGE_CONFLICT dispatch records`);
760
+ }
761
+ pr.baseRefName = nextBaseRefName;
762
+ updated = true;
763
+ }
764
+
751
765
  // P-w1a3f9b2 — Phase 1.1: plumb mergeable / isDraft / mergeStateStatus /
752
766
  // headRefOid onto the PR object so watches captureState (engine/watches.js)
753
767
  // and future predicates (Phase 2.1: head-commit-change, mergeable-flipped,
@@ -2291,6 +2291,15 @@ function recordPrNoOpFixAttempt(target, cause, source, dispatchItem, branchChang
2291
2291
  return out;
2292
2292
  })()
2293
2293
  : {}),
2294
+ // #3079 — MERGE_CONFLICT noops record the composite guard key (source
2295
+ // head + base SHA + target ref name). The same-head guard at
2296
+ // engine.js:5676 reads this back and compares against the current
2297
+ // PR's prMergeConflictGuardKey, so a retarget (target ref change) or
2298
+ // a parent-merge (base SHA change) naturally releases the pause even
2299
+ // when the source head didn't move.
2300
+ ...(cause === shared.PR_FIX_CAUSE.MERGE_CONFLICT
2301
+ ? { mergeConflictKey: shared.prMergeConflictGuardKey(target) }
2302
+ : {}),
2294
2303
  };
2295
2304
  target.lastDispatchedAt = now;
2296
2305
  target.lastDispatchOutcome = 'noop';
@@ -2664,6 +2673,14 @@ function updatePrAfterFixError(pr, project, source, options = {}) {
2664
2673
  || '');
2665
2674
  if (commentKey) next.lastProcessedCommentKey = commentKey;
2666
2675
  }
2676
+ // #3079 — Refresh mergeConflictKey from live target state so a
2677
+ // mid-flight retarget reflects in the agent-error record too. The
2678
+ // engine.js skipConflictFix guard prefers mergeConflictKey over
2679
+ // legacy headSha; without this refresh, the prior record's stale
2680
+ // key could re-fire the suppression even after a retarget.
2681
+ if (cause === shared.PR_FIX_CAUSE.MERGE_CONFLICT) {
2682
+ next.mergeConflictKey = shared.prMergeConflictGuardKey(target);
2683
+ }
2667
2684
  target._lastDispatchByCause[cause] = next;
2668
2685
  result = { cause, indeterminate: true, errorClass };
2669
2686
  log('warn', `Updated ${pr.id} → recorded ${cause} agent-error fix attempt (indeterminate=true) — same-head guard relaxed for next tick${errorMessage ? ` (${errorMessage.slice(0, 80)})` : ''}`);
package/engine/shared.js CHANGED
@@ -2352,6 +2352,30 @@ const ENGINE_DEFAULTS = {
2352
2352
  autoConsolidateMemory: false, // opt-in: periodically spawn engine/kb-sweep-runner.js from the tick loop (4h cadence). Inbox→notes consolidation already runs every tick via consolidateInbox; this flag only controls the KB sweep.
2353
2353
  prNoOpFixPauseAttempts: 2, // pause one PR automation cause after repeated no-op fixes for unchanged evidence
2354
2354
  quarantineAutoRecoveryMax: 2, // #2996 follow-up: cap on auto-flipping WORKTREE_DIRTY/WORKTREE_DIVERGENT failures back to pending (the quarantine is self-healing so the next dispatch starts clean; the cap prevents infinite loops if quarantine itself keeps failing).
2355
+ // W-mq5n1zx5 — Layer 1a/2b: harden the quarantine rename path against
2356
+ // Windows EBUSY/EPERM races where a lingering `git.exe` descendant (leaked
2357
+ // by a status-probe timeout) still holds packfile handles when we try to
2358
+ // rename the worktree dir out of the way. With these on, the engine
2359
+ // retries the rename with jittered backoff (worst case ≈ baseMs * 2^N + N
2360
+ // jitter), then — if everything still fails — asks git to tear down its
2361
+ // own worktree record via `git worktree remove --force` so the next
2362
+ // dispatch isn't blocked by a half-renamed tree. Each knob is overridable
2363
+ // via `config.engine.<name>`.
2364
+ quarantineRenameRetryAttempts: 6,
2365
+ quarantineRenameRetryBaseMs: 250,
2366
+ quarantineForceRemoveFallback: true,
2367
+ // W-mq5n1zx5 — Layer 1b/2a: pre-empt the EBUSY race itself. (1b)
2368
+ // `--no-optional-locks` tells git to skip the .git/index.lock acquire
2369
+ // around the untracked-cache refresh in status; typical probe duration
2370
+ // under AV drops from 6–12s to <500ms, removing the timeout that
2371
+ // creates the leaked child in the first place. (2a) On Windows only,
2372
+ // before attempting the rename we walk live git.exe processes whose
2373
+ // command line points at the worktree path and Stop-Process them, so
2374
+ // a still-alive descendant from a previous status probe doesn't pin
2375
+ // packfile handles. Both gates are belt-and-suspenders — most
2376
+ // incidents are closed by (1b) alone — but both are cheap and idempotent.
2377
+ statusProbeUseNoOptionalLocks: true,
2378
+ statusProbeKillDescendantsWin32: true,
2355
2379
  completionReportRetentionDays: 90, // retain completion report sidecars beyond capped dispatch history
2356
2380
  completionReportMaxFiles: 5000, // hard cap for completion report sidecars during cleanup
2357
2381
  // P-bfa2c-cors-wildcard: extra Origins permitted to receive an
@@ -2538,6 +2562,17 @@ const ENGINE_DEFAULTS = {
2538
2562
  worktreeStuckThreshold: 10, // consecutive removal failures before escalation
2539
2563
  worktreeStuckSuppressMs: 60 * 60 * 1000, // 60min — suppress per-tick warn after escalation
2540
2564
  worktreeStuckSlowRetryMs: 30 * 60 * 1000, // 30min — slow-cadence retry window after escalation
2565
+ // W-mq6f2fe0000557fa — orphan-worktree GC: identify the OS process holding
2566
+ // a stuck worktree's cwd and, behind this opt-in flag, kill it. Default OFF
2567
+ // because killing a foreign process is destructive; flipping on rescues the
2568
+ // engine from the W-mq1k8z6o003acd89 / PID 16828 lock-spin pattern automatically.
2569
+ // Auto-reap only fires for processes whose cmdline matches `spawn-agent.js`
2570
+ // AND whose age exceeds `agentTimeout * 2` AND that have no live dispatch row
2571
+ // — i.e. unambiguous orphan agents the engine forgot about. The holder scan
2572
+ // itself ALWAYS runs on every orphan-sweep escalation (the flag only gates
2573
+ // the kill); holder details are appended to the inbox note either way.
2574
+ autoReapOrphanWorktreeHolders: false,
2575
+ orphanHolderScanTimeoutMs: 5000, // 5s ceiling for the cross-platform holder scan (PowerShell / /proc walk / lsof)
2541
2576
  ccMaxTurns: 50, // max tool-use turns per CC/doc-chat call before CLI stops (per response, not per session)
2542
2577
  ccTurnTimeoutMs: 300000, // W-mpmwxni2000c25c7-b/-d: 5min per-turn no-progress watchdog. The window resets on every liveness signal — token chunk, tool-call notification, tool-update — so an actively-streaming CC/doc-chat turn (long shell command, deep search, sub-agent loop) survives indefinitely up to the outer CC_CALL_TIMEOUT_MS (~1h) ceiling. Only true silence past this window with no progress fires the cancel: the in-flight LLM call is aborted and the handler surfaces `{code:'cc-turn-timeout', retryable:true}` via the typed error envelope so the UI can stop the spinner and offer Retry. Clamped to [10000, 3600000] in the settings POST handler. Independent of CC_CALL_TIMEOUT_MS. Non-streaming doc-chat is the lone wall-clock exception (no progress hooks); see _raceCcDocChatTimeout in dashboard.js for the dual factory/promise shape.
2543
2578
  docSessionMaxEntries: 200, // cap doc-chat session map/disk store by least-recent activity (LRU; sessions are non-expiring otherwise)
@@ -3682,6 +3717,7 @@ const FAILURE_CLASS = {
3682
3717
  WORKTREE_PREFLIGHT: 'worktree-preflight', // Pre-spawn worktree validation rejected (nested-in-project, drive-root collapse) — never retryable
3683
3718
  WORKTREE_DIRTY: 'worktree-dirty', // #2996: reused worktree had uncommitted edits and the engine could not auto-heal (or already quarantined). Non-retryable for this dispatch — next discovery cycle creates a fresh worktree.
3684
3719
  WORKTREE_DIVERGENT: 'worktree-divergent', // #2996: reused worktree's local branch was N commits ahead of origin (unsafe to reset, may contain unpushed agent work). Engine quarantined the worktree + backed up the local ref; non-retryable for this dispatch.
3720
+ WORKTREE_QUARANTINE_ENV_BLOCKED: 'worktree-quarantine-env-blocked', // W-mq5n1zx5: quarantine rename was attempted but the OS refused to release the worktree dir (Windows EBUSY/EPERM/EACCES from a lingering git.exe descendant). Engine retried with jittered backoff and (when enabled) `git worktree remove --force`; if all paths failed we surface this dedicated class so the WI auto-recovery loop can re-queue WITHOUT bumping the per-agent retry counter (the failure is environmental, not the agent's fault).
3685
3721
  DEPENDENCY_MERGE_SETUP: 'dependency-merge-setup', // Dependency pre-merge plumbing (stash/status/reset) failed before a real file conflict was verified. Retryable so a fresh worktree can recover.
3686
3722
  INVALID_KEEP_PROCESSES_WORKDIR: 'invalid-keep-processes-workdir', // W-mp6k7ywi000fa33c: keep-pids.json declared a cwd that is not a real git worktree (likely a selective copy of the repo) — never retryable; agent must rerun in a real worktree
3687
3723
  INVALID_KEEP_PROCESSES_SCHEMA: 'invalid-keep-processes-schema', // W-mp7i902u000l991f: keep-pids.json failed validation for a reason other than workdir (pids-missing, ttl-too-long, expires_at-missing, pids-too-many, port-invalid, etc.) — agent wrote the wrong shape; never retryable until they fix the file
@@ -6239,6 +6275,189 @@ function listAllProcesses() {
6239
6275
  return process.platform === 'win32' ? _winListProcesses() : _unixListProcesses();
6240
6276
  }
6241
6277
 
6278
+ // W-mq6f2fe0000557fa — orphan-worktree GC: identify OS processes whose cwd
6279
+ // is at or inside `dir`. Cross-platform with a fail-open contract: on any
6280
+ // error or timeout we return [] (the caller continues with its existing
6281
+ // escalation path and an unenriched note).
6282
+ //
6283
+ // Return shape: [{ pid, cmdline, startedAt, ageMs }, ...]
6284
+ // - pid: numeric OS pid
6285
+ // - cmdline: full command line as a single string (best-effort; possibly truncated)
6286
+ // - startedAt: ms-since-epoch process start time (0 when unknown)
6287
+ // - ageMs: Date.now() - startedAt (0 when startedAt is 0)
6288
+ //
6289
+ // Platform notes:
6290
+ // - Windows: PowerShell `Get-CimInstance Win32_Process` filtered by the
6291
+ // worktree basename appearing in CommandLine. (Win32_Process does not
6292
+ // expose the working directory; the basename match is the most-reliable
6293
+ // heuristic — every spawn-agent invocation embeds the dispatch id, which
6294
+ // is the worktree basename, in its argv.)
6295
+ // - Linux: walk `/proc/*/cwd` and keep PIDs whose readlinkSync resolves
6296
+ // under `dir`. Cmdline read from `/proc/<pid>/cmdline` (NUL-separated).
6297
+ // Start time derived from stat() mtime of the cwd entry as a proxy.
6298
+ // - macOS: `lsof -F p -d cwd` filtered by directory, then `ps -p <pid>` for
6299
+ // cmdline + start time.
6300
+ function findProcessesWithCwdInside(dir, opts = {}) {
6301
+ if (!dir || typeof dir !== 'string') return [];
6302
+ let resolved;
6303
+ try { resolved = path.resolve(dir); } catch { return []; }
6304
+ if (!resolved) return [];
6305
+ const timeoutMs = Number(opts.timeoutMs) > 0
6306
+ ? Number(opts.timeoutMs)
6307
+ : (ENGINE_DEFAULTS.orphanHolderScanTimeoutMs || 5000);
6308
+ const now = Date.now();
6309
+
6310
+ try {
6311
+ if (process.platform === 'win32') {
6312
+ return _findWindowsProcessesWithCwdInside(resolved, timeoutMs, now);
6313
+ }
6314
+ if (process.platform === 'linux') {
6315
+ return _findLinuxProcessesWithCwdInside(resolved, timeoutMs, now);
6316
+ }
6317
+ return _findMacProcessesWithCwdInside(resolved, timeoutMs, now);
6318
+ } catch { return []; }
6319
+ }
6320
+
6321
+ function _findWindowsProcessesWithCwdInside(resolved, timeoutMs, now) {
6322
+ // Win32_Process exposes CommandLine but not the cwd. Match by basename of
6323
+ // the worktree dir (the dispatch id like W-mq1k8z6o003acd89) appearing in
6324
+ // the cmdline — every spawn-agent prompt-file path embeds it.
6325
+ const basename = path.basename(resolved);
6326
+ if (!basename || basename.length < 3) return [];
6327
+ // PowerShell-quote: outer escape ' as ''
6328
+ const psBasename = basename.replace(/'/g, "''");
6329
+ const psResolved = resolved.replace(/'/g, "''");
6330
+ // Use -like with wildcards on BOTH basename and full path so a process
6331
+ // whose cmdline carries the worktree dir (but a different basename
6332
+ // anywhere in argv) also matches.
6333
+ const script = `Get-CimInstance Win32_Process | Where-Object { ($_.CommandLine -like '*${psBasename}*') -or ($_.CommandLine -like '*${psResolved}*') } | ForEach-Object { [PSCustomObject]@{ pid = $_.ProcessId; cmdline = $_.CommandLine; startedAt = if ($_.CreationDate) { [int64](([datetimeoffset]$_.CreationDate).ToUnixTimeMilliseconds()) } else { 0 } } } | ConvertTo-Json -Compress -Depth 2`;
6334
+ let raw;
6335
+ try {
6336
+ raw = _execSync(
6337
+ `powershell -NoProfile -NonInteractive -Command "${script}"`,
6338
+ { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: timeoutMs, windowsHide: true, maxBuffer: 4 * 1024 * 1024 }
6339
+ );
6340
+ } catch { return []; }
6341
+ if (!raw || !String(raw).trim()) return [];
6342
+ let parsed;
6343
+ try { parsed = JSON.parse(raw); }
6344
+ catch { return []; }
6345
+ const arr = Array.isArray(parsed) ? parsed : [parsed];
6346
+ const out = [];
6347
+ for (const r of arr) {
6348
+ if (!r) continue;
6349
+ const pid = Number(r.pid);
6350
+ if (!Number.isInteger(pid) || pid <= 0) continue;
6351
+ const startedAt = Number(r.startedAt) || 0;
6352
+ out.push({
6353
+ pid,
6354
+ cmdline: r.cmdline ? String(r.cmdline) : '',
6355
+ startedAt,
6356
+ ageMs: startedAt > 0 ? Math.max(0, now - startedAt) : 0,
6357
+ });
6358
+ }
6359
+ return out;
6360
+ }
6361
+
6362
+ function _findLinuxProcessesWithCwdInside(resolved, _timeoutMs, now) {
6363
+ // /proc walk: scan numeric entries, readlink cwd, keep matches.
6364
+ let entries;
6365
+ try { entries = fs.readdirSync('/proc'); }
6366
+ catch { return []; }
6367
+ const prefix = resolved + path.sep;
6368
+ const out = [];
6369
+ for (const e of entries) {
6370
+ if (!/^\d+$/.test(e)) continue;
6371
+ const pid = Number(e);
6372
+ let cwd;
6373
+ try { cwd = fs.readlinkSync(`/proc/${pid}/cwd`); }
6374
+ catch { continue; }
6375
+ if (!cwd) continue;
6376
+ if (cwd !== resolved && !cwd.startsWith(prefix)) continue;
6377
+ let cmdline = '';
6378
+ try {
6379
+ const buf = fs.readFileSync(`/proc/${pid}/cmdline`);
6380
+ cmdline = buf.toString('utf8').replace(/\0/g, ' ').trim();
6381
+ } catch { /* cmdline read can fail on race */ }
6382
+ let startedAt = 0;
6383
+ try {
6384
+ const st = fs.statSync(`/proc/${pid}`);
6385
+ startedAt = st && st.ctimeMs ? Math.floor(st.ctimeMs) : 0;
6386
+ } catch { /* stat can fail on race */ }
6387
+ out.push({
6388
+ pid,
6389
+ cmdline,
6390
+ startedAt,
6391
+ ageMs: startedAt > 0 ? Math.max(0, now - startedAt) : 0,
6392
+ });
6393
+ }
6394
+ return out;
6395
+ }
6396
+
6397
+ function _findMacProcessesWithCwdInside(resolved, timeoutMs, now) {
6398
+ // `lsof -a -d cwd -F pn` emits PID then cwd path. Filter for dir matches,
6399
+ // then run a single `ps` pass to enrich cmdline + start time.
6400
+ let raw;
6401
+ try {
6402
+ raw = _execSync('lsof -a -d cwd -F pn 2>/dev/null', {
6403
+ encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: timeoutMs, maxBuffer: 4 * 1024 * 1024,
6404
+ });
6405
+ } catch { return []; }
6406
+ if (!raw) return [];
6407
+ const prefix = resolved + path.sep;
6408
+ const pids = new Set();
6409
+ let currentPid = null;
6410
+ for (const line of String(raw).split(/\r?\n/)) {
6411
+ if (!line) continue;
6412
+ const tag = line[0];
6413
+ const val = line.slice(1);
6414
+ if (tag === 'p') {
6415
+ const n = Number(val);
6416
+ currentPid = Number.isInteger(n) && n > 0 ? n : null;
6417
+ } else if (tag === 'n' && currentPid != null) {
6418
+ if (val === resolved || val.startsWith(prefix)) pids.add(currentPid);
6419
+ }
6420
+ }
6421
+ if (pids.size === 0) return [];
6422
+ const out = [];
6423
+ for (const pid of pids) {
6424
+ let line;
6425
+ try {
6426
+ line = String(_execSync(`ps -p ${pid} -o lstart=,command=`, {
6427
+ stdio: ['ignore', 'pipe', 'ignore'], timeout: 2000, encoding: 'utf8',
6428
+ }) || '').trim();
6429
+ } catch { line = ''; }
6430
+ let startedAt = 0;
6431
+ let cmdline = '';
6432
+ if (line) {
6433
+ // ps lstart= emits "Mon Jun 9 07:04:33 2026" (24 chars), then cmdline.
6434
+ const lstart = line.slice(0, 24).trim();
6435
+ cmdline = line.slice(24).trim();
6436
+ const parsed = Date.parse(lstart);
6437
+ if (!Number.isNaN(parsed)) startedAt = parsed;
6438
+ }
6439
+ out.push({
6440
+ pid,
6441
+ cmdline,
6442
+ startedAt,
6443
+ ageMs: startedAt > 0 ? Math.max(0, now - startedAt) : 0,
6444
+ });
6445
+ }
6446
+ return out;
6447
+ }
6448
+
6449
+ // W-mq6f2fe0000557fa — clear the per-path failure cooldown entry so the
6450
+ // post-holder-reap retry can attempt removeWorktree even though the path
6451
+ // has already failed >= 3 times. Without this, the cooldown silently
6452
+ // suppresses the retry and the orphan dir is left stuck.
6453
+ function clearWorktreeFailureCache(wtPath) {
6454
+ if (!wtPath) return false;
6455
+ try {
6456
+ const resolved = path.resolve(wtPath);
6457
+ return _removeWorktreeFailures.delete(resolved);
6458
+ } catch { return false; }
6459
+ }
6460
+
6242
6461
  // Cross-check a single PID's command line for a Minions agent invocation
6243
6462
  // (`claude` or `copilot`, including the `node spawn-agent.js --runtime <name>`
6244
6463
  // wrapper and `gh copilot` fallback). Used by orphan/recycled-PID safety:
@@ -6573,6 +6792,61 @@ function _prHeadSha(pr) {
6573
6792
  return String(pr?.headRefOid || pr?.headSha || pr?._adoSourceCommit || pr?._adoHeadCommit || '').trim();
6574
6793
  }
6575
6794
 
6795
+ // Target-branch base SHA, normalized across hosts. GitHub PRs carry `baseSha`
6796
+ // (engine/github.js:746-749); ADO PRs carry `_adoTargetCommit`
6797
+ // (engine/ado.js:1168-1172, inline in pollPrStatus).
6798
+ function _prBaseSha(pr) {
6799
+ return String(pr?.baseSha || pr?._adoTargetCommit || '').trim();
6800
+ }
6801
+
6802
+ // Target ref name, normalized across hosts. GitHub PRs carry `baseRefName`
6803
+ // (e.g. "master"); ADO PRs carry `targetRefName` (e.g. "refs/heads/main").
6804
+ function _prTargetRefName(pr) {
6805
+ return String(pr?.baseRefName || pr?.targetRefName || '').trim();
6806
+ }
6807
+
6808
+ // #3079 — Composite key the engine uses to gate merge-conflict re-dispatch.
6809
+ // The same-head guard at engine.js:5676 previously compared only source head
6810
+ // SHA, missing the case where a PR is retargeted (e.g. parent branch → main)
6811
+ // without the source moving. Including the base SHA and target ref name makes
6812
+ // the guard target-aware so retargets naturally release the pause.
6813
+ function prMergeConflictGuardKey(pr) {
6814
+ return `${_prHeadSha(pr)}|${_prBaseSha(pr)}|${_prTargetRefName(pr)}`;
6815
+ }
6816
+
6817
+ // #3079 — Helper invoked by the ADO + GitHub poller metadata-apply paths.
6818
+ // When the PR's target ref changes (user retargeted via gh pr edit --base or
6819
+ // ADO PR edit), clear the stale MERGE_CONFLICT records so the next tick can
6820
+ // re-evaluate the conflict against the new target. Other causes
6821
+ // (BUILD_FAILURE, REVIEW_FEEDBACK, HUMAN_FEEDBACK) are untouched — they are
6822
+ // not target-sensitive. First-poll seeding (no prior target ref recorded)
6823
+ // is a no-op so we don't strip state immediately after an upsert.
6824
+ //
6825
+ // Returns true when state was actually mutated (callers may use this to set
6826
+ // the surrounding "updated" flag and trigger a persist).
6827
+ function resetMergeConflictStateOnRetarget(pr, newTargetRef) {
6828
+ if (!pr) return false;
6829
+ const next = String(newTargetRef || '').trim();
6830
+ if (!next) return false;
6831
+ const prior = _prTargetRefName(pr);
6832
+ // First-poll seeding: no prior value → record new value via the caller,
6833
+ // do NOT clear merge-conflict state.
6834
+ if (!prior) return false;
6835
+ if (prior === next) return false;
6836
+ let changed = false;
6837
+ if (pr._lastDispatchByCause && pr._lastDispatchByCause[PR_FIX_CAUSE.MERGE_CONFLICT]) {
6838
+ delete pr._lastDispatchByCause[PR_FIX_CAUSE.MERGE_CONFLICT];
6839
+ if (Object.keys(pr._lastDispatchByCause).length === 0) delete pr._lastDispatchByCause;
6840
+ changed = true;
6841
+ }
6842
+ if (pr._noOpFixes && pr._noOpFixes[PR_FIX_CAUSE.MERGE_CONFLICT]) {
6843
+ delete pr._noOpFixes[PR_FIX_CAUSE.MERGE_CONFLICT];
6844
+ if (Object.keys(pr._noOpFixes).length === 0) delete pr._noOpFixes;
6845
+ changed = true;
6846
+ }
6847
+ return changed;
6848
+ }
6849
+
6576
6850
  function prFixEvidenceFingerprint(pr, cause = PR_FIX_CAUSE.UNKNOWN) {
6577
6851
  const review = pr?.minionsReview || {};
6578
6852
  const feedback = pr?.humanFeedback || {};
@@ -6596,6 +6870,16 @@ function prFixEvidenceFingerprint(pr, cause = PR_FIX_CAUSE.UNKNOWN) {
6596
6870
  evidence.mergeConflict = !!pr?._mergeConflict;
6597
6871
  evidence.mergeStatus = pr?.mergeStatus || '';
6598
6872
  evidence.mergeConflictDetail = pr?._mergeConflictDetail || '';
6873
+ // #3079 — same rationale as BUILD_FAILURE / REVIEW_FEEDBACK in #2979:
6874
+ // without head/base/target fields the fingerprint was sticky across a
6875
+ // rebase + force-push AND across a PR retarget, so existing
6876
+ // _noOpFixes[MERGE_CONFLICT] pauses never released. Adding head + base
6877
+ // SHA + target ref name gives MERGE_CONFLICT the same natural-unsticking
6878
+ // property the other auto-fix causes already have.
6879
+ evidence.headRefOid = _prHeadSha(pr);
6880
+ evidence.baseSha = _prBaseSha(pr);
6881
+ evidence.targetRefName = _prTargetRefName(pr);
6882
+ evidence.lastPushedAt = pr?.lastPushedAt || '';
6599
6883
  } else {
6600
6884
  evidence.reviewStatus = pr?.reviewStatus || '';
6601
6885
  evidence.lastReviewedAt = pr?.lastReviewedAt || '';
@@ -7221,6 +7505,8 @@ module.exports = {
7221
7505
  PR_FIX_CAUSE,
7222
7506
  getPrFixAutomationCause,
7223
7507
  prFixEvidenceFingerprint,
7508
+ prMergeConflictGuardKey,
7509
+ resetMergeConflictStateOnRetarget,
7224
7510
  getPrNoOpFixRecord,
7225
7511
  isPrNoOpFixCausePaused,
7226
7512
  getPrPausedCauses,
@@ -7233,6 +7519,8 @@ module.exports = {
7233
7519
  killByPidsImmediate,
7234
7520
  isProcessCommandLineMatchingAgent,
7235
7521
  listAllProcesses,
7522
+ findProcessesWithCwdInside,
7523
+ clearWorktreeFailureCache,
7236
7524
  listProcessDescendants,
7237
7525
  listProcessReachable,
7238
7526
  removeWorktree,
@@ -79,18 +79,60 @@ function _markStuckSuccess(resolvedPath, opts = {}) {
79
79
  try {
80
80
  const writer = typeof opts.writeToInbox === 'function' ? opts.writeToInbox : shared.writeToInbox;
81
81
  const basename = path.basename(resolvedPath);
82
- const body = [
83
- `# Worktree recovered: ${basename}`,
84
- '',
85
- `Previously stuck worktree dir **${resolvedPath}** has been successfully removed`,
86
- `after escalation. Suppression cleared.`,
87
- ].join('\n');
88
- writer('engine', `worktree-recovered-${basename}`, body);
82
+ // W-mq6f2fe0000557fa when the recovery is the direct result of an
83
+ // auto-reap kill, write a distinct slug + body so operators can see
84
+ // which orphan-sweep escalations were resolved by the engine itself
85
+ // vs. naturally by an external holder release.
86
+ const viaReap = opts.recoveryReason === 'holder-reap';
87
+ const reapedPids = Array.isArray(opts.reapedPids) ? opts.reapedPids : [];
88
+ const slug = viaReap
89
+ ? `worktree-recovered-${basename}-via-holder-reap`
90
+ : `worktree-recovered-${basename}`;
91
+ const bodyLines = viaReap
92
+ ? [
93
+ `# Worktree recovered via holder-reap: ${basename}`,
94
+ '',
95
+ `Previously stuck worktree dir **${resolvedPath}** was successfully removed`,
96
+ `after the engine auto-killed the orphan spawn-agent holder(s)`,
97
+ `${reapedPids.length > 0 ? `(PID${reapedPids.length > 1 ? 's' : ''} ${reapedPids.join(', ')})` : ''}`.trim(),
98
+ `under the \`engine.autoReapOrphanWorktreeHolders\` flag (W-mq6f2fe0000557fa).`,
99
+ `Suppression cleared.`,
100
+ ]
101
+ : [
102
+ `# Worktree recovered: ${basename}`,
103
+ '',
104
+ `Previously stuck worktree dir **${resolvedPath}** has been successfully removed`,
105
+ `after escalation. Suppression cleared.`,
106
+ ];
107
+ writer('engine', slug, bodyLines.join('\n'));
89
108
  } catch { /* note best-effort */ }
90
109
  }
91
110
  return wasEscalated;
92
111
  }
93
112
 
113
+ // W-mq6f2fe0000557fa — gate: is this holder an unambiguous orphan spawn-agent
114
+ // process we can safely kill? Requires ALL of:
115
+ // (a) cmdline contains 'spawn-agent.js' (only minions agent wrappers qualify)
116
+ // (b) cmdline references the worktree basename (extra confirmation it's
117
+ // OUR spawn-agent and not an unrelated process that happens to mention
118
+ // the path)
119
+ // (c) ageMs > agentTimeoutMs * 2 (the worktree was definitely abandoned by
120
+ // its dispatch; agentTimeout default is 5h, so 10h)
121
+ //
122
+ // The orphan-sweep code path already guarantees no live dispatch row claims
123
+ // the path — we'd never be here otherwise — so the "no live dispatch" check
124
+ // is implicit, not duplicated here.
125
+ function _isSafeReapHolder(holder, basename, minAgeMs) {
126
+ if (!holder || !holder.pid || !holder.cmdline) return false;
127
+ const cmd = String(holder.cmdline);
128
+ if (!/spawn-agent\.js/i.test(cmd)) return false;
129
+ if (basename && !cmd.includes(basename)) return false;
130
+ const age = Number(holder.ageMs) || 0;
131
+ if (age <= 0) return false; // unknown age — refuse to kill
132
+ if (age < minAgeMs) return false;
133
+ return true;
134
+ }
135
+
94
136
  function _maybeEscalateStuck(resolvedPath, errorMsg, opts = {}) {
95
137
  const cfgEng = (opts.config && opts.config.engine) || {};
96
138
  const threshold = cfgEng.worktreeStuckThreshold || shared.ENGINE_DEFAULTS.worktreeStuckThreshold || 10;
@@ -100,7 +142,7 @@ function _maybeEscalateStuck(resolvedPath, errorMsg, opts = {}) {
100
142
  rec.failures++;
101
143
  rec.lastAttempt = now;
102
144
  rec.lastError = errorMsg || rec.lastError || 'unknown';
103
- if (rec.failures < threshold) return { escalated: false, alreadyEscalated: false };
145
+ if (rec.failures < threshold) return { escalated: false, alreadyEscalated: false, holders: [], reapedPids: [] };
104
146
 
105
147
  const alreadyEscalated = rec.escalatedAt > 0;
106
148
  if (!alreadyEscalated) {
@@ -112,15 +154,57 @@ function _maybeEscalateStuck(resolvedPath, errorMsg, opts = {}) {
112
154
  rec.suppressedUntil = now + suppressMs;
113
155
  }
114
156
 
157
+ // W-mq6f2fe0000557fa — orphan-sweep enrichment + opt-in auto-reap. Scans
158
+ // and reap-attempts run on EVERY escalated tick (no per-day dedup) so that
159
+ // a fresh holder appearing after the first escalation still gets killed.
160
+ // The inbox NOTE is still dedup'd per UTC day below.
161
+ let holders = [];
162
+ let reapedPids = [];
163
+ const isOrphanSweep = opts.reason === 'orphan-sweep';
164
+ if (isOrphanSweep) {
165
+ try {
166
+ const findHoldersFn = typeof opts.findProcessesWithCwdInside === 'function'
167
+ ? opts.findProcessesWithCwdInside
168
+ : shared.findProcessesWithCwdInside;
169
+ const scanTimeoutMs = cfgEng.orphanHolderScanTimeoutMs
170
+ ?? shared.ENGINE_DEFAULTS.orphanHolderScanTimeoutMs
171
+ ?? 5000;
172
+ holders = findHoldersFn(resolvedPath, { timeoutMs: scanTimeoutMs }) || [];
173
+ } catch { holders = []; }
174
+
175
+ if (cfgEng.autoReapOrphanWorktreeHolders === true && holders.length > 0) {
176
+ const basename = path.basename(resolvedPath);
177
+ const agentTimeoutMs = Number(cfgEng.agentTimeout)
178
+ || Number(shared.ENGINE_DEFAULTS.agentTimeout)
179
+ || (5 * 60 * 60 * 1000);
180
+ const minAgeMs = agentTimeoutMs * 2;
181
+ const killFn = typeof opts.killImmediate === 'function'
182
+ ? opts.killImmediate
183
+ : shared.killImmediate;
184
+ for (const h of holders) {
185
+ if (!_isSafeReapHolder(h, basename, minAgeMs)) continue;
186
+ try {
187
+ killFn({ pid: h.pid });
188
+ reapedPids.push(h.pid);
189
+ } catch { /* fail-open — keep trying other holders */ }
190
+ }
191
+ if (reapedPids.length > 0) {
192
+ try { shared.bumpWorktreeGcMetric('orphanHoldersReaped'); } catch { /* optional */ }
193
+ }
194
+ }
195
+ }
196
+
115
197
  // Dedup inbox note per UTC day.
116
198
  const today = new Date(now).toISOString().slice(0, 10);
117
- if (rec.lastNoteDate === today) return { escalated: !alreadyEscalated, alreadyEscalated };
199
+ if (rec.lastNoteDate === today) {
200
+ return { escalated: !alreadyEscalated, alreadyEscalated, holders, reapedPids };
201
+ }
118
202
  rec.lastNoteDate = today;
119
203
 
120
204
  try {
121
205
  const writer = typeof opts.writeToInbox === 'function' ? opts.writeToInbox : shared.writeToInbox;
122
206
  const basename = path.basename(resolvedPath);
123
- const body = [
207
+ const bodyParts = [
124
208
  `# Worktree stuck: ${basename}`,
125
209
  '',
126
210
  `Worktree dir **${resolvedPath}** has failed removal ${rec.failures} consecutive times.`,
@@ -153,16 +237,116 @@ function _maybeEscalateStuck(resolvedPath, errorMsg, opts = {}) {
153
237
  '',
154
238
  `Per-tick warn spam is suppressed for the next`,
155
239
  `\`ENGINE_DEFAULTS.worktreeStuckSuppressMs\` (default 60min).`,
156
- ].join('\n');
157
- writer('engine', `worktree-stuck-${basename}`, body);
240
+ ];
241
+
242
+ // W-mq6f2fe0000557fa — append the holder section for orphan-sweep callers.
243
+ if (isOrphanSweep) {
244
+ bodyParts.push('', '## Live holders');
245
+ bodyParts.push('');
246
+ if (holders.length === 0) {
247
+ bodyParts.push(
248
+ '_No holder process found via cwd scan._',
249
+ '',
250
+ 'If the dir is still stuck, the holder may be a non-process resource (',
251
+ 'antivirus scan-in-progress, OneDrive sync, indexer); try again after',
252
+ '~60s, or run `handle.exe` interactively for a deeper search.'
253
+ );
254
+ } else {
255
+ for (const h of holders) {
256
+ const tail = String(h.cmdline || '').replace(/\r?\n/g, ' ').slice(0, 300);
257
+ const ageStr = h.ageMs ? `${Math.floor(h.ageMs / 1000)}s` : 'unknown';
258
+ bodyParts.push(`- **PID ${h.pid}** (age: ${ageStr})`);
259
+ bodyParts.push(' ```');
260
+ bodyParts.push(` ${tail}`);
261
+ bodyParts.push(' ```');
262
+ }
263
+ bodyParts.push('');
264
+ if (cfgEng.autoReapOrphanWorktreeHolders === true) {
265
+ if (reapedPids.length > 0) {
266
+ bodyParts.push(
267
+ `## Auto-reap`,
268
+ '',
269
+ `Engine auto-killed ${reapedPids.length} unambiguous orphan`,
270
+ `spawn-agent process${reapedPids.length === 1 ? '' : 'es'}: PID${reapedPids.length === 1 ? '' : 's'} ${reapedPids.join(', ')}.`,
271
+ `If the kill released the cwd lock, the next sweep tick will`,
272
+ `successfully remove the worktree and emit a`,
273
+ `\`worktree-recovered-${basename}-via-holder-reap\` recovery note.`
274
+ );
275
+ } else {
276
+ bodyParts.push(
277
+ `## Auto-reap`,
278
+ '',
279
+ `\`engine.autoReapOrphanWorktreeHolders\` is ON but no holder`,
280
+ `qualified for auto-kill (cmdline must match \`spawn-agent.js\`,`,
281
+ `reference the worktree basename, and process age must exceed`,
282
+ `\`engine.agentTimeout * 2\`). Manual intervention may be needed.`
283
+ );
284
+ }
285
+ } else {
286
+ bodyParts.push(
287
+ `_To auto-kill orphan spawn-agent processes holding this worktree,_`,
288
+ `_flip \`engine.autoReapOrphanWorktreeHolders\` ON in dashboard Settings._`
289
+ );
290
+ }
291
+ }
292
+ }
293
+
294
+ writer('engine', `worktree-stuck-${basename}`, bodyParts.join('\n'));
158
295
  } catch { /* note best-effort */ }
159
296
 
160
- return { escalated: !alreadyEscalated, alreadyEscalated };
297
+ return { escalated: !alreadyEscalated, alreadyEscalated, holders, reapedPids };
161
298
  }
162
299
 
163
300
  // Test-only reset hook. Production code never calls this.
164
301
  function _resetStuckPathsForTesting() { _stuckPaths.clear(); }
165
302
 
303
+ // W-mq6f2fe0000557fa — post-auto-reap retry. Called only when
304
+ // `_maybeEscalateStuck` reports `reapedPids.length > 0`. Clears the per-path
305
+ // failure cooldown (otherwise the post-3-strike suppression in
306
+ // `shared.removeWorktree` silently skips the retry), waits briefly for the
307
+ // OS to release the killed processes' file handles, then attempts ONE more
308
+ // removal. On success, writes the recovered-via-holder-reap note via
309
+ // `_markStuckSuccess` with the reaped-pid attribution.
310
+ function _postReapRetry(wtPath, gitRoot, parentDir, resolvedPath, _removeWorktree, opts, projStats, result, log, reapedPids) {
311
+ try { shared.clearWorktreeFailureCache(resolvedPath); } catch { /* optional */ }
312
+ // Brief settle window for OS to release file handles from killed processes.
313
+ try {
314
+ const sleepFn = typeof opts.sleepSyncFn === 'function'
315
+ ? opts.sleepSyncFn
316
+ : (ms) => {
317
+ try {
318
+ require('child_process').execFileSync(process.execPath, ['-e', `setTimeout(()=>process.exit(0), ${Number(ms) || 0})`], { stdio: 'ignore' });
319
+ } catch { /* timeout / spawn fail — proceed anyway */ }
320
+ };
321
+ sleepFn(2000);
322
+ } catch { /* sleep failures should not block the retry */ }
323
+
324
+ try {
325
+ const removed = _removeWorktree(wtPath, gitRoot, parentDir);
326
+ if (removed) {
327
+ // Down-count the failure we recorded just before; this dispatch
328
+ // ultimately succeeded after the auto-reap.
329
+ if (projStats) { projStats.failed = Math.max(0, projStats.failed - 1); projStats.evicted++; }
330
+ if (result) { result.failed = Math.max(0, result.failed - 1); result.evicted++; }
331
+ _markStuckSuccess(resolvedPath, {
332
+ writeToInbox: opts.writeToInbox,
333
+ recoveryReason: 'holder-reap',
334
+ reapedPids,
335
+ });
336
+ try { shared.bumpWorktreeGcMetric('recoveredViaHolderReap'); } catch { /* optional */ }
337
+ if (typeof log === 'function') {
338
+ log('info', `worktree-gc: removed ${wtPath} after auto-reaping holder(s) ${reapedPids.join(', ')}`);
339
+ }
340
+ return true;
341
+ }
342
+ } catch (retryErr) {
343
+ if (typeof log === 'function') {
344
+ log('warn', `worktree-gc: post-reap retry threw for ${wtPath}: ${retryErr && retryErr.message}`);
345
+ }
346
+ }
347
+ return false;
348
+ }
349
+
166
350
  /**
167
351
  * Decide whether a dispatch-end worktree should be GC'd.
168
352
  *
@@ -494,19 +678,39 @@ function pruneOrphanWorktrees(opts) {
494
678
  log('info', `worktree-gc: boot-evicted orphan ${name} for project ${project.name || 'default'}`);
495
679
  } else {
496
680
  projStats.failed++; result.failed++;
497
- const { alreadyEscalated } = _maybeEscalateStuck(wtResolved, 'remove returned false', { config: opts.config, writeToInbox: opts.writeToInbox });
498
- if (!alreadyEscalated && !suppressed) {
681
+ const escResult = _maybeEscalateStuck(wtResolved, 'remove returned false', {
682
+ config: opts.config, writeToInbox: opts.writeToInbox,
683
+ reason: 'orphan-sweep',
684
+ findProcessesWithCwdInside: opts.findProcessesWithCwdInside,
685
+ killImmediate: opts.killImmediate,
686
+ });
687
+ if (!escResult.alreadyEscalated && !suppressed) {
499
688
  log('warn', `worktree-gc: boot-evict returned false for ${wtPath}`);
500
689
  }
690
+ // W-mq6f2fe0000557fa — if auto-reap killed any holders, retry the
691
+ // removal once after a brief settle window. Clear the failure
692
+ // cache entry so the post-reap retry isn't suppressed by the
693
+ // 3-strike cooldown.
694
+ if (escResult.reapedPids && escResult.reapedPids.length > 0) {
695
+ _postReapRetry(wtPath, rootDir, wtParent, wtResolved, _removeWorktree, opts, projStats, result, log, escResult.reapedPids);
696
+ }
501
697
  }
502
698
  } catch (rmErr) {
503
699
  projStats.failed++; result.failed++;
504
700
  const wtResolved = path.resolve(wtPath);
505
701
  const suppressed = _isStuckPathSuppressed(wtResolved);
506
- const { alreadyEscalated } = _maybeEscalateStuck(wtResolved, rmErr && rmErr.message, { config: opts.config, writeToInbox: opts.writeToInbox });
507
- if (!alreadyEscalated && !suppressed) {
702
+ const escResult = _maybeEscalateStuck(wtResolved, rmErr && rmErr.message, {
703
+ config: opts.config, writeToInbox: opts.writeToInbox,
704
+ reason: 'orphan-sweep',
705
+ findProcessesWithCwdInside: opts.findProcessesWithCwdInside,
706
+ killImmediate: opts.killImmediate,
707
+ });
708
+ if (!escResult.alreadyEscalated && !suppressed) {
508
709
  log('warn', `worktree-gc: boot-evict threw for ${wtPath}: ${rmErr.message}`);
509
710
  }
711
+ if (escResult.reapedPids && escResult.reapedPids.length > 0) {
712
+ _postReapRetry(wtPath, rootDir, wtParent, wtResolved, _removeWorktree, opts, projStats, result, log, escResult.reapedPids);
713
+ }
510
714
  }
511
715
  }
512
716
  result.perProject[project.name || rootDir] = projStats;
@@ -693,17 +897,33 @@ function pruneOrphanWorktreesFromGitRegistry(opts) {
693
897
  log('info', `worktree-gc: out-of-root evicted ${wtAbs} for project ${project.name || 'default'}`);
694
898
  } else {
695
899
  projStats.failed++; result.failed++;
696
- const { alreadyEscalated } = _maybeEscalateStuck(wtAbs, 'remove returned false', { config: opts.config, writeToInbox: opts.writeToInbox });
697
- if (!alreadyEscalated && !suppressed) {
900
+ const escResult = _maybeEscalateStuck(wtAbs, 'remove returned false', {
901
+ config: opts.config, writeToInbox: opts.writeToInbox,
902
+ reason: 'orphan-sweep',
903
+ findProcessesWithCwdInside: opts.findProcessesWithCwdInside,
904
+ killImmediate: opts.killImmediate,
905
+ });
906
+ if (!escResult.alreadyEscalated && !suppressed) {
698
907
  log('warn', `worktree-gc: out-of-root remove returned false for ${wtAbs}`);
699
908
  }
909
+ if (escResult.reapedPids && escResult.reapedPids.length > 0) {
910
+ _postReapRetry(wtAbs, rootDir, parentDir, wtAbs, _removeWorktree, opts, projStats, result, log, escResult.reapedPids);
911
+ }
700
912
  }
701
913
  } catch (rmErr) {
702
914
  projStats.failed++; result.failed++;
703
- const { alreadyEscalated } = _maybeEscalateStuck(wtAbs, rmErr && rmErr.message, { config: opts.config, writeToInbox: opts.writeToInbox });
704
- if (!alreadyEscalated && !suppressed) {
915
+ const escResult = _maybeEscalateStuck(wtAbs, rmErr && rmErr.message, {
916
+ config: opts.config, writeToInbox: opts.writeToInbox,
917
+ reason: 'orphan-sweep',
918
+ findProcessesWithCwdInside: opts.findProcessesWithCwdInside,
919
+ killImmediate: opts.killImmediate,
920
+ });
921
+ if (!escResult.alreadyEscalated && !suppressed) {
705
922
  log('warn', `worktree-gc: out-of-root remove threw for ${wtAbs}: ${rmErr.message}`);
706
923
  }
924
+ if (escResult.reapedPids && escResult.reapedPids.length > 0) {
925
+ _postReapRetry(wtAbs, rootDir, parentDir, wtAbs, _removeWorktree, opts, projStats, result, log, escResult.reapedPids);
926
+ }
707
927
  }
708
928
  }
709
929
 
package/engine.js CHANGED
@@ -918,6 +918,109 @@ async function pruneStaleWorktreeForBranch(rootDir, branchName, gitOpts) {
918
918
  return removed;
919
919
  }
920
920
 
921
+ // W-mq5n1zx5 — Layer 1b: status-probe command builder. Honors the
922
+ // ENGINE_DEFAULTS.statusProbeUseNoOptionalLocks toggle so operators can
923
+ // disable the flag if a future git release breaks something. Default ON:
924
+ // `--no-optional-locks` tells git to skip the .git/index.lock acquire
925
+ // around the untracked-cache refresh that runs inside `status`. Without
926
+ // the flag, a 6–12s probe under aggressive AV scanning is normal; with
927
+ // it, typical probes drop to <500ms, which removes the timeout that
928
+ // leaks the git.exe descendant that later pins packfile handles and
929
+ // breaks the quarantine rename.
930
+ function _statusPorcelainCmd() {
931
+ return ENGINE_DEFAULTS.statusProbeUseNoOptionalLocks
932
+ ? 'git --no-optional-locks status --porcelain'
933
+ : 'git status --porcelain';
934
+ }
935
+
936
+ // W-mq5n1zx5 — Layer 1a: rename worktree dir with jittered backoff. The
937
+ // raw `fs.renameSync` used to throw Windows EBUSY/EPERM/EACCES if a
938
+ // lingering `git.exe` descendant (typically leaked by a status-probe
939
+ // timeout) still held a packfile handle. We retry with capped exponential
940
+ // backoff + random jitter so the descendant has time to exit on its own.
941
+ // Worst-case wall time ≈ baseMs * (2^attempts) ≈ 16s when attempts=6,
942
+ // baseMs=250 — small enough not to wedge a tick, large enough to clear
943
+ // the typical race. Throws the LAST error on exhaustion so the caller
944
+ // can decide whether to fall back to `git worktree remove --force`.
945
+ async function _renameWithRetry(src, dst, opts = {}) {
946
+ const attempts = Number(opts.attempts) > 0
947
+ ? Number(opts.attempts)
948
+ : ENGINE_DEFAULTS.quarantineRenameRetryAttempts;
949
+ const baseMs = Number(opts.baseMs) > 0
950
+ ? Number(opts.baseMs)
951
+ : ENGINE_DEFAULTS.quarantineRenameRetryBaseMs;
952
+ let lastErr;
953
+ for (let i = 0; i < attempts; i++) {
954
+ try { fs.renameSync(src, dst); return { attempts: i + 1 }; }
955
+ catch (e) {
956
+ if (!['EBUSY', 'EPERM', 'EACCES', 'ENOTEMPTY'].includes(e.code)) throw e;
957
+ lastErr = e;
958
+ if (i < attempts - 1) {
959
+ const delay = baseMs * (2 ** i) + Math.random() * 200;
960
+ await new Promise(r => setTimeout(r, delay));
961
+ }
962
+ }
963
+ }
964
+ throw lastErr;
965
+ }
966
+
967
+ // W-mq5n1zx5 — Layer 2a: on Windows, kill any live `git.exe` descendants
968
+ // whose command line points at the quarantine target path. We never
969
+ // tracked the PID of the `git status --porcelain` child that the probe
970
+ // timed out on, so we can't kill by PID — instead we shell out to
971
+ // PowerShell's CIM cmdlets to find matching processes and Stop-Process
972
+ // them. Cheap (<2s) and idempotent — if no descendants are alive, the
973
+ // CIM query returns nothing and exits 0. POSIX is a no-op (the EBUSY
974
+ // race is Windows-specific). Best-effort; failure is logged but never
975
+ // blocks the quarantine.
976
+ function _killGitDescendantsForWorktree(worktreePath) {
977
+ if (process.platform !== 'win32') return { killed: 0, skipped: true };
978
+ if (!ENGINE_DEFAULTS.statusProbeKillDescendantsWin32) return { killed: 0, skipped: true };
979
+ if (!worktreePath) return { killed: 0, skipped: true };
980
+ // PowerShell expects single-quoted literals; escape any embedded single
981
+ // quote by doubling it (PowerShell's standard single-quote escape).
982
+ const safePath = String(worktreePath).replace(/'/g, "''");
983
+ const ps = [
984
+ "$ErrorActionPreference='SilentlyContinue';",
985
+ `$matches = Get-CimInstance Win32_Process | Where-Object { $_.Name -eq 'git.exe' -and $_.CommandLine -like '*${safePath}*' };`,
986
+ "if ($matches) { $matches | ForEach-Object { Stop-Process -Id $_.ProcessId -Force; $_.ProcessId }; }",
987
+ ].join(' ');
988
+ try {
989
+ const out = shared.execSilent(`powershell -NoProfile -Command "${ps.replace(/"/g, '\\"')}"`, { timeout: 2000, encoding: 'utf8' });
990
+ const killed = String(out || '').split(/\r?\n/).map(s => s.trim()).filter(Boolean).length;
991
+ if (killed > 0) log('info', `_killGitDescendantsForWorktree: killed ${killed} git.exe descendant(s) holding ${worktreePath}`);
992
+ return { killed };
993
+ } catch (e) {
994
+ log('warn', `_killGitDescendantsForWorktree: powershell probe failed: ${e.message}`);
995
+ return { killed: 0, error: e.message };
996
+ }
997
+ }
998
+
999
+ // W-mq5n1zx5 — Layer 3a: bump the rolling counters for quarantine rename
1000
+ // outcomes (attempts, success, successAfterRetry, fallbackForceRemove,
1001
+ // totalFailure). Best-effort; metrics failures must not break the
1002
+ // quarantine path. The shape lives under metrics._engine.worktreeQuarantineOutcomes
1003
+ // so it sits beside the other _engine.* engine-internal telemetry.
1004
+ function _bumpQuarantineOutcome(key, delta = 1) {
1005
+ try {
1006
+ shared.mutateMetrics((metrics) => {
1007
+ if (!metrics._engine) metrics._engine = {};
1008
+ if (!metrics._engine.worktreeQuarantineOutcomes) {
1009
+ metrics._engine.worktreeQuarantineOutcomes = {
1010
+ attempts: 0, success: 0, successAfterRetry: 0,
1011
+ fallbackForceRemove: 0, totalFailure: 0,
1012
+ };
1013
+ }
1014
+ const o = metrics._engine.worktreeQuarantineOutcomes;
1015
+ o[key] = (o[key] || 0) + delta;
1016
+ return metrics;
1017
+ });
1018
+ } catch (e) {
1019
+ log('warn', `_bumpQuarantineOutcome(${key}): ${e.message}`);
1020
+ }
1021
+ }
1022
+
1023
+
921
1024
  // ─── assertCleanSharedWorktree (#2439) ──────────────────────────────────────
922
1025
  // Engine-side preflight that prevents shared-branch (and PR-targeted reused —
923
1026
  // see opts.quarantineOnUnsafe, issue #2996) dispatches from spawning into a
@@ -980,7 +1083,7 @@ async function assertCleanSharedWorktree(rootDir, worktreePath, branchName, disp
980
1083
  // 1. Status probe (filesystem)
981
1084
  let statusOut = '';
982
1085
  try {
983
- const r = await execAsync('git status --porcelain', { ...gitOpts, cwd: worktreePath, timeout: statusTimeoutMs });
1086
+ const r = await execAsync(_statusPorcelainCmd(), { ...gitOpts, cwd: worktreePath, timeout: statusTimeoutMs });
984
1087
  statusOut = (r || '').toString().trim();
985
1088
  } catch (e) {
986
1089
  // W-mq1habhf: previously this bailed out with the bad worktree intact,
@@ -1186,7 +1289,7 @@ async function assertCleanSharedWorktree(rootDir, worktreePath, branchName, disp
1186
1289
 
1187
1290
  // 7. Re-verify
1188
1291
  try {
1189
- const r2 = await execAsync('git status --porcelain', { ...gitOpts, cwd: worktreePath, timeout: statusTimeoutMs });
1292
+ const r2 = await execAsync(_statusPorcelainCmd(), { ...gitOpts, cwd: worktreePath, timeout: statusTimeoutMs });
1190
1293
  const after = (r2 || '').toString().trim();
1191
1294
  if (after) {
1192
1295
  result.reason = 'dirty-after-clean';
@@ -1218,9 +1321,23 @@ async function assertCleanSharedWorktree(rootDir, worktreePath, branchName, disp
1218
1321
  // branch (assertCleanSharedWorktree returns 'other-dispatch-active' before
1219
1322
  // invoking this). Renaming an active worktree out from under a running
1220
1323
  // agent would be catastrophic.
1324
+ //
1325
+ // W-mq5n1zx5 — layered defense against Windows EBUSY: (Layer 2a) kill
1326
+ // lingering git.exe descendants whose command-line points at the worktree
1327
+ // before touching the path; (Layer 1a) retry the rename with jittered
1328
+ // backoff; (Layer 2b) if every retry still fails, fall back to
1329
+ // `git worktree remove --force` so git tears down its own metadata + dir
1330
+ // rather than leaving a half-quarantined tree behind; (Layer 3a) emit
1331
+ // outcome counters to metrics._engine.worktreeQuarantineOutcomes;
1332
+ // (Layer 3b) when EVERY path fails, write a dedicated inbox alert naming
1333
+ // the manual recovery command. The throw-or-return contract is unchanged
1334
+ // for callers: a thrown error means the caller's `result.quarantineError`
1335
+ // field is populated and `result.quarantined` stays false (env-blocked
1336
+ // path); a normal return means quarantine succeeded.
1221
1337
  async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOpts, diag = {}) {
1222
1338
  const ts = Date.now();
1223
1339
  const quarantinedPath = `${worktreePath}-quarantine-${ts}`;
1340
+ _bumpQuarantineOutcome('attempts', 1);
1224
1341
 
1225
1342
  // Capture HEAD sha BEFORE renaming so we can back up the local branch ref.
1226
1343
  let headSha = '';
@@ -1231,24 +1348,112 @@ async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOp
1231
1348
  log('warn', `_quarantineDirtyWorktree: rev-parse HEAD failed for ${worktreePath}: ${e.message} — backup ref will be skipped`);
1232
1349
  }
1233
1350
 
1234
- // Rename the worktree dir. Once this succeeds the worktree is functionally
1235
- // quarantined; subsequent failures only affect ref bookkeeping.
1236
1351
  // W-mq5rwwss000f30a7 — never quarantine (=rename out from under) a worktree
1237
1352
  // that a live dispatch still claims. Quarantine breaks the agent's cwd just
1238
- // as completely as removeWorktree would.
1353
+ // as completely as removeWorktree would. Check before any destructive work
1354
+ // (including the kill-descendants sweep below) so we don't disturb a live
1355
+ // agent's git child processes either.
1239
1356
  if (shared.isWorktreePathLive(worktreePath)) {
1240
1357
  log('warn', `_quarantineDirtyWorktree: skip — live dispatch in ${worktreePath}`);
1241
1358
  shared._writeWorktreeSkipLiveInboxNote(worktreePath, '_quarantineDirtyWorktree');
1242
1359
  return { quarantinedPath: null, backupRef: null, skipped: true };
1243
1360
  }
1244
- fs.renameSync(worktreePath, quarantinedPath);
1245
1361
 
1246
- // Prune git's stale worktree metadata so the next `git worktree add` for
1247
- // the same branch isn't blocked by "branch is already used by worktree".
1362
+ // W-mq5n1zx5 Layer 2a: pre-emptively kill any git.exe descendants whose
1363
+ // command line points at the worktree. The status-probe child that timed
1364
+ // out earlier may still be alive holding packfile handles; if so, the
1365
+ // rename below will fail with EBUSY/EPERM until the descendant exits.
1366
+ // POSIX is a no-op. Best-effort; failure here is non-fatal.
1367
+ _killGitDescendantsForWorktree(worktreePath);
1368
+
1369
+ // W-mq5n1zx5 Layer 1a: rename with jittered backoff. Replaces the bare
1370
+ // `fs.renameSync(worktreePath, quarantinedPath)` that used to throw
1371
+ // Windows EBUSY uncatchably and burn the WI's auto-recovery budget.
1372
+ let renameAttempts = 0;
1373
+ let renameError = null;
1248
1374
  try {
1249
- await shared.shellSafeGit(['worktree', 'prune'], { ...gitOpts, cwd: rootDir, timeout: 15000 });
1375
+ const r = await _renameWithRetry(worktreePath, quarantinedPath);
1376
+ renameAttempts = r.attempts || 1;
1250
1377
  } catch (e) {
1251
- log('warn', `_quarantineDirtyWorktree: worktree prune after rename: ${e.message}`);
1378
+ renameError = e;
1379
+ }
1380
+
1381
+ // W-mq5n1zx5 Layer 2b: if the rename retries exhausted, fall back to
1382
+ // `git worktree remove --force`. This DESTROYS the worktree contents
1383
+ // (no forensics dir), so it's last-resort only — gated on the
1384
+ // quarantineForceRemoveFallback engine default. The next dispatch will
1385
+ // create a fresh worktree on the origin tip, which is the same outcome
1386
+ // the rename path would produce, just without the recovery dir.
1387
+ let forceRemoved = false;
1388
+ let forceRemoveError = null;
1389
+ if (renameError && ENGINE_DEFAULTS.quarantineForceRemoveFallback) {
1390
+ log('warn', `_quarantineDirtyWorktree: rename failed after ${renameAttempts || ENGINE_DEFAULTS.quarantineRenameRetryAttempts} attempt(s) (${renameError.code || renameError.message}); falling back to git worktree remove --force`);
1391
+ try {
1392
+ await shared.shellSafeGit(['worktree', 'remove', '--force', worktreePath], { ...gitOpts, cwd: rootDir, timeout: 30000 });
1393
+ forceRemoved = true;
1394
+ log('warn', `_quarantineDirtyWorktree: git worktree remove --force succeeded for ${worktreePath} (no quarantine dir preserved)`);
1395
+ } catch (e) {
1396
+ forceRemoveError = e;
1397
+ log('error', `_quarantineDirtyWorktree: git worktree remove --force ALSO failed: ${e.message}`);
1398
+ }
1399
+ }
1400
+
1401
+ // If neither the rename retries nor the force-remove fallback succeeded,
1402
+ // we are env-blocked: the worktree dir is still on disk and git still
1403
+ // thinks it owns it. Write a dedicated inbox alert (Layer 3b) and rethrow
1404
+ // the original rename error so the caller's quarantineError branch fires.
1405
+ if (renameError && !forceRemoved) {
1406
+ _bumpQuarantineOutcome('totalFailure', 1);
1407
+ const sanitizedRefSegmentEnv = sanitizeBranch(branchName).replace(/\//g, '-');
1408
+ const failBody = [
1409
+ '# Engine quarantine TOTAL FAILURE — manual recovery required (W-mq5n1zx5)',
1410
+ '',
1411
+ `- Dispatch: ${diag.dispatchId || '<unknown>'}`,
1412
+ `- Branch: ${branchName}`,
1413
+ `- Worktree path (STILL PRESENT): ${worktreePath}`,
1414
+ `- Reason: ${diag.reason || 'unsafe'}`,
1415
+ `- Rename attempts: ${renameAttempts || ENGINE_DEFAULTS.quarantineRenameRetryAttempts}`,
1416
+ `- Last rename error: ${renameError.code || ''} ${renameError.message}`,
1417
+ `- Force-remove fallback: ${ENGINE_DEFAULTS.quarantineForceRemoveFallback ? `tried, failed (${forceRemoveError ? forceRemoveError.message : 'unknown'})` : 'disabled'}`,
1418
+ '',
1419
+ '## Manual recovery',
1420
+ '',
1421
+ '1. Find any live `git.exe` (or other) processes still holding the dir:',
1422
+ ` \`Get-CimInstance Win32_Process | Where-Object { $_.CommandLine -like '*${worktreePath}*' }\``,
1423
+ '2. Stop them: `Stop-Process -Id <pid> -Force`',
1424
+ '3. Force-remove the dir:',
1425
+ ` Windows: \`Remove-Item -Recurse -Force "${worktreePath}"\` (or \`rmdir /s /q "${worktreePath}"\` from cmd.exe)`,
1426
+ ` POSIX: \`rm -rf "${worktreePath}"\``,
1427
+ `4. Prune git's stale worktree metadata: \`git -C "${rootDir}" worktree prune\``,
1428
+ '',
1429
+ 'After this, the next dispatch for the branch will create a fresh worktree.',
1430
+ ].join('\n');
1431
+ try {
1432
+ shared.writeToInbox('engine', `worktree-quarantine-failed-${diag.dispatchId || sanitizedRefSegmentEnv}`, failBody);
1433
+ } catch (e) {
1434
+ log('warn', `_quarantineDirtyWorktree: totalFailure writeToInbox failed: ${e.message}`);
1435
+ }
1436
+ throw renameError;
1437
+ }
1438
+
1439
+ // Track outcome bucket for the successful path (rename or force-remove).
1440
+ if (forceRemoved) {
1441
+ _bumpQuarantineOutcome('fallbackForceRemove', 1);
1442
+ } else if (renameAttempts > 1) {
1443
+ _bumpQuarantineOutcome('successAfterRetry', 1);
1444
+ }
1445
+ _bumpQuarantineOutcome('success', 1);
1446
+
1447
+ // Prune git's stale worktree metadata so the next `git worktree add` for
1448
+ // the same branch isn't blocked by "branch is already used by worktree".
1449
+ // Skipped on the force-removed path — `git worktree remove --force` already
1450
+ // wipes its own metadata entry.
1451
+ if (!forceRemoved) {
1452
+ try {
1453
+ await shared.shellSafeGit(['worktree', 'prune'], { ...gitOpts, cwd: rootDir, timeout: 15000 });
1454
+ } catch (e) {
1455
+ log('warn', `_quarantineDirtyWorktree: worktree prune after rename: ${e.message}`);
1456
+ }
1252
1457
  }
1253
1458
 
1254
1459
  // Backup the local branch HEAD to refs/minions/quarantine/<sanitized>/<ts>
@@ -1293,7 +1498,9 @@ async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOp
1293
1498
  `- Dispatch: ${diag.dispatchId || '<unknown>'}`,
1294
1499
  `- Branch: ${branchName}`,
1295
1500
  `- Original worktree: ${worktreePath}`,
1296
- `- Quarantined to: ${quarantinedPath}`,
1501
+ forceRemoved
1502
+ ? `- Force-removed via \`git worktree remove --force\` (no quarantine dir preserved — W-mq5n1zx5 Layer 2b)`
1503
+ : `- Quarantined to: ${quarantinedPath}${renameAttempts > 1 ? ` (rename took ${renameAttempts} attempt(s) — W-mq5n1zx5 Layer 1a)` : ''}`,
1297
1504
  `- Reason: ${diag.reason || 'unsafe'}`,
1298
1505
  `- Local commits ahead of origin: ${diag.ahead || 0}`,
1299
1506
  `- Local commits behind origin: ${diag.behind || 0}`,
@@ -1305,12 +1512,20 @@ async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOp
1305
1512
  '',
1306
1513
  backupRefCreated
1307
1514
  ? `The local branch HEAD was backed up to \`${backupRef}\` (${headSha.slice(0, 12)}) and \`refs/heads/${branchName}\` was reset to \`refs/remotes/origin/${branchName}\`.`
1308
- : `Backup ref was NOT created (HEAD sha unavailable). The local branch ref was still reset to \`refs/remotes/origin/${branchName}\` if possible. Inspect the quarantined directory directly to recover unpushed work.`,
1515
+ : `Backup ref was NOT created (HEAD sha unavailable). The local branch ref was still reset to \`refs/remotes/origin/${branchName}\` if possible.${forceRemoved ? ' Worktree contents were destroyed by the force-remove fallback — no quarantine dir to inspect.' : ' Inspect the quarantined directory directly to recover unpushed work.'}`,
1309
1516
  '',
1310
- `1. Inspect the quarantined diff: \`git -C "${rootDir}" diff refs/remotes/origin/${branchName}${backupRefCreated ? ` ${backupRef}` : ''} -- .\``,
1311
- `2. Cherry-pick the commits you want from \`${backupRefCreated ? backupRef : `<inspect ${quarantinedPath}>`}\` onto a fresh ${branchName} worktree.`,
1312
- `3. Inspect uncommitted edits inside the quarantined dir directly: \`${quarantinedPath}\`.`,
1313
- `4. Delete the quarantined dir when done: \`Remove-Item -Recurse -Force "${quarantinedPath}"\` (Windows) or \`rm -rf "${quarantinedPath}"\` (POSIX).`,
1517
+ forceRemoved
1518
+ ? `1. The worktree dir was force-removed; there is no quarantine dir to inspect. If the local branch ref still has divergent commits, recover from \`${backupRefCreated ? backupRef : 'reflog'}\`.`
1519
+ : `1. Inspect the quarantined diff: \`git -C "${rootDir}" diff refs/remotes/origin/${branchName}${backupRefCreated ? ` ${backupRef}` : ''} -- .\``,
1520
+ forceRemoved
1521
+ ? ''
1522
+ : `2. Cherry-pick the commits you want from \`${backupRefCreated ? backupRef : `<inspect ${quarantinedPath}>`}\` onto a fresh ${branchName} worktree.`,
1523
+ forceRemoved
1524
+ ? ''
1525
+ : `3. Inspect uncommitted edits inside the quarantined dir directly: \`${quarantinedPath}\`.`,
1526
+ forceRemoved
1527
+ ? ''
1528
+ : `4. Delete the quarantined dir when done: \`Remove-Item -Recurse -Force "${quarantinedPath}"\` (Windows) or \`rm -rf "${quarantinedPath}"\` (POSIX).`,
1314
1529
  backupRefCreated ? `5. Delete the backup ref when done: \`git -C "${rootDir}" update-ref -d ${backupRef}\`.` : '',
1315
1530
  ].filter(Boolean).join('\n');
1316
1531
  try {
@@ -1319,8 +1534,13 @@ async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOp
1319
1534
  log('warn', `_quarantineDirtyWorktree: writeToInbox failed: ${e.message}`);
1320
1535
  }
1321
1536
 
1322
- log('warn', `Quarantined dirty worktree ${worktreePath} → ${quarantinedPath} (branch ${branchName}, ${diag.ahead || 0} ahead, ${diag.behind || 0} behind, ${dirtyFiles.length} dirty files)`);
1323
- return { quarantinedPath, backupRef: backupRefCreated ? backupRef : null };
1537
+ log('warn', `Quarantined dirty worktree ${worktreePath} → ${forceRemoved ? '<force-removed>' : quarantinedPath} (branch ${branchName}, ${diag.ahead || 0} ahead, ${diag.behind || 0} behind, ${dirtyFiles.length} dirty files${renameAttempts > 1 ? `, rename retries=${renameAttempts}` : ''}${forceRemoved ? ', force-remove fallback' : ''})`);
1538
+ return {
1539
+ quarantinedPath: forceRemoved ? null : quarantinedPath,
1540
+ backupRef: backupRefCreated ? backupRef : null,
1541
+ forceRemoved,
1542
+ renameAttempts,
1543
+ };
1324
1544
  }
1325
1545
 
1326
1546
  async function recoverPartialWorktree(rootDir, worktreePath, branchName, gitOpts) {
@@ -2070,8 +2290,23 @@ async function spawnAgent(dispatchItem, config) {
2070
2290
  // no-upstream / dirty-after-clean stay non-retryable because they
2071
2291
  // protect potentially-unpushed agent work.
2072
2292
  const isStatusProbeFailed = cleanResult.reason === 'status-failed';
2073
- const failureClassValue = isDivergent ? FAILURE_CLASS.WORKTREE_DIVERGENT : FAILURE_CLASS.WORKTREE_DIRTY;
2074
- const failureClassName = isDivergent ? 'WORKTREE_DIVERGENT' : 'WORKTREE_DIRTY';
2293
+ // W-mq5n1zx5 Layer 1c: when quarantine itself errored (rename retries
2294
+ // + force-remove fallback all failed) the worktree dir is still on
2295
+ // disk and the failure is purely environmental — a lingering git.exe
2296
+ // descendant or AV scan holding handles. Route through a dedicated
2297
+ // WORKTREE_QUARANTINE_ENV_BLOCKED class so the dispatch-side retry
2298
+ // counter doesn't bump (agentRetryable:false → non-retryable branch
2299
+ // → no _retriesByAgent / _retryCount mutation) and the WI auto-
2300
+ // recovery loop in discoverFromWorkItems re-queues this item under
2301
+ // the existing _quarantineRecoveryCount cap. This stops env failures
2302
+ // from burning the per-agent retry budget the way they did before.
2303
+ const isQuarantineEnvBlocked = !!(cleanResult.quarantineError && !cleanResult.quarantined);
2304
+ const failureClassValue = isQuarantineEnvBlocked
2305
+ ? FAILURE_CLASS.WORKTREE_QUARANTINE_ENV_BLOCKED
2306
+ : (isDivergent ? FAILURE_CLASS.WORKTREE_DIVERGENT : FAILURE_CLASS.WORKTREE_DIRTY);
2307
+ const failureClassName = isQuarantineEnvBlocked
2308
+ ? 'WORKTREE_QUARANTINE_ENV_BLOCKED'
2309
+ : (isDivergent ? 'WORKTREE_DIVERGENT' : 'WORKTREE_DIRTY');
2075
2310
  const reasonMsg = cleanResult.quarantined
2076
2311
  ? `${failureClassName}: reused worktree at ${worktreePath} was dirty/divergent (${cleanResult.reason}; ${cleanResult.ahead || 0} ahead, ${cleanResult.behind || 0} behind, ${cleanResult.dirtyFiles?.length || 0} dirty file(s)${previewFiles ? ': ' + previewFiles : ''}) — quarantined to ${cleanResult.quarantinedPath}. Next dispatch will start fresh.`
2077
2312
  : `${failureClassName}: reused worktree at ${worktreePath} is dirty/divergent (${cleanResult.reason}; ${cleanResult.ahead || 0} ahead, ${cleanResult.behind || 0} behind, ${cleanResult.dirtyFiles?.length || 0} file(s)${previewFiles ? ': ' + previewFiles : ''}). Quarantine ${cleanResult.quarantineError ? 'errored: ' + cleanResult.quarantineError : (cleanResult.quarantineSkipped ? 'was skipped — another live dispatch claims the worktree (see notes/inbox/ engine-worktree-skip-live note).' : 'was not attempted (' + cleanResult.reason + ').')}`;
@@ -2081,7 +2316,7 @@ async function spawnAgent(dispatchItem, config) {
2081
2316
  id,
2082
2317
  DISPATCH_RESULT.ERROR,
2083
2318
  reasonMsg.slice(0, 500),
2084
- `Engine preflight refused to dispatch into a dirty/divergent reused worktree (#2996). Reason: ${cleanResult.reason}.${cleanResult.quarantined ? ` Worktree quarantined to ${cleanResult.quarantinedPath}; backup ref ${cleanResult.backupRef || '(skipped)'}. See notes/inbox/ for recovery instructions.` : (cleanResult.quarantineSkipped ? ' Quarantine was skipped because another live dispatch claims this worktree path; this dispatch will not auto-retry until the live claimant clears.' : '')}`,
2319
+ `Engine preflight refused to dispatch into a dirty/divergent reused worktree (#2996). Reason: ${cleanResult.reason}.${cleanResult.quarantined ? ` Worktree quarantined to ${cleanResult.quarantinedPath}; backup ref ${cleanResult.backupRef || '(skipped)'}. See notes/inbox/ for recovery instructions.` : (cleanResult.quarantineSkipped ? ' Quarantine was skipped because another live dispatch claims this worktree path; this dispatch will not auto-retry until the live claimant clears.' : '')}${isQuarantineEnvBlocked ? ' Environmental quarantine failure (Windows EBUSY); WI auto-recovery loop will re-queue without bumping per-agent retry counter.' : ''}`,
2085
2320
  { agentRetryable: isStatusProbeFailed && cleanResult.quarantined, failureClass: failureClassValue },
2086
2321
  );
2087
2322
  cleanupTempAgent(agentId);
@@ -2260,7 +2495,10 @@ async function spawnAgent(dispatchItem, config) {
2260
2495
  let stashed = false;
2261
2496
  if (!depMergeFailed && !skipDepMerge && prunedDeps.length > 0) {
2262
2497
  try {
2263
- const statusOut = gitOutputToString(await shared.shellSafeGit(['status', '--porcelain'], { ..._gitOpts, cwd: worktreePath })).trim();
2498
+ const statusArgs = ENGINE_DEFAULTS.statusProbeUseNoOptionalLocks
2499
+ ? ['--no-optional-locks', 'status', '--porcelain']
2500
+ : ['status', '--porcelain'];
2501
+ const statusOut = gitOutputToString(await shared.shellSafeGit(statusArgs, { ..._gitOpts, cwd: worktreePath })).trim();
2264
2502
  if (statusOut) {
2265
2503
  await shared.shellSafeGit(['stash', 'push', '--include-untracked', '-m', 'engine: stash before dep re-merge'], { ..._gitOpts, cwd: worktreePath });
2266
2504
  stashed = true;
@@ -2501,7 +2739,7 @@ async function spawnAgent(dispatchItem, config) {
2501
2739
  if (worktreePath && fs.existsSync(worktreePath)) {
2502
2740
  _phaseT.dirtyProbeStart = Date.now();
2503
2741
  try {
2504
- const dirtyResult = await execAsync('git status --porcelain', { ..._gitOpts, cwd: worktreePath, timeout: 10000 });
2742
+ const dirtyResult = await execAsync(_statusPorcelainCmd(), { ..._gitOpts, cwd: worktreePath, timeout: 10000 });
2505
2743
  const dirtyOutput = (dirtyResult.stdout || '').trim();
2506
2744
  if (dirtyOutput) {
2507
2745
  const dirtyFiles = dirtyOutput.split('\n').map(l => l.trim()).filter(Boolean);
@@ -5666,19 +5904,28 @@ async function discoverFromPrs(config, project) {
5666
5904
  // starvation guarantee.
5667
5905
  const conflictCauseKey = getPrAutomationCauseKey('merge-conflict', pr);
5668
5906
  const key = getPrAutomationDispatchKey(`conflict-fix-${project?.name || 'default'}-${prDisplayId}`, conflictCauseKey);
5669
- // W-mpritzcr0004afc5 (#2955): per-cause same-head guard mirroring the
5670
- // build-failure block above. `_conflictFixedAt` is a 10-min wall-clock
5671
- // suppression for ADO/GH mergeStatus lag; `_lastDispatchByCause` is a
5672
- // headSha-pinned suppression for repeated agent noops on an unchanged
5673
- // base+head pair. Both must fire (their windows are independent).
5907
+ // W-mpritzcr0004afc5 (#2955) + #3079: per-cause same-head guard mirroring
5908
+ // the build-failure block above. `_conflictFixedAt` is a 10-min wall-clock
5909
+ // suppression for ADO/GH mergeStatus lag; `_lastDispatchByCause` is now
5910
+ // a (head+base+targetRef)-pinned suppression for repeated agent noops on
5911
+ // an unchanged source+base+target tuple. #3079: prior version compared
5912
+ // only headSha, so retargeting a PR (e.g. parent-branch → main after
5913
+ // parent merge) never released the pause even though the base SHA and
5914
+ // target ref had changed and conflicts may have become resolvable.
5915
+ const currentGuardKey = shared.prMergeConflictGuardKey(pr);
5674
5916
  const currentHeadSha = String(pr.headSha || pr._adoSourceCommit || pr._adoHeadCommit || '').trim();
5675
5917
  const lastConflictDispatch = pr._lastDispatchByCause?.[shared.PR_FIX_CAUSE.MERGE_CONFLICT];
5676
- const skipConflictFix = !!(lastConflictDispatch?.outcome === 'noop'
5677
- && lastConflictDispatch.headSha
5918
+ // Prefer the new mergeConflictKey (post-#3079); fall back to legacy
5919
+ // headSha compare so PRs paused before this fix don't loop forever.
5920
+ const lastGuardKey = lastConflictDispatch?.mergeConflictKey;
5921
+ const guardKeyMatches = !!(lastGuardKey && currentGuardKey && lastGuardKey === currentGuardKey);
5922
+ const legacyHeadMatches = !lastGuardKey && !!(lastConflictDispatch?.headSha
5678
5923
  && currentHeadSha
5679
5924
  && lastConflictDispatch.headSha === currentHeadSha);
5925
+ const skipConflictFix = !!(lastConflictDispatch?.outcome === 'noop'
5926
+ && (guardKeyMatches || legacyHeadMatches));
5680
5927
  if (skipConflictFix) {
5681
- log('info', `Skipping conflict-fix for ${pr.id}: last merge-conflict dispatch was noop on the same head ${currentHeadSha.slice(0, 8)} (${(lastConflictDispatch.reason || '').slice(0, 120)})`);
5928
+ log('info', `Skipping conflict-fix for ${pr.id}: last merge-conflict dispatch was noop on the same source+base+target (${(lastConflictDispatch.reason || '').slice(0, 120)})`);
5682
5929
  continue;
5683
5930
  }
5684
5931
  // Suppress re-dispatch for 10 min after last attempt — ADO/GitHub recomputes
@@ -6151,8 +6398,10 @@ function discoverFromWorkItems(config, project) {
6151
6398
  const fr = String(item.failReason || '');
6152
6399
  const isQuarantineFail = item._failureClass === FAILURE_CLASS.WORKTREE_DIRTY
6153
6400
  || item._failureClass === FAILURE_CLASS.WORKTREE_DIVERGENT
6401
+ || item._failureClass === FAILURE_CLASS.WORKTREE_QUARANTINE_ENV_BLOCKED
6154
6402
  || /\bWORKTREE_DIRTY\b/.test(fr)
6155
- || /\bWORKTREE_DIVERGENT\b/.test(fr);
6403
+ || /\bWORKTREE_DIVERGENT\b/.test(fr)
6404
+ || /\bWORKTREE_QUARANTINE_ENV_BLOCKED\b/.test(fr);
6156
6405
  if (isQuarantineFail) {
6157
6406
  item._quarantineRecoveryCount = (item._quarantineRecoveryCount || 0) + 1;
6158
6407
  const cap = ENGINE_DEFAULTS.quarantineAutoRecoveryMax || 2;
@@ -8282,6 +8531,7 @@ module.exports = {
8282
8531
  gitOutputToString, gitErrorOutput, classifyDepMergeFailureOutput, listUnmergedFiles, // exported for testing
8283
8532
  buildDepConflictFixItem, deriveConflictFixKey, // exported for testing (W-mpcwojgr000a0244)
8284
8533
  isWorktreeRetryableError, removeStaleIndexLock, syncReusedWorktree, assertCleanSharedWorktree, _quarantineDirtyWorktree, // exported for testing
8534
+ _renameWithRetry, _statusPorcelainCmd, _killGitDescendantsForWorktree, _bumpQuarantineOutcome, // exported for testing (W-mq5n1zx5)
8285
8535
  pruneStaleWorktreeForBranch, // exported for testing
8286
8536
  findExistingWorktree, // exported for testing
8287
8537
  probeBranchOnRemote, // exported for testing (W-mphnm6a1000281b8)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@yemi33/minions",
3
- "version": "0.1.2148",
3
+ "version": "0.1.2149",
4
4
  "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
5
5
  "bin": {
6
6
  "minions": "bin/minions.js"