@yemi33/minions 0.1.2356 → 0.1.2357

package/engine.js

@@ -168,7 +168,8 @@ const { runPostCompletionHooks, updateWorkItemStatus, syncPrdItemStatus, reconci
   syncPrsFromOutput, updatePrAfterReview, updatePrAfterFix, checkForLearnings, extractSkillsFromOutput,
   updateAgentHistory, updateMetrics, createReviewFeedbackForAuthor, parseAgentOutput, syncPrdFromPrs, persistVerifyPrsToPrd,
   isItemCompleted, classifyFailure: classifyFailureFallback, diagnoseEmptyOutput, processPendingRebases, resolveWorkItemPath,
-  mergeArtifactNotes, promoteCompletionArtifacts, pruneScopeMismatchDuplicatePrs, collapseAllDuplicatePrRecords } = require('./engine/lifecycle');
+  mergeArtifactNotes, promoteCompletionArtifacts, pruneScopeMismatchDuplicatePrs, collapseAllDuplicatePrRecords,
+  recordWorktreeHeldPause, clearWorktreeHeldPause } = require('./engine/lifecycle');
 
 // ─── Diagnostics: memory + event-loop + GC sampler (P-a1b2c3d4 / P-b2c3d4e5) ─
 
@@ -1335,7 +1336,7 @@ function _bumpQuarantineOutcome(key, delta = 1) {
       if (!metrics._engine.worktreeQuarantineOutcomes) {
         metrics._engine.worktreeQuarantineOutcomes = {
           attempts: 0, success: 0, successAfterRetry: 0,
-          fallbackForceRemove: 0, totalFailure: 0,
+          fallbackForceRemove: 0, totalFailure: 0, alreadyQuarantinedConcurrent: 0,
         };
       }
       const o = metrics._engine.worktreeQuarantineOutcomes;
@@ -1347,6 +1348,28 @@ function _bumpQuarantineOutcome(key, delta = 1) {
   }
 }
 
+// W-mrcllbrx000oec11 — generic "serialize calls keyed by a string" helper.
+// Used to prevent two overlapping quarantine recovery attempts against the
+// SAME worktree path from running their destructive steps (holder-reap,
+// rename, force-remove) concurrently. `map` holds the tail promise of the
+// most recent caller for a given key; each new caller chains behind it and
+// replaces the map entry, so callers for the same key run strictly one at a
+// time while callers for different keys are unaffected. The map entry is
+// removed once a call's turn finishes, unless another caller has already
+// queued behind it (in which case that caller owns the cleanup).
+function _withPathLock(map, key, fn) {
+  const prior = map.get(key) || Promise.resolve();
+  const ourTurn = prior.catch(() => {}).then(fn, fn);
+  const tail = ourTurn.catch(() => {});
+  map.set(key, tail);
+  tail.finally(() => {
+    if (map.get(key) === tail) map.delete(key);
+  });
+  return ourTurn;
+}
+
+const _quarantinePathLocks = new Map();
+
 
 // ─── assertCleanSharedWorktree (#2439) ──────────────────────────────────────
 // Engine-side preflight that prevents shared-branch (and PR-targeted reused —
@@ -1673,14 +1696,30 @@ async function assertCleanSharedWorktree(rootDir, worktreePath, branchName, disp
 // for callers: a thrown error means the caller's `result.quarantineError`
 // field is populated and `result.quarantined` stays false (env-blocked
 // path); a normal return means quarantine succeeded.
+// W-mrcllbrx000oec11 — public entry point: serializes overlapping calls for
+// the SAME worktreePath through _withPathLock before delegating to the real
+// implementation below. See _quarantineDirtyWorktreeCore for the quarantine
+// logic itself, including the ENOENT idempotency check that this locking
+// layer works alongside (belt-and-suspenders: the lock stops concurrent
+// destructive work; the ENOENT check makes a still-overlapping caller — e.g.
+// from a process that doesn't share this in-memory map — resolve cleanly
+// instead of surfacing a false WORKTREE_QUARANTINE_ENV_BLOCKED failure).
 async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOpts, diag = {}) {
+  return _withPathLock(
+    _quarantinePathLocks,
+    worktreePath,
+    () => _quarantineDirtyWorktreeCore(rootDir, worktreePath, branchName, gitOpts, diag),
+  );
+}
+
+async function _quarantineDirtyWorktreeCore(rootDir, worktreePath, branchName, gitOpts, diag = {}) {
   // P-c7e2b405 — INTENTIONAL divergence from worktree-gc.reapAndRemoveWorktree:
   // this is the bespoke dirty-worktree quarantine flow (rename → branch backup-ref
   // → reset → fetch), NOT a plain reap-then-remove. It is deliberately NOT folded
   // into the shared helper. It does share the same live-guard + holder-reap
   // primitives (isWorktreePathLive + _reapWorktreeHolders below).
   const ts = Date.now();
-  const quarantinedPath = `${worktreePath}-quarantine-${ts}`;
+  let quarantinedPath = `${worktreePath}-quarantine-${ts}`;
   _bumpQuarantineOutcome('attempts', 1);
 
   // Capture HEAD sha BEFORE renaming so we can back up the local branch ref.
@@ -1704,6 +1743,48 @@ async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOp
     return { quarantinedPath: null, backupRef: null, skipped: true };
   }
 
+  // W-mrcqzuxy000w676e — PRESERVE UNCOMMITTED WORK BEFORE QUARANTINE.
+  // The backup ref written near the end of this function only captures the
+  // worktree's COMMITTED HEAD (via `headSha` above). It does NOT stash,
+  // commit, or otherwise persist UNCOMMITTED modified/untracked files —
+  // so a prior agent's real, on-task, uncommitted fix + test were silently
+  // and permanently destroyed once the dir was renamed/pruned
+  // (W-mrciooy1000e2ac3). Snapshot any uncommitted changes to a dedicated
+  // backup ref BEFORE any destructive step below (holder-reap/rename/
+  // force-remove) so they survive regardless of which path the quarantine
+  // takes. Best-effort and non-blocking: a failure here never stops the
+  // quarantine, but always logs one of three distinct, unambiguous outcomes
+  // (clean / preserved / failed-to-preserve) instead of silently no-opping.
+  const sanitizedRefSegment = sanitizeBranch(branchName).replace(/\//g, '-');
+  let wipRef = null;
+  let wipOutcome = 'clean';
+  try {
+    const statusOut = ((await execAsync('git status --porcelain', { ...gitOpts, cwd: worktreePath, timeout: 15000 })) || '').toString();
+    if (statusOut.trim()) {
+      try {
+        await shared.shellSafeGit(['add', '-A'], { ...gitOpts, cwd: worktreePath, timeout: 30000 });
+        await shared.shellSafeGit(
+          ['commit', '--no-verify', '-m', `quarantine: uncommitted WIP snapshot (${diag.dispatchId || 'unknown'})`],
+          { ...gitOpts, cwd: worktreePath, timeout: 30000 },
+        );
+        const wipSha = ((await execAsync('git rev-parse HEAD', { ...gitOpts, cwd: worktreePath, timeout: 10000 })) || '').toString().trim();
+        const candidateWipRef = `refs/minions/quarantine-wip/${sanitizedRefSegment}/${ts}`;
+        await shared.shellSafeGit(['update-ref', candidateWipRef, wipSha], { ...gitOpts, cwd: rootDir, timeout: 10000 });
+        wipRef = candidateWipRef;
+        wipOutcome = 'preserved';
+        log('warn', `_quarantineDirtyWorktree: preserved uncommitted WIP for ${worktreePath} at ${wipRef} (${wipSha.slice(0, 12)})`);
+      } catch (e) {
+        wipOutcome = 'failed';
+        log('error', `_quarantineDirtyWorktree: FAILED to preserve uncommitted WIP for ${worktreePath}: ${e.message} — uncommitted changes may be lost`);
+      }
+    }
+  } catch (e) {
+    // The status probe itself failed. Treat as failed-to-preserve (never
+    // block the quarantine) rather than silently assuming the tree was clean.
+    wipOutcome = 'failed';
+    log('warn', `_quarantineDirtyWorktree: git status --porcelain probe failed for ${worktreePath}: ${e.message} — cannot determine whether uncommitted WIP needs preservation`);
+  }
+
   // W-mq5n1zx5 / W-mqila0t5 Layer 0–2: pre-emptively reap any process holding
   // the worktree dir before we rename it. The status-probe child that timed out
   // earlier may still hold packfile handles (Layer 0: git.exe by cmdline), and —
@@ -1742,6 +1823,44 @@ async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOp
     renameError = e;
   }
 
+  // W-mrcllbrx000oec11 — IDEMPOTENT-QUARANTINE GUARD: an ENOENT on the
+  // rename means the source path no longer exists. `_retryFsOp` treats
+  // ENOENT as non-retryable and throws on the very first attempt (it isn't
+  // in shared._WORKTREE_RETRYABLE_CODES), so this fires immediately rather
+  // than after a backoff loop. The only way the source can vanish between
+  // our dirty-check (which confirmed the worktree existed) and this rename
+  // is that another quarantine attempt against the SAME path already won
+  // the race and renamed it away first (observed ~96ms apart in production,
+  // W-mrciooy1000e2ac3 — 12 recovery cycles burned before the WI was
+  // wrongly marked terminally failed even though the tree was already
+  // clean). Recognize "source is gone" as quarantine having already
+  // succeeded (from a concurrent caller) instead of a genuine environment
+  // block, so we don't increment _quarantineRecoveryCount or surface
+  // WORKTREE_QUARANTINE_ENV_BLOCKED for a race that already resolved
+  // successfully. Best-effort: look for the sibling <path>-quarantine-*
+  // directory the winning attempt created so we can still report a path;
+  // if none is found (e.g. it was later cleaned up), source-gone alone is
+  // still treated as success.
+  let alreadyQuarantinedConcurrently = false;
+  if (renameError && renameError.code === 'ENOENT' && !fs.existsSync(worktreePath)) {
+    let winningPath = null;
+    try {
+      const parentDir = path.dirname(worktreePath);
+      const prefix = `${path.basename(worktreePath)}-quarantine-`;
+      const candidates = fs.readdirSync(parentDir)
+        .filter((name) => name.startsWith(prefix))
+        .sort(); // timestamp suffix sorts lexicographically == chronologically
+      if (candidates.length > 0) winningPath = path.join(parentDir, candidates[candidates.length - 1]);
+    } catch (e) {
+      log('warn', `_quarantineDirtyWorktree: sibling quarantine-dir lookup after ENOENT failed for ${worktreePath}: ${e.message}`);
+    }
+    log('warn', `_quarantineDirtyWorktree: rename ENOENT for ${worktreePath} but source no longer exists — treating as already quarantined by a concurrent attempt${winningPath ? ` (found ${winningPath})` : ' (no sibling quarantine dir found)'}`);
+    _bumpQuarantineOutcome('alreadyQuarantinedConcurrent', 1);
+    alreadyQuarantinedConcurrently = true;
+    renameError = null; // clear so the force-remove / env-blocked branches below don't fire
+    quarantinedPath = winningPath;
+  }
+
   // W-mq5n1zx5 Layer 2b: if the rename retries exhausted, fall back to
   // `git worktree remove --force`. This DESTROYS the worktree contents
   // (no forensics dir), so it's last-resort only — gated on the
@@ -1832,7 +1951,6 @@ async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOp
   // starts on a clean origin tip rather than recreating the divergent state.
   // Both ref ops are best-effort: if the local ref machinery is in a weird
   // state we still want the worktree quarantine to stick.
-  const sanitizedRefSegment = sanitizeBranch(branchName).replace(/\//g, '-');
   const backupRef = `refs/minions/quarantine/${sanitizedRefSegment}/${ts}`;
   let backupRefCreated = false;
   if (headSha) {
@@ -1871,7 +1989,9 @@ async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOp
     `- Original worktree: ${worktreePath}`,
     forceRemoved
       ? `- Force-removed via \`git worktree remove --force\` (no quarantine dir preserved — W-mq5n1zx5 Layer 2b)`
-      : `- Quarantined to: ${quarantinedPath}${renameAttempts > 1 ? ` (rename took ${renameAttempts} attempt(s) — W-mq5n1zx5 Layer 1a)` : ''}`,
+      : alreadyQuarantinedConcurrently
+        ? `- Already quarantined by a concurrent attempt${quarantinedPath ? ` — quarantined to: ${quarantinedPath}` : ' (source dir was gone by the time this recovery attempt ran; no sibling quarantine dir found)'} (W-mrcllbrx000oec11)`
+        : `- Quarantined to: ${quarantinedPath}${renameAttempts > 1 ? ` (rename took ${renameAttempts} attempt(s) — W-mq5n1zx5 Layer 1a)` : ''}`,
     `- Reason: ${diag.reason || 'unsafe'}`,
     `- Local commits ahead of origin: ${diag.ahead || 0}`,
     `- Local commits behind origin: ${diag.behind || 0}`,
@@ -1884,6 +2004,11 @@ async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOp
     backupRefCreated
       ? `The local branch HEAD was backed up to \`${backupRef}\` (${headSha.slice(0, 12)}) and \`refs/heads/${branchName}\` was reset to \`refs/remotes/origin/${branchName}\`.`
       : `Backup ref was NOT created (HEAD sha unavailable). The local branch ref was still reset to \`refs/remotes/origin/${branchName}\` if possible.${forceRemoved ? ' Worktree contents were destroyed by the force-remove fallback — no quarantine dir to inspect.' : ' Inspect the quarantined directory directly to recover unpushed work.'}`,
+    wipOutcome === 'preserved'
+      ? `Uncommitted WIP (modified/untracked files at quarantine time) was committed and backed up to \`${wipRef}\`. Recover with \`git -C "${rootDir}" show ${wipRef}\` or \`git -C "${rootDir}" cherry-pick ${wipRef}\` onto a fresh ${branchName} worktree.`
+      : wipOutcome === 'failed'
+        ? `⚠️ Uncommitted WIP preservation FAILED — see engine log for details. Any modified/untracked files that were NOT part of \`${backupRefCreated ? backupRef : 'the backup ref'}\` may be unrecoverable once the quarantine dir is pruned.`
+        : 'No uncommitted WIP was present (working tree was clean at quarantine time) — nothing extra to preserve.',
     '',
     forceRemoved
       ? `1. The worktree dir was force-removed; there is no quarantine dir to inspect. If the local branch ref still has divergent commits, recover from \`${backupRefCreated ? backupRef : 'reflog'}\`.`
@@ -1905,12 +2030,15 @@ async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOp
     log('warn', `_quarantineDirtyWorktree: writeToInbox failed: ${e.message}`);
   }
 
-  log('warn', `Quarantined dirty worktree ${worktreePath} → ${forceRemoved ? '<force-removed>' : quarantinedPath} (branch ${branchName}, ${diag.ahead || 0} ahead, ${diag.behind || 0} behind, ${dirtyFiles.length} dirty files${renameAttempts > 1 ? `, rename retries=${renameAttempts}` : ''}${forceRemoved ? ', force-remove fallback' : ''})`);
+  log('warn', `Quarantined dirty worktree ${worktreePath} → ${forceRemoved ? '<force-removed>' : (quarantinedPath || '<already quarantined concurrently>')} (branch ${branchName}, ${diag.ahead || 0} ahead, ${diag.behind || 0} behind, ${dirtyFiles.length} dirty files${renameAttempts > 1 ? `, rename retries=${renameAttempts}` : ''}${forceRemoved ? ', force-remove fallback' : ''}${alreadyQuarantinedConcurrently ? ', already-quarantined-concurrently' : ''}, wip=${wipOutcome})`);
   return {
     quarantinedPath: forceRemoved ? null : quarantinedPath,
     backupRef: backupRefCreated ? backupRef : null,
     forceRemoved,
     renameAttempts,
+    alreadyQuarantinedConcurrently,
+    wipRef,
+    wipOutcome,
   };
 }
 
@@ -2395,6 +2523,31 @@ async function spawnAgent(dispatchItem, config) {
     if (rawErr && rawErr.message) {
       log('debug', `spawnAgent: raw worktree-add failure for ${id}: ${String(rawErr.message).split('\n')[0]}`);
     }
+    // #764 — PR-level circuit breaker. `agentRetryable: false` above only
+    // stops retries of THIS WI instance; it carries no memory at the PR
+    // level, so the human-feedback / review / build-fix / merge-conflict
+    // pollers in discoverFromPrs see the PR is still pendingFix (or has
+    // coalesced feedback) and create a brand-new work item next cooldown
+    // window — forever, until a human manually releases the branch (live
+    // repro: ado:office/office/1js#5429466, 7 wasted dispatches in ~4.5h).
+    // Stamp `_worktreeHeldPaused` on the live PR record for PR-sourced
+    // dispatches (source is 'pr' or 'pr-human-feedback' — see buildPrDispatch
+    // call sites) so discoverFromPrs's isWorktreeHeldPauseSuppressed gate can
+    // suppress further dispatch until the holder releases the branch.
+    if ((meta?.source === 'pr' || meta?.source === 'pr-human-feedback') && meta?.pr) {
+      try {
+        const pauseProject = meta.project || project;
+        mutatePullRequests(projectPrPath(pauseProject), (prs) => {
+          if (!Array.isArray(prs)) return prs;
+          const target = shared.findPrRecord(prs, meta.pr, pauseProject);
+          if (!target) return prs;
+          recordWorktreeHeldPause(target, { holderPath, isProjectRoot, branch: branchName, dispatchItem });
+          return prs;
+        });
+      } catch (e) {
+        log('warn', `spawnAgent: failed to stamp worktree-held pause for ${meta.pr.id || meta.pr.url || '(unknown PR)'}: ${e.message}`);
+      }
+    }
   };
   _phaseT.afterPrompt = Date.now();
 
@@ -2417,15 +2570,23 @@ async function spawnAgent(dispatchItem, config) {
   //   cwd = project.localPath, worktreeRootDir = null, liveMode = true.
   // Instead of creating an engine-managed worktree we run prepareLiveCheckout
   // in-place: it validates a clean tree and then checks out (or creates) the
-  // target branch. On a CONFIRMED dirty refusal (the helper returns
-  // { ok:false, reason:'dirty', dirtyFiles:[…] }) we write an inbox alert, stamp
-  // `_pendingReason: 'live_checkout_dirty'` on the WI, complete the dispatch
-  // non-retryably with FAILURE_CLASS.LIVE_CHECKOUT_DIRTY, and return — the
-  // engine never runs `git reset`/`git clean` against the operator tree. If the
-  // helper instead THROWS (guard/ref-validation/transient git error), that is
-  // NOT proof of a dirty tree (#305): we complete with the separate
-  // FAILURE_CLASS.LIVE_CHECKOUT_FAILED, which is retryable with bounded backoff
-  // so racy branch-lock handoff or a transient git failure auto-recovers.
+  // target branch. W-mrcifup2000c218a: the dirty/stale-base conditions below
+  // are now caught EARLIER, at allocation time, by the pre-dispatch checks in
+  // the dispatch-existing-pending loop (`checkLiveCheckoutDirty` /
+  // `checkLiveCheckoutStaleBase` in engine/live-checkout.js) — a positive hit
+  // there leaves the WI pending with `_pendingReason` stamped and never reaches
+  // spawnAgent at all, so this block is now the EXCEPTION path (defense in
+  // depth for the race window between that probe and the git operations
+  // below), not the common path. On a CONFIRMED dirty refusal (the helper
+  // returns { ok:false, reason:'dirty', dirtyFiles:[…] }) we write an inbox
+  // alert, stamp `_pendingReason: 'live_checkout_dirty'` on the WI, complete
+  // the dispatch non-retryably with FAILURE_CLASS.LIVE_CHECKOUT_DIRTY, and
+  // return — the engine never runs `git reset`/`git clean` against the
+  // operator tree. If the helper instead THROWS (guard/ref-validation/transient
+  // git error), that is NOT proof of a dirty tree (#305): we complete with the
+  // separate FAILURE_CLASS.LIVE_CHECKOUT_FAILED, which is retryable with
+  // bounded backoff so racy branch-lock handoff or a transient git failure
+  // auto-recovers.
   // On success we fall through; the worktree-creation block below is gated
   // on `!liveMode`, so it never runs, and downstream cleanup paths
   // (worktreePool.returnToPool, worktree-gc.gcDispatchWorktreeIfOrphan) are
@@ -3019,6 +3180,15 @@ async function spawnAgent(dispatchItem, config) {
     // so it gets its own non-retryable FAILURE_CLASS instead of LIVE_CHECKOUT_FAILED's
     // bounded retry-storm. The engine NEVER resets/cleans the operator's local
     // mainRef; the operator must reconcile it themselves, then re-dispatch.
+    // W-mrcifup2000c218a: this condition is now caught EARLIER, at allocation
+    // time, by `checkLiveCheckoutStaleBase` in the dispatch-existing-pending
+    // loop — a positive hit there leaves the WI pending (`_pendingReason:
+    // 'live_checkout_stale_base'`, no retry consumed, no failure) and this WI
+    // never reaches spawnAgent. This in-spawn refusal is now the EXCEPTION path
+    // (the race window between that probe and the checkout below — e.g. the
+    // base was pushed-then-re-diverged between allocation and spawn), not the
+    // common path; it still fails non-retryably because a stale base found
+    // HERE is still deterministic and un-recoverable by a bare retry.
     if (_liveResult && _liveResult.ok === false && _liveResult.reason === 'stale-main') {
       const _staleMainRef = typeof _liveResult.mainRef === 'string' && _liveResult.mainRef ? _liveResult.mainRef : 'main';
       const _staleAhead = Number.isFinite(_liveResult.ahead) ? _liveResult.ahead : 0;
@@ -6960,6 +7130,72 @@ function isBuildFixIneffectiveSuppressed(pr) {
   return true;
 }
 
+// #764 — PR-level circuit breaker for the WORKTREE_PREFLIGHT
+// branch-held-externally failure class. `spawnAgent`'s
+// `_failBranchHeldByExternalWorktree` stamps `_worktreeHeldPaused` on the
+// live PR record (via `recordWorktreeHeldPause`) whenever a PR-sourced
+// dispatch (review, re-review, human-feedback fix, review-feedback fix,
+// build-fix, merge-conflict fix — all of which create a worktree on the PR's
+// branch) hits this failure. Without this gate, `discoverFromPrs` sees the
+// PR is still `pendingFix` (or has coalesced feedback) every tick and
+// re-dispatches a brand-new work item against the identical unresolved
+// cause, forever (live repro: ado:office/office/1js#5429466, 7 wasted
+// dispatches in ~4.5h — see issue #764).
+//
+// Unlike the no-op-fix pauses above, this one performs a cheap live re-check
+// (a single `git worktree list --porcelain` against the project root) before
+// deciding to keep suppressing: the whole point of the breaker is to avoid a
+// full wasted agent dispatch, and one `git worktree list` call is orders of
+// magnitude cheaper than that. When the holder no longer holds the paused
+// branch, the pause auto-clears (no manual step needed in the common case,
+// per the issue's proposed fix) and dispatch is allowed to proceed. Any
+// failure re-checking live state is treated conservatively (stays paused) —
+// a false "still held" merely costs one more skipped tick, whereas a false
+// "released" would resume the exact dispatch storm this breaker exists to
+// stop.
+async function isWorktreeHeldPauseSuppressed(pr, project) {
+  const record = pr?._worktreeHeldPaused;
+  if (!record || typeof record !== 'object') return false;
+  let stillHeld = true;
+  try {
+    const rootDir = shared.resolveProjectRootDir(project?.localPath, MINIONS_DIR);
+    const out = await shared.shellSafeGit(['worktree', 'list', '--porcelain'], { cwd: rootDir, timeout: 10000 });
+    const list = shared.parseWorktreePorcelain(out);
+    // Compare via realpath, not a bare path.resolve() string match: git
+    // resolves `worktree list` entries to their real (subst/junction-free)
+    // filesystem path, but `record.holderPath` may have been captured from a
+    // substituted drive letter (common on Windows CI runners that `subst` a
+    // drive for TEMP) or a symlinked path. A raw string comparison then
+    // false-mismatches two identical directories, wrongly auto-clearing the
+    // pause and letting the exact dispatch storm this breaker exists to stop
+    // resume (#764 regression — see W-mrdiq0jp000bb541).
+    const holderReal = record.holderPath ? shared.realPathForComparison(record.holderPath) : null;
+    const holderEntry = holderReal
+      ? list.find(w => shared.realPathForComparison(w.path) === holderReal)
+      : null;
+    stillHeld = !!holderEntry && holderEntry.branch === record.branch;
+  } catch (e) {
+    log('warn', `PR ${pr.id}: worktree-held pause live re-check failed (${e.message}) — keeping automation paused`);
+  }
+  if (!stillHeld) {
+    try {
+      mutatePullRequests(projectPrPath(project), (prs) => {
+        if (!Array.isArray(prs)) return prs;
+        const target = shared.findPrRecord(prs, pr, project);
+        if (!target) return prs;
+        clearWorktreeHeldPause(target);
+        return prs;
+      });
+      log('info', `PR ${pr.id}: worktree-held pause auto-cleared — ${record.holderPath} no longer holds branch ${record.branch}`);
+    } catch (e) {
+      log('warn', `PR ${pr.id}: failed to auto-clear worktree-held pause: ${e.message}`);
+    }
+    return false;
+  }
+  log('warn', `PR ${pr.id}: suppressing PR-driven review/fix automation — branch ${record.branch} still held ${record.isProjectRoot ? 'by the project root' : 'by an external worktree'} at ${record.holderPath}; waiting for the holder to release it`);
+  return true;
+}
+
 const PR_PENDING_MISSING_BRANCH = shared.PR_PENDING_REASON.MISSING_BRANCH;
 
 function normalizePrBranch(value) {
@@ -7350,6 +7586,15 @@ async function discoverFromPrs(config, project) {
       _logPrDispatchSkipOnce(pr, 'not-auto-managed');
       continue;
     }
+    // #764 — PR-level circuit breaker: a prior dispatch against this PR's
+    // branch hit WORKTREE_PREFLIGHT because the branch is held externally
+    // (project root or an unreachable sibling worktree). Every dispatch path
+    // below (review, re-review, human-feedback fix, review-feedback fix,
+    // build-fix, merge-conflict fix) creates a worktree on the SAME branch,
+    // so skip the whole PR for this tick rather than re-litigating the same
+    // doomed preflight per-cause. Auto-clears itself once the holder
+    // releases the branch — see isWorktreeHeldPauseSuppressed.
+    if (await isWorktreeHeldPauseSuppressed(pr, project)) continue;
 
     const prNumber = shared.getPrNumber(pr);
     // Use reviewStatus as single source of truth (synced from ADO/GitHub votes)
@@ -9679,7 +9924,7 @@ function sweepStaleArchivedPrdBackups(prdDir, prdArchiveDir) {
 async function discoverWork(config) {
   resetClaimedAgents(); // Reset per-tick agent claims for fair distribution
   const projects = getProjects(config);
-  let allFixes = [], allReviews = [], allWorkItems = [];
+  let persistedFixes = 0, persistedReviews = 0, persistedOther = 0;
 
   // Side-effect passes: materialize plans and design docs into work-items.json
   // These write to project work queues — picked up by discoverFromWorkItems below.
@@ -9692,19 +9937,109 @@ async function discoverWork(config) {
     materializePlansAsWorkItems(config);
   }
 
+  const evalConcurrency = resolvePreDispatchEvalConcurrency(config);
+
+  // #768 — persist each batch of discovered items to the dispatch queue as
+  // soon as it is found, instead of accumulating everything across every
+  // project/source into local arrays that are only persisted once this whole
+  // function returns. discoverWork() is invoked inside withTickTimeout(...)
+  // AND the tick's outer hard timeout (TICK_TIMEOUT_MS / tickHardTimeoutMs) —
+  // neither one actually cancels the in-flight promise, it just stops
+  // awaiting it. A pathologically slow (or genuinely hung) per-project
+  // git/worktree operation later in the sequential `for (const project of
+  // projects)` loop below leaves this whole async function permanently
+  // unresolved, so anything only collected into a local array and persisted
+  // at the very end (the pre-fix allFixes/allReviews/allWorkItems) never
+  // reaches dispatch.json — silently discarding valid work already
+  // discovered for unrelated, healthy projects processed earlier in the same
+  // loop. Persisting per batch means a healthy project's discoveries land in
+  // dispatch.json immediately, before the loop ever reaches a slow project.
+  async function persistBatch(items) {
+    if (!items || items.length === 0) return;
+    // Gate reviews and fixes: only when at max concurrency — idle agents should pick up reviews
+    // even if implement items are in progress (implements get priority via sort order, not by blocking)
+    const hasIncompleteImplements = queries.getWorkItems(config).some(i =>
+      ['queued', 'pending', 'dispatched'].includes(i.status) && (i.type || '').startsWith('implement')
+    );
+    let toDispatch = items;
+    if (hasIncompleteImplements) {
+      const activeCount = (getDispatch().active || []).length;
+      const maxConcurrent = resolveMaxConcurrent(config);
+      const freeSlots = Math.max(0, maxConcurrent - activeCount);
+      if (freeSlots === 0) {
+        // PR review feedback (#771): the pre-fix code only ever gated
+        // allFixes/allReviews, which were populated exclusively from
+        // discoverFromPrs() output (`meta.source === 'pr'`). FIX/REVIEW-typed
+        // items surfaced by discoverFromWorkItems (`meta.source ===
+        // 'work-item'`), discoverCentralWorkItems (`'central-work-item'` /
+        // `'central-work-item-fanout'`), or discoverMeetingWork (`'meeting'`)
+        // were NEVER gated — allWorkItems/centralWork bypassed the gate
+        // entirely. persistBatch() now runs on every source's batch, so the
+        // type-only filter below would silently start gating those
+        // previously-ungated sources too. Scope the gate back down to
+        // `meta.source === 'pr'` to preserve the original behavior exactly.
+        const isGateEligible = (w) => w.meta?.source === 'pr' && (w.type === WORK_TYPE.FIX || w.type === WORK_TYPE.REVIEW);
+        const gated = items.filter(isGateEligible);
+        if (gated.length > 0) {
+          log('info', `Gating ${gated.length} fixes/reviews — implement items in progress and no free slots`);
+        }
+        toDispatch = items.filter(w => !isGateEligible(w));
+      }
+    }
+    if (toDispatch.length === 0) return;
+
+    // W-mq9acoo800177bcb — bounded-concurrency pre-dispatch eval.
+    // The validator inside addToDispatchWithValidation runs a ~25-30s LLM
+    // call per item; an approved 11-item PRD used to take ~5 min to queue
+    // because we awaited each item in series. Process items in chunks of
+    // `preDispatchEvalConcurrency` (default 6, clamp 1-20) via
+    // Promise.allSettled so a rejected branch in one item still fails open
+    // and the rest of the chunk continues. Iteration order within a chunk
+    // is irrelevant: addToDispatch's dedup gates are mutateDispatch-locked,
+    // and the only per-item side effect (`clearPendingHumanFeedbackFlag`)
+    // is an idempotent flag-clear on a single PR record.
+    for (let i = 0; i < toDispatch.length; i += evalConcurrency) {
+      const chunk = toDispatch.slice(i, i + evalConcurrency);
+      await Promise.allSettled(chunk.map(async (item) => {
+        try {
+          await addToDispatchWithValidation(item, { config });
+        } catch (e) {
+          // Fail-open contract preserved: any validator throw here means the
+          // item is dropped from this discovery batch rather than wedging the
+          // whole chunk. The item stays `pending` in work-items.json and the
+          // next tick re-discovers it.
+          log('warn', `pre-dispatch-eval: addToDispatchWithValidation threw for ${item?.meta?.item?.id || item?.id || 'unknown'}: ${e?.message || e}`);
+        }
+        // W-mph8xt88000ke0fc — Cooldowns are stamped by addToDispatch ONLY on
+        // successful append (post-dedup). Stamping unconditionally here used to
+        // leave orphan cooldowns for items collapsed by prDedupeKey / workItem-id
+        // / dispatchKey dedup or routed to the pre-dispatch review queue.
+        if (item.meta?.source === 'pr-human-feedback') {
+          clearPendingHumanFeedbackFlag(item.meta.project, item.meta.pr?.id);
+        }
+      }));
+    }
+    for (const w of toDispatch) {
+      if (w.type === WORK_TYPE.FIX) persistedFixes++;
+      else if (w.type === WORK_TYPE.REVIEW) persistedReviews++;
+      else persistedOther++;
+    }
+  }
+
   for (const project of projects) {
     const root = project.localPath ? path.resolve(project.localPath) : null;
     if (!root || !fs.existsSync(root)) continue;
 
+    const projectWork = [];
+
     // Source 1: Pull Requests → fixes, reviews, build-test
     // Gated by config.engine?.prDiscoveryEnabled !== false (P-d6f0a2b5) —
     // per-project project.workSources.pullRequests.enabled still composes
     // inside discoverFromPrs; a `false` at either level skips the call.
     if (config.engine?.prDiscoveryEnabled !== false) {
       const prWork = await discoverFromPrs(config, project);
-      allFixes.push(...prWork.filter(w => w.type === WORK_TYPE.FIX));
-      allReviews.push(...prWork.filter(w => w.type === WORK_TYPE.REVIEW));
-      allWorkItems.push(...prWork.filter(w => w.type === WORK_TYPE.TEST));
+      projectWork.push(...prWork.filter(w =>
+        w.type === WORK_TYPE.FIX || w.type === WORK_TYPE.REVIEW || w.type === WORK_TYPE.TEST));
     }
 
     // Side-effect: specs → work items (picked up below)
@@ -9715,8 +10050,14 @@ async function discoverWork(config) {
     // per-project project.workSources.workItems.enabled still composes
     // inside discoverFromWorkItems; a `false` at either level skips the call.
     if (config.engine?.workItemsDiscoveryEnabled !== false) {
-      allWorkItems.push(...discoverFromWorkItems(config, project));
+      projectWork.push(...discoverFromWorkItems(config, project));
     }
+
+    // #768 — persist this project's discovered items right now rather than
+    // waiting for every remaining project in the loop to also be walked. See
+    // the persistBatch() comment above for why this matters on a tick
+    // hard-timeout.
+    await persistBatch(projectWork);
   }
 
   // Source 2: Minions-level PRD → implements (multi-project, called once outside project loop)
@@ -9730,6 +10071,8 @@ async function discoverWork(config) {
   if (config.engine?.centralWorkDiscoveryEnabled !== false) {
     centralWork = discoverCentralWorkItems(config);
   }
+  // #768 — persist immediately, same rationale as the per-project loop above.
+  await persistBatch([...centralWork]);
 
   // Scheduled tasks (cron-style recurring work)
   // Gated by config.engine?.scheduledWorkDiscoveryEnabled !== false (P-d6f0a2b5).
@@ -9798,7 +10141,8 @@ async function discoverWork(config) {
   try {
     const { discoverMeetingWork } = require('./engine/meeting');
     const meetingWork = discoverMeetingWork(config);
-    allWorkItems.push(...meetingWork);
+    // #768 — persist immediately, same rationale as the per-project loop above.
+    await persistBatch(meetingWork);
   } catch (e) { log('warn', 'discover meeting work: ' + e.message); }
 
   // Pipeline orchestration — check stage completions and start ready stages
@@ -9862,67 +10206,12 @@ async function discoverWork(config) {
     } catch (e) { log('warn', 'plan completion sweep: ' + e.message); }
   }
 
-  // Gate reviews and fixes: only when at max concurrency — idle agents should pick up reviews
-  // even if implement items are in progress (implements get priority via sort order, not by blocking)
-  const hasIncompleteImplements = queries.getWorkItems(config).some(i =>
-    ['queued', 'pending', 'dispatched'].includes(i.status) && (i.type || '').startsWith('implement')
-  );
-  if (hasIncompleteImplements) {
-    const activeCount = (getDispatch().active || []).length;
-    const maxConcurrent = resolveMaxConcurrent(config);
-    const freeSlots = Math.max(0, maxConcurrent - activeCount);
-    if (freeSlots === 0) {
-      if (allReviews.length > 0) {
-        log('info', `Gating ${allReviews.length} reviews — implement items in progress and no free slots`);
-        allReviews = [];
-      }
-      if (allFixes.length > 0) {
-        log('info', `Gating ${allFixes.length} fixes — implement items in progress and no free slots`);
-        allFixes = [];
-      }
-    }
-  }
-
-  const allWork = [...allFixes, ...allReviews, ...allWorkItems, ...centralWork];
-
-  // W-mq9acoo800177bcb — bounded-concurrency pre-dispatch eval.
-  // The validator inside addToDispatchWithValidation runs a ~25-30s LLM
-  // call per item; an approved 11-item PRD used to take ~5 min to queue
-  // because we awaited each item in series. Process items in chunks of
-  // `preDispatchEvalConcurrency` (default 6, clamp 1-20) via
-  // Promise.allSettled so a rejected branch in one item still fails open
-  // and the rest of the chunk continues. Iteration order within a chunk
-  // is irrelevant: addToDispatch's dedup gates are mutateDispatch-locked,
-  // and the only per-item side effect (`clearPendingHumanFeedbackFlag`)
-  // is an idempotent flag-clear on a single PR record.
-  const evalConcurrency = resolvePreDispatchEvalConcurrency(config);
-  for (let i = 0; i < allWork.length; i += evalConcurrency) {
-    const chunk = allWork.slice(i, i + evalConcurrency);
-    await Promise.allSettled(chunk.map(async (item) => {
-      try {
-        await addToDispatchWithValidation(item, { config });
-      } catch (e) {
-        // Fail-open contract preserved: any validator throw here means the
-        // item is dropped from this discovery tick rather than wedging the
-        // whole chunk. The item stays `pending` in work-items.json and the
-        // next tick re-discovers it.
-        log('warn', `pre-dispatch-eval: addToDispatchWithValidation threw for ${item?.meta?.item?.id || item?.id || 'unknown'}: ${e?.message || e}`);
-      }
-      // W-mph8xt88000ke0fc — Cooldowns are stamped by addToDispatch ONLY on
-      // successful append (post-dedup). Stamping unconditionally here used to
-      // leave orphan cooldowns for items collapsed by prDedupeKey / workItem-id
-      // / dispatchKey dedup or routed to the pre-dispatch review queue.
-      if (item.meta?.source === 'pr-human-feedback') {
-        clearPendingHumanFeedbackFlag(item.meta.project, item.meta.pr?.id);
-      }
-    }));
-  }
-
-  if (allWork.length > 0) {
-    log('info', `Discovered ${allWork.length} new work items: ${allFixes.length} fixes, ${allReviews.length} reviews, ${allWorkItems.length} work-items`);
+  const totalPersisted = persistedFixes + persistedReviews + persistedOther;
+  if (totalPersisted > 0) {
+    log('info', `Discovered ${totalPersisted} new work items: ${persistedFixes} fixes, ${persistedReviews} reviews, ${persistedOther} work-items`);
   }
 
-  return allWork.length;
+  return totalPersisted;
 }
 
 function getPendingDispatchRoutingOpts(item) {
@@ -10916,6 +11205,123 @@ async function tickInner() {
         continue;
       }
     }
+    // W-mrcifup2000c218a: Live-checkout dirty / stale-base PRE-DISPATCH checks.
+    // Mirrors the live_checkout_busy gate directly above: dirty-tree and
+    // stale-base are ENVIRONMENTAL blocking conditions on the operator's live
+    // checkout, not failures of the task itself, so they get detected here at
+    // ALLOCATION time (before spawnAgent runs) and leave the item pending with
+    // a _pendingReason stamp — no retry consumed, no terminal failure — same
+    // as the busy gate. Previously these were only caught INSIDE spawnAgent
+    // (prepareLiveCheckout's dirty two-strike guard and STALE-BASE GUARD),
+    // which meant every hit burned a dispatch attempt (dirty) or failed
+    // non-retryably on the first hit (stale-base), requiring manual cleanup
+    // even after the operator fixed the underlying condition. The in-spawn
+    // guards (engine.js ~line 2617 dirty, ~line 3017 stale-base) remain in
+    // place as a defense-in-depth fallback for the race window between this
+    // probe and the actual git operations inside spawnAgent — they should now
+    // fire rarely, as the exception path rather than the common path.
+    if (
+      itemProjName
+      && !READ_ONLY_ROOT_TASK_TYPES.has(item.type)
+    ) {
+      const _precheckProjCfg = shared.findProjectByName(shared.getProjects(config), itemProjName);
+      if (_precheckProjCfg && shared.resolveCheckoutMode(_precheckProjCfg, item.type) === 'live' && _precheckProjCfg.localPath) {
+        const _precheckLocalPath = path.resolve(String(_precheckProjCfg.localPath));
+        const _liveCheckoutPrecheck = require('./engine/live-checkout');
+        const _persistPendingReason = (reason) => {
+          item._pendingReason = reason;
+          try {
+            if (_isTickStale(myGeneration)) return;
+            mutateDispatch((dp) => {
+              const p = (dp.pending || []).find(d => d.id === item.id);
+              if (p) {
+                if (reason) p._pendingReason = reason;
+                else delete p._pendingReason;
+              }
+              return dp;
+            });
+          } catch (e) { log('warn', `Persist live-checkout precheck pendingReason for ${item.id} failed: ${e.message}`); }
+        };
+        let _dirtyResult = null;
+        try {
+          _dirtyResult = await _liveCheckoutPrecheck.checkLiveCheckoutDirty({
+            localPath: _precheckLocalPath,
+            skipDirtyCheck: !!_precheckProjCfg.skipLiveCheckoutDirtyCheck,
+          });
+        } catch (e) { log('warn', `live-checkout precheck (dirty) failed for ${item.id}: ${e.message}`); }
+        if (_dirtyResult && _dirtyResult.dirty) {
+          _persistPendingReason('live_checkout_dirty');
+          try {
+            const _alertBody = [
+              '# Live-checkout pending: dirty worktree (pre-dispatch check)',
+              '',
+              `**Project:** ${_precheckProjCfg.name || itemProjName}`,
+              `**Local path:** ${_precheckLocalPath}`,
+              `**Work item:** ${item.id}`,
+              ...(_dirtyResult.branchInfo ? ['', `**Current checkout:** \`${_dirtyResult.branchInfo}\``] : []),
+              '',
+              `The engine detected uncommitted changes in \`${_precheckLocalPath}\` before dispatching ${item.id}. Live-checkout mode runs in your project directly — the engine never \`git reset\` or \`git clean\` your tree. The work item stays pending (no retry consumed, no failure) and will be re-checked automatically once the tree is clean.`,
+              '',
+              '## Dirty files (`git status --porcelain=v1 -b`)',
+              '',
+              '```',
+              ...(Array.isArray(_dirtyResult.dirtyFiles) && _dirtyResult.dirtyFiles.length > 0
+                ? _liveCheckoutPrecheck.capDirtyFileLines(_dirtyResult.dirtyFiles)
+                : ['(none reported)']),
+              '```',
+              '',
+              'Recovery: commit, stash, or discard the changes in the project checkout — the work item will dispatch automatically on a later tick once the tree is clean.',
+            ].join('\n');
+            writeInboxAlert(`live-checkout-dirty-${item.id}`, _alertBody);
+          } catch (e) { log('warn', `live-checkout precheck: writeInboxAlert failed: ${e.message}`); }
+          continue;
+        }
+        // Stale-base only matters when this item would fork a NEW branch off
+        // mainRef — mirror spawnAgent's allowNonMainBase resolution exactly
+        // (W-mr3lunnq000o9f41: PR-targeted / shared-branch continuations opt out).
+        const _itemMeta = item.meta || {};
+        const _precheckAllowNonMainBase = !!(_itemMeta.useExistingBranch || _itemMeta.branchStrategy === 'shared-branch');
+        const _precheckMainRef = sanitizeBranch(shared.resolveMainBranch(_precheckLocalPath, _precheckProjCfg.mainBranch));
+        let _staleResult = null;
+        try {
+          _staleResult = await _liveCheckoutPrecheck.checkLiveCheckoutStaleBase({
+            localPath: _precheckLocalPath,
+            mainRef: _precheckMainRef,
+            allowNonMainBase: _precheckAllowNonMainBase,
+          });
+        } catch (e) { log('warn', `live-checkout precheck (stale-base) failed for ${item.id}: ${e.message}`); }
+        if (_staleResult && _staleResult.stale) {
+          _persistPendingReason('live_checkout_stale_base');
+          try {
+            const _alertBody = [
+              '# Live-checkout pending: local base branch has diverged (pre-dispatch check)',
+              '',
+              `**Project:** ${_precheckProjCfg.name || itemProjName}`,
+              `**Local path:** ${_precheckLocalPath}`,
+              `**Base ref:** ${_staleResult.mainRef}`,
+              `**Local ${_staleResult.mainRef}:** ${_staleResult.localMainSha || '(unknown)'}`,
+              `**origin/${_staleResult.mainRef}:** ${_staleResult.originMainSha || '(unknown)'}`,
+              `**Unpushed commits on local ${_staleResult.mainRef}:** ${_staleResult.ahead}`,
+              `**Work item:** ${item.id}`,
+              '',
+              `The engine detected that local \`${_staleResult.mainRef}\` is ${_staleResult.ahead} commit(s) ahead of \`origin/${_staleResult.mainRef}\` before dispatching ${item.id} — forking a new branch off it would silently inherit those unpushed commits into the new branch and its PR. The work item stays pending (no retry consumed, no failure) and will be re-checked automatically once the base is reconciled.`,
+              '',
+              '## Recovery',
+              '',
+              `Reconcile local \`${_staleResult.mainRef}\` so it matches \`origin/${_staleResult.mainRef}\` (move any real work onto its own branch first, then \`git reset --hard origin/${_staleResult.mainRef}\`). The engine never resets it for you.`,
+            ].join('\n');
+            writeInboxAlert(`live-checkout-stale-base-${item.id}`, _alertBody);
+          } catch (e) { log('warn', `live-checkout precheck: writeInboxAlert failed: ${e.message}`); }
+          continue;
+        }
+        // Checks passed — if a prior tick stamped a dirty/stale-base pending
+        // reason on this item, clear it so the dashboard chip doesn't show a
+        // stale condition once the tree is clean / base is reconciled again.
+        if (item._pendingReason === 'live_checkout_dirty' || item._pendingReason === 'live_checkout_stale_base') {
+          _persistPendingReason(null);
+        }
+      }
+    }
     if (generalSlots <= 0) continue;
     seenPendingIds.add(item.id);
     toDispatch.push(item);