@yemi33/minions 0.1.2356 → 0.1.2358

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -561,7 +561,12 @@ async function _refreshPipelineDetail(id) {
561
561
  _pipelinesData = _pipelinesData.map(function(x) { return x.id === id ? fresh : x; });
562
562
  _pipelinePollHash = _computePipelineDetailHash(fresh);
563
563
  renderPipelines(_pipelinesData);
564
- openPipelineDetail(id);
564
+ // Only refresh the detail modal when it's already open for this pipeline —
565
+ // callers triggered from the card list (modal closed / different id) must not
566
+ // pop the modal open. See same guard pattern above at :625, :640, :649.
567
+ if (document.getElementById('modal')?.classList?.contains('open') && _pipelinePollId === id) {
568
+ openPipelineDetail(id);
569
+ }
565
570
  }
566
571
  } catch (e) { /* silent — next poll will catch up */ }
567
572
  }
@@ -569,12 +574,16 @@ async function _refreshPipelineDetail(id) {
569
574
  async function _triggerPipeline(id, btn) {
570
575
  if (btn) { btn.textContent = 'Starting...'; btn.style.pointerEvents = 'none'; btn.style.opacity = '0.6'; }
571
576
  showToast('cmd-toast', 'Pipeline triggered', true);
572
- // Optimistic: inject a running state so the UI updates immediately
577
+ // Optimistic: inject a running state so the card list updates immediately.
578
+ // Do NOT open the modal here — this function is shared by the CARD's Run Now
579
+ // button (modal must stay closed) and the modal's own Run Now button (the
580
+ // modal, if open for this id, is refreshed exactly once below via the
581
+ // guarded _refreshPipelineDetail() call once the trigger POST resolves).
573
582
  var p = _pipelinesData.find(function(x) { return x.id === id; });
574
583
  if (p) {
575
584
  if (!p.runs) p.runs = [];
576
585
  p.runs.push({ status: 'running', startedAt: new Date().toISOString(), stages: {} });
577
- openPipelineDetail(id);
586
+ renderPipelines(_pipelinesData);
578
587
  }
579
588
  try {
580
589
  var res = await fetch('/api/pipelines/trigger', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ id: id }) });
@@ -225,6 +225,26 @@ function prRow(pr) {
225
225
  + 'onclick="event.stopPropagation();resumeBuildFixIneffective(this)">'
226
226
  + '⏸ infra issue suspected ▶</button>';
227
227
  }
228
+ // Issue #764 — worktree-held-externally chip. pr._worktreeHeldPaused is set
229
+ // when a PR-driven review/fix dispatch hit WORKTREE_PREFLIGHT because this
230
+ // PR's branch is checked out at the project root or in an external
231
+ // worktree the engine can't reach (see engine/lifecycle.js
232
+ // recordWorktreeHeldPause). engine.js auto-clears this pause once it
233
+ // detects the holder no longer holds the branch; this chip lets an
234
+ // operator force-clear it immediately via
235
+ // /api/pull-requests/clear-worktree-held-pause (resumeWorktreeHeldPause
236
+ // below) without waiting for the next PR poll tick.
237
+ var worktreeHeldPaused = pr._worktreeHeldPaused;
238
+ if (worktreeHeldPaused && typeof worktreeHeldPaused === 'object') {
239
+ var whpTitle = (worktreeHeldPaused.reason || 'Branch is held externally; PR automation paused')
240
+ + '. Auto-clears once the holder releases the branch, or click to resume now.';
241
+ pausedChips += ' <button class="pr-badge rejected pr-worktree-held-chip" '
242
+ + 'style="font-size:var(--text-xs);cursor:pointer;border:none" '
243
+ + 'title="' + escapeHtml(whpTitle) + '" '
244
+ + 'data-pr-id="' + escapeHtml(String(pr.id || '')) + '" '
245
+ + 'onclick="event.stopPropagation();resumeWorktreeHeldPause(this)">'
246
+ + '⏸ branch held externally ▶</button>';
247
+ }
228
248
  const titleText = pr.title || 'Untitled';
229
249
  // Polish (W-mqv5ccn7): flatten Markdown in the body so the preview shows
230
250
  // plain text instead of raw `##` / `**…**` / backtick source.
@@ -791,4 +811,38 @@ async function resumeBuildFixIneffective(btn) {
791
811
  }
792
812
  }
793
813
 
794
- window.MinionsPrs = { prRow, prTableHtml, renderPrs, prPrev, prNext, openAllPrs, openModal, openAddPrModal, openPrDetail, unlinkPr, togglePrObserve, resumePausedCause, resumeBuildFixIneffective };
814
+ // Issue #764 resume a paused worktree-held-externally pause. Mirrors
815
+ // resumeBuildFixIneffective above but targets the holder-path-keyed
816
+ // _worktreeHeldPaused record via its own endpoint. engine.js already
817
+ // auto-clears this pause the next time it detects the holder released the
818
+ // branch; this handler exists for operators who don't want to wait for the
819
+ // next PR poll tick.
820
+ async function resumeWorktreeHeldPause(btn) {
821
+ if (!btn || btn.disabled) return;
822
+ const prId = btn.dataset.prId;
823
+ if (!prId) return;
824
+ if (!confirm('Resume PR automation on ' + prId + '?\n\nThis clears the "branch held externally" pause so the engine can re-dispatch review/fix work on the next discovery pass.')) return;
825
+ btn.disabled = true;
826
+ const prevDisplay = btn.style.display;
827
+ btn.style.display = 'none';
828
+ showToast('pr-toast', 'Resumed PR automation on ' + prId, true);
829
+ try {
830
+ const res = await fetch('/api/pull-requests/clear-worktree-held-pause', {
831
+ method: 'POST', headers: { 'Content-Type': 'application/json' },
832
+ body: JSON.stringify({ prId })
833
+ });
834
+ if (!res.ok) {
835
+ const d = await res.json().catch(() => ({}));
836
+ btn.style.display = prevDisplay;
837
+ btn.disabled = false;
838
+ showToast('pr-toast', 'Failed: ' + (d.error || 'unknown'), false);
839
+ return;
840
+ }
841
+ } catch (e) {
842
+ btn.style.display = prevDisplay;
843
+ btn.disabled = false;
844
+ showToast('pr-toast', 'Error: ' + e.message, false);
845
+ }
846
+ }
847
+
848
+ window.MinionsPrs = { prRow, prTableHtml, renderPrs, prPrev, prNext, openAllPrs, openModal, openAddPrModal, openPrDetail, unlinkPr, togglePrObserve, resumePausedCause, resumeBuildFixIneffective, resumeWorktreeHeldPause };
package/dashboard.js CHANGED
@@ -6434,6 +6434,14 @@ const server = http.createServer(async (req, res) => {
6434
6434
  found = true;
6435
6435
  item.status = WI_STATUS.PENDING;
6436
6436
  item._retryCount = 0; // Reset retry counter on manual retry
6437
+ // W-mrcqzuxy000w676e — quarantineRecoveryCount is a lifetime cumulative
6438
+ // counter (engine.js discoverWork auto-recovery loop), so unrelated prior
6439
+ // quarantine noise can permanently give up on a WI even when a later
6440
+ // attempt would recover cleanly. A manual retry is an explicit human
6441
+ // signal to try again — reset the counter/give-up flag so it isn't
6442
+ // punished by history.
6443
+ delete item._quarantineRecoveryCount;
6444
+ delete item._quarantineRecoveryGaveUp;
6437
6445
  delete item.dispatched_at;
6438
6446
  delete item.dispatched_to;
6439
6447
  delete item.failReason;
@@ -7097,6 +7105,16 @@ const server = http.createServer(async (req, res) => {
7097
7105
  // committed. Compensate by removing the item from the target
7098
7106
  // store again, so the item ends up in exactly one store (its
7099
7107
  // original) instead of duplicated across two.
7108
+ //
7109
+ // W-mrciooy1000e2ac3 — the compensating rollback write can ITSELF
7110
+ // fail (double-failure: destination insert succeeded, source
7111
+ // splice failed, and the compensating removal from destination
7112
+ // also failed). Swallowing that second error and unconditionally
7113
+ // reporting "move was rolled back" is a lie: the item is still
7114
+ // duplicated in both stores. Track the rollback outcome and
7115
+ // surface the double-failure distinctly so the caller/UI never
7116
+ // believes a rollback happened when it didn't.
7117
+ let rollbackErr = null;
7100
7118
  try {
7101
7119
  mutateJsonFileLocked(projectMove.wiPath, (targetItems) => {
7102
7120
  if (!Array.isArray(targetItems)) targetItems = [];
@@ -7104,7 +7122,30 @@ const server = http.createServer(async (req, res) => {
7104
7122
  if (ti !== -1) targetItems.splice(ti, 1);
7105
7123
  return targetItems;
7106
7124
  });
7107
- } catch { /* best-effort rollback; surfaced via the 500 below regardless */ }
7125
+ } catch (rbErr) { rollbackErr = rbErr; }
7126
+
7127
+ if (rollbackErr) {
7128
+ console.error(
7129
+ `[work-items/update] DOUBLE FAILURE on cross-project move of ${id}: ` +
7130
+ `source-splice failed (${spliceErr.message}) AND compensating rollback ` +
7131
+ `from target store also failed (${rollbackErr.message}). Item may now be ` +
7132
+ `duplicated across source (${wiPath}) and target (${projectMove.wiPath}) ` +
7133
+ `— manual reconciliation required.`
7134
+ );
7135
+ return jsonReply(res, 500, {
7136
+ error: `project move failed while removing from source store, and the compensating rollback also failed: ${rollbackErr.message}. ` +
7137
+ `Item may be duplicated across both stores — manual reconciliation required.`,
7138
+ reconciliation: {
7139
+ required: true,
7140
+ id,
7141
+ sourceStore: wiPath,
7142
+ targetStore: projectMove.wiPath,
7143
+ spliceError: spliceErr.message,
7144
+ rollbackError: rollbackErr.message,
7145
+ },
7146
+ });
7147
+ }
7148
+
7108
7149
  return jsonReply(res, 500, {
7109
7150
  error: `project move failed while removing from source store; move was rolled back: ${spliceErr.message}`,
7110
7151
  });
@@ -14167,6 +14208,42 @@ What would you like to discuss or change? When you're happy, say "approve" and I
14167
14208
  return jsonReply(res, 200, { ok: true, cleared: 'build-fix-ineffective', prId });
14168
14209
  }},
14169
14210
 
14211
+ // Issue #764 — sibling to the two clear endpoints above, for the
14212
+ // holder-path-keyed `_worktreeHeldPaused` circuit breaker (see
14213
+ // engine/lifecycle.js#recordWorktreeHeldPause / engine.js's
14214
+ // isWorktreeHeldPauseSuppressed). engine.js already auto-clears this the
14215
+ // next time it detects the holder no longer holds the branch, but this
14216
+ // endpoint lets an operator force a clear immediately (e.g. they just
14217
+ // released the branch and don't want to wait for the next PR poll tick).
14218
+ { method: 'POST', path: '/api/pull-requests/clear-worktree-held-pause', desc: 'Resume a paused worktree-held-externally pause (issue #764) — clears _worktreeHeldPaused so PR-driven review/fix auto-dispatch can resume', params: 'prId (canonical host:slug#number)', handler: async (req, res) => {
14219
+ const body = await readBody(req);
14220
+ const prId = typeof body?.prId === 'string' ? body.prId.trim() : '';
14221
+ if (!prId) return jsonReply(res, 400, { error: 'prId required' });
14222
+ reloadConfig();
14223
+ const lifecycle = require('./engine/lifecycle');
14224
+ const prPaths = [
14225
+ ...shared.getProjects(CONFIG).map(p => shared.projectPrPath(p)),
14226
+ shared.centralPullRequestsPath(MINIONS_DIR),
14227
+ ];
14228
+ let prFound = false;
14229
+ let recordCleared = false;
14230
+ for (const prPath of prPaths) {
14231
+ if (recordCleared) break;
14232
+ shared.mutatePullRequests(prPath, (prs) => {
14233
+ if (!Array.isArray(prs)) return prs;
14234
+ const target = prs.find(p => p && p.id === prId);
14235
+ if (!target) return prs;
14236
+ prFound = true;
14237
+ recordCleared = lifecycle.clearWorktreeHeldPause(target);
14238
+ return prs;
14239
+ });
14240
+ }
14241
+ if (!prFound) return jsonReply(res, 404, { error: `PR ${prId} not found` });
14242
+ if (!recordCleared) return jsonReply(res, 400, { error: `PR ${prId} has no tracked worktree-held pause` });
14243
+ invalidateStatusCache();
14244
+ return jsonReply(res, 200, { ok: true, cleared: 'worktree-held-pause', prId });
14245
+ }},
14246
+
14170
14247
  // ─── P-f3c9d0e7: convenience pause/resume endpoints ─────────────────────
14171
14248
  // Single-call wrappers over `POST /api/settings { engine: { <flag>: bool } }`
14172
14249
  // for the two operator kill-switches:
package/docs/README.md CHANGED
@@ -31,6 +31,7 @@ Architecture, design proposals, and lifecycle references for people working on t
31
31
  - [harness-propagation.md](harness-propagation.md) — How user-level and project-local harness assets (skills, slash-commands, MCP config, `CLAUDE.md` / `AGENTS.md`) propagate into an agent's worktree via `--add-dir`, the `harnessPropagateProjectLocal` flag, and the project-local-on-main worktree-visibility footgun.
32
32
  - [harness-transparency.md](harness-transparency.md) — The `harnessUsed` self-report contract: capture (agent reports the skills / MCPs / commands / docs it used) → ground (engine cross-checks against `_harnessPropagated` and annotates `grounded:true\|false`, never dropping) → surface (PR comment, final agent note, work-item modal).
33
33
  - [kb-dedup-duplicate-pair-investigation.md](kb-dedup-duplicate-pair-investigation.md) — Investigation-only pass diffing suspected duplicate KB filename groups (content-hash dedup for agent-authored reviews/build-reports) to confirm true full-body duplication before any fix is proposed.
34
+ - [kb-pr3223-cascade-archiving.md](kb-pr3223-cascade-archiving.md) — Findings from PR #3223: cascade-archiving PRDs when their linked markdown plan is archived, including the per-concern try/catch pattern and `source_plan` basename-matching invariant in `handlePlansArchive` (`dashboard.js`).
34
35
  - [kb-sweep.md](kb-sweep.md) — Knowledge-base consolidation sweep (hash dedup → LLM batch dedup/reclassify → per-entry compress) and the detached runner that keeps it alive across `minions restart`.
35
36
  - [keep-processes.md](keep-processes.md) — `meta.keep_processes` sidecar contract: when to use it vs managed-spawn, sidecar schema, caps, and the [`engine/keep-process-sweep.js`](../engine/keep-process-sweep.js) lifecycle.
36
37
  - [live-checkout-mode.md](live-checkout-mode.md) — Per-project opt-in `checkoutMode: 'live'`: skips `git worktree add` and dispatches in-place inside `project.localPath` for `repo`-managed trees, submodule-heavy repos, deep Windows paths, and native build state. Includes the refuse-on-dirty contract and the per-project mutating-concurrency cap of 1.
@@ -30,6 +30,10 @@ Implementation: `engine.js` builds a `liveProjectsInUse` set from the active dis
30
30
 
31
31
  ### 2. Refuse-on-dirty (engine never resets the operator tree)
32
32
 
33
+ **W-mrcifup2000c218a — detected at ALLOCATION time first.** Before a live-mode work item is even dispatched, `engine.js`'s dispatch-existing-pending loop calls `engine/live-checkout.js#checkLiveCheckoutDirty` (a cheap, read-only, non-mutating `git status --porcelain=v1 -b` probe — same command, same auto-cleanable-artifact allowance as the in-spawn check below) in the same pass that already gates on `live_checkout_busy`. A dirty hit stamps `_pendingReason: 'live_checkout_dirty'` on the pending item and `continue`s — **no dispatch attempt, no retry consumed, no failure recorded** — exactly like the busy gate. The item is re-checked every tick and dispatches normally the moment the tree is clean. A `live-checkout-dirty-<wi-id>` inbox alert is written (deduped per day, same as the in-spawn alert below) so a persistent dirty condition is still surfaced to a human even though the WI itself stays pending.
34
+
35
+ The in-spawn check below remains as a **defense-in-depth fallback** for the race window between the allocation-time probe and the actual git operations inside `spawnAgent` (e.g. the tree goes dirty in the few seconds between the pre-check and the spawn) — it is now the *exception* path, not the common path.
36
+
33
37
  Before spawning, `engine/live-checkout.js#prepareLiveCheckout` runs `git status --porcelain` from `project.localPath`. Any output (staged, unstaged, untracked) **returns** an explicit `{ ok:false, reason:'dirty', dirtyFiles:[…] }` result (it does NOT throw), which refuses the dispatch immediately:
34
38
 
35
39
  - Non-retryable `FAILURE_CLASS.LIVE_CHECKOUT_DIRTY` (added to `engine/dispatch.js`'s `neverRetry` set so the dispatcher never re-spawns mechanically). **Reserved for this confirmed-dirty result only** — a thrown helper/git error is `LIVE_CHECKOUT_FAILED`, see below (#305). **Auto-cleanup exception (W-mqzmkoqt000hbca2, #582):** when `liveCheckoutAutoReset` or `liveCheckoutAutoStash` is enabled (per-project or fleet-wide), `spawnAgent` treats the dirty refusal as retryable instead — the engine re-cleans the tree on every attempt, so the two-strike cap below is skipped and the dispatch keeps retrying up to the normal `maxRetries` cap. The `dispatch.js` `neverRetry` entry is kept only as a defensive fallback for callers that omit the explicit `agentRetryable` override.
@@ -129,6 +133,8 @@ Either condition fails the dispatch non-retryably with `FAILURE_CLASS.LIVE_CHECK
129
133
 
130
134
  ### 6. STALE-BASE GUARD — refuse to fork off a diverged local base (W-mr98op8w000ma4ad)
131
135
 
136
+ **W-mrcifup2000c218a — detected at ALLOCATION time first.** Same as Guarantee 2, the stale-base condition is now checked BEFORE dispatch by `engine/live-checkout.js#checkLiveCheckoutStaleBase` (a cheap, read-only, fetch-free `git rev-list --count origin/<mainRef>..<mainRef>` probe, mirroring the in-spawn check below) in the dispatch-existing-pending loop. A stale hit stamps `_pendingReason: 'live_checkout_stale_base'` and leaves the item pending — no retry consumed, no terminal failure — re-checked every tick, instead of the pre-fix behavior of failing non-retryably on the very first hit. A `live-checkout-stale-base-<wi-id>` inbox alert is written (deduped per day) so the condition is still surfaced to a human. The in-spawn refusal documented below is now a defense-in-depth fallback for the race window between the pre-check and the actual `git checkout -b`, not the common path — it still fails non-retryably when it fires, because a stale base found there is still deterministic and un-recoverable by a bare retry.
137
+
132
138
  A **clean** tree can still hide **committed** contamination. When `prepareLiveCheckout` is about to create a **new** branch (`git checkout -b <branch>`) off the operator's current HEAD, and HEAD is sitting on the project base ref (`mainRef`, the common case), the local `mainRef` branch may carry commits that were never pushed to `origin/<mainRef>` — e.g. leftover in-progress work from a prior dispatch on a live-mode project (capped to one mutating dispatch, but shared across many sequential dispatches), an interrupted process, or a bad rebase. Forking a new branch off that base silently inherits those commits into the branch **and its PR**. This is exactly the scope-contamination class first seen on ADO PR 5411214 and recurring on AB#12016662 / ADO PR 5419146 (~20 unrelated OCM files baked into an otherwise-clean 2-file a11y fix).
133
139
 
134
140
  It survives the dirty check (Guarantee 2) because the contamination is **committed**, not uncommitted, and it cannot be caught by a "HEAD is on the wrong branch" check because here HEAD **is** on `mainRef`.
@@ -84,6 +84,47 @@ recreate a clean worktree. Six layered defenses cover the Windows EBUSY
84
84
  race when git status descendants still hold packfile handles
85
85
  (W-mq5n1zx5000hcfb5; PR #3156).
86
86
 
87
+ ### Uncommitted-work preservation (W-mrcqzuxy000w676e)
88
+
89
+ The committed-HEAD backup ref (`refs/minions/quarantine/<sanitized-branch>/<ts>`,
90
+ written near the end of `_quarantineDirtyWorktreeCore`) only ever captured
91
+ `git rev-parse HEAD` — it never covered **uncommitted** modified/untracked
92
+ files. A prior incident (W-mrciooy1000e2ac3) lost a real, on-task fix + a new
93
+ regression test that a previous agent had written but not committed: the
94
+ worktree was quarantined cleanly, but the uncommitted edits were gone the
95
+ moment the dir was renamed away.
96
+
97
+ `_quarantineDirtyWorktreeCore` now runs a **separate, best-effort WIP-snapshot
98
+ step** immediately after the live-dispatch guard and before any destructive
99
+ work (holder-reap / rename / force-remove):
100
+
101
+ 1. Probe `git status --porcelain` in the worktree.
102
+ 2. If it reports any entries, run `git add -A && git commit --no-verify -m
103
+ "quarantine: uncommitted WIP snapshot (<dispatchId>)"` in the worktree
104
+ (captures both modified tracked files and new untracked files), then
105
+ `git update-ref refs/minions/quarantine-wip/<sanitized-branch>/<ts> <sha>`
106
+ in the shared repo to pin the WIP commit under its own ref — independent of
107
+ the plain `refs/minions/quarantine/...` backup ref.
108
+ 3. Never blocks the quarantine: any failure at any point (status probe, add,
109
+ commit, update-ref) is caught, logged, and the quarantine proceeds
110
+ regardless.
111
+
112
+ The result object carries this as `result.wipRef` (the ref path, or `null`)
113
+ and `result.wipOutcome`, one of exactly three values so a "did we lose
114
+ anything?" question always has an unambiguous, logged answer:
115
+
116
+ - `'clean'` — no uncommitted changes existed; nothing to preserve.
117
+ - `'preserved'` — uncommitted changes were committed and backed up to `wipRef`.
118
+ - `'failed'` — uncommitted changes existed (or status was undeterminable) but
119
+ the snapshot attempt itself failed; check the engine log for the specific
120
+ git error.
121
+
122
+ The quarantine inbox note (`notes/inbox/engine-worktree-quarantine-*.md`) always
123
+ includes a `wipOutcome`-specific recovery line — the `wipRef` cherry-pick/show
124
+ command when `preserved`, a `⚠️` warning when `failed`, or an explicit "nothing
125
+ to preserve" note when `clean` — so a human never has to guess which bucket a
126
+ given quarantine landed in.
127
+
87
128
  ### Layer 1b — `--no-optional-locks` on every status probe (load-bearing)
88
129
 
89
130
  `_statusPorcelainCmd()` emits `git --no-optional-locks status …`. Skips
@@ -105,6 +146,33 @@ budget explicitly. Toggles:
105
146
  `ENGINE_DEFAULTS.quarantineRenameRetryAttempts` (6),
106
147
  `quarantineRenameRetryBaseMs` (250).
107
148
 
149
+ ### Layer 1a′ — concurrent-quarantine race guard (W-mrcllbrx000oec11)
150
+
151
+ Two overlapping recovery attempts against the SAME dirty worktree path can
152
+ both reach the rename step (observed ~96ms apart in production — see
153
+ W-mrciooy1000e2ac3, 12 recovery cycles burned before the WI was wrongly
154
+ marked terminally failed even though the tree was already clean). ENOENT is
155
+ non-retryable in `_retryFsOp`, so the loser's rename throws immediately once
156
+ the winner has already moved the source dir aside. Two layers address this:
157
+
158
+ 1. **In-process serialization.** `engine.js#_quarantineDirtyWorktree` is now a
159
+ thin wrapper that runs the real implementation
160
+ (`_quarantineDirtyWorktreeCore`) through `_withPathLock`, keyed by
161
+ `worktreePath`. Overlapping calls for the SAME path queue and run their
162
+ destructive steps (holder-reap, rename, force-remove) strictly one at a
163
+ time; calls for different paths are unaffected.
164
+ 2. **ENOENT idempotency check.** If the rename still fails with `ENOENT` AND
165
+ `worktreePath` no longer exists on disk, the source was already renamed
166
+ away by a concurrent/prior attempt — this is quarantine having *already
167
+ succeeded*, not an environment block. `_quarantineDirtyWorktreeCore` best-
168
+ effort locates the sibling `<worktreePath>-quarantine-*` dir the winning
169
+ attempt created (used for `result.quarantinedPath`), sets
170
+ `result.alreadyQuarantinedConcurrently = true`, clears the rename error,
171
+ and proceeds through the normal success path (backup-ref, `git worktree
172
+ prune`, inbox note) instead of falling into the force-remove fallback or
173
+ the `WORKTREE_QUARANTINE_ENV_BLOCKED` total-failure branch. Counter:
174
+ `metrics._engine.worktreeQuarantineOutcomes.alreadyQuarantinedConcurrent`.
175
+
108
176
  ### Layer 2a — `_killGitDescendantsForWorktree` (Windows-only)
109
177
 
110
178
  PowerShell sweep, 2 s timeout. Shell-out built via single-quoted literal
@@ -128,10 +196,18 @@ the new class via both enum check and regex on legacy `failReason`
128
196
  strings, then re-queues under the existing
129
197
  `ENGINE_DEFAULTS.quarantineAutoRecoveryMax` (default 2) cap.
130
198
 
199
+ `_quarantineRecoveryCount` is a **lifetime cumulative** counter — it is never
200
+ reset by a clean recovery, only by a manual retry. `POST /api/work-items/retry`
201
+ (`dashboard.js#handleWorkItemsRetry`) deletes both `_quarantineRecoveryCount`
202
+ and `_quarantineRecoveryGaveUp` on every manual retry (W-mrcqzuxy000w676e), so
203
+ an explicit human "try again" isn't punished by unrelated prior quarantine
204
+ noise that would otherwise leave `_quarantineRecoveryGaveUp: true` permanently
205
+ set even after a later attempt recovers cleanly.
206
+
131
207
  ### Layer 3a/3b — metrics + inbox alert
132
208
 
133
209
  Counters at
134
- `metrics._engine.worktreeQuarantineOutcomes.{attempts, success, successAfterRetry, fallbackForceRemove, totalFailure}`.
210
+ `metrics._engine.worktreeQuarantineOutcomes.{attempts, success, successAfterRetry, fallbackForceRemove, totalFailure, alreadyQuarantinedConcurrent}`.
135
211
  Total-failure path writes a structured inbox alert with the exact
136
212
  PowerShell/POSIX recovery commands and `git worktree prune` instructions.
137
213
 
package/engine/ado.js CHANGED
@@ -279,7 +279,7 @@ function repairAdoProjectConfig(project, purpose, prs = null) {
279
279
  const missingBefore = getMissingAdoProjectConfigFields(project);
280
280
  if (missingBefore.length === 0) return false;
281
281
 
282
- const trackedPrs = prs || shared.safeJson(shared.projectPrPath(project)) || [];
282
+ const trackedPrs = prs || shared.safeJsonArr(shared.projectPrPath(project));
283
283
  for (const candidate of collectAdoRepairCandidates(project, trackedPrs)) {
284
284
  const repairs = {};
285
285
  if (!project.adoOrg && candidate.adoOrg) repairs.adoOrg = candidate.adoOrg;
package/engine/cleanup.js CHANGED
@@ -9,7 +9,7 @@ const shared = require('./shared');
9
9
  const queries = require('./queries');
10
10
 
11
11
  const { exec, execAsync, execSilent, log, ts, ENGINE_DEFAULTS } = shared;
12
- const { safeJson, safeJsonNoRestore, safeWrite, safeReadDir, mutateCooldowns, mutateWorkItems, mutateJsonFileLocked, getProjects, projectWorkItemsPath, projectPrPath,
12
+ const { safeJson, safeJsonArr, safeJsonNoRestore, safeWrite, safeReadDir, mutateCooldowns, mutateWorkItems, mutateJsonFileLocked, getProjects, projectWorkItemsPath, projectPrPath,
13
13
  sanitizeBranch, KB_CATEGORIES } = shared;
14
14
  const { getDispatch, getAgentStatus } = queries;
15
15
 
@@ -98,7 +98,7 @@ function localBranchWorktreeInUse(root, branch) {
98
98
  function collectPhantomBranchesForProject(project) {
99
99
  const branches = new Set();
100
100
  try {
101
- const items = safeJson(projectWorkItemsPath(project)) || [];
101
+ const items = safeJsonArr(projectWorkItemsPath(project));
102
102
  if (!Array.isArray(items)) return branches;
103
103
  for (const w of items) {
104
104
  if (w && w._phantomCompletion === true && w._phantomBranch) {
@@ -1427,11 +1427,11 @@ async function runCleanup(config, verbose = false) {
1427
1427
  try {
1428
1428
  const wiIds = new Set();
1429
1429
  for (const project of projects) {
1430
- const items = safeJson(projectWorkItemsPath(project)) || [];
1430
+ const items = safeJsonArr(projectWorkItemsPath(project));
1431
1431
  for (const wi of items) { if (wi?.id) wiIds.add(wi.id); }
1432
1432
  }
1433
1433
  try {
1434
- const centralWi = safeJson(path.join(MINIONS_DIR, 'work-items.json')) || [];
1434
+ const centralWi = safeJsonArr(path.join(MINIONS_DIR, 'work-items.json'));
1435
1435
  for (const wi of centralWi) { if (wi?.id) wiIds.add(wi.id); }
1436
1436
  } catch { /* optional */ }
1437
1437
 
@@ -310,25 +310,53 @@ function addToDispatch(item) {
310
310
  function _persistInvalidWorkItem(item, evaluation) {
311
311
  const meta = item?.meta;
312
312
  const itemId = meta?.item?.id;
313
- if (!itemId) return;
313
+ if (!itemId) return { stuck: false };
314
314
  let wiPath;
315
315
  try { wiPath = lifecycle().resolveWorkItemPath(meta); } catch { wiPath = null; }
316
- if (!wiPath) return;
316
+ if (!wiPath) return { stuck: false };
317
+ // W-mrcrh5hj0017d22f — repeat-rejection cap. Compare against the
318
+ // description snapshot from the LAST rejection so an operator edit to the
319
+ // WI resets the counter (it deserves a fresh evaluation), while an
320
+ // unchanged, permanently-unactionable description accumulates toward
321
+ // the cap instead of being silently re-evaluated forever.
322
+ const maxRejections = queries.getConfig()?.engine?.preDispatchEvalMaxRejections
323
+ ?? ENGINE_DEFAULTS.preDispatchEvalMaxRejections;
324
+ const currentDescription = String((meta.item && meta.item.description) || '');
325
+ let stuck = false;
317
326
  try {
318
327
  mutateWorkItems(wiPath, (items) => {
319
328
  if (!Array.isArray(items)) return items;
320
329
  const idx = items.findIndex(w => w && w.id === itemId);
321
330
  if (idx === -1) return items;
322
- items[idx]._preDispatchEval = {
331
+ const wi = items[idx];
332
+ const prior = wi._preDispatchEval;
333
+ const sameDescription = prior && prior.description === currentDescription;
334
+ const rejectCount = (sameDescription ? (prior.rejectCount || 0) : 0) + 1;
335
+ wi._preDispatchEval = {
323
336
  valid: false,
324
337
  reason: evaluation.reason || '',
325
338
  evaluatedAt: ts(),
339
+ rejectCount,
340
+ description: currentDescription,
326
341
  };
342
+ if (rejectCount >= maxRejections) {
343
+ stuck = true;
344
+ wi.status = WI_STATUS.FAILED;
345
+ wi._failureClass = FAILURE_CLASS.PRE_DISPATCH_EVAL_STUCK;
346
+ wi.failReason = `pre-dispatch-eval rejected this work item ${rejectCount} times in a row ` +
347
+ `with an unchanged description — last reason: ${evaluation.reason || 'criteria not actionable'}`;
348
+ wi.failedAt = ts();
349
+ wi._pendingReason = 'pre_dispatch_eval_stuck';
350
+ }
327
351
  return items;
328
352
  }, { skipWriteIfUnchanged: true });
329
353
  } catch (e) {
330
354
  log('warn', `pre-dispatch-eval: failed to persist reason on ${itemId}: ${e.message}`);
331
355
  }
356
+ if (stuck) {
357
+ log('warn', `pre-dispatch-eval: ${itemId} rejected ${maxRejections}+ times with unchanged description — flipped to failed (PRE_DISPATCH_EVAL_STUCK) instead of re-evaluating forever`);
358
+ }
359
+ return { stuck };
332
360
  }
333
361
 
334
362
  function _routeToReviewQueue(item, evaluation) {
@@ -455,8 +483,12 @@ async function addToDispatchWithValidation(item, opts = {}) {
455
483
 
456
484
  if (!evaluation || evaluation.valid !== false) return addToDispatch(item);
457
485
 
458
- _persistInvalidWorkItem(item, evaluation);
459
- _routeToReviewQueue(item, evaluation);
486
+ const { stuck } = _persistInvalidWorkItem(item, evaluation);
487
+ // W-mrcrh5hj0017d22f: once the WI has been flipped to failed for
488
+ // repeat-rejection, don't also push it into the review queue — the
489
+ // failed status already stops future discovery ticks from re-evaluating
490
+ // it, and a review-queue entry would just be a stale duplicate signal.
491
+ if (!stuck) _routeToReviewQueue(item, evaluation);
460
492
  log('warn', `pre-dispatch-eval: blocked work item ${wi.id} — ${evaluation.reason || 'criteria not actionable'}`);
461
493
  return null;
462
494
  }
@@ -3054,6 +3054,71 @@ function clearBuildFixIneffective(target) {
3054
3054
  return true;
3055
3055
  }
3056
3056
 
3057
+ // #764 — PR-level circuit breaker for the WORKTREE_PREFLIGHT
3058
+ // branch-held-externally failure class (engine.js `_failBranchHeldByExternalWorktree`).
3059
+ // That failure already stamps the WI `agentRetryable: false`, which stops the
3060
+ // immediate quarantine-recovery-loop storm for that one dispatch instance, but
3061
+ // it carries no memory at the PR level — the human-feedback / review / build /
3062
+ // conflict fix dispatch pollers see the PR is still `pendingFix` (or has
3063
+ // coalesced feedback) and re-dispatch a brand-new work item next cooldown
3064
+ // window, forever, until a human manually releases the branch. Mirrors the
3065
+ // `_buildFixIneffective` shape (see `recordBuildFixIneffective` above) so the
3066
+ // dashboard/pause-inspection conventions stay consistent, but is keyed on the
3067
+ // holder path + branch rather than a head SHA — the signal that should clear
3068
+ // this pause is "the holder no longer holds the branch", not "a new commit
3069
+ // landed" (see `engine.js#isWorktreeHeldPauseSuppressed`, which performs that
3070
+ // live re-check and calls `clearWorktreeHeldPause` when the holder has moved
3071
+ // on).
3072
+ function recordWorktreeHeldPause(target, { holderPath, isProjectRoot, branch, dispatchItem } = {}) {
3073
+ if (!target) return null;
3074
+ const now = ts();
3075
+ const prior = target._worktreeHeldPaused && typeof target._worktreeHeldPaused === 'object'
3076
+ ? target._worktreeHeldPaused : null;
3077
+ target._worktreeHeldPaused = {
3078
+ holderPath: holderPath || null,
3079
+ isProjectRoot: !!isProjectRoot,
3080
+ branch: branch || null,
3081
+ since: prior?.since || now,
3082
+ lastAt: now,
3083
+ dispatchId: dispatchItem?.id || null,
3084
+ reason: isProjectRoot
3085
+ ? `Branch ${branch || '(unknown)'} is held by the project root (${holderPath}). PR-driven review/fix automation is paused until the root releases the branch.`
3086
+ : `Branch ${branch || '(unknown)'} is held by an external worktree at ${holderPath} the engine can't reach. PR-driven review/fix automation is paused until that worktree releases the branch.`,
3087
+ };
3088
+ if (!prior) {
3089
+ try {
3090
+ const prNumber = target.prNumber || shared.getPrNumber(target) || target.id || 'unknown';
3091
+ const noteBody = `# Worktree-held pause — PR automation stopped: ${target.id || prNumber}\n\n`
3092
+ + `**PR:** ${target.url || target.id || '(unknown)'}\n`
3093
+ + `**Branch:** ${branch || target.branch || '(unknown)'}\n`
3094
+ + `**Holder:** ${holderPath || '(unknown)'}${isProjectRoot ? ' (project root)' : ' (external worktree)'}\n\n`
3095
+ + `A WORKTREE_PREFLIGHT dispatch failed because this branch is checked out ${isProjectRoot ? 'at the project root' : 'in an external worktree'} the engine can't reach. `
3096
+ + `Review/human-feedback/build-fix/merge-conflict automation for this PR is now paused so the poller stops re-dispatching against the same unresolved cause every cooldown window (issue #764).\n\n`
3097
+ + `**Recovery:** ${isProjectRoot ? `\`git -C ${holderPath} checkout <mainBranch>\` to release the branch` : 'finish or remove the external worktree holding this branch'} — the engine auto-clears this pause the next time it detects the holder no longer holds the branch.\n`;
3098
+ shared.writeToInbox(
3099
+ 'engine',
3100
+ `worktree-held-paused-${prNumber}`,
3101
+ noteBody,
3102
+ null,
3103
+ { wi: dispatchItem?.meta?.item?.id || null, pr: target.id || null },
3104
+ );
3105
+ } catch (err) {
3106
+ log('warn', `worktree-held-paused inbox alert for ${target.id || '(unknown)'}: ${err.message}`);
3107
+ }
3108
+ }
3109
+ return target._worktreeHeldPaused;
3110
+ }
3111
+
3112
+ // Clears the `_worktreeHeldPaused` circuit breaker (issue #764) so PR-driven
3113
+ // review/fix dispatch can resume. Returns true when a record was actually
3114
+ // removed so callers (the dashboard resume endpoint, engine.js's live
3115
+ // re-check) can distinguish "cleared" from "nothing to clear".
3116
+ function clearWorktreeHeldPause(target) {
3117
+ if (!target?._worktreeHeldPaused) return false;
3118
+ delete target._worktreeHeldPaused;
3119
+ return true;
3120
+ }
3121
+
3057
3122
  function clearPrNoOpFixAttempt(target, cause) {
3058
3123
  if (!target?._noOpFixes || !target._noOpFixes[cause]) return;
3059
3124
  delete target._noOpFixes[cause];
@@ -7011,6 +7076,13 @@ module.exports = {
7011
7076
  // Issue #671 — exported so dashboard.js can clear the build-fix-ineffective
7012
7077
  // ("infra issue suspected") pause from the resume endpoint.
7013
7078
  clearBuildFixIneffective,
7079
+ // Issue #764 — PR-level circuit breaker for WORKTREE_PREFLIGHT
7080
+ // branch-held-externally failures. Exported so engine.js can stamp the
7081
+ // pause at dispatch-failure time and auto-clear it via the live re-check,
7082
+ // and so dashboard.js can expose a manual clear endpoint + unit tests can
7083
+ // exercise both paths directly.
7084
+ recordWorktreeHeldPause,
7085
+ clearWorktreeHeldPause,
7014
7086
  // W-mpx44p05000ze8d8 — exported so engine/ado.js + engine/github.js can
7015
7087
  // drop stale `_noOpFixes` records during their PR status poll, and so
7016
7088
  // unit tests can exercise the sweep helper directly.