@yemi33/minions 0.1.2355 → 0.1.2357
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/js/render-pipelines.js +12 -3
- package/dashboard/js/render-prs.js +55 -1
- package/dashboard/js/settings.js +30 -10
- package/dashboard.js +102 -2
- package/docs/README.md +1 -0
- package/docs/live-checkout-mode.md +6 -0
- package/docs/worktree-lifecycle.md +77 -1
- package/engine/ado.js +1 -1
- package/engine/cleanup.js +4 -4
- package/engine/db/migrations/016-pr-mirror-hash.js +31 -0
- package/engine/lifecycle.js +93 -5
- package/engine/live-checkout.js +141 -0
- package/engine/pipeline.js +87 -10
- package/engine/playbook.js +6 -5
- package/engine/pull-requests-store.js +70 -9
- package/engine/queries.js +44 -29
- package/engine/shared.js +208 -3
- package/engine.js +519 -87
- package/package.json +1 -1
package/engine.js
CHANGED
|
@@ -168,7 +168,8 @@ const { runPostCompletionHooks, updateWorkItemStatus, syncPrdItemStatus, reconci
|
|
|
168
168
|
syncPrsFromOutput, updatePrAfterReview, updatePrAfterFix, checkForLearnings, extractSkillsFromOutput,
|
|
169
169
|
updateAgentHistory, updateMetrics, createReviewFeedbackForAuthor, parseAgentOutput, syncPrdFromPrs, persistVerifyPrsToPrd,
|
|
170
170
|
isItemCompleted, classifyFailure: classifyFailureFallback, diagnoseEmptyOutput, processPendingRebases, resolveWorkItemPath,
|
|
171
|
-
mergeArtifactNotes, promoteCompletionArtifacts, pruneScopeMismatchDuplicatePrs, collapseAllDuplicatePrRecords
|
|
171
|
+
mergeArtifactNotes, promoteCompletionArtifacts, pruneScopeMismatchDuplicatePrs, collapseAllDuplicatePrRecords,
|
|
172
|
+
recordWorktreeHeldPause, clearWorktreeHeldPause } = require('./engine/lifecycle');
|
|
172
173
|
|
|
173
174
|
// ─── Diagnostics: memory + event-loop + GC sampler (P-a1b2c3d4 / P-b2c3d4e5) ─
|
|
174
175
|
|
|
@@ -710,6 +711,12 @@ function resolveDependencyBranches(depIds, sourcePlan, project, config) {
|
|
|
710
711
|
projectName: p.name || null,
|
|
711
712
|
projectRoot: depRoot,
|
|
712
713
|
isCrossRepo: !!currentProjectName && p.name !== currentProjectName,
|
|
714
|
+
// W-mrchrfe100067d3a — same-project deps whose PR is scoped to one
|
|
715
|
+
// of `p`'s additionalRepoScopes entries (e.g. a backport PR opened
|
|
716
|
+
// against yemi33/minions from the minions-opg checkout) live on a
|
|
717
|
+
// different git remote than 'origin'. Resolved once here so the
|
|
718
|
+
// fetch/push loop below doesn't hardcode 'origin' for those.
|
|
719
|
+
remote: shared.resolveGitRemoteForPrScope(p, pr),
|
|
713
720
|
});
|
|
714
721
|
}
|
|
715
722
|
}
|
|
@@ -1329,7 +1336,7 @@ function _bumpQuarantineOutcome(key, delta = 1) {
|
|
|
1329
1336
|
if (!metrics._engine.worktreeQuarantineOutcomes) {
|
|
1330
1337
|
metrics._engine.worktreeQuarantineOutcomes = {
|
|
1331
1338
|
attempts: 0, success: 0, successAfterRetry: 0,
|
|
1332
|
-
fallbackForceRemove: 0, totalFailure: 0,
|
|
1339
|
+
fallbackForceRemove: 0, totalFailure: 0, alreadyQuarantinedConcurrent: 0,
|
|
1333
1340
|
};
|
|
1334
1341
|
}
|
|
1335
1342
|
const o = metrics._engine.worktreeQuarantineOutcomes;
|
|
@@ -1341,6 +1348,28 @@ function _bumpQuarantineOutcome(key, delta = 1) {
|
|
|
1341
1348
|
}
|
|
1342
1349
|
}
|
|
1343
1350
|
|
|
1351
|
+
// W-mrcllbrx000oec11 — generic "serialize calls keyed by a string" helper.
|
|
1352
|
+
// Used to prevent two overlapping quarantine recovery attempts against the
|
|
1353
|
+
// SAME worktree path from running their destructive steps (holder-reap,
|
|
1354
|
+
// rename, force-remove) concurrently. `map` holds the tail promise of the
|
|
1355
|
+
// most recent caller for a given key; each new caller chains behind it and
|
|
1356
|
+
// replaces the map entry, so callers for the same key run strictly one at a
|
|
1357
|
+
// time while callers for different keys are unaffected. The map entry is
|
|
1358
|
+
// removed once a call's turn finishes, unless another caller has already
|
|
1359
|
+
// queued behind it (in which case that caller owns the cleanup).
|
|
1360
|
+
function _withPathLock(map, key, fn) {
|
|
1361
|
+
const prior = map.get(key) || Promise.resolve();
|
|
1362
|
+
const ourTurn = prior.catch(() => {}).then(fn, fn);
|
|
1363
|
+
const tail = ourTurn.catch(() => {});
|
|
1364
|
+
map.set(key, tail);
|
|
1365
|
+
tail.finally(() => {
|
|
1366
|
+
if (map.get(key) === tail) map.delete(key);
|
|
1367
|
+
});
|
|
1368
|
+
return ourTurn;
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
const _quarantinePathLocks = new Map();
|
|
1372
|
+
|
|
1344
1373
|
|
|
1345
1374
|
// ─── assertCleanSharedWorktree (#2439) ──────────────────────────────────────
|
|
1346
1375
|
// Engine-side preflight that prevents shared-branch (and PR-targeted reused —
|
|
@@ -1667,14 +1696,30 @@ async function assertCleanSharedWorktree(rootDir, worktreePath, branchName, disp
|
|
|
1667
1696
|
// for callers: a thrown error means the caller's `result.quarantineError`
|
|
1668
1697
|
// field is populated and `result.quarantined` stays false (env-blocked
|
|
1669
1698
|
// path); a normal return means quarantine succeeded.
|
|
1699
|
+
// W-mrcllbrx000oec11 — public entry point: serializes overlapping calls for
|
|
1700
|
+
// the SAME worktreePath through _withPathLock before delegating to the real
|
|
1701
|
+
// implementation below. See _quarantineDirtyWorktreeCore for the quarantine
|
|
1702
|
+
// logic itself, including the ENOENT idempotency check that this locking
|
|
1703
|
+
// layer works alongside (belt-and-suspenders: the lock stops concurrent
|
|
1704
|
+
// destructive work; the ENOENT check makes a still-overlapping caller — e.g.
|
|
1705
|
+
// from a process that doesn't share this in-memory map — resolve cleanly
|
|
1706
|
+
// instead of surfacing a false WORKTREE_QUARANTINE_ENV_BLOCKED failure).
|
|
1670
1707
|
async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOpts, diag = {}) {
|
|
1708
|
+
return _withPathLock(
|
|
1709
|
+
_quarantinePathLocks,
|
|
1710
|
+
worktreePath,
|
|
1711
|
+
() => _quarantineDirtyWorktreeCore(rootDir, worktreePath, branchName, gitOpts, diag),
|
|
1712
|
+
);
|
|
1713
|
+
}
|
|
1714
|
+
|
|
1715
|
+
async function _quarantineDirtyWorktreeCore(rootDir, worktreePath, branchName, gitOpts, diag = {}) {
|
|
1671
1716
|
// P-c7e2b405 — INTENTIONAL divergence from worktree-gc.reapAndRemoveWorktree:
|
|
1672
1717
|
// this is the bespoke dirty-worktree quarantine flow (rename → branch backup-ref
|
|
1673
1718
|
// → reset → fetch), NOT a plain reap-then-remove. It is deliberately NOT folded
|
|
1674
1719
|
// into the shared helper. It does share the same live-guard + holder-reap
|
|
1675
1720
|
// primitives (isWorktreePathLive + _reapWorktreeHolders below).
|
|
1676
1721
|
const ts = Date.now();
|
|
1677
|
-
|
|
1722
|
+
let quarantinedPath = `${worktreePath}-quarantine-${ts}`;
|
|
1678
1723
|
_bumpQuarantineOutcome('attempts', 1);
|
|
1679
1724
|
|
|
1680
1725
|
// Capture HEAD sha BEFORE renaming so we can back up the local branch ref.
|
|
@@ -1698,6 +1743,48 @@ async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOp
|
|
|
1698
1743
|
return { quarantinedPath: null, backupRef: null, skipped: true };
|
|
1699
1744
|
}
|
|
1700
1745
|
|
|
1746
|
+
// W-mrcqzuxy000w676e — PRESERVE UNCOMMITTED WORK BEFORE QUARANTINE.
|
|
1747
|
+
// The backup ref written near the end of this function only captures the
|
|
1748
|
+
// worktree's COMMITTED HEAD (via `headSha` above). It does NOT stash,
|
|
1749
|
+
// commit, or otherwise persist UNCOMMITTED modified/untracked files —
|
|
1750
|
+
// so a prior agent's real, on-task, uncommitted fix + test were silently
|
|
1751
|
+
// and permanently destroyed once the dir was renamed/pruned
|
|
1752
|
+
// (W-mrciooy1000e2ac3). Snapshot any uncommitted changes to a dedicated
|
|
1753
|
+
// backup ref BEFORE any destructive step below (holder-reap/rename/
|
|
1754
|
+
// force-remove) so they survive regardless of which path the quarantine
|
|
1755
|
+
// takes. Best-effort and non-blocking: a failure here never stops the
|
|
1756
|
+
// quarantine, but always logs one of three distinct, unambiguous outcomes
|
|
1757
|
+
// (clean / preserved / failed-to-preserve) instead of silently no-opping.
|
|
1758
|
+
const sanitizedRefSegment = sanitizeBranch(branchName).replace(/\//g, '-');
|
|
1759
|
+
let wipRef = null;
|
|
1760
|
+
let wipOutcome = 'clean';
|
|
1761
|
+
try {
|
|
1762
|
+
const statusOut = ((await execAsync('git status --porcelain', { ...gitOpts, cwd: worktreePath, timeout: 15000 })) || '').toString();
|
|
1763
|
+
if (statusOut.trim()) {
|
|
1764
|
+
try {
|
|
1765
|
+
await shared.shellSafeGit(['add', '-A'], { ...gitOpts, cwd: worktreePath, timeout: 30000 });
|
|
1766
|
+
await shared.shellSafeGit(
|
|
1767
|
+
['commit', '--no-verify', '-m', `quarantine: uncommitted WIP snapshot (${diag.dispatchId || 'unknown'})`],
|
|
1768
|
+
{ ...gitOpts, cwd: worktreePath, timeout: 30000 },
|
|
1769
|
+
);
|
|
1770
|
+
const wipSha = ((await execAsync('git rev-parse HEAD', { ...gitOpts, cwd: worktreePath, timeout: 10000 })) || '').toString().trim();
|
|
1771
|
+
const candidateWipRef = `refs/minions/quarantine-wip/${sanitizedRefSegment}/${ts}`;
|
|
1772
|
+
await shared.shellSafeGit(['update-ref', candidateWipRef, wipSha], { ...gitOpts, cwd: rootDir, timeout: 10000 });
|
|
1773
|
+
wipRef = candidateWipRef;
|
|
1774
|
+
wipOutcome = 'preserved';
|
|
1775
|
+
log('warn', `_quarantineDirtyWorktree: preserved uncommitted WIP for ${worktreePath} at ${wipRef} (${wipSha.slice(0, 12)})`);
|
|
1776
|
+
} catch (e) {
|
|
1777
|
+
wipOutcome = 'failed';
|
|
1778
|
+
log('error', `_quarantineDirtyWorktree: FAILED to preserve uncommitted WIP for ${worktreePath}: ${e.message} — uncommitted changes may be lost`);
|
|
1779
|
+
}
|
|
1780
|
+
}
|
|
1781
|
+
} catch (e) {
|
|
1782
|
+
// The status probe itself failed. Treat as failed-to-preserve (never
|
|
1783
|
+
// block the quarantine) rather than silently assuming the tree was clean.
|
|
1784
|
+
wipOutcome = 'failed';
|
|
1785
|
+
log('warn', `_quarantineDirtyWorktree: git status --porcelain probe failed for ${worktreePath}: ${e.message} — cannot determine whether uncommitted WIP needs preservation`);
|
|
1786
|
+
}
|
|
1787
|
+
|
|
1701
1788
|
// W-mq5n1zx5 / W-mqila0t5 Layer 0–2: pre-emptively reap any process holding
|
|
1702
1789
|
// the worktree dir before we rename it. The status-probe child that timed out
|
|
1703
1790
|
// earlier may still hold packfile handles (Layer 0: git.exe by cmdline), and —
|
|
@@ -1736,6 +1823,44 @@ async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOp
|
|
|
1736
1823
|
renameError = e;
|
|
1737
1824
|
}
|
|
1738
1825
|
|
|
1826
|
+
// W-mrcllbrx000oec11 — IDEMPOTENT-QUARANTINE GUARD: an ENOENT on the
|
|
1827
|
+
// rename means the source path no longer exists. `_retryFsOp` treats
|
|
1828
|
+
// ENOENT as non-retryable and throws on the very first attempt (it isn't
|
|
1829
|
+
// in shared._WORKTREE_RETRYABLE_CODES), so this fires immediately rather
|
|
1830
|
+
// than after a backoff loop. The only way the source can vanish between
|
|
1831
|
+
// our dirty-check (which confirmed the worktree existed) and this rename
|
|
1832
|
+
// is that another quarantine attempt against the SAME path already won
|
|
1833
|
+
// the race and renamed it away first (observed ~96ms apart in production,
|
|
1834
|
+
// W-mrciooy1000e2ac3 — 12 recovery cycles burned before the WI was
|
|
1835
|
+
// wrongly marked terminally failed even though the tree was already
|
|
1836
|
+
// clean). Recognize "source is gone" as quarantine having already
|
|
1837
|
+
// succeeded (from a concurrent caller) instead of a genuine environment
|
|
1838
|
+
// block, so we don't increment _quarantineRecoveryCount or surface
|
|
1839
|
+
// WORKTREE_QUARANTINE_ENV_BLOCKED for a race that already resolved
|
|
1840
|
+
// successfully. Best-effort: look for the sibling <path>-quarantine-*
|
|
1841
|
+
// directory the winning attempt created so we can still report a path;
|
|
1842
|
+
// if none is found (e.g. it was later cleaned up), source-gone alone is
|
|
1843
|
+
// still treated as success.
|
|
1844
|
+
let alreadyQuarantinedConcurrently = false;
|
|
1845
|
+
if (renameError && renameError.code === 'ENOENT' && !fs.existsSync(worktreePath)) {
|
|
1846
|
+
let winningPath = null;
|
|
1847
|
+
try {
|
|
1848
|
+
const parentDir = path.dirname(worktreePath);
|
|
1849
|
+
const prefix = `${path.basename(worktreePath)}-quarantine-`;
|
|
1850
|
+
const candidates = fs.readdirSync(parentDir)
|
|
1851
|
+
.filter((name) => name.startsWith(prefix))
|
|
1852
|
+
.sort(); // timestamp suffix sorts lexicographically == chronologically
|
|
1853
|
+
if (candidates.length > 0) winningPath = path.join(parentDir, candidates[candidates.length - 1]);
|
|
1854
|
+
} catch (e) {
|
|
1855
|
+
log('warn', `_quarantineDirtyWorktree: sibling quarantine-dir lookup after ENOENT failed for ${worktreePath}: ${e.message}`);
|
|
1856
|
+
}
|
|
1857
|
+
log('warn', `_quarantineDirtyWorktree: rename ENOENT for ${worktreePath} but source no longer exists — treating as already quarantined by a concurrent attempt${winningPath ? ` (found ${winningPath})` : ' (no sibling quarantine dir found)'}`);
|
|
1858
|
+
_bumpQuarantineOutcome('alreadyQuarantinedConcurrent', 1);
|
|
1859
|
+
alreadyQuarantinedConcurrently = true;
|
|
1860
|
+
renameError = null; // clear so the force-remove / env-blocked branches below don't fire
|
|
1861
|
+
quarantinedPath = winningPath;
|
|
1862
|
+
}
|
|
1863
|
+
|
|
1739
1864
|
// W-mq5n1zx5 Layer 2b: if the rename retries exhausted, fall back to
|
|
1740
1865
|
// `git worktree remove --force`. This DESTROYS the worktree contents
|
|
1741
1866
|
// (no forensics dir), so it's last-resort only — gated on the
|
|
@@ -1826,7 +1951,6 @@ async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOp
|
|
|
1826
1951
|
// starts on a clean origin tip rather than recreating the divergent state.
|
|
1827
1952
|
// Both ref ops are best-effort: if the local ref machinery is in a weird
|
|
1828
1953
|
// state we still want the worktree quarantine to stick.
|
|
1829
|
-
const sanitizedRefSegment = sanitizeBranch(branchName).replace(/\//g, '-');
|
|
1830
1954
|
const backupRef = `refs/minions/quarantine/${sanitizedRefSegment}/${ts}`;
|
|
1831
1955
|
let backupRefCreated = false;
|
|
1832
1956
|
if (headSha) {
|
|
@@ -1865,7 +1989,9 @@ async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOp
|
|
|
1865
1989
|
`- Original worktree: ${worktreePath}`,
|
|
1866
1990
|
forceRemoved
|
|
1867
1991
|
? `- Force-removed via \`git worktree remove --force\` (no quarantine dir preserved — W-mq5n1zx5 Layer 2b)`
|
|
1868
|
-
:
|
|
1992
|
+
: alreadyQuarantinedConcurrently
|
|
1993
|
+
? `- Already quarantined by a concurrent attempt${quarantinedPath ? ` — quarantined to: ${quarantinedPath}` : ' (source dir was gone by the time this recovery attempt ran; no sibling quarantine dir found)'} (W-mrcllbrx000oec11)`
|
|
1994
|
+
: `- Quarantined to: ${quarantinedPath}${renameAttempts > 1 ? ` (rename took ${renameAttempts} attempt(s) — W-mq5n1zx5 Layer 1a)` : ''}`,
|
|
1869
1995
|
`- Reason: ${diag.reason || 'unsafe'}`,
|
|
1870
1996
|
`- Local commits ahead of origin: ${diag.ahead || 0}`,
|
|
1871
1997
|
`- Local commits behind origin: ${diag.behind || 0}`,
|
|
@@ -1878,6 +2004,11 @@ async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOp
|
|
|
1878
2004
|
backupRefCreated
|
|
1879
2005
|
? `The local branch HEAD was backed up to \`${backupRef}\` (${headSha.slice(0, 12)}) and \`refs/heads/${branchName}\` was reset to \`refs/remotes/origin/${branchName}\`.`
|
|
1880
2006
|
: `Backup ref was NOT created (HEAD sha unavailable). The local branch ref was still reset to \`refs/remotes/origin/${branchName}\` if possible.${forceRemoved ? ' Worktree contents were destroyed by the force-remove fallback — no quarantine dir to inspect.' : ' Inspect the quarantined directory directly to recover unpushed work.'}`,
|
|
2007
|
+
wipOutcome === 'preserved'
|
|
2008
|
+
? `Uncommitted WIP (modified/untracked files at quarantine time) was committed and backed up to \`${wipRef}\`. Recover with \`git -C "${rootDir}" show ${wipRef}\` or \`git -C "${rootDir}" cherry-pick ${wipRef}\` onto a fresh ${branchName} worktree.`
|
|
2009
|
+
: wipOutcome === 'failed'
|
|
2010
|
+
? `⚠️ Uncommitted WIP preservation FAILED — see engine log for details. Any modified/untracked files that were NOT part of \`${backupRefCreated ? backupRef : 'the backup ref'}\` may be unrecoverable once the quarantine dir is pruned.`
|
|
2011
|
+
: 'No uncommitted WIP was present (working tree was clean at quarantine time) — nothing extra to preserve.',
|
|
1881
2012
|
'',
|
|
1882
2013
|
forceRemoved
|
|
1883
2014
|
? `1. The worktree dir was force-removed; there is no quarantine dir to inspect. If the local branch ref still has divergent commits, recover from \`${backupRefCreated ? backupRef : 'reflog'}\`.`
|
|
@@ -1899,12 +2030,15 @@ async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOp
|
|
|
1899
2030
|
log('warn', `_quarantineDirtyWorktree: writeToInbox failed: ${e.message}`);
|
|
1900
2031
|
}
|
|
1901
2032
|
|
|
1902
|
-
log('warn', `Quarantined dirty worktree ${worktreePath} → ${forceRemoved ? '<force-removed>' : quarantinedPath} (branch ${branchName}, ${diag.ahead || 0} ahead, ${diag.behind || 0} behind, ${dirtyFiles.length} dirty files${renameAttempts > 1 ? `, rename retries=${renameAttempts}` : ''}${forceRemoved ? ', force-remove fallback' : ''})`);
|
|
2033
|
+
log('warn', `Quarantined dirty worktree ${worktreePath} → ${forceRemoved ? '<force-removed>' : (quarantinedPath || '<already quarantined concurrently>')} (branch ${branchName}, ${diag.ahead || 0} ahead, ${diag.behind || 0} behind, ${dirtyFiles.length} dirty files${renameAttempts > 1 ? `, rename retries=${renameAttempts}` : ''}${forceRemoved ? ', force-remove fallback' : ''}${alreadyQuarantinedConcurrently ? ', already-quarantined-concurrently' : ''}, wip=${wipOutcome})`);
|
|
1903
2034
|
return {
|
|
1904
2035
|
quarantinedPath: forceRemoved ? null : quarantinedPath,
|
|
1905
2036
|
backupRef: backupRefCreated ? backupRef : null,
|
|
1906
2037
|
forceRemoved,
|
|
1907
2038
|
renameAttempts,
|
|
2039
|
+
alreadyQuarantinedConcurrently,
|
|
2040
|
+
wipRef,
|
|
2041
|
+
wipOutcome,
|
|
1908
2042
|
};
|
|
1909
2043
|
}
|
|
1910
2044
|
|
|
@@ -2389,6 +2523,31 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
2389
2523
|
if (rawErr && rawErr.message) {
|
|
2390
2524
|
log('debug', `spawnAgent: raw worktree-add failure for ${id}: ${String(rawErr.message).split('\n')[0]}`);
|
|
2391
2525
|
}
|
|
2526
|
+
// #764 — PR-level circuit breaker. `agentRetryable: false` above only
|
|
2527
|
+
// stops retries of THIS WI instance; it carries no memory at the PR
|
|
2528
|
+
// level, so the human-feedback / review / build-fix / merge-conflict
|
|
2529
|
+
// pollers in discoverFromPrs see the PR is still pendingFix (or has
|
|
2530
|
+
// coalesced feedback) and create a brand-new work item next cooldown
|
|
2531
|
+
// window — forever, until a human manually releases the branch (live
|
|
2532
|
+
// repro: ado:office/office/1js#5429466, 7 wasted dispatches in ~4.5h).
|
|
2533
|
+
// Stamp `_worktreeHeldPaused` on the live PR record for PR-sourced
|
|
2534
|
+
// dispatches (source is 'pr' or 'pr-human-feedback' — see buildPrDispatch
|
|
2535
|
+
// call sites) so discoverFromPrs's isWorktreeHeldPauseSuppressed gate can
|
|
2536
|
+
// suppress further dispatch until the holder releases the branch.
|
|
2537
|
+
if ((meta?.source === 'pr' || meta?.source === 'pr-human-feedback') && meta?.pr) {
|
|
2538
|
+
try {
|
|
2539
|
+
const pauseProject = meta.project || project;
|
|
2540
|
+
mutatePullRequests(projectPrPath(pauseProject), (prs) => {
|
|
2541
|
+
if (!Array.isArray(prs)) return prs;
|
|
2542
|
+
const target = shared.findPrRecord(prs, meta.pr, pauseProject);
|
|
2543
|
+
if (!target) return prs;
|
|
2544
|
+
recordWorktreeHeldPause(target, { holderPath, isProjectRoot, branch: branchName, dispatchItem });
|
|
2545
|
+
return prs;
|
|
2546
|
+
});
|
|
2547
|
+
} catch (e) {
|
|
2548
|
+
log('warn', `spawnAgent: failed to stamp worktree-held pause for ${meta.pr.id || meta.pr.url || '(unknown PR)'}: ${e.message}`);
|
|
2549
|
+
}
|
|
2550
|
+
}
|
|
2392
2551
|
};
|
|
2393
2552
|
_phaseT.afterPrompt = Date.now();
|
|
2394
2553
|
|
|
@@ -2411,15 +2570,23 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
2411
2570
|
// cwd = project.localPath, worktreeRootDir = null, liveMode = true.
|
|
2412
2571
|
// Instead of creating an engine-managed worktree we run prepareLiveCheckout
|
|
2413
2572
|
// in-place: it validates a clean tree and then checks out (or creates) the
|
|
2414
|
-
// target branch.
|
|
2415
|
-
//
|
|
2416
|
-
//
|
|
2417
|
-
//
|
|
2418
|
-
//
|
|
2419
|
-
//
|
|
2420
|
-
//
|
|
2421
|
-
//
|
|
2422
|
-
//
|
|
2573
|
+
// target branch. W-mrcifup2000c218a: the dirty/stale-base conditions below
|
|
2574
|
+
// are now caught EARLIER, at allocation time, by the pre-dispatch checks in
|
|
2575
|
+
// the dispatch-existing-pending loop (`checkLiveCheckoutDirty` /
|
|
2576
|
+
// `checkLiveCheckoutStaleBase` in engine/live-checkout.js) — a positive hit
|
|
2577
|
+
// there leaves the WI pending with `_pendingReason` stamped and never reaches
|
|
2578
|
+
// spawnAgent at all, so this block is now the EXCEPTION path (defense in
|
|
2579
|
+
// depth for the race window between that probe and the git operations
|
|
2580
|
+
// below), not the common path. On a CONFIRMED dirty refusal (the helper
|
|
2581
|
+
// returns { ok:false, reason:'dirty', dirtyFiles:[…] }) we write an inbox
|
|
2582
|
+
// alert, stamp `_pendingReason: 'live_checkout_dirty'` on the WI, complete
|
|
2583
|
+
// the dispatch non-retryably with FAILURE_CLASS.LIVE_CHECKOUT_DIRTY, and
|
|
2584
|
+
// return — the engine never runs `git reset`/`git clean` against the
|
|
2585
|
+
// operator tree. If the helper instead THROWS (guard/ref-validation/transient
|
|
2586
|
+
// git error), that is NOT proof of a dirty tree (#305): we complete with the
|
|
2587
|
+
// separate FAILURE_CLASS.LIVE_CHECKOUT_FAILED, which is retryable with
|
|
2588
|
+
// bounded backoff so racy branch-lock handoff or a transient git failure
|
|
2589
|
+
// auto-recovers.
|
|
2423
2590
|
// On success we fall through; the worktree-creation block below is gated
|
|
2424
2591
|
// on `!liveMode`, so it never runs, and downstream cleanup paths
|
|
2425
2592
|
// (worktreePool.returnToPool, worktree-gc.gcDispatchWorktreeIfOrphan) are
|
|
@@ -3013,6 +3180,15 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
3013
3180
|
// so it gets its own non-retryable FAILURE_CLASS instead of LIVE_CHECKOUT_FAILED's
|
|
3014
3181
|
// bounded retry-storm. The engine NEVER resets/cleans the operator's local
|
|
3015
3182
|
// mainRef; the operator must reconcile it themselves, then re-dispatch.
|
|
3183
|
+
// W-mrcifup2000c218a: this condition is now caught EARLIER, at allocation
|
|
3184
|
+
// time, by `checkLiveCheckoutStaleBase` in the dispatch-existing-pending
|
|
3185
|
+
// loop — a positive hit there leaves the WI pending (`_pendingReason:
|
|
3186
|
+
// 'live_checkout_stale_base'`, no retry consumed, no failure) and this WI
|
|
3187
|
+
// never reaches spawnAgent. This in-spawn refusal is now the EXCEPTION path
|
|
3188
|
+
// (the race window between that probe and the checkout below — e.g. the
|
|
3189
|
+
// base was pushed-then-re-diverged between allocation and spawn), not the
|
|
3190
|
+
// common path; it still fails non-retryably because a stale base found
|
|
3191
|
+
// HERE is still deterministic and un-recoverable by a bare retry.
|
|
3016
3192
|
if (_liveResult && _liveResult.ok === false && _liveResult.reason === 'stale-main') {
|
|
3017
3193
|
const _staleMainRef = typeof _liveResult.mainRef === 'string' && _liveResult.mainRef ? _liveResult.mainRef : 'main';
|
|
3018
3194
|
const _staleAhead = Number.isFinite(_liveResult.ahead) ? _liveResult.ahead : 0;
|
|
@@ -3758,12 +3934,26 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
3758
3934
|
depMergeFailed = true;
|
|
3759
3935
|
}
|
|
3760
3936
|
const fetchResults = await Promise.allSettled(
|
|
3761
|
-
fetchable.map(({ branch: depBranch }) =>
|
|
3937
|
+
fetchable.map(({ branch: depBranch, remote: depRemote }) => {
|
|
3762
3938
|
// runAdoGit for ADO projects auto-retries once on auth failure
|
|
3763
3939
|
// with a refreshed bearer token (handles mid-dispatch expiry).
|
|
3764
3940
|
// For non-ADO projects it's a thin pass-through to shellSafeGit.
|
|
3765
|
-
|
|
3766
|
-
|
|
3941
|
+
// W-mrchrfe100067d3a — depRemote resolves to a non-'origin'
|
|
3942
|
+
// name only when this dep's PR is scoped to one of the
|
|
3943
|
+
// project's allowlisted additionalRepoScopes AND a local git
|
|
3944
|
+
// remote actually maps to it. In that case fetch with an
|
|
3945
|
+
// explicit refspec that writes into refs/remotes/origin/* —
|
|
3946
|
+
// an aliased local ref path — so every downstream merge-base /
|
|
3947
|
+
// merge-tree / merge step that reads `origin/<depBranch>`
|
|
3948
|
+
// (below, and in pruneAncestorDeps / preflightMergeSimulation)
|
|
3949
|
+
// keeps working unchanged, regardless of which real remote the
|
|
3950
|
+
// branch actually lives on.
|
|
3951
|
+
const remote = depRemote || 'origin';
|
|
3952
|
+
const fetchArgs = remote === 'origin'
|
|
3953
|
+
? ['fetch', 'origin', depBranch]
|
|
3954
|
+
: ['fetch', remote, `+${depBranch}:refs/remotes/origin/${depBranch}`];
|
|
3955
|
+
return adoGitAuth.runAdoGit(project, fetchArgs, { ..._gitOpts, cwd: rootDir }).then(() => depBranch);
|
|
3956
|
+
})
|
|
3767
3957
|
);
|
|
3768
3958
|
const hasFetchFailures = fetchResults.some(r => r.status === 'rejected');
|
|
3769
3959
|
const allPrsForFetch = hasFetchFailures ? shared.getProjects(config).reduce((acc, p) => acc.concat(safeJsonArr(shared.projectPrPath(p))), []) : [];
|
|
@@ -3773,6 +3963,7 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
3773
3963
|
if (fetchResults[i].status === 'rejected') {
|
|
3774
3964
|
const failedBranch = fetchable[i].branch;
|
|
3775
3965
|
const failedPrId = fetchable[i].prId;
|
|
3966
|
+
const failedRemote = fetchable[i].remote || 'origin';
|
|
3776
3967
|
const errMsg = fetchResults[i].reason?.message || '';
|
|
3777
3968
|
const pr = allPrsForFetch.find(p => p.id === failedPrId);
|
|
3778
3969
|
if (pr && (pr.status === PR_STATUS.MERGED || pr.status === PR_STATUS.CLOSED)) {
|
|
@@ -3783,10 +3974,15 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
3783
3974
|
if (errMsg.includes('couldn\'t find remote ref') || errMsg.includes('not found in upstream')) {
|
|
3784
3975
|
try {
|
|
3785
3976
|
await shared.shellSafeGit(['rev-parse', '--verify', `refs/heads/${failedBranch}`], { ..._gitOpts, cwd: rootDir });
|
|
3786
|
-
// Branch exists locally — push it to
|
|
3787
|
-
log('info', `Dependency ${failedBranch} exists locally but not on remote — pushing to
|
|
3788
|
-
await shared.shellSafeGit(['push',
|
|
3789
|
-
|
|
3977
|
+
// Branch exists locally — push it to its resolved remote
|
|
3978
|
+
log('info', `Dependency ${failedBranch} exists locally but not on remote — pushing to ${failedRemote}`);
|
|
3979
|
+
await shared.shellSafeGit(['push', failedRemote, failedBranch], { ..._gitOpts, cwd: rootDir, timeout: 60000 });
|
|
3980
|
+
// Keep the refs/remotes/origin/* alias in sync with the
|
|
3981
|
+
// local branch so downstream `origin/<branch>` reads
|
|
3982
|
+
// (merge-base/merge-tree/merge) resolve correctly even
|
|
3983
|
+
// though the push landed on a non-origin remote.
|
|
3984
|
+
await shared.shellSafeGit(['update-ref', `refs/remotes/origin/${failedBranch}`, `refs/heads/${failedBranch}`], { ..._gitOpts, cwd: rootDir });
|
|
3985
|
+
log('info', `Successfully pushed local-only dependency branch ${failedBranch} to ${failedRemote}`);
|
|
3790
3986
|
recoveredBranches.add(failedBranch);
|
|
3791
3987
|
continue;
|
|
3792
3988
|
} catch (localErr) {
|
|
@@ -6934,6 +7130,72 @@ function isBuildFixIneffectiveSuppressed(pr) {
|
|
|
6934
7130
|
return true;
|
|
6935
7131
|
}
|
|
6936
7132
|
|
|
7133
|
+
// #764 — PR-level circuit breaker for the WORKTREE_PREFLIGHT
|
|
7134
|
+
// branch-held-externally failure class. `spawnAgent`'s
|
|
7135
|
+
// `_failBranchHeldByExternalWorktree` stamps `_worktreeHeldPaused` on the
|
|
7136
|
+
// live PR record (via `recordWorktreeHeldPause`) whenever a PR-sourced
|
|
7137
|
+
// dispatch (review, re-review, human-feedback fix, review-feedback fix,
|
|
7138
|
+
// build-fix, merge-conflict fix — all of which create a worktree on the PR's
|
|
7139
|
+
// branch) hits this failure. Without this gate, `discoverFromPrs` sees the
|
|
7140
|
+
// PR is still `pendingFix` (or has coalesced feedback) every tick and
|
|
7141
|
+
// re-dispatches a brand-new work item against the identical unresolved
|
|
7142
|
+
// cause, forever (live repro: ado:office/office/1js#5429466, 7 wasted
|
|
7143
|
+
// dispatches in ~4.5h — see issue #764).
|
|
7144
|
+
//
|
|
7145
|
+
// Unlike the no-op-fix pauses above, this one performs a cheap live re-check
|
|
7146
|
+
// (a single `git worktree list --porcelain` against the project root) before
|
|
7147
|
+
// deciding to keep suppressing: the whole point of the breaker is to avoid a
|
|
7148
|
+
// full wasted agent dispatch, and one `git worktree list` call is orders of
|
|
7149
|
+
// magnitude cheaper than that. When the holder no longer holds the paused
|
|
7150
|
+
// branch, the pause auto-clears (no manual step needed in the common case,
|
|
7151
|
+
// per the issue's proposed fix) and dispatch is allowed to proceed. Any
|
|
7152
|
+
// failure re-checking live state is treated conservatively (stays paused) —
|
|
7153
|
+
// a false "still held" merely costs one more skipped tick, whereas a false
|
|
7154
|
+
// "released" would resume the exact dispatch storm this breaker exists to
|
|
7155
|
+
// stop.
|
|
7156
|
+
async function isWorktreeHeldPauseSuppressed(pr, project) {
|
|
7157
|
+
const record = pr?._worktreeHeldPaused;
|
|
7158
|
+
if (!record || typeof record !== 'object') return false;
|
|
7159
|
+
let stillHeld = true;
|
|
7160
|
+
try {
|
|
7161
|
+
const rootDir = shared.resolveProjectRootDir(project?.localPath, MINIONS_DIR);
|
|
7162
|
+
const out = await shared.shellSafeGit(['worktree', 'list', '--porcelain'], { cwd: rootDir, timeout: 10000 });
|
|
7163
|
+
const list = shared.parseWorktreePorcelain(out);
|
|
7164
|
+
// Compare via realpath, not a bare path.resolve() string match: git
|
|
7165
|
+
// resolves `worktree list` entries to their real (subst/junction-free)
|
|
7166
|
+
// filesystem path, but `record.holderPath` may have been captured from a
|
|
7167
|
+
// substituted drive letter (common on Windows CI runners that `subst` a
|
|
7168
|
+
// drive for TEMP) or a symlinked path. A raw string comparison then
|
|
7169
|
+
// false-mismatches two identical directories, wrongly auto-clearing the
|
|
7170
|
+
// pause and letting the exact dispatch storm this breaker exists to stop
|
|
7171
|
+
// resume (#764 regression — see W-mrdiq0jp000bb541).
|
|
7172
|
+
const holderReal = record.holderPath ? shared.realPathForComparison(record.holderPath) : null;
|
|
7173
|
+
const holderEntry = holderReal
|
|
7174
|
+
? list.find(w => shared.realPathForComparison(w.path) === holderReal)
|
|
7175
|
+
: null;
|
|
7176
|
+
stillHeld = !!holderEntry && holderEntry.branch === record.branch;
|
|
7177
|
+
} catch (e) {
|
|
7178
|
+
log('warn', `PR ${pr.id}: worktree-held pause live re-check failed (${e.message}) — keeping automation paused`);
|
|
7179
|
+
}
|
|
7180
|
+
if (!stillHeld) {
|
|
7181
|
+
try {
|
|
7182
|
+
mutatePullRequests(projectPrPath(project), (prs) => {
|
|
7183
|
+
if (!Array.isArray(prs)) return prs;
|
|
7184
|
+
const target = shared.findPrRecord(prs, pr, project);
|
|
7185
|
+
if (!target) return prs;
|
|
7186
|
+
clearWorktreeHeldPause(target);
|
|
7187
|
+
return prs;
|
|
7188
|
+
});
|
|
7189
|
+
log('info', `PR ${pr.id}: worktree-held pause auto-cleared — ${record.holderPath} no longer holds branch ${record.branch}`);
|
|
7190
|
+
} catch (e) {
|
|
7191
|
+
log('warn', `PR ${pr.id}: failed to auto-clear worktree-held pause: ${e.message}`);
|
|
7192
|
+
}
|
|
7193
|
+
return false;
|
|
7194
|
+
}
|
|
7195
|
+
log('warn', `PR ${pr.id}: suppressing PR-driven review/fix automation — branch ${record.branch} still held ${record.isProjectRoot ? 'by the project root' : 'by an external worktree'} at ${record.holderPath}; waiting for the holder to release it`);
|
|
7196
|
+
return true;
|
|
7197
|
+
}
|
|
7198
|
+
|
|
6937
7199
|
const PR_PENDING_MISSING_BRANCH = shared.PR_PENDING_REASON.MISSING_BRANCH;
|
|
6938
7200
|
|
|
6939
7201
|
function normalizePrBranch(value) {
|
|
@@ -7324,6 +7586,15 @@ async function discoverFromPrs(config, project) {
|
|
|
7324
7586
|
_logPrDispatchSkipOnce(pr, 'not-auto-managed');
|
|
7325
7587
|
continue;
|
|
7326
7588
|
}
|
|
7589
|
+
// #764 — PR-level circuit breaker: a prior dispatch against this PR's
|
|
7590
|
+
// branch hit WORKTREE_PREFLIGHT because the branch is held externally
|
|
7591
|
+
// (project root or an unreachable sibling worktree). Every dispatch path
|
|
7592
|
+
// below (review, re-review, human-feedback fix, review-feedback fix,
|
|
7593
|
+
// build-fix, merge-conflict fix) creates a worktree on the SAME branch,
|
|
7594
|
+
// so skip the whole PR for this tick rather than re-litigating the same
|
|
7595
|
+
// doomed preflight per-cause. Auto-clears itself once the holder
|
|
7596
|
+
// releases the branch — see isWorktreeHeldPauseSuppressed.
|
|
7597
|
+
if (await isWorktreeHeldPauseSuppressed(pr, project)) continue;
|
|
7327
7598
|
|
|
7328
7599
|
const prNumber = shared.getPrNumber(pr);
|
|
7329
7600
|
// Use reviewStatus as single source of truth (synced from ADO/GitHub votes)
|
|
@@ -9653,7 +9924,7 @@ function sweepStaleArchivedPrdBackups(prdDir, prdArchiveDir) {
|
|
|
9653
9924
|
async function discoverWork(config) {
|
|
9654
9925
|
resetClaimedAgents(); // Reset per-tick agent claims for fair distribution
|
|
9655
9926
|
const projects = getProjects(config);
|
|
9656
|
-
let
|
|
9927
|
+
let persistedFixes = 0, persistedReviews = 0, persistedOther = 0;
|
|
9657
9928
|
|
|
9658
9929
|
// Side-effect passes: materialize plans and design docs into work-items.json
|
|
9659
9930
|
// These write to project work queues — picked up by discoverFromWorkItems below.
|
|
@@ -9666,19 +9937,109 @@ async function discoverWork(config) {
|
|
|
9666
9937
|
materializePlansAsWorkItems(config);
|
|
9667
9938
|
}
|
|
9668
9939
|
|
|
9940
|
+
const evalConcurrency = resolvePreDispatchEvalConcurrency(config);
|
|
9941
|
+
|
|
9942
|
+
// #768 — persist each batch of discovered items to the dispatch queue as
|
|
9943
|
+
// soon as it is found, instead of accumulating everything across every
|
|
9944
|
+
// project/source into local arrays that are only persisted once this whole
|
|
9945
|
+
// function returns. discoverWork() is invoked inside withTickTimeout(...)
|
|
9946
|
+
// AND the tick's outer hard timeout (TICK_TIMEOUT_MS / tickHardTimeoutMs) —
|
|
9947
|
+
// neither one actually cancels the in-flight promise, it just stops
|
|
9948
|
+
// awaiting it. A pathologically slow (or genuinely hung) per-project
|
|
9949
|
+
// git/worktree operation later in the sequential `for (const project of
|
|
9950
|
+
// projects)` loop below leaves this whole async function permanently
|
|
9951
|
+
// unresolved, so anything only collected into a local array and persisted
|
|
9952
|
+
// at the very end (the pre-fix allFixes/allReviews/allWorkItems) never
|
|
9953
|
+
// reaches dispatch.json — silently discarding valid work already
|
|
9954
|
+
// discovered for unrelated, healthy projects processed earlier in the same
|
|
9955
|
+
// loop. Persisting per batch means a healthy project's discoveries land in
|
|
9956
|
+
// dispatch.json immediately, before the loop ever reaches a slow project.
|
|
9957
|
+
async function persistBatch(items) {
|
|
9958
|
+
if (!items || items.length === 0) return;
|
|
9959
|
+
// Gate reviews and fixes: only when at max concurrency — idle agents should pick up reviews
|
|
9960
|
+
// even if implement items are in progress (implements get priority via sort order, not by blocking)
|
|
9961
|
+
const hasIncompleteImplements = queries.getWorkItems(config).some(i =>
|
|
9962
|
+
['queued', 'pending', 'dispatched'].includes(i.status) && (i.type || '').startsWith('implement')
|
|
9963
|
+
);
|
|
9964
|
+
let toDispatch = items;
|
|
9965
|
+
if (hasIncompleteImplements) {
|
|
9966
|
+
const activeCount = (getDispatch().active || []).length;
|
|
9967
|
+
const maxConcurrent = resolveMaxConcurrent(config);
|
|
9968
|
+
const freeSlots = Math.max(0, maxConcurrent - activeCount);
|
|
9969
|
+
if (freeSlots === 0) {
|
|
9970
|
+
// PR review feedback (#771): the pre-fix code only ever gated
|
|
9971
|
+
// allFixes/allReviews, which were populated exclusively from
|
|
9972
|
+
// discoverFromPrs() output (`meta.source === 'pr'`). FIX/REVIEW-typed
|
|
9973
|
+
// items surfaced by discoverFromWorkItems (`meta.source ===
|
|
9974
|
+
// 'work-item'`), discoverCentralWorkItems (`'central-work-item'` /
|
|
9975
|
+
// `'central-work-item-fanout'`), or discoverMeetingWork (`'meeting'`)
|
|
9976
|
+
// were NEVER gated — allWorkItems/centralWork bypassed the gate
|
|
9977
|
+
// entirely. persistBatch() now runs on every source's batch, so the
|
|
9978
|
+
// type-only filter below would silently start gating those
|
|
9979
|
+
// previously-ungated sources too. Scope the gate back down to
|
|
9980
|
+
// `meta.source === 'pr'` to preserve the original behavior exactly.
|
|
9981
|
+
const isGateEligible = (w) => w.meta?.source === 'pr' && (w.type === WORK_TYPE.FIX || w.type === WORK_TYPE.REVIEW);
|
|
9982
|
+
const gated = items.filter(isGateEligible);
|
|
9983
|
+
if (gated.length > 0) {
|
|
9984
|
+
log('info', `Gating ${gated.length} fixes/reviews — implement items in progress and no free slots`);
|
|
9985
|
+
}
|
|
9986
|
+
toDispatch = items.filter(w => !isGateEligible(w));
|
|
9987
|
+
}
|
|
9988
|
+
}
|
|
9989
|
+
if (toDispatch.length === 0) return;
|
|
9990
|
+
|
|
9991
|
+
// W-mq9acoo800177bcb — bounded-concurrency pre-dispatch eval.
|
|
9992
|
+
// The validator inside addToDispatchWithValidation runs a ~25-30s LLM
|
|
9993
|
+
// call per item; an approved 11-item PRD used to take ~5 min to queue
|
|
9994
|
+
// because we awaited each item in series. Process items in chunks of
|
|
9995
|
+
// `preDispatchEvalConcurrency` (default 6, clamp 1-20) via
|
|
9996
|
+
// Promise.allSettled so a rejected branch in one item still fails open
|
|
9997
|
+
// and the rest of the chunk continues. Iteration order within a chunk
|
|
9998
|
+
// is irrelevant: addToDispatch's dedup gates are mutateDispatch-locked,
|
|
9999
|
+
// and the only per-item side effect (`clearPendingHumanFeedbackFlag`)
|
|
10000
|
+
// is an idempotent flag-clear on a single PR record.
|
|
10001
|
+
for (let i = 0; i < toDispatch.length; i += evalConcurrency) {
|
|
10002
|
+
const chunk = toDispatch.slice(i, i + evalConcurrency);
|
|
10003
|
+
await Promise.allSettled(chunk.map(async (item) => {
|
|
10004
|
+
try {
|
|
10005
|
+
await addToDispatchWithValidation(item, { config });
|
|
10006
|
+
} catch (e) {
|
|
10007
|
+
// Fail-open contract preserved: any validator throw here means the
|
|
10008
|
+
// item is dropped from this discovery batch rather than wedging the
|
|
10009
|
+
// whole chunk. The item stays `pending` in work-items.json and the
|
|
10010
|
+
// next tick re-discovers it.
|
|
10011
|
+
log('warn', `pre-dispatch-eval: addToDispatchWithValidation threw for ${item?.meta?.item?.id || item?.id || 'unknown'}: ${e?.message || e}`);
|
|
10012
|
+
}
|
|
10013
|
+
// W-mph8xt88000ke0fc — Cooldowns are stamped by addToDispatch ONLY on
|
|
10014
|
+
// successful append (post-dedup). Stamping unconditionally here used to
|
|
10015
|
+
// leave orphan cooldowns for items collapsed by prDedupeKey / workItem-id
|
|
10016
|
+
// / dispatchKey dedup or routed to the pre-dispatch review queue.
|
|
10017
|
+
if (item.meta?.source === 'pr-human-feedback') {
|
|
10018
|
+
clearPendingHumanFeedbackFlag(item.meta.project, item.meta.pr?.id);
|
|
10019
|
+
}
|
|
10020
|
+
}));
|
|
10021
|
+
}
|
|
10022
|
+
for (const w of toDispatch) {
|
|
10023
|
+
if (w.type === WORK_TYPE.FIX) persistedFixes++;
|
|
10024
|
+
else if (w.type === WORK_TYPE.REVIEW) persistedReviews++;
|
|
10025
|
+
else persistedOther++;
|
|
10026
|
+
}
|
|
10027
|
+
}
|
|
10028
|
+
|
|
9669
10029
|
for (const project of projects) {
|
|
9670
10030
|
const root = project.localPath ? path.resolve(project.localPath) : null;
|
|
9671
10031
|
if (!root || !fs.existsSync(root)) continue;
|
|
9672
10032
|
|
|
10033
|
+
const projectWork = [];
|
|
10034
|
+
|
|
9673
10035
|
// Source 1: Pull Requests → fixes, reviews, build-test
|
|
9674
10036
|
// Gated by config.engine?.prDiscoveryEnabled !== false (P-d6f0a2b5) —
|
|
9675
10037
|
// per-project project.workSources.pullRequests.enabled still composes
|
|
9676
10038
|
// inside discoverFromPrs; a `false` at either level skips the call.
|
|
9677
10039
|
if (config.engine?.prDiscoveryEnabled !== false) {
|
|
9678
10040
|
const prWork = await discoverFromPrs(config, project);
|
|
9679
|
-
|
|
9680
|
-
|
|
9681
|
-
allWorkItems.push(...prWork.filter(w => w.type === WORK_TYPE.TEST));
|
|
10041
|
+
projectWork.push(...prWork.filter(w =>
|
|
10042
|
+
w.type === WORK_TYPE.FIX || w.type === WORK_TYPE.REVIEW || w.type === WORK_TYPE.TEST));
|
|
9682
10043
|
}
|
|
9683
10044
|
|
|
9684
10045
|
// Side-effect: specs → work items (picked up below)
|
|
@@ -9689,8 +10050,14 @@ async function discoverWork(config) {
|
|
|
9689
10050
|
// per-project project.workSources.workItems.enabled still composes
|
|
9690
10051
|
// inside discoverFromWorkItems; a `false` at either level skips the call.
|
|
9691
10052
|
if (config.engine?.workItemsDiscoveryEnabled !== false) {
|
|
9692
|
-
|
|
10053
|
+
projectWork.push(...discoverFromWorkItems(config, project));
|
|
9693
10054
|
}
|
|
10055
|
+
|
|
10056
|
+
// #768 — persist this project's discovered items right now rather than
|
|
10057
|
+
// waiting for every remaining project in the loop to also be walked. See
|
|
10058
|
+
// the persistBatch() comment above for why this matters on a tick
|
|
10059
|
+
// hard-timeout.
|
|
10060
|
+
await persistBatch(projectWork);
|
|
9694
10061
|
}
|
|
9695
10062
|
|
|
9696
10063
|
// Source 2: Minions-level PRD → implements (multi-project, called once outside project loop)
|
|
@@ -9704,6 +10071,8 @@ async function discoverWork(config) {
|
|
|
9704
10071
|
if (config.engine?.centralWorkDiscoveryEnabled !== false) {
|
|
9705
10072
|
centralWork = discoverCentralWorkItems(config);
|
|
9706
10073
|
}
|
|
10074
|
+
// #768 — persist immediately, same rationale as the per-project loop above.
|
|
10075
|
+
await persistBatch([...centralWork]);
|
|
9707
10076
|
|
|
9708
10077
|
// Scheduled tasks (cron-style recurring work)
|
|
9709
10078
|
// Gated by config.engine?.scheduledWorkDiscoveryEnabled !== false (P-d6f0a2b5).
|
|
@@ -9772,7 +10141,8 @@ async function discoverWork(config) {
|
|
|
9772
10141
|
try {
|
|
9773
10142
|
const { discoverMeetingWork } = require('./engine/meeting');
|
|
9774
10143
|
const meetingWork = discoverMeetingWork(config);
|
|
9775
|
-
|
|
10144
|
+
// #768 — persist immediately, same rationale as the per-project loop above.
|
|
10145
|
+
await persistBatch(meetingWork);
|
|
9776
10146
|
} catch (e) { log('warn', 'discover meeting work: ' + e.message); }
|
|
9777
10147
|
|
|
9778
10148
|
// Pipeline orchestration — check stage completions and start ready stages
|
|
@@ -9836,67 +10206,12 @@ async function discoverWork(config) {
|
|
|
9836
10206
|
} catch (e) { log('warn', 'plan completion sweep: ' + e.message); }
|
|
9837
10207
|
}
|
|
9838
10208
|
|
|
9839
|
-
|
|
9840
|
-
|
|
9841
|
-
|
|
9842
|
-
['queued', 'pending', 'dispatched'].includes(i.status) && (i.type || '').startsWith('implement')
|
|
9843
|
-
);
|
|
9844
|
-
if (hasIncompleteImplements) {
|
|
9845
|
-
const activeCount = (getDispatch().active || []).length;
|
|
9846
|
-
const maxConcurrent = resolveMaxConcurrent(config);
|
|
9847
|
-
const freeSlots = Math.max(0, maxConcurrent - activeCount);
|
|
9848
|
-
if (freeSlots === 0) {
|
|
9849
|
-
if (allReviews.length > 0) {
|
|
9850
|
-
log('info', `Gating ${allReviews.length} reviews — implement items in progress and no free slots`);
|
|
9851
|
-
allReviews = [];
|
|
9852
|
-
}
|
|
9853
|
-
if (allFixes.length > 0) {
|
|
9854
|
-
log('info', `Gating ${allFixes.length} fixes — implement items in progress and no free slots`);
|
|
9855
|
-
allFixes = [];
|
|
9856
|
-
}
|
|
9857
|
-
}
|
|
9858
|
-
}
|
|
9859
|
-
|
|
9860
|
-
const allWork = [...allFixes, ...allReviews, ...allWorkItems, ...centralWork];
|
|
9861
|
-
|
|
9862
|
-
// W-mq9acoo800177bcb — bounded-concurrency pre-dispatch eval.
|
|
9863
|
-
// The validator inside addToDispatchWithValidation runs a ~25-30s LLM
|
|
9864
|
-
// call per item; an approved 11-item PRD used to take ~5 min to queue
|
|
9865
|
-
// because we awaited each item in series. Process items in chunks of
|
|
9866
|
-
// `preDispatchEvalConcurrency` (default 6, clamp 1-20) via
|
|
9867
|
-
// Promise.allSettled so a rejected branch in one item still fails open
|
|
9868
|
-
// and the rest of the chunk continues. Iteration order within a chunk
|
|
9869
|
-
// is irrelevant: addToDispatch's dedup gates are mutateDispatch-locked,
|
|
9870
|
-
// and the only per-item side effect (`clearPendingHumanFeedbackFlag`)
|
|
9871
|
-
// is an idempotent flag-clear on a single PR record.
|
|
9872
|
-
const evalConcurrency = resolvePreDispatchEvalConcurrency(config);
|
|
9873
|
-
for (let i = 0; i < allWork.length; i += evalConcurrency) {
|
|
9874
|
-
const chunk = allWork.slice(i, i + evalConcurrency);
|
|
9875
|
-
await Promise.allSettled(chunk.map(async (item) => {
|
|
9876
|
-
try {
|
|
9877
|
-
await addToDispatchWithValidation(item, { config });
|
|
9878
|
-
} catch (e) {
|
|
9879
|
-
// Fail-open contract preserved: any validator throw here means the
|
|
9880
|
-
// item is dropped from this discovery tick rather than wedging the
|
|
9881
|
-
// whole chunk. The item stays `pending` in work-items.json and the
|
|
9882
|
-
// next tick re-discovers it.
|
|
9883
|
-
log('warn', `pre-dispatch-eval: addToDispatchWithValidation threw for ${item?.meta?.item?.id || item?.id || 'unknown'}: ${e?.message || e}`);
|
|
9884
|
-
}
|
|
9885
|
-
// W-mph8xt88000ke0fc — Cooldowns are stamped by addToDispatch ONLY on
|
|
9886
|
-
// successful append (post-dedup). Stamping unconditionally here used to
|
|
9887
|
-
// leave orphan cooldowns for items collapsed by prDedupeKey / workItem-id
|
|
9888
|
-
// / dispatchKey dedup or routed to the pre-dispatch review queue.
|
|
9889
|
-
if (item.meta?.source === 'pr-human-feedback') {
|
|
9890
|
-
clearPendingHumanFeedbackFlag(item.meta.project, item.meta.pr?.id);
|
|
9891
|
-
}
|
|
9892
|
-
}));
|
|
9893
|
-
}
|
|
9894
|
-
|
|
9895
|
-
if (allWork.length > 0) {
|
|
9896
|
-
log('info', `Discovered ${allWork.length} new work items: ${allFixes.length} fixes, ${allReviews.length} reviews, ${allWorkItems.length} work-items`);
|
|
10209
|
+
const totalPersisted = persistedFixes + persistedReviews + persistedOther;
|
|
10210
|
+
if (totalPersisted > 0) {
|
|
10211
|
+
log('info', `Discovered ${totalPersisted} new work items: ${persistedFixes} fixes, ${persistedReviews} reviews, ${persistedOther} work-items`);
|
|
9897
10212
|
}
|
|
9898
10213
|
|
|
9899
|
-
return
|
|
10214
|
+
return totalPersisted;
|
|
9900
10215
|
}
|
|
9901
10216
|
|
|
9902
10217
|
function getPendingDispatchRoutingOpts(item) {
|
|
@@ -10890,6 +11205,123 @@ async function tickInner() {
|
|
|
10890
11205
|
continue;
|
|
10891
11206
|
}
|
|
10892
11207
|
}
|
|
11208
|
+
// W-mrcifup2000c218a: Live-checkout dirty / stale-base PRE-DISPATCH checks.
|
|
11209
|
+
// Mirrors the live_checkout_busy gate directly above: dirty-tree and
|
|
11210
|
+
// stale-base are ENVIRONMENTAL blocking conditions on the operator's live
|
|
11211
|
+
// checkout, not failures of the task itself, so they get detected here at
|
|
11212
|
+
// ALLOCATION time (before spawnAgent runs) and leave the item pending with
|
|
11213
|
+
// a _pendingReason stamp — no retry consumed, no terminal failure — same
|
|
11214
|
+
// as the busy gate. Previously these were only caught INSIDE spawnAgent
|
|
11215
|
+
// (prepareLiveCheckout's dirty two-strike guard and STALE-BASE GUARD),
|
|
11216
|
+
// which meant every hit burned a dispatch attempt (dirty) or failed
|
|
11217
|
+
// non-retryably on the first hit (stale-base), requiring manual cleanup
|
|
11218
|
+
// even after the operator fixed the underlying condition. The in-spawn
|
|
11219
|
+
// guards (engine.js ~line 2617 dirty, ~line 3017 stale-base) remain in
|
|
11220
|
+
// place as a defense-in-depth fallback for the race window between this
|
|
11221
|
+
// probe and the actual git operations inside spawnAgent — they should now
|
|
11222
|
+
// fire rarely, as the exception path rather than the common path.
|
|
11223
|
+
if (
|
|
11224
|
+
itemProjName
|
|
11225
|
+
&& !READ_ONLY_ROOT_TASK_TYPES.has(item.type)
|
|
11226
|
+
) {
|
|
11227
|
+
const _precheckProjCfg = shared.findProjectByName(shared.getProjects(config), itemProjName);
|
|
11228
|
+
if (_precheckProjCfg && shared.resolveCheckoutMode(_precheckProjCfg, item.type) === 'live' && _precheckProjCfg.localPath) {
|
|
11229
|
+
const _precheckLocalPath = path.resolve(String(_precheckProjCfg.localPath));
|
|
11230
|
+
const _liveCheckoutPrecheck = require('./engine/live-checkout');
|
|
11231
|
+
const _persistPendingReason = (reason) => {
|
|
11232
|
+
item._pendingReason = reason;
|
|
11233
|
+
try {
|
|
11234
|
+
if (_isTickStale(myGeneration)) return;
|
|
11235
|
+
mutateDispatch((dp) => {
|
|
11236
|
+
const p = (dp.pending || []).find(d => d.id === item.id);
|
|
11237
|
+
if (p) {
|
|
11238
|
+
if (reason) p._pendingReason = reason;
|
|
11239
|
+
else delete p._pendingReason;
|
|
11240
|
+
}
|
|
11241
|
+
return dp;
|
|
11242
|
+
});
|
|
11243
|
+
} catch (e) { log('warn', `Persist live-checkout precheck pendingReason for ${item.id} failed: ${e.message}`); }
|
|
11244
|
+
};
|
|
11245
|
+
let _dirtyResult = null;
|
|
11246
|
+
try {
|
|
11247
|
+
_dirtyResult = await _liveCheckoutPrecheck.checkLiveCheckoutDirty({
|
|
11248
|
+
localPath: _precheckLocalPath,
|
|
11249
|
+
skipDirtyCheck: !!_precheckProjCfg.skipLiveCheckoutDirtyCheck,
|
|
11250
|
+
});
|
|
11251
|
+
} catch (e) { log('warn', `live-checkout precheck (dirty) failed for ${item.id}: ${e.message}`); }
|
|
11252
|
+
if (_dirtyResult && _dirtyResult.dirty) {
|
|
11253
|
+
_persistPendingReason('live_checkout_dirty');
|
|
11254
|
+
try {
|
|
11255
|
+
const _alertBody = [
|
|
11256
|
+
'# Live-checkout pending: dirty worktree (pre-dispatch check)',
|
|
11257
|
+
'',
|
|
11258
|
+
`**Project:** ${_precheckProjCfg.name || itemProjName}`,
|
|
11259
|
+
`**Local path:** ${_precheckLocalPath}`,
|
|
11260
|
+
`**Work item:** ${item.id}`,
|
|
11261
|
+
...(_dirtyResult.branchInfo ? ['', `**Current checkout:** \`${_dirtyResult.branchInfo}\``] : []),
|
|
11262
|
+
'',
|
|
11263
|
+
`The engine detected uncommitted changes in \`${_precheckLocalPath}\` before dispatching ${item.id}. Live-checkout mode runs in your project directly — the engine never \`git reset\` or \`git clean\` your tree. The work item stays pending (no retry consumed, no failure) and will be re-checked automatically once the tree is clean.`,
|
|
11264
|
+
'',
|
|
11265
|
+
'## Dirty files (`git status --porcelain=v1 -b`)',
|
|
11266
|
+
'',
|
|
11267
|
+
'```',
|
|
11268
|
+
...(Array.isArray(_dirtyResult.dirtyFiles) && _dirtyResult.dirtyFiles.length > 0
|
|
11269
|
+
? _liveCheckoutPrecheck.capDirtyFileLines(_dirtyResult.dirtyFiles)
|
|
11270
|
+
: ['(none reported)']),
|
|
11271
|
+
'```',
|
|
11272
|
+
'',
|
|
11273
|
+
'Recovery: commit, stash, or discard the changes in the project checkout — the work item will dispatch automatically on a later tick once the tree is clean.',
|
|
11274
|
+
].join('\n');
|
|
11275
|
+
writeInboxAlert(`live-checkout-dirty-${item.id}`, _alertBody);
|
|
11276
|
+
} catch (e) { log('warn', `live-checkout precheck: writeInboxAlert failed: ${e.message}`); }
|
|
11277
|
+
continue;
|
|
11278
|
+
}
|
|
11279
|
+
// Stale-base only matters when this item would fork a NEW branch off
|
|
11280
|
+
// mainRef — mirror spawnAgent's allowNonMainBase resolution exactly
|
|
11281
|
+
// (W-mr3lunnq000o9f41: PR-targeted / shared-branch continuations opt out).
|
|
11282
|
+
const _itemMeta = item.meta || {};
|
|
11283
|
+
const _precheckAllowNonMainBase = !!(_itemMeta.useExistingBranch || _itemMeta.branchStrategy === 'shared-branch');
|
|
11284
|
+
const _precheckMainRef = sanitizeBranch(shared.resolveMainBranch(_precheckLocalPath, _precheckProjCfg.mainBranch));
|
|
11285
|
+
let _staleResult = null;
|
|
11286
|
+
try {
|
|
11287
|
+
_staleResult = await _liveCheckoutPrecheck.checkLiveCheckoutStaleBase({
|
|
11288
|
+
localPath: _precheckLocalPath,
|
|
11289
|
+
mainRef: _precheckMainRef,
|
|
11290
|
+
allowNonMainBase: _precheckAllowNonMainBase,
|
|
11291
|
+
});
|
|
11292
|
+
} catch (e) { log('warn', `live-checkout precheck (stale-base) failed for ${item.id}: ${e.message}`); }
|
|
11293
|
+
if (_staleResult && _staleResult.stale) {
|
|
11294
|
+
_persistPendingReason('live_checkout_stale_base');
|
|
11295
|
+
try {
|
|
11296
|
+
const _alertBody = [
|
|
11297
|
+
'# Live-checkout pending: local base branch has diverged (pre-dispatch check)',
|
|
11298
|
+
'',
|
|
11299
|
+
`**Project:** ${_precheckProjCfg.name || itemProjName}`,
|
|
11300
|
+
`**Local path:** ${_precheckLocalPath}`,
|
|
11301
|
+
`**Base ref:** ${_staleResult.mainRef}`,
|
|
11302
|
+
`**Local ${_staleResult.mainRef}:** ${_staleResult.localMainSha || '(unknown)'}`,
|
|
11303
|
+
`**origin/${_staleResult.mainRef}:** ${_staleResult.originMainSha || '(unknown)'}`,
|
|
11304
|
+
`**Unpushed commits on local ${_staleResult.mainRef}:** ${_staleResult.ahead}`,
|
|
11305
|
+
`**Work item:** ${item.id}`,
|
|
11306
|
+
'',
|
|
11307
|
+
`The engine detected that local \`${_staleResult.mainRef}\` is ${_staleResult.ahead} commit(s) ahead of \`origin/${_staleResult.mainRef}\` before dispatching ${item.id} — forking a new branch off it would silently inherit those unpushed commits into the new branch and its PR. The work item stays pending (no retry consumed, no failure) and will be re-checked automatically once the base is reconciled.`,
|
|
11308
|
+
'',
|
|
11309
|
+
'## Recovery',
|
|
11310
|
+
'',
|
|
11311
|
+
`Reconcile local \`${_staleResult.mainRef}\` so it matches \`origin/${_staleResult.mainRef}\` (move any real work onto its own branch first, then \`git reset --hard origin/${_staleResult.mainRef}\`). The engine never resets it for you.`,
|
|
11312
|
+
].join('\n');
|
|
11313
|
+
writeInboxAlert(`live-checkout-stale-base-${item.id}`, _alertBody);
|
|
11314
|
+
} catch (e) { log('warn', `live-checkout precheck: writeInboxAlert failed: ${e.message}`); }
|
|
11315
|
+
continue;
|
|
11316
|
+
}
|
|
11317
|
+
// Checks passed — if a prior tick stamped a dirty/stale-base pending
|
|
11318
|
+
// reason on this item, clear it so the dashboard chip doesn't show a
|
|
11319
|
+
// stale condition once the tree is clean / base is reconciled again.
|
|
11320
|
+
if (item._pendingReason === 'live_checkout_dirty' || item._pendingReason === 'live_checkout_stale_base') {
|
|
11321
|
+
_persistPendingReason(null);
|
|
11322
|
+
}
|
|
11323
|
+
}
|
|
11324
|
+
}
|
|
10893
11325
|
if (generalSlots <= 0) continue;
|
|
10894
11326
|
seenPendingIds.add(item.id);
|
|
10895
11327
|
toDispatch.push(item);
|