@yemi33/minions 0.1.2238 → 0.1.2239
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/js/command-center.js +25 -2
- package/dashboard/js/refresh.js +18 -3
- package/dashboard/js/render-other.js +19 -5
- package/dashboard/js/render-plans.js +8 -6
- package/dashboard/js/render-work-items.js +5 -3
- package/dashboard/js/utils.js +23 -13
- package/dashboard.js +2 -2
- package/docs/harness-transparency.md +6 -5
- package/docs/live-checkout-mode.md +37 -7
- package/engine/cli.js +26 -0
- package/engine/comment-classifier.js +1 -1
- package/engine/consolidation.js +119 -24
- package/engine/dispatch-store.js +27 -4
- package/engine/dispatch.js +1 -0
- package/engine/github.js +4 -1
- package/engine/live-checkout.js +292 -10
- package/engine/queries.js +115 -14
- package/engine/shared.js +3 -1
- package/engine/timeout.js +10 -1
- package/engine/watchdog.js +75 -0
- package/engine.js +153 -5
- package/package.json +1 -1
package/engine/queries.js
CHANGED
|
@@ -526,6 +526,61 @@ function _wiIdFromNoteFrontmatter(fm) {
|
|
|
526
526
|
return fm.work_item || fm.wi || fm.sourceItem || null;
|
|
527
527
|
}
|
|
528
528
|
|
|
529
|
+
// Archive notes-by-WI cache. The archive dir accumulates thousands of
|
|
530
|
+
// immutable note files (2000+ and growing); reading + frontmatter-parsing all
|
|
531
|
+
// of them was the single dominant SYNCHRONOUS cost inside getWorkItems()
|
|
532
|
+
// (measured: multiple seconds with a cold OS file cache, a few hundred ms warm)
|
|
533
|
+
// — and getWorkItems runs on a 1 s TTL, so the dashboard re-read the whole
|
|
534
|
+
// archive every few seconds. That blocked the dashboard event loop and starved
|
|
535
|
+
// the Command Center SSE heartbeat into stalls + reconnects. Archive files are
|
|
536
|
+
// written once (consolidation moves inbox → archive and never edits in place),
|
|
537
|
+
// so an mtime-gated cache on the archive DIRECTORY is safe: any add/remove
|
|
538
|
+
// bumps the dir mtime and busts the cache; the TTL is a belt-and-suspenders
|
|
539
|
+
// backstop. Inbox is NOT cached here — it is tiny and churns constantly, so
|
|
540
|
+
// _buildNotesByWiMap scans it fresh every call (see below).
|
|
541
|
+
let _archiveNotesByWiCache = null;
|
|
542
|
+
let _archiveNotesByWiCacheAt = 0;
|
|
543
|
+
let _archiveNotesByWiCacheMtime = null;
|
|
544
|
+
// The dir-mtime gate is the PRIMARY freshness signal (NTFS/POSIX bump the
|
|
545
|
+
// archive dir mtime on every add/remove/rename, so a newly-archived note busts
|
|
546
|
+
// the cache on the next call). The TTL is only a belt-and-suspenders backstop
|
|
547
|
+
// for the rare FS that misses an mtime update — kept long so we don't pay the
|
|
548
|
+
// full multi-thousand-file re-read on a fixed cadence when nothing changed.
|
|
549
|
+
const ARCHIVE_NOTES_BY_WI_TTL = 300000; // 5 min
|
|
550
|
+
|
|
551
|
+
function _buildArchiveNotesByWiMap() {
|
|
552
|
+
let dirMtime = null;
|
|
553
|
+
try { dirMtime = fs.statSync(ARCHIVE_DIR).mtimeMs; } catch { dirMtime = null; }
|
|
554
|
+
const now = Date.now();
|
|
555
|
+
if (_archiveNotesByWiCache
|
|
556
|
+
&& (now - _archiveNotesByWiCacheAt) < ARCHIVE_NOTES_BY_WI_TTL
|
|
557
|
+
&& _archiveNotesByWiCacheMtime === dirMtime) {
|
|
558
|
+
return _archiveNotesByWiCache;
|
|
559
|
+
}
|
|
560
|
+
const out = Object.create(null);
|
|
561
|
+
for (const f of safeReadDir(ARCHIVE_DIR)) {
|
|
562
|
+
if (!f.endsWith('.md')) continue;
|
|
563
|
+
const fm = _parseNoteFrontmatter(safeRead(path.join(ARCHIVE_DIR, f)));
|
|
564
|
+
const wiId = _wiIdFromNoteFrontmatter(fm);
|
|
565
|
+
if (!wiId) continue;
|
|
566
|
+
if (!out[wiId]) out[wiId] = [];
|
|
567
|
+
out[wiId].push('archive:' + f);
|
|
568
|
+
}
|
|
569
|
+
_archiveNotesByWiCache = out;
|
|
570
|
+
_archiveNotesByWiCacheAt = now;
|
|
571
|
+
_archiveNotesByWiCacheMtime = dirMtime;
|
|
572
|
+
return out;
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
// Test/maintenance hook — drop the archive notes cache (mirrors the other
|
|
576
|
+
// invalidate* helpers). Not normally needed: the cache self-heals via the
|
|
577
|
+
// dir-mtime gate + TTL.
|
|
578
|
+
function invalidateArchiveNotesByWiCache() {
|
|
579
|
+
_archiveNotesByWiCache = null;
|
|
580
|
+
_archiveNotesByWiCacheAt = 0;
|
|
581
|
+
_archiveNotesByWiCacheMtime = null;
|
|
582
|
+
}
|
|
583
|
+
|
|
529
584
|
// Build a wiId → [filename, ...] map by scanning the inbox + archive dirs.
|
|
530
585
|
// Archive entries are prefixed with `archive:` so the dashboard renderer can
|
|
531
586
|
// tell them apart (matches the convention already used by _artifacts.notes).
|
|
@@ -537,17 +592,18 @@ function _buildNotesByWiMap() {
|
|
|
537
592
|
if (!out[wiId]) out[wiId] = [];
|
|
538
593
|
if (!out[wiId].includes(token)) out[wiId].push(token);
|
|
539
594
|
};
|
|
595
|
+
// Inbox: tiny and churns constantly — scan fresh every call.
|
|
540
596
|
for (const f of safeReadDir(INBOX_DIR)) {
|
|
541
597
|
if (!f.endsWith('.md')) continue;
|
|
542
598
|
const fm = _parseNoteFrontmatter(safeRead(path.join(INBOX_DIR, f)));
|
|
543
599
|
const wiId = _wiIdFromNoteFrontmatter(fm);
|
|
544
600
|
if (wiId) addNote(wiId, f);
|
|
545
601
|
}
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
602
|
+
// Archive: large + immutable — served from the mtime-gated cache so we don't
|
|
603
|
+
// re-read thousands of files on every getWorkItems() rebuild.
|
|
604
|
+
const archiveMap = _buildArchiveNotesByWiMap();
|
|
605
|
+
for (const wiId in archiveMap) {
|
|
606
|
+
for (const token of archiveMap[wiId]) addNote(wiId, token);
|
|
551
607
|
}
|
|
552
608
|
return out;
|
|
553
609
|
}
|
|
@@ -635,7 +691,11 @@ function getAgentStatus(agentId) {
|
|
|
635
691
|
const staleOrphanTimeout = config.engine?.heartbeatTimeout || ENGINE_DEFAULTS.heartbeatTimeout;
|
|
636
692
|
const staleThresholdMs = staleOrphanTimeout * 2;
|
|
637
693
|
const now = Date.now();
|
|
638
|
-
|
|
694
|
+
// Lean read — this fallback only inspects core fields (dispatched_to,
|
|
695
|
+
// status, dispatched_at, …), never _pr/_artifacts/_notes. Skipping
|
|
696
|
+
// enrichment keeps the per-idle-agent getAgents() roster build off the
|
|
697
|
+
// heavy detail-modal path. (Fix #3 — dashboard event-loop perf.)
|
|
698
|
+
const allItems = getWorkItems(config, { enrich: false });
|
|
639
699
|
const latestInFlight = allItems
|
|
640
700
|
.filter(w => {
|
|
641
701
|
if ((w.dispatched_to || '').toLowerCase() !== String(agentId).toLowerCase()) return false;
|
|
@@ -1711,10 +1771,24 @@ function getKnowledgeBaseIndex() {
|
|
|
1711
1771
|
let _workItemsCache = null;
|
|
1712
1772
|
let _workItemsCacheAt = 0;
|
|
1713
1773
|
let _workItemsCacheMtimes = null;
|
|
1774
|
+
// Lean cache for getWorkItems(config, { enrich: false }) — core fields only,
|
|
1775
|
+
// skipping the PR cross-reference + per-item _artifacts/_notes detail-modal
|
|
1776
|
+
// enrichment (the synchronous KB-scan + agent-dir reads that the detail modal
|
|
1777
|
+
// needs but hot callers don't). getAgentStatus()'s per-idle-agent fallback and
|
|
1778
|
+
// count-only callers use this so the agent roster / CC preamble never pay the
|
|
1779
|
+
// enrichment cost. Held separately so an enrich:false caller can't poison the
|
|
1780
|
+
// enriched cache (and vice-versa) — readAllWorkItems() returns fresh row
|
|
1781
|
+
// objects each call, so the two caches never share mutable references.
|
|
1782
|
+
let _workItemsLeanCache = null;
|
|
1783
|
+
let _workItemsLeanCacheAt = 0;
|
|
1784
|
+
let _workItemsLeanCacheMtimes = null;
|
|
1714
1785
|
function invalidateWorkItemsCache() {
|
|
1715
1786
|
_workItemsCache = null;
|
|
1716
1787
|
_workItemsCacheAt = 0;
|
|
1717
1788
|
_workItemsCacheMtimes = null;
|
|
1789
|
+
_workItemsLeanCache = null;
|
|
1790
|
+
_workItemsLeanCacheAt = 0;
|
|
1791
|
+
_workItemsLeanCacheMtimes = null;
|
|
1718
1792
|
}
|
|
1719
1793
|
|
|
1720
1794
|
function _workItemsFilePaths(config) {
|
|
@@ -1744,14 +1818,29 @@ function _workItemsMtimesDiffer(prev, curr) {
|
|
|
1744
1818
|
return false;
|
|
1745
1819
|
}
|
|
1746
1820
|
|
|
1747
|
-
|
|
1821
|
+
// getWorkItems(config, { enrich })
|
|
1822
|
+
// enrich (default true): also cross-reference PRs and populate the
|
|
1823
|
+
// detail-modal _artifacts/_notes fields. The /api/work-items list payload
|
|
1824
|
+
// needs these (PR badges + note chips).
|
|
1825
|
+
// enrich: false — core fields only (status, dispatched_to, depends_on, …)
|
|
1826
|
+
// plus the cheap dispatch cross-reference. Skips the synchronous PR
|
|
1827
|
+
// cross-ref and per-item KB/agent-dir enrichment. Use from hot callers that
|
|
1828
|
+
// never read _artifacts/_notes (getAgentStatus fallback, count-only callers)
|
|
1829
|
+
// so the agent roster / CC preamble don't pay the enrichment cost on every
|
|
1830
|
+
// cold rebuild.
|
|
1831
|
+
function getWorkItems(config, opts) {
|
|
1832
|
+
const enrich = !(opts && opts.enrich === false);
|
|
1748
1833
|
const now = Date.now();
|
|
1749
1834
|
config = config || getConfig();
|
|
1750
|
-
if (
|
|
1835
|
+
if (enrich) {
|
|
1836
|
+
if (_workItemsCache && (now - _workItemsCacheAt) < 1000) {
|
|
1837
|
+
const currMtimes = _snapshotWorkItemsMtimes(config);
|
|
1838
|
+
if (!_workItemsMtimesDiffer(_workItemsCacheMtimes, currMtimes)) return _workItemsCache;
|
|
1839
|
+
}
|
|
1840
|
+
} else if (_workItemsLeanCache && (now - _workItemsLeanCacheAt) < 1000) {
|
|
1751
1841
|
const currMtimes = _snapshotWorkItemsMtimes(config);
|
|
1752
|
-
if (!_workItemsMtimesDiffer(
|
|
1842
|
+
if (!_workItemsMtimesDiffer(_workItemsLeanCacheMtimes, currMtimes)) return _workItemsLeanCache;
|
|
1753
1843
|
}
|
|
1754
|
-
const projects = getProjects(config);
|
|
1755
1844
|
let allItems = [];
|
|
1756
1845
|
|
|
1757
1846
|
// SQL is the canonical (and only) work-items store after Phase 9.
|
|
@@ -1795,6 +1884,10 @@ function getWorkItems(config) {
|
|
|
1795
1884
|
}
|
|
1796
1885
|
}
|
|
1797
1886
|
|
|
1887
|
+
// Detail-modal enrichment (PR cross-ref + _artifacts/_notes). Skipped for
|
|
1888
|
+
// enrich:false callers — see the getWorkItems header.
|
|
1889
|
+
if (enrich) {
|
|
1890
|
+
const projects = getProjects(config);
|
|
1798
1891
|
// Cross-reference with PRs
|
|
1799
1892
|
const allPrs = getPullRequests(config);
|
|
1800
1893
|
for (const item of allItems) {
|
|
@@ -1890,6 +1983,7 @@ function getWorkItems(config) {
|
|
|
1890
1983
|
const mentions = _notesByWi[item.id];
|
|
1891
1984
|
if (mentions && mentions.length > 0) item._notes = mentions;
|
|
1892
1985
|
}
|
|
1986
|
+
} // end if (enrich)
|
|
1893
1987
|
|
|
1894
1988
|
const statusOrder = {
|
|
1895
1989
|
pending: 0,
|
|
@@ -1906,9 +2000,16 @@ function getWorkItems(config) {
|
|
|
1906
2000
|
return (b.created || '').localeCompare(a.created || '');
|
|
1907
2001
|
});
|
|
1908
2002
|
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
2003
|
+
const cacheMtimes = _snapshotWorkItemsMtimes(config);
|
|
2004
|
+
if (enrich) {
|
|
2005
|
+
_workItemsCache = allItems;
|
|
2006
|
+
_workItemsCacheAt = now;
|
|
2007
|
+
_workItemsCacheMtimes = cacheMtimes;
|
|
2008
|
+
} else {
|
|
2009
|
+
_workItemsLeanCache = allItems;
|
|
2010
|
+
_workItemsLeanCacheAt = now;
|
|
2011
|
+
_workItemsLeanCacheMtimes = cacheMtimes;
|
|
2012
|
+
}
|
|
1912
2013
|
return allItems;
|
|
1913
2014
|
}
|
|
1914
2015
|
|
|
@@ -3103,7 +3204,7 @@ module.exports = {
|
|
|
3103
3204
|
getKnowledgeBaseEntries, getKnowledgeBaseEntriesSnapshot, getKnowledgeBaseIndex,
|
|
3104
3205
|
|
|
3105
3206
|
// Work items & PRD
|
|
3106
|
-
getWorkItems, invalidateWorkItemsCache, getPrdInfo,
|
|
3207
|
+
getWorkItems, invalidateWorkItemsCache, invalidateArchiveNotesByWiCache, getPrdInfo,
|
|
3107
3208
|
|
|
3108
3209
|
// W-mq5uzmc6001d708f — test hooks for the defensive PR enrichment cache.
|
|
3109
3210
|
_resetPrEnrichmentCacheForTest,
|
package/engine/shared.js
CHANGED
|
@@ -4297,7 +4297,9 @@ const FAILURE_CLASS = {
|
|
|
4297
4297
|
INVALID_MANAGED_SPAWN: 'invalid-managed-spawn', // P-7a3b1c92: agents/<id>/managed-spawn.json failed validator (bad schema, broken workdir, executable/env not on allowlist, healthcheck shape wrong). Engine refuses to spawn any spec — agent must fix file; never retryable as-is.
|
|
4298
4298
|
MANAGED_SPAWN_HEALTHCHECK_FAILED: 'managed-spawn-healthcheck-failed', // P-7a3b1c92: at least one managed-spawn spec was spawned but failed its healthcheck within timeout_s. Engine killed the failing PIDs; siblings stay alive. Dispatch ERROR with the failing spec name + log tail surfaced in the inbox alert.
|
|
4299
4299
|
INJECTION_FLAGGED: 'injection-flagged', // F5 (W-mpeklod3000we69c): the agent set `securityFlags.injectionAttempt:true` in its completion report after spotting a prompt-injection attempt inside an <UNTRUSTED-INPUT> fence. Engine writes a security inbox note + stamps `_securityFlag` on the WI and treats the dispatch as non-retryable so a human can review the source before the agent re-runs.
|
|
4300
|
-
LIVE_CHECKOUT_DIRTY: 'live-checkout-dirty', // P-a3f9b204 (live-checkout dispatch mode): spawnAgent ran prepareLiveCheckout against project.localPath and `git status --porcelain` reported uncommitted changes. Engine refused to spawn (it never runs `git reset`/`git clean` against the operator tree). Inbox alert lists dirty files; WI is stamped `_pendingReason: 'live_checkout_dirty'`. Non-retryable — operator must commit/stash/discard before re-dispatch.
|
|
4300
|
+
LIVE_CHECKOUT_DIRTY: 'live-checkout-dirty', // P-a3f9b204 (live-checkout dispatch mode): spawnAgent ran prepareLiveCheckout against project.localPath and `git status --porcelain` reported uncommitted changes. Engine refused to spawn (it never runs `git reset`/`git clean` against the operator tree). Inbox alert lists dirty files; WI is stamped `_pendingReason: 'live_checkout_dirty'`. Non-retryable — operator must commit/stash/discard before re-dispatch. RESERVED for confirmed dirty status results only — a thrown helper/git exception is LIVE_CHECKOUT_FAILED, not this (#305).
|
|
4301
|
+
LIVE_CHECKOUT_FAILED: 'live-checkout-failed', // #305 (live-checkout dispatch mode): prepareLiveCheckout THREW before agent spawn (helper guard, ref validation, or a transient `git status`/`rev-parse`/`checkout` failure) — distinct from the confirmed-dirty result (LIVE_CHECKOUT_DIRTY) which the helper returns, not throws. A thrown error is NOT proof the tree is dirty, so it must not be over-classified as dirty. Retryable with bounded backoff (NOT in dispatch.js neverRetry): racy branch-lock handoff, just-finished sibling dispatch, or transient git errors frequently clear on the next attempt; the engine auto-retries up to ENGINE_DEFAULTS.maxRetries before giving up. Genuinely terminal underlying reasons (auth, validation) still short-circuit via the reason-string check in isRetryableFailureReason.
|
|
4302
|
+
LIVE_CHECKOUT_MID_OPERATION: 'live-checkout-mid-operation', // P-a7f3c1d9 (live-checkout dispatch mode): spawnAgent could not switch/create the target branch in project.localPath because the operator tree is mid-operation — an in-progress merge/rebase/cherry-pick/bisect or a detached HEAD. Distinct from LIVE_CHECKOUT_DIRTY (uncommitted changes): here the tree may be clean but the branch op cannot proceed. Engine refuses to spawn (it never runs `git reset`/`git clean`/`git rebase --abort` against the operator tree). Non-retryable — operator must finish or abort the in-progress operation, or checkout a branch, before re-dispatch.
|
|
4301
4303
|
INVALID_WORKDIR: 'invalid-workdir', // P-714ef144: dispatch carried a meta.workdir override that failed validation — non-string, absolute path, drive-letter prefix, null byte, ".." segment, or post-resolve containment escape against project.localPath / worktree root. Engine refuses to spawn (the subpath would either be unreachable on disk or point outside the operator's allowed surface). Non-retryable — operator must fix the WI's meta.workdir before re-dispatch. Inbox alert lists the offending value + the resolved-vs-base mismatch.
|
|
4302
4304
|
MODEL_UNAVAILABLE: 'model-unavailable', // W-mpg6isvy000xca4d: requested model returned overloaded_error / 503 / service_unavailable. Retriable — engine swaps in the runtime-appropriate fallback model on next spawn (Claude leans on --fallback-model already plumbed; Copilot overrides --model with engine.copilotFallbackModel).
|
|
4303
4305
|
WORKSPACE_MANIFEST_REPO: 'workspace-manifest-repo-forbidden', // W-mq07avbk000m5543: dispatch routed an agent to a project/repo not present in its workspace_manifest.allowed_repos. Structural — never retryable until the manifest is widened or a different agent is chosen.
|
package/engine/timeout.js
CHANGED
|
@@ -748,7 +748,16 @@ function checkTimeouts(config) {
|
|
|
748
748
|
// mid-multi-turn session. Confirmed 2026-05-11: agents on both Copilot and
|
|
749
749
|
// Claude died at turn boundaries with claude/copilot.exe still actively
|
|
750
750
|
// making API calls, with the sentinel + claude events interleaved in the log.
|
|
751
|
-
|
|
751
|
+
//
|
|
752
|
+
// PID requirement: the alive guard is only reliable when a verifiable OS PID
|
|
753
|
+
// is present. A PID-less tracked object (e.g. stale in-memory handle after
|
|
754
|
+
// engine restart, or a Copilot spawn that never resolved its PID) has no
|
|
755
|
+
// real process behind it — isTrackedProcessAlive falls back to `!!proc &&
|
|
756
|
+
// !proc.killed` which is unverifiable. Trusting it loops forever (#319).
|
|
757
|
+
// When the PID is null, the terminal sentinel is the strongest available
|
|
758
|
+
// signal; complete from output unconditionally.
|
|
759
|
+
const pidVerifiable = trackedProcessPid(procInfo) !== null;
|
|
760
|
+
if (processAlive && pidVerifiable) {
|
|
752
761
|
log('warn', `${item.id}: [process-exit] code=${processExitCode} sentinel found in live-output.log but tracked process (pid=${trackedProcessPid(procInfo) || '?'}) is still alive — treating sentinel as stale/premature, skipping completion`);
|
|
753
762
|
} else {
|
|
754
763
|
completeFromOutput(item, liveLogPath, processExitCode, liveLogTail, hasProcess);
|
package/engine/watchdog.js
CHANGED
|
@@ -46,6 +46,54 @@ const WATCHDOG_LOG_NAME = 'watchdog-stdio.log';
|
|
|
46
46
|
// over months. Truncates to half-size on overflow (oldest lines dropped).
|
|
47
47
|
const WATCHDOG_LOG_MAX_BYTES = 2 * 1024 * 1024;
|
|
48
48
|
|
|
49
|
+
// Post-spawn grace beacon (P-b2d6c14a). Each watchdog tick is a fresh OS-
|
|
50
|
+
// scheduled process with NO in-memory memory of the previous tick, and the
|
|
51
|
+
// engine writes its PID to control.json LATE in boot. So on a slow Windows
|
|
52
|
+
// cold-start the next scheduled tick can read a not-yet-written PID, conclude
|
|
53
|
+
// "dead", and fire a SECOND `minions start` before the first finished binding
|
|
54
|
+
// — re-introducing the documented duplicate-engine/dashboard storm class. The
|
|
55
|
+
// in-process supervisor avoids this with an in-memory POST_SPAWN_GRACE_MS
|
|
56
|
+
// (engine/supervisor.js:76); the watchdog can't keep state across ticks, so it
|
|
57
|
+
// persists a tiny timestamp beacon to disk and stands down inside the window.
|
|
58
|
+
const WATCHDOG_SPAWN_BEACON_NAME = 'watchdog-spawn.json';
|
|
59
|
+
// Floor for the grace window. The effective window is max(2 × interval, floor)
|
|
60
|
+
// so a fast cadence (e.g. --interval=1) still leaves a slow cold-start time to
|
|
61
|
+
// finish, while a long cadence scales the window up with the interval.
|
|
62
|
+
const POST_SPAWN_GRACE_FLOOR_MS = 90 * 1000;
|
|
63
|
+
|
|
64
|
+
function watchdogSpawnBeaconPath(minionsHome) {
|
|
65
|
+
return path.join(minionsHome, 'engine', WATCHDOG_SPAWN_BEACON_NAME);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// Effective grace window: max(2 × interval, 90s). intervalMin defaults to the
|
|
69
|
+
// scheduler's default cadence when the caller doesn't thread one through.
|
|
70
|
+
function resolvePostSpawnGraceMs(intervalMin) {
|
|
71
|
+
const iv = Number.isFinite(intervalMin) && intervalMin > 0 ? intervalMin : DEFAULT_INTERVAL_MIN;
|
|
72
|
+
return Math.max(2 * iv * 60 * 1000, POST_SPAWN_GRACE_FLOOR_MS);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Read the last-spawn timestamp (ms epoch) from the beacon. Returns null when
|
|
76
|
+
// the beacon is missing, unreadable, or malformed — any of which means "no
|
|
77
|
+
// recent spawn on record", so recovery proceeds (fail-open).
|
|
78
|
+
function readSpawnBeaconTs(minionsHome) {
|
|
79
|
+
try {
|
|
80
|
+
const obj = JSON.parse(fs.readFileSync(watchdogSpawnBeaconPath(minionsHome), 'utf8'));
|
|
81
|
+
const ts = obj && Number(obj.ts);
|
|
82
|
+
return Number.isFinite(ts) ? ts : null;
|
|
83
|
+
} catch { return null; }
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// Stamp the beacon just before we spawn. Failure is swallowed (fail-open): a
|
|
87
|
+
// beacon we couldn't write must never block the recovery spawn — at worst we
|
|
88
|
+
// lose the double-fire guard for one window, which is the pre-existing risk.
|
|
89
|
+
function writeSpawnBeacon(minionsHome, nowMs, action) {
|
|
90
|
+
try {
|
|
91
|
+
const dir = path.join(minionsHome, 'engine');
|
|
92
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
93
|
+
fs.writeFileSync(watchdogSpawnBeaconPath(minionsHome), JSON.stringify({ ts: nowMs, action }));
|
|
94
|
+
} catch { /* fail-open — recovery still proceeds */ }
|
|
95
|
+
}
|
|
96
|
+
|
|
49
97
|
function logLine(minionsHome, msg) {
|
|
50
98
|
try {
|
|
51
99
|
const dir = path.join(minionsHome, 'engine');
|
|
@@ -155,6 +203,24 @@ async function tick(opts) {
|
|
|
155
203
|
|
|
156
204
|
const partial = engineAlive || dashUp;
|
|
157
205
|
const action = partial ? 'restart' : 'start';
|
|
206
|
+
|
|
207
|
+
// Post-spawn grace: if a previous tick already issued a spawn within the
|
|
208
|
+
// grace window, stand down instead of double-firing. Each tick is a fresh
|
|
209
|
+
// stateless process, so the only memory is the on-disk beacon. This closes
|
|
210
|
+
// the slow-cold-start race where control.json's late PID write lets the next
|
|
211
|
+
// scheduled tick spawn a second daemon stack.
|
|
212
|
+
const graceMs = resolvePostSpawnGraceMs(opts.intervalMin);
|
|
213
|
+
const nowMs = now().getTime();
|
|
214
|
+
const lastSpawnTs = readSpawnBeaconTs(minionsHome);
|
|
215
|
+
if (lastSpawnTs != null && nowMs - lastSpawnTs < graceMs) {
|
|
216
|
+
logLine(
|
|
217
|
+
minionsHome,
|
|
218
|
+
`unhealthy but recent spawn ${nowMs - lastSpawnTs}ms ago (< grace ${graceMs}ms) ` +
|
|
219
|
+
`→ standing down (post-spawn grace, avoiding double-fire)`
|
|
220
|
+
);
|
|
221
|
+
return { healthy: false, action: 'skip', reason: 'post-spawn-grace' };
|
|
222
|
+
}
|
|
223
|
+
|
|
158
224
|
logLine(
|
|
159
225
|
minionsHome,
|
|
160
226
|
`unhealthy engine=${enginePid || '-'}(${engineAlive ? 'alive' : 'dead'}) ` +
|
|
@@ -162,6 +228,11 @@ async function tick(opts) {
|
|
|
162
228
|
);
|
|
163
229
|
|
|
164
230
|
try {
|
|
231
|
+
// Stamp the spawn beacon BEFORE spawning so the next tick (which may fire
|
|
232
|
+
// before the new stack has bound its port / written control.json) sees the
|
|
233
|
+
// in-flight spawn and stands down. Write failure is swallowed — recovery
|
|
234
|
+
// proceeds regardless (fail-open).
|
|
235
|
+
writeSpawnBeacon(minionsHome, nowMs, action);
|
|
165
236
|
// Detach the child so its lifetime is independent of this tick process.
|
|
166
237
|
// The OS scheduler closes our stdio shortly after exit; we don't want
|
|
167
238
|
// that closure to cascade into the new daemon stack.
|
|
@@ -477,6 +548,10 @@ module.exports = {
|
|
|
477
548
|
MAC_PLIST_LABEL,
|
|
478
549
|
LINUX_UNIT_NAME,
|
|
479
550
|
WATCHDOG_LOG_NAME,
|
|
551
|
+
WATCHDOG_SPAWN_BEACON_NAME,
|
|
552
|
+
POST_SPAWN_GRACE_FLOOR_MS,
|
|
553
|
+
watchdogSpawnBeaconPath,
|
|
554
|
+
resolvePostSpawnGraceMs,
|
|
480
555
|
buildMacPlist,
|
|
481
556
|
buildLinuxService,
|
|
482
557
|
buildLinuxTimer,
|
package/engine.js
CHANGED
|
@@ -2257,10 +2257,15 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
2257
2257
|
// cwd = project.localPath, worktreeRootDir = null, liveMode = true.
|
|
2258
2258
|
// Instead of creating an engine-managed worktree we run prepareLiveCheckout
|
|
2259
2259
|
// in-place: it validates a clean tree and then checks out (or creates) the
|
|
2260
|
-
// target branch. On dirty refusal
|
|
2260
|
+
// target branch. On a CONFIRMED dirty refusal (the helper returns
|
|
2261
|
+
// { ok:false, reason:'dirty', dirtyFiles:[…] }) we write an inbox alert, stamp
|
|
2261
2262
|
// `_pendingReason: 'live_checkout_dirty'` on the WI, complete the dispatch
|
|
2262
2263
|
// non-retryably with FAILURE_CLASS.LIVE_CHECKOUT_DIRTY, and return — the
|
|
2263
|
-
// engine never runs `git reset`/`git clean` against the operator tree.
|
|
2264
|
+
// engine never runs `git reset`/`git clean` against the operator tree. If the
|
|
2265
|
+
// helper instead THROWS (guard/ref-validation/transient git error), that is
|
|
2266
|
+
// NOT proof of a dirty tree (#305): we complete with the separate
|
|
2267
|
+
// FAILURE_CLASS.LIVE_CHECKOUT_FAILED, which is retryable with bounded backoff
|
|
2268
|
+
// so racy branch-lock handoff or a transient git failure auto-recovers.
|
|
2264
2269
|
// On success we fall through; the worktree-creation block below is gated
|
|
2265
2270
|
// on `!liveMode`, so it never runs, and downstream cleanup paths
|
|
2266
2271
|
// (worktreePool.returnToPool, worktree-gc.gcDispatchWorktreeIfOrphan) are
|
|
@@ -2284,14 +2289,22 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
2284
2289
|
log: (msg, lvl) => log(lvl || 'info', msg),
|
|
2285
2290
|
});
|
|
2286
2291
|
} catch (liveErr) {
|
|
2287
|
-
|
|
2292
|
+
// #305: A THROWN helper/git error is NOT proof the worktree is dirty.
|
|
2293
|
+
// Classify it as LIVE_CHECKOUT_FAILED (transient/pre-spawn) and let the
|
|
2294
|
+
// dispatch retry path decide retryability via isRetryableFailureReason —
|
|
2295
|
+
// racy branch-lock handoff, a just-finished sibling dispatch, or a
|
|
2296
|
+
// transient `git status`/`rev-parse`/`checkout` failure usually clears on
|
|
2297
|
+
// the next attempt (bounded by ENGINE_DEFAULTS.maxRetries). Do NOT stamp
|
|
2298
|
+
// _pendingReason: 'live_checkout_dirty' or write a dirty-files alert here —
|
|
2299
|
+
// those belong to the confirmed-dirty result path below.
|
|
2300
|
+
log('error', `spawnAgent: live-checkout helper threw for ${id}: ${liveErr.message}`);
|
|
2288
2301
|
_cleanupPromptFiles();
|
|
2289
2302
|
completeDispatch(
|
|
2290
2303
|
id,
|
|
2291
2304
|
DISPATCH_RESULT.ERROR,
|
|
2292
2305
|
`live-checkout failed: ${liveErr.message}`.slice(0, 800),
|
|
2293
|
-
'Live-checkout helper threw before agent spawn
|
|
2294
|
-
{ failureClass: FAILURE_CLASS.
|
|
2306
|
+
'Live-checkout helper threw before agent spawn (not proof of a dirty tree). Auto-retried with bounded backoff; transient git/branch-lock state usually clears on the next attempt.',
|
|
2307
|
+
{ failureClass: FAILURE_CLASS.LIVE_CHECKOUT_FAILED },
|
|
2295
2308
|
);
|
|
2296
2309
|
cleanupTempAgent(agentId);
|
|
2297
2310
|
return null;
|
|
@@ -2342,7 +2355,96 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
2342
2355
|
cleanupTempAgent(agentId);
|
|
2343
2356
|
return null;
|
|
2344
2357
|
}
|
|
2358
|
+
// P-c5a1f3b8: mid-operation / detached-HEAD refusal. The tree is clean
|
|
2359
|
+
// (the dirty branch above already bailed) but the branch op cannot proceed
|
|
2360
|
+
// because the operator is mid-merge/rebase/cherry-pick/revert or sitting on
|
|
2361
|
+
// a detached HEAD. Distinct operator condition from `dirty` → distinct
|
|
2362
|
+
// FAILURE_CLASS (LIVE_CHECKOUT_MID_OPERATION) and distinct recovery
|
|
2363
|
+
// guidance. The engine NEVER runs git reset/clean/stash or aborts the
|
|
2364
|
+
// operator's in-progress operation — it refuses and tells the operator how
|
|
2365
|
+
// to recover with their own commands.
|
|
2366
|
+
if (_liveResult && _liveResult.ok === false
|
|
2367
|
+
&& (_liveResult.reason === 'mid-operation' || _liveResult.reason === 'detached-head')) {
|
|
2368
|
+
const _isMidOp = _liveResult.reason === 'mid-operation';
|
|
2369
|
+
const _op = _isMidOp ? (_liveResult.op || 'operation') : null;
|
|
2370
|
+
const _sha = _isMidOp ? null : (_liveResult.sha || '(unknown)');
|
|
2371
|
+
const _alertBody = [
|
|
2372
|
+
_isMidOp
|
|
2373
|
+
? '# Live-checkout blocked: in-progress git operation'
|
|
2374
|
+
: '# Live-checkout blocked: detached HEAD',
|
|
2375
|
+
'',
|
|
2376
|
+
`**Project:** ${project.name || '(unknown)'}`,
|
|
2377
|
+
`**Local path:** ${cwd}`,
|
|
2378
|
+
`**Branch:** ${branchName}`,
|
|
2379
|
+
`**Work item:** ${_wiIdForAlert}`,
|
|
2380
|
+
`**Dispatch:** ${id}`,
|
|
2381
|
+
'',
|
|
2382
|
+
_isMidOp
|
|
2383
|
+
? `The engine refused to spawn the agent because \`${cwd}\` has an in-progress **${_op}**. Live-checkout mode runs in your project directly — the engine never \`git reset\`, \`git clean\`, or aborts an operation in your tree.`
|
|
2384
|
+
: `The engine refused to spawn the agent because \`${cwd}\` is in a **detached-HEAD** state (HEAD at \`${_sha}\`). Branching off a detached HEAD would strand your anonymous commits, so the engine refuses rather than risk losing work.`,
|
|
2385
|
+
'',
|
|
2386
|
+
'## Recovery',
|
|
2387
|
+
'',
|
|
2388
|
+
_isMidOp
|
|
2389
|
+
? `Finish or abort the in-progress ${_op} with your own git commands (e.g. \`git ${_op} --continue\` or \`git ${_op} --abort\`), or checkout a branch, then re-dispatch the work item.`
|
|
2390
|
+
: 'Checkout a branch (e.g. `git checkout <branch>`) so HEAD points at a named ref, then re-dispatch the work item. If you have commits on the detached HEAD you want to keep, create a branch first (`git branch <name>`).',
|
|
2391
|
+
].join('\n');
|
|
2392
|
+
try { writeInboxAlert(`live-checkout-blocked-${_wiIdForAlert}`, _alertBody); }
|
|
2393
|
+
catch (e) { log('warn', `live-checkout: writeInboxAlert failed: ${e.message}`); }
|
|
2394
|
+
const _pendingReason = _isMidOp ? 'live_checkout_mid_operation' : 'live_checkout_detached_head';
|
|
2395
|
+
try {
|
|
2396
|
+
const _wiPath = resolveWorkItemPath(dispatchItem.meta);
|
|
2397
|
+
if (_wiPath && dispatchItem.meta?.item?.id) {
|
|
2398
|
+
mutateJsonFileLocked(_wiPath, (data) => {
|
|
2399
|
+
if (!Array.isArray(data)) return data;
|
|
2400
|
+
const wi = data.find(i => i && i.id === dispatchItem.meta.item.id);
|
|
2401
|
+
if (wi) wi._pendingReason = _pendingReason;
|
|
2402
|
+
return data;
|
|
2403
|
+
});
|
|
2404
|
+
}
|
|
2405
|
+
} catch (e) { log('warn', `live-checkout: failed to stamp _pendingReason: ${e.message}`); }
|
|
2406
|
+
const _shortMsg = _isMidOp
|
|
2407
|
+
? `live-checkout blocked: in-progress ${_op} in ${cwd}`
|
|
2408
|
+
: `live-checkout blocked: detached HEAD (${_sha}) in ${cwd}`;
|
|
2409
|
+
const _guidance = _isMidOp
|
|
2410
|
+
? `Live-checkout mode cannot switch/create a branch while a ${_op} is in progress; the engine never aborts operator operations. Finish/abort the ${_op} or checkout a branch, then re-dispatch.`
|
|
2411
|
+
: 'Live-checkout mode cannot branch off a detached HEAD without risking operator commits; the engine never moves HEAD for you. Checkout a branch, then re-dispatch.';
|
|
2412
|
+
log('error', `spawnAgent: ${_shortMsg}`);
|
|
2413
|
+
_cleanupPromptFiles();
|
|
2414
|
+
completeDispatch(
|
|
2415
|
+
id,
|
|
2416
|
+
DISPATCH_RESULT.ERROR,
|
|
2417
|
+
_shortMsg.slice(0, 800),
|
|
2418
|
+
_guidance,
|
|
2419
|
+
{ failureClass: FAILURE_CLASS.LIVE_CHECKOUT_MID_OPERATION, agentRetryable: false },
|
|
2420
|
+
);
|
|
2421
|
+
cleanupTempAgent(agentId);
|
|
2422
|
+
return null;
|
|
2423
|
+
}
|
|
2345
2424
|
log('info', `live-checkout: ${_liveResult.created ? 'created' : 'switched to'} branch ${branchName} in ${cwd} (in-place; no worktree)`);
|
|
2425
|
+
// P-c5a1f3b8: persist the operator's original ref on the dispatch record so
|
|
2426
|
+
// the dispatch-end auto-restore (P-d9e6b2c4) can return the tree to where the
|
|
2427
|
+
// operator started — even after an engine restart + re-attach, where the
|
|
2428
|
+
// spawnAgent closure is gone and only the persisted dispatch record survives
|
|
2429
|
+
// (same pattern as resolveHarnessPropagated at engine/lifecycle.js:3576).
|
|
2430
|
+
if (_liveResult.originalRef) {
|
|
2431
|
+
dispatchItem.originalRef = _liveResult.originalRef;
|
|
2432
|
+
dispatchItem.originalRefType = _liveResult.originalRefType || 'branch';
|
|
2433
|
+
try {
|
|
2434
|
+
mutateDispatch((dispatch) => {
|
|
2435
|
+
for (const queue of ['pending', 'active', 'completed']) {
|
|
2436
|
+
const arr = Array.isArray(dispatch?.[queue]) ? dispatch[queue] : null;
|
|
2437
|
+
if (!arr) continue;
|
|
2438
|
+
const found = arr.find(d => d && d.id === id);
|
|
2439
|
+
if (found) {
|
|
2440
|
+
found.originalRef = _liveResult.originalRef;
|
|
2441
|
+
found.originalRefType = _liveResult.originalRefType || 'branch';
|
|
2442
|
+
}
|
|
2443
|
+
}
|
|
2444
|
+
return dispatch;
|
|
2445
|
+
});
|
|
2446
|
+
} catch (e) { log('warn', `live-checkout: failed to persist originalRef for ${id}: ${e.message}`); }
|
|
2447
|
+
}
|
|
2346
2448
|
}
|
|
2347
2449
|
|
|
2348
2450
|
if (branchName && !liveMode) {
|
|
@@ -3814,6 +3916,15 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
3814
3916
|
}
|
|
3815
3917
|
|
|
3816
3918
|
_phaseT.spawnCallStart = Date.now();
|
|
3919
|
+
// P-c5a1f3b8 (Item D — live-mode push/PR-create cwd guard): in live mode
|
|
3920
|
+
// worktreePath stays null, so `cwd` here is project.localPath (it is never
|
|
3921
|
+
// reassigned to a worktree by the `if (worktreePath ...)` block above). The
|
|
3922
|
+
// agent's `git push -u origin <branch>` and PR-create therefore run in-place
|
|
3923
|
+
// against the operator's checkout. This cwd MUST NEVER be null for a mutating
|
|
3924
|
+
// live dispatch — shared.resolveSpawnPaths returns cwd === project.localPath
|
|
3925
|
+
// for live mode (regression-guarded in
|
|
3926
|
+
// test/unit/spawn-agent-live-mode-wiring.test.js). A null cwd would push from
|
|
3927
|
+
// the engine's own process dir and open the PR off the wrong tree.
|
|
3817
3928
|
// `detached: true` puts the agent in its own process group (POSIX) / job
|
|
3818
3929
|
// object (Windows), so when the engine dies — gracefully via stop, abruptly
|
|
3819
3930
|
// via taskkill, or because of a crash — the agent keeps running and can be
|
|
@@ -4946,6 +5057,43 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
4946
5057
|
}
|
|
4947
5058
|
}
|
|
4948
5059
|
|
|
5060
|
+
// ── P-d9e6b2c4 — live-mode dispatch-end auto-restore + terminal-failure
|
|
5061
|
+
// notify. Runs on EVERY terminal result for a live-mode dispatch: the
|
|
5062
|
+
// agent ran in-place in the operator's checkout, so switch the tree back
|
|
5063
|
+
// to the ref it was on before (see engine/live-checkout.js#
|
|
5064
|
+
// restoreLiveCheckoutAtDispatchEnd). The worktree-GC block above no-ops
|
|
5065
|
+
// in live mode (worktreePath===null); this is its live-mode counterpart.
|
|
5066
|
+
// originalRef was captured by prepareLiveCheckout and persisted on the
|
|
5067
|
+
// dispatch record at branch-resolution time (P-c5a1f3b8); the in-memory
|
|
5068
|
+
// closure copy is authoritative here. The engine-restart reattach path
|
|
5069
|
+
// fires the same helper from the persisted record in engine/cli.js. The
|
|
5070
|
+
// restore is a plain `git checkout <originalRef>` — never --force/reset/
|
|
5071
|
+
// clean/stash — and is best-effort: a thrown restore never alters the
|
|
5072
|
+
// dispatch result.
|
|
5073
|
+
if (liveMode && branchName) {
|
|
5074
|
+
try {
|
|
5075
|
+
const _liveRestore = require('./engine/live-checkout');
|
|
5076
|
+
await _liveRestore.restoreLiveCheckoutAtDispatchEnd({
|
|
5077
|
+
localPath: cwd,
|
|
5078
|
+
branchName,
|
|
5079
|
+
originalRef: dispatchItem.originalRef || '',
|
|
5080
|
+
originalRefType: dispatchItem.originalRefType || 'branch',
|
|
5081
|
+
dispatchId: id,
|
|
5082
|
+
projectName: project?.name,
|
|
5083
|
+
isTerminalFailure: effectiveResult !== DISPATCH_RESULT.SUCCESS,
|
|
5084
|
+
resultLabel: errorReason || effectiveResult,
|
|
5085
|
+
gitOpts: _gitOpts,
|
|
5086
|
+
log,
|
|
5087
|
+
writeInboxAlert,
|
|
5088
|
+
});
|
|
5089
|
+
} catch (restoreErr) {
|
|
5090
|
+
// restoreLiveCheckoutAtDispatchEnd swallows its own errors; this guard
|
|
5091
|
+
// is belt-and-suspenders so a require() hiccup can never break the
|
|
5092
|
+
// dispatch-completion path that follows.
|
|
5093
|
+
log('warn', `live-checkout: dispatch-end restore wiring threw for ${id}: ${restoreErr.message}`);
|
|
5094
|
+
}
|
|
5095
|
+
}
|
|
5096
|
+
|
|
4949
5097
|
completeDispatch(id, effectiveResult, errorReason, resultSummary, completeOpts);
|
|
4950
5098
|
|
|
4951
5099
|
// W-mpbpexrg00110661 — surface managed-spawn partial-healthcheck failures
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@yemi33/minions",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2239",
|
|
4
4
|
"description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
|
|
5
5
|
"bin": {
|
|
6
6
|
"minions": "bin/minions.js"
|