@yemi33/minions 0.1.2305 → 0.1.2306
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/js/refresh.js +17 -13
- package/dashboard/js/render-other.js +251 -3
- package/dashboard/js/render-plans.js +1 -1
- package/dashboard/js/render-prs.js +4 -1
- package/dashboard/js/render-utils.js +13 -0
- package/dashboard/js/render-work-items.js +11 -13
- package/dashboard/js/settings.js +3 -3
- package/dashboard/styles.css +58 -0
- package/dashboard.js +32 -9
- package/docs/completion-reports.md +25 -0
- package/docs/copilot-cli-schema.md +31 -1
- package/docs/design-state-storage.md +1 -1
- package/docs/live-checkout-mode.md +5 -3
- package/engine/ado.js +37 -0
- package/engine/consolidation.js +59 -2
- package/engine/lifecycle.js +149 -0
- package/engine/live-checkout.js +128 -7
- package/engine/playbook.js +2 -2
- package/engine/queries.js +11 -7
- package/engine/shared.js +6 -5
- package/engine/supervisor.js +131 -19
- package/engine.js +2 -2
- package/package.json +1 -1
package/engine/live-checkout.js
CHANGED
|
@@ -188,6 +188,65 @@ function _collectAutoCleanablePaths(dirtyFiles) {
|
|
|
188
188
|
return cleanable.length > 0 ? cleanable : null;
|
|
189
189
|
}
|
|
190
190
|
|
|
191
|
+
// issue #608 — detects a branch name that the ENGINE created for a live-mode
|
|
192
|
+
// dispatch (never the operator's own branch). Matches ONLY the literal
|
|
193
|
+
// `work/<wi-id>` convention produced by `shared.deriveWorkItemBranchName` —
|
|
194
|
+
// deliberately narrow so an operator's own `user/...`/`feature/...` branch is
|
|
195
|
+
// never mistaken for agent litter. Used by the orphaned-agent-branch self-heal
|
|
196
|
+
// below (prepareLiveCheckout) to recognize a stray branch left behind by an
|
|
197
|
+
// earlier dispatch whose dispatch-end restore did not complete.
|
|
198
|
+
function _looksLikeAgentBranch(name) {
|
|
199
|
+
return typeof name === 'string' && /^work\//.test(name.trim());
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// issue #608 — pull the current local branch name out of the `## <branch>...`
|
|
203
|
+
// header line that `git status --porcelain=v1 -b` already returns, so the
|
|
204
|
+
// orphaned-agent-branch self-heal below needs ZERO extra git calls on the
|
|
205
|
+
// common (non-stray) dirty path — preserving the existing "no extra git calls
|
|
206
|
+
// when the tree isn't all-artifact-cleanable" contract asserted by
|
|
207
|
+
// test/unit/prepare-live-checkout.test.js. Returns '' for a detached HEAD
|
|
208
|
+
// (`## HEAD (no branch)`) or an unparsable header.
|
|
209
|
+
function _extractBranchFromPorcelainHeader(branchInfo) {
|
|
210
|
+
if (!branchInfo || typeof branchInfo !== 'string') return '';
|
|
211
|
+
const body = branchInfo.replace(/^##\s*/, '');
|
|
212
|
+
const firstToken = body.split(' ')[0] || '';
|
|
213
|
+
const localBranch = firstToken.split('...')[0];
|
|
214
|
+
return (!localBranch || localBranch === 'HEAD') ? '' : localBranch;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// issue #608 — commit whatever is staged/unstaged onto the CURRENT branch
|
|
218
|
+
// with bounded retry+backoff, specifically to survive a transient
|
|
219
|
+
// `.git/index.lock` collision against a sibling git process (the root cause
|
|
220
|
+
// of the 2026-06-30 incident: two live-checkout dispatches raced `git add -A`
|
|
221
|
+
// / `git commit` against the same working tree, the commit lost the lock race
|
|
222
|
+
// once, and the prior code gave up on the first failure with no retry). Only
|
|
223
|
+
// RETRIES when the failure looks like transient lock contention; any other
|
|
224
|
+
// failure (e.g. "nothing to commit") returns immediately so callers fall
|
|
225
|
+
// through to their existing best-effort handling. Never throws — callers get
|
|
226
|
+
// `{ ok:false, error }` on exhaustion.
|
|
227
|
+
async function _commitAgentWipWithRetry(git, baseOpts, message, options = {}) {
|
|
228
|
+
const attempts = Number.isFinite(options.attempts) ? options.attempts : 3;
|
|
229
|
+
const baseDelayMs = Number.isFinite(options.baseDelayMs) ? options.baseDelayMs : 300;
|
|
230
|
+
const sleep = typeof options.sleep === 'function'
|
|
231
|
+
? options.sleep
|
|
232
|
+
: (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
233
|
+
let lastErr = null;
|
|
234
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
235
|
+
try {
|
|
236
|
+
await git(['add', '-A'], baseOpts);
|
|
237
|
+
await git(['commit', '--no-verify', '-m', message], baseOpts);
|
|
238
|
+
return { ok: true, attempts: attempt };
|
|
239
|
+
} catch (e) {
|
|
240
|
+
lastErr = e;
|
|
241
|
+
const msg = String((e && e.message) || '');
|
|
242
|
+
const transientLock = /index\.lock/i.test(msg) || /another git process/i.test(msg);
|
|
243
|
+
if (!transientLock || attempt === attempts) break;
|
|
244
|
+
await sleep(baseDelayMs * attempt);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
return { ok: false, error: lastErr, attempts };
|
|
248
|
+
}
|
|
249
|
+
|
|
191
250
|
// W-mqvejug6000eeb20 — production default for resolving the live-checkout
|
|
192
251
|
// auto-reset decision when the caller (engine.js spawnAgent) does not pass an
|
|
193
252
|
// explicit `autoReset` boolean. engine.js is intentionally not threading the
|
|
@@ -237,6 +296,7 @@ async function prepareLiveCheckout(opts = {}) {
|
|
|
237
296
|
_rm, // private injection for testing — defaults to fs.rmSync (recursive, force)
|
|
238
297
|
_resolveAutoReset, // private injection for testing — defaults to config-based resolver
|
|
239
298
|
_writeInboxNote, // private injection for testing — defaults to dispatch.writeInboxAlert
|
|
299
|
+
_sleep, // private injection for testing — defaults to a real setTimeout-based sleep
|
|
240
300
|
} = opts;
|
|
241
301
|
|
|
242
302
|
// ── Required-arg guards. Throw rather than return {ok:false} so a
|
|
@@ -319,6 +379,52 @@ async function prepareLiveCheckout(opts = {}) {
|
|
|
319
379
|
const reRaw = await git(['status', '--porcelain=v1', '-b'], baseOpts);
|
|
320
380
|
({ branchInfo, dirtyFiles } = parsePorcelain(reRaw));
|
|
321
381
|
}
|
|
382
|
+
// issue #608 — orphaned-agent-branch self-heal. If the tree is STILL dirty
|
|
383
|
+
// and HEAD is sitting on a DIFFERENT engine-created branch than the one
|
|
384
|
+
// THIS dispatch is about to use, the dirt is provably leftover from an
|
|
385
|
+
// earlier dispatch whose dispatch-end restore never completed (e.g. an
|
|
386
|
+
// `.git/index.lock` race against a sibling dispatch — see
|
|
387
|
+
// restoreLiveCheckoutAtDispatchEnd). Left alone, the ORIGINAL-REF CAPTURE
|
|
388
|
+
// below (Step 3) would adopt that stray branch as `originalRef`, and every
|
|
389
|
+
// future dispatch would inherit it in turn — the tree never stops
|
|
390
|
+
// orbiting agent branches and every dispatch fails LIVE_CHECKOUT_DIRTY
|
|
391
|
+
// forever (321 incidents / 12 days). Since the branch was created BY the
|
|
392
|
+
// engine (matched narrowly via `_looksLikeAgentBranch` — the literal
|
|
393
|
+
// `work/<id>` convention from shared.deriveWorkItemBranchName, never the
|
|
394
|
+
// operator's own branch), committing the leftover WIP there is exactly as
|
|
395
|
+
// safe as the existing dispatch-end self-heal: additive, fully
|
|
396
|
+
// recoverable from that branch's own history, and never touches the
|
|
397
|
+
// operator's real branch. Unconditional (not opt-in) because it is
|
|
398
|
+
// non-destructive, unlike the auto-reset path below. After committing,
|
|
399
|
+
// switch back to `mainRef` — the one ref that is always a real,
|
|
400
|
+
// non-agent baseline — so the tree stops drifting onto agent branches.
|
|
401
|
+
if (dirtyFiles.length > 0) {
|
|
402
|
+
const strayBranch = _extractBranchFromPorcelainHeader(branchInfo);
|
|
403
|
+
if (strayBranch && strayBranch !== branchName && _looksLikeAgentBranch(strayBranch)) {
|
|
404
|
+
const commitRes = await _commitAgentWipWithRetry(
|
|
405
|
+
git, baseOpts,
|
|
406
|
+
`minions: auto-save orphaned WIP from ${strayBranch} (dispatch ${dispatchId || '(unknown)'})`,
|
|
407
|
+
{ sleep: _sleep },
|
|
408
|
+
);
|
|
409
|
+
if (commitRes.ok) {
|
|
410
|
+
logFn(`[live-checkout] self-healed orphaned agent branch '${strayBranch}' — committed WIP` +
|
|
411
|
+
(commitRes.attempts > 1 ? ` after ${commitRes.attempts} attempts` : '') +
|
|
412
|
+
`, switching back to '${mainRef}'`, 'info');
|
|
413
|
+
try {
|
|
414
|
+
await git(['checkout', mainRef], baseOpts);
|
|
415
|
+
} catch (backErr) {
|
|
416
|
+
logFn(`[live-checkout] could not switch back to '${mainRef}' after self-healing '${strayBranch}': ${backErr && backErr.message}`, 'warn');
|
|
417
|
+
}
|
|
418
|
+
// Re-run the dirty check. Either the tree is now clean (self-heal
|
|
419
|
+
// succeeded end-to-end) or something new appeared (rare) — either
|
|
420
|
+
// way the normal auto-reset / dirty-bail logic below sees fresh state.
|
|
421
|
+
const healRaw = await git(['status', '--porcelain=v1', '-b'], baseOpts);
|
|
422
|
+
({ branchInfo, dirtyFiles } = parsePorcelain(healRaw));
|
|
423
|
+
} else {
|
|
424
|
+
logFn(`[live-checkout] could not self-heal orphaned agent branch '${strayBranch}' (commit failed): ${commitRes.error && commitRes.error.message}`, 'warn');
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
}
|
|
322
428
|
if (dirtyFiles.length > 0) {
|
|
323
429
|
// W-mqvejug6000eeb20 — opt-in auto-reset. Default behavior is to bail with
|
|
324
430
|
// reason:'dirty' (spawnAgent translates to non-retryable LIVE_CHECKOUT_DIRTY
|
|
@@ -641,6 +747,7 @@ async function restoreLiveCheckoutAtDispatchEnd(opts = {}) {
|
|
|
641
747
|
log,
|
|
642
748
|
writeInboxAlert,
|
|
643
749
|
_git, // private injection for testing — defaults to shared.shellSafeGit
|
|
750
|
+
_sleep, // private injection for testing — defaults to a real setTimeout-based sleep
|
|
644
751
|
} = opts;
|
|
645
752
|
|
|
646
753
|
const logFn = (typeof log === 'function') ? log : () => {};
|
|
@@ -750,16 +857,27 @@ async function restoreLiveCheckoutAtDispatchEnd(opts = {}) {
|
|
|
750
857
|
onAgentBranch = cur && cur === branchName;
|
|
751
858
|
} catch { /* best-effort */ }
|
|
752
859
|
if (onAgentBranch) {
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
860
|
+
// issue #608 — bounded retry+backoff on the commit. The 2026-06-30
|
|
861
|
+
// incident was caused by exactly this commit losing a transient
|
|
862
|
+
// `.git/index.lock` race against a sibling dispatch's git process;
|
|
863
|
+
// the previous single-attempt code gave up immediately and left the
|
|
864
|
+
// stage half-committed, which then perpetuated as a stray agent
|
|
865
|
+
// branch for every subsequent dispatch (see the orphaned-agent-branch
|
|
866
|
+
// self-heal in prepareLiveCheckout). `--no-verify` skips operator
|
|
867
|
+
// commit hooks that may fail headless.
|
|
868
|
+
const commitRes = await _commitAgentWipWithRetry(
|
|
869
|
+
git, baseOpts,
|
|
870
|
+
`minions: auto-save agent WIP (dispatch ${did})`,
|
|
871
|
+
{ sleep: _sleep },
|
|
872
|
+
);
|
|
873
|
+
if (commitRes.ok) {
|
|
757
874
|
result.autoCommitted = true;
|
|
758
|
-
logFn('info', `live-checkout: auto-committed agent WIP onto ${branchName} before restore (dispatch ${did})`
|
|
759
|
-
|
|
875
|
+
logFn('info', `live-checkout: auto-committed agent WIP onto ${branchName} before restore (dispatch ${did})` +
|
|
876
|
+
(commitRes.attempts > 1 ? ` after ${commitRes.attempts} attempts` : ''));
|
|
877
|
+
} else {
|
|
760
878
|
// Leave the tree as-is; the plain checkout below will refuse and the
|
|
761
879
|
// fallback alert fires. Never force, never discard.
|
|
762
|
-
logFn('warn', `live-checkout: could not auto-save agent WIP onto ${branchName} for ${did}: ${
|
|
880
|
+
logFn('warn', `live-checkout: could not auto-save agent WIP onto ${branchName} for ${did}: ${commitRes.error && commitRes.error.message}`);
|
|
763
881
|
}
|
|
764
882
|
}
|
|
765
883
|
}
|
|
@@ -906,5 +1024,8 @@ module.exports = {
|
|
|
906
1024
|
_isPartialCloneBlobError,
|
|
907
1025
|
_isAutoCleanableArtifact,
|
|
908
1026
|
_collectAutoCleanablePaths,
|
|
1027
|
+
_looksLikeAgentBranch,
|
|
1028
|
+
_extractBranchFromPorcelainHeader,
|
|
1029
|
+
_commitAgentWipWithRetry,
|
|
909
1030
|
LIVE_CHECKOUT_AUTO_CLEAN_PATTERNS,
|
|
910
1031
|
};
|
package/engine/playbook.js
CHANGED
|
@@ -13,7 +13,7 @@ const { wrapUntrusted, buildSource } = require('./untrusted-fence');
|
|
|
13
13
|
const { MINIONS_BRAND_URL } = require('./comment-format');
|
|
14
14
|
|
|
15
15
|
const { safeJson, safeRead, getProjects, log, ts, dateStamp, truncateTextBytes, ENGINE_DEFAULTS, WI_STATUS, WORK_TYPE, PR_STATUS, DISPATCH_RESULT, getProjectOrg } = shared;
|
|
16
|
-
const { getConfig, getDispatch, getNotes,
|
|
16
|
+
const { getConfig, getDispatch, getNotes, getPrs, getKnowledgeBaseIndex, AGENTS_DIR } = queries;
|
|
17
17
|
|
|
18
18
|
const MINIONS_DIR = shared.MINIONS_DIR;
|
|
19
19
|
const PLAYBOOKS_DIR = path.join(MINIONS_DIR, 'playbooks');
|
|
@@ -1052,7 +1052,7 @@ function _checkPlaybookSections(content, context = '') {
|
|
|
1052
1052
|
// Lean system prompt: agent identity + rules only (~2-4KB, never grows)
|
|
1053
1053
|
function buildSystemPrompt(agentId, config, project) {
|
|
1054
1054
|
const agent = config.agents[agentId] || tempAgents.get(agentId) || { name: agentId, role: 'Temporary Agent', expertise: [] };
|
|
1055
|
-
const charter =
|
|
1055
|
+
const charter = safeRead(path.join(AGENTS_DIR, agentId, 'charter.md')); // returns '' for temp agents (no charter file)
|
|
1056
1056
|
project = project || getProjects(config)[0] || {};
|
|
1057
1057
|
|
|
1058
1058
|
let prompt = '';
|
package/engine/queries.js
CHANGED
|
@@ -722,10 +722,6 @@ function getAgentStatus(agentId) {
|
|
|
722
722
|
// setAgentStatus removed — agent status is derived from dispatch.json.
|
|
723
723
|
// Status.json files no longer exist.
|
|
724
724
|
|
|
725
|
-
function getAgentCharter(agentId) {
|
|
726
|
-
return safeRead(path.join(AGENTS_DIR, agentId, 'charter.md'));
|
|
727
|
-
}
|
|
728
|
-
|
|
729
725
|
function getAgents(config) {
|
|
730
726
|
config = config || getConfig();
|
|
731
727
|
// Fall back to DEFAULT_AGENTS if config has no agents (uninitialized repo)
|
|
@@ -2063,13 +2059,21 @@ function getWorkItems(config, opts) {
|
|
|
2063
2059
|
});
|
|
2064
2060
|
|
|
2065
2061
|
const cacheMtimes = _snapshotWorkItemsMtimes(config);
|
|
2062
|
+
// Use Date.now() here (not the `now` captured at function entry) so the
|
|
2063
|
+
// 1 s TTL is measured from when the cache was BUILT, not from when the
|
|
2064
|
+
// call started. On a slow or loaded machine the first cold call can take
|
|
2065
|
+
// > 1 s while DB migrations run; if we used the entry-time `now`, the
|
|
2066
|
+
// very next call would see (entry_time_of_call_2 − entry_time_of_call_1)
|
|
2067
|
+
// >= 1 000 ms and miss the cache, defeating the cache entirely.
|
|
2068
|
+
// (W-mr00aooc000i1d6b — queries-work-items-cache CI failure on slow runners)
|
|
2069
|
+
const cacheSetAt = Date.now();
|
|
2066
2070
|
if (enrich) {
|
|
2067
2071
|
_workItemsCache = allItems;
|
|
2068
|
-
_workItemsCacheAt =
|
|
2072
|
+
_workItemsCacheAt = cacheSetAt;
|
|
2069
2073
|
_workItemsCacheMtimes = cacheMtimes;
|
|
2070
2074
|
} else {
|
|
2071
2075
|
_workItemsLeanCache = allItems;
|
|
2072
|
-
_workItemsLeanCacheAt =
|
|
2076
|
+
_workItemsLeanCacheAt = cacheSetAt;
|
|
2073
2077
|
_workItemsLeanCacheMtimes = cacheMtimes;
|
|
2074
2078
|
}
|
|
2075
2079
|
return allItems;
|
|
@@ -3320,7 +3324,7 @@ module.exports = {
|
|
|
3320
3324
|
_buildNotesByWiMap, // exported for testing
|
|
3321
3325
|
|
|
3322
3326
|
// Agents
|
|
3323
|
-
getAgentStatus,
|
|
3327
|
+
getAgentStatus, getAgents, getAgentDetail,
|
|
3324
3328
|
|
|
3325
3329
|
// Pull requests
|
|
3326
3330
|
getPrs, getPullRequests,
|
package/engine/shared.js
CHANGED
|
@@ -3254,11 +3254,12 @@ const ENGINE_DEFAULTS = {
|
|
|
3254
3254
|
// When ON, a dirty/broken live-checkout tree is `git fetch origin` +
|
|
3255
3255
|
// `git reset --hard origin/<branch>`'d before dispatch instead of failing
|
|
3256
3256
|
// LIVE_CHECKOUT_DIRTY. Per-project `project.liveCheckoutAutoReset` overrides
|
|
3257
|
-
// this. Default
|
|
3258
|
-
//
|
|
3259
|
-
//
|
|
3260
|
-
//
|
|
3261
|
-
|
|
3257
|
+
// this. Default ON — discarding uncommitted dirt via reset-to-remote keeps
|
|
3258
|
+
// live-checkout branches from diverging (the old `false` default accumulated
|
|
3259
|
+
// `minions: auto-save agent WIP` commits on every dirty exit). Resolution
|
|
3260
|
+
// precedence lives in `resolveLiveCheckoutAutoReset`. Fires ONLY on confirmed
|
|
3261
|
+
// dirty-tree detection, never on mid-operation / blob-fetch / tooling failures.
|
|
3262
|
+
liveCheckoutAutoReset: true,
|
|
3262
3263
|
orphanHolderScanTimeoutMs: 5000, // 5s ceiling for the cross-platform holder scan (PowerShell / /proc walk / lsof)
|
|
3263
3264
|
ccMaxTurns: 50, // max tool-use turns per CC/doc-chat call before CLI stops (per response, not per session)
|
|
3264
3265
|
ccWorkerIdleTimeoutMs: 30 * 60 * 1000, // W-mr0qs0vw: idle-reaper window for the persistent `copilot --acp` worker pool (engine/cc-worker-pool.js). After this much inactivity with no in-flight turn the warm ACP process is killed; the next message cold-spawns a fresh session with NO memory of prior turns (CC shows a "context cleared after inactivity" notice). Tradeoff: shorter = less idle memory/process footprint, longer = more context durability across gaps between messages. Wired into the pool via ccWorkerPool.setIdleTimeoutMs() on every reloadConfig(); clamped to [60000, 28800000] (1min–8h) in the settings POST handler.
|
package/engine/supervisor.js
CHANGED
|
@@ -5,7 +5,14 @@
|
|
|
5
5
|
* Spawned `detached: true` by `minions start`/`restart` after the engine and
|
|
6
6
|
* dashboard are up. Polls every SUPERVISOR_INTERVAL_MS:
|
|
7
7
|
* - engine PID from `engine/control.json`
|
|
8
|
-
* - dashboard
|
|
8
|
+
* - dashboard via a port-listener probe (port 7331 by default) AND, when the
|
|
9
|
+
* port is bound, an HTTP `/api/health` probe. The HTTP layer catches the
|
|
10
|
+
* "frozen but still listening" dashboard (wedged event loop) that the
|
|
11
|
+
* port-listen probe alone reports as healthy — it respawns only after
|
|
12
|
+
* DASH_HEALTH_MAX_FAILS consecutive failed probes so a dashboard merely
|
|
13
|
+
* busy with a heavy synchronous getStatus() rebuild isn't killed. What
|
|
14
|
+
* counts as "responsive" (and why engine 'degraded'/'stopped' does NOT mean
|
|
15
|
+
* the dashboard is frozen) is documented at _probeDashboardHealth.
|
|
9
16
|
*
|
|
10
17
|
* When either is dead AND the stop-intent flag is NOT set, respawns the dead
|
|
11
18
|
* one in the same way the CLI does (detached, stdio routed to the engine-
|
|
@@ -76,6 +83,16 @@ function _resolveDashPort() {
|
|
|
76
83
|
const POST_SPAWN_GRACE_MS = Number(process.env.MINIONS_SUPERVISOR_GRACE_MS) || 15000;
|
|
77
84
|
// #421 — heartbeat age threshold before supervisor considers the engine event loop frozen.
|
|
78
85
|
const SUPERVISOR_STALE_ENGINE_HEARTBEAT_MS = Number(process.env.MINIONS_SUPERVISOR_STALE_HEARTBEAT_MS) || 180000; // 3 min
|
|
86
|
+
// Dashboard HTTP liveness (frozen-but-listening detection). A dashboard whose
|
|
87
|
+
// event loop is wedged keeps the socket LISTENING, so the port-listen probe
|
|
88
|
+
// alone reports it healthy while the in-browser SPA shows the "unreachable"
|
|
89
|
+
// banner forever. We layer an HTTP /api/health probe on top: only respawn after
|
|
90
|
+
// DASH_HEALTH_MAX_FAILS *consecutive* failed probes so a dashboard that is
|
|
91
|
+
// merely busy with a heavy synchronous getStatus() rebuild (documented to block
|
|
92
|
+
// the loop 15-25s) isn't mistaken for a hang. With the default 30s interval,
|
|
93
|
+
// 3 strikes ≈ 90s of continuous unresponsiveness before we act.
|
|
94
|
+
const DASH_HEALTH_TIMEOUT_MS = Number(process.env.MINIONS_SUPERVISOR_DASH_HEALTH_TIMEOUT_MS) || 5000;
|
|
95
|
+
const DASH_HEALTH_MAX_FAILS = Number(process.env.MINIONS_SUPERVISOR_DASH_HEALTH_FAILS) || 3;
|
|
79
96
|
const isWin = process.platform === 'win32';
|
|
80
97
|
|
|
81
98
|
function safeReadJson(p) {
|
|
@@ -354,6 +371,39 @@ function spawnDashboard() {
|
|
|
354
371
|
// the engine.
|
|
355
372
|
let _lastEngineRespawnAt = 0;
|
|
356
373
|
let _lastDashboardRespawnAt = 0;
|
|
374
|
+
// Consecutive failed dashboard HTTP health probes. Reset on any healthy probe,
|
|
375
|
+
// after a respawn, and inside the post-spawn grace window.
|
|
376
|
+
let _dashHealthFailStreak = 0;
|
|
377
|
+
|
|
378
|
+
// Lazy, defensive HTTP health probe. Reuses restart-health.js's httpGetJson
|
|
379
|
+
// (rule: reuse before re-implementing) but never lets a load failure (e.g. a
|
|
380
|
+
// mid-upgrade broken shared.js that restart-health requires at module top)
|
|
381
|
+
// break the supervisor. Returns null when the probe helper is unavailable so
|
|
382
|
+
// the caller can FALL BACK to port-only liveness instead of false-respawning a
|
|
383
|
+
// healthy dashboard.
|
|
384
|
+
function _probeDashboardHealth(port, timeoutMs = DASH_HEALTH_TIMEOUT_MS) {
|
|
385
|
+
let getJson = null;
|
|
386
|
+
try {
|
|
387
|
+
const rh = require('./restart-health');
|
|
388
|
+
if (rh && rh._private && typeof rh._private.httpGetJson === 'function') {
|
|
389
|
+
getJson = rh._private.httpGetJson;
|
|
390
|
+
}
|
|
391
|
+
} catch { /* restart-health/shared unavailable — fall through */ }
|
|
392
|
+
if (!getJson) return Promise.resolve(null);
|
|
393
|
+
return getJson(`http://127.0.0.1:${port}/api/health`, timeoutMs)
|
|
394
|
+
// "Responsive" == the dashboard event loop executed handleHealth and returned
|
|
395
|
+
// a well-formed 2xx envelope, REGARDLESS of the engine-derived status value.
|
|
396
|
+
// /api/health reports 'degraded' when the engine is paused/stopping and
|
|
397
|
+
// 'stopped' when the engine is down — those are legitimate ENGINE states
|
|
398
|
+
// (owned by checkEngine/checkEngineHung), not evidence the DASHBOARD process
|
|
399
|
+
// is frozen. Gating on status==='healthy' would respawn a perfectly
|
|
400
|
+
// responsive dashboard every time an operator pauses the engine, tearing
|
|
401
|
+
// down the socket and tripping the in-browser "unreachable" banner for no
|
|
402
|
+
// reason. Only a failed/timed-out request (res.ok false or a rejection)
|
|
403
|
+
// means the dashboard's own event loop is actually wedged.
|
|
404
|
+
.then(res => !!(res && res.ok && res.json && typeof res.json.status === 'string'))
|
|
405
|
+
.catch(() => false);
|
|
406
|
+
}
|
|
357
407
|
|
|
358
408
|
// Window during which a `null` pid + recent `restarted_at` is interpreted as
|
|
359
409
|
// "another watchdog is currently respawning the engine — don't double-spawn."
|
|
@@ -364,12 +414,29 @@ let _lastDashboardRespawnAt = 0;
|
|
|
364
414
|
// to enter cli.js:443 and write its own PID on cold Windows boots.
|
|
365
415
|
const RESPAWN_IN_PROGRESS_WINDOW_MS = Number(process.env.MINIONS_SUPERVISOR_RESPAWN_WINDOW_MS) || 60000;
|
|
366
416
|
|
|
417
|
+
// #3758 — 'degraded' is a FAILURE state (set by dashboard.js's frozen-tick
|
|
418
|
+
// watchdog, see dashboard.js#_markEngineAsDegradedIfFrozen), not a deliberate
|
|
419
|
+
// user-intent state like 'paused'/'stopping'/'stopped'. When a frozen tick loop
|
|
420
|
+
// finally unblocks, engine.js#tickInner reads control.state !== 'running' and
|
|
421
|
+
// calls process.exit(0) — self-exiting exactly BECAUSE it saw 'degraded'. The
|
|
422
|
+
// dashboard.js comment documents the expected recovery: "the PID-dead watchdog
|
|
423
|
+
// then auto-restarts it." Excluding 'degraded' here left that promise unmet —
|
|
424
|
+
// the engine PID goes away and the control plane silently stays down until a
|
|
425
|
+
// manual `minions restart`. Respawn eligibility must include both 'running'
|
|
426
|
+
// (still up) and 'degraded' (crashed/crashing out of a real failure); only
|
|
427
|
+
// 'paused', 'stopping', and 'stopped' reflect actual user intent to stay down.
|
|
428
|
+
function _isRespawnEligibleEngineState(state) {
|
|
429
|
+
return state === 'running' || state === 'degraded';
|
|
430
|
+
}
|
|
431
|
+
|
|
367
432
|
function checkEngine(now) {
|
|
368
433
|
if (now - _lastEngineRespawnAt < POST_SPAWN_GRACE_MS) return;
|
|
369
434
|
const control = safeReadJson(CONTROL_PATH_FN());
|
|
370
|
-
//
|
|
371
|
-
//
|
|
372
|
-
|
|
435
|
+
// Respawn when control.json says "running" (PID died unexpectedly) or
|
|
436
|
+
// "degraded" (watchdog-flagged failure state — see _isRespawnEligibleEngineState
|
|
437
|
+
// above). "paused"/"stopped"/"stopping" are legitimate user-intent states the
|
|
438
|
+
// supervisor must not override.
|
|
439
|
+
if (!control || !_isRespawnEligibleEngineState(control.state)) return;
|
|
373
440
|
if (control.pid && isPidAlive(control.pid)) return;
|
|
374
441
|
|
|
375
442
|
// Cross-watchdog race guard: dashboard.js's in-process engine watchdog
|
|
@@ -403,7 +470,11 @@ function checkEngine(now) {
|
|
|
403
470
|
function checkEngineHung(now) {
|
|
404
471
|
if (now - _lastEngineRespawnAt < POST_SPAWN_GRACE_MS) return;
|
|
405
472
|
const control = safeReadJson(CONTROL_PATH_FN());
|
|
406
|
-
|
|
473
|
+
// Include 'degraded' (see _isRespawnEligibleEngineState) so a permanently
|
|
474
|
+
// frozen tick loop that never gets a chance to self-exit (tickInner never
|
|
475
|
+
// re-enters) still gets force-restarted once its own heartbeat goes stale,
|
|
476
|
+
// instead of being ignored forever the moment the state flips to 'degraded'.
|
|
477
|
+
if (!control || !_isRespawnEligibleEngineState(control.state)) return;
|
|
407
478
|
// Only acts when the PID is alive — a dead PID is handled by checkEngine().
|
|
408
479
|
if (!control.pid || !isPidAlive(control.pid)) return;
|
|
409
480
|
|
|
@@ -424,31 +495,69 @@ function checkEngineHung(now) {
|
|
|
424
495
|
console.log(`[supervisor] Engine restarted due to stale heartbeat (new PID: ${newPid})`);
|
|
425
496
|
}
|
|
426
497
|
|
|
427
|
-
function checkDashboard(now) {
|
|
428
|
-
if (now - _lastDashboardRespawnAt < POST_SPAWN_GRACE_MS) return;
|
|
498
|
+
async function checkDashboard(now) {
|
|
499
|
+
if (now - _lastDashboardRespawnAt < POST_SPAWN_GRACE_MS) { _dashHealthFailStreak = 0; return; }
|
|
429
500
|
const dashPort = _resolveDashPort();
|
|
430
501
|
const pids = listeningPidsForPort(dashPort);
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
502
|
+
|
|
503
|
+
const respawn = (why) => {
|
|
504
|
+
console.log(`[supervisor] Dashboard ${why} on port ${dashPort} — respawning...`);
|
|
505
|
+
// Reap stray dashboards first. Besides bounding the count, this collapses the
|
|
506
|
+
// port-desync case: orphan dashboards holding other ports are killed, freeing
|
|
507
|
+
// the canonical port for the fresh one whose beacon _resolveDashPort() reads.
|
|
508
|
+
// Critically, for the frozen-but-listening case the reap also KILLS the wedged
|
|
509
|
+
// process still holding the port so spawnDashboard() can bind it.
|
|
510
|
+
reapStrayProcesses(path.join(MINIONS_DIR, 'dashboard.js'), 'dashboard');
|
|
511
|
+
const newPid = spawnDashboard();
|
|
512
|
+
_lastDashboardRespawnAt = now;
|
|
513
|
+
_dashHealthFailStreak = 0;
|
|
514
|
+
console.log(`[supervisor] Dashboard respawned (new PID: ${newPid})`);
|
|
515
|
+
};
|
|
516
|
+
|
|
517
|
+
// Hard-down: nothing bound to the port → respawn immediately.
|
|
518
|
+
if (pids.length === 0) {
|
|
519
|
+
_dashHealthFailStreak = 0;
|
|
520
|
+
respawn('not listening');
|
|
521
|
+
return;
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
// Port is bound. Verify the event loop is actually responsive — a frozen-but-
|
|
525
|
+
// listening dashboard keeps the socket LISTENING yet serves the in-browser
|
|
526
|
+
// "unreachable" banner. Probe /api/health; only respawn after a streak of
|
|
527
|
+
// failures so a legitimately busy getStatus() rebuild isn't mistaken for a hang.
|
|
528
|
+
const healthy = await _probeDashboardHealth(dashPort);
|
|
529
|
+
if (healthy === null) {
|
|
530
|
+
// Probe helper unavailable — fall back to port-only liveness (the pre-existing
|
|
531
|
+
// behavior). Don't count this as a failure; we won't respawn a bound dashboard
|
|
532
|
+
// we can't actually prove is frozen.
|
|
533
|
+
_dashHealthFailStreak = 0;
|
|
534
|
+
return;
|
|
535
|
+
}
|
|
536
|
+
if (healthy) { _dashHealthFailStreak = 0; return; }
|
|
537
|
+
|
|
538
|
+
_dashHealthFailStreak++;
|
|
539
|
+
console.log(`[supervisor] Dashboard bound but unresponsive on port ${dashPort} (${_dashHealthFailStreak}/${DASH_HEALTH_MAX_FAILS} strikes)`);
|
|
540
|
+
if (_dashHealthFailStreak < DASH_HEALTH_MAX_FAILS) return;
|
|
541
|
+
respawn('frozen (health probe failed)');
|
|
441
542
|
}
|
|
442
543
|
|
|
443
|
-
|
|
544
|
+
let _tickInFlight = false;
|
|
545
|
+
async function tick() {
|
|
546
|
+
// The dashboard health probe is async; guard against a slow probe overlapping
|
|
547
|
+
// the next interval fire. setInterval invoking an async fn is fire-and-forget,
|
|
548
|
+
// so this flag is the only thing preventing concurrent ticks.
|
|
549
|
+
if (_tickInFlight) return;
|
|
550
|
+
_tickInFlight = true;
|
|
444
551
|
try {
|
|
445
552
|
if (isStopIntentSet()) return;
|
|
446
553
|
const now = Date.now();
|
|
447
554
|
checkEngine(now);
|
|
448
555
|
checkEngineHung(now);
|
|
449
|
-
checkDashboard(now);
|
|
556
|
+
await checkDashboard(now);
|
|
450
557
|
} catch (e) {
|
|
451
558
|
console.error(`[supervisor] tick error: ${e && e.message}`);
|
|
559
|
+
} finally {
|
|
560
|
+
_tickInFlight = false;
|
|
452
561
|
}
|
|
453
562
|
}
|
|
454
563
|
|
|
@@ -517,8 +626,11 @@ module.exports = {
|
|
|
517
626
|
openAppendFd,
|
|
518
627
|
checkEngine,
|
|
519
628
|
checkEngineHung,
|
|
629
|
+
_isRespawnEligibleEngineState,
|
|
520
630
|
checkDashboard,
|
|
631
|
+
_probeDashboardHealth,
|
|
521
632
|
SUPERVISOR_STALE_ENGINE_HEARTBEAT_MS,
|
|
633
|
+
DASH_HEALTH_MAX_FAILS,
|
|
522
634
|
tick,
|
|
523
635
|
// Path getters honor MINIONS_TEST_DIR via shared.ENGINE_DIR, so test
|
|
524
636
|
// isolation correctly redirects writes/reads under createTestMinionsDir.
|
package/engine.js
CHANGED
|
@@ -138,7 +138,7 @@ const worktreePool = require('./engine/worktree-pool');
|
|
|
138
138
|
// ─── State Readers (delegated to engine/queries.js) ─────────────────────────
|
|
139
139
|
|
|
140
140
|
const { getConfig, getControl, getDispatch, getNotes,
|
|
141
|
-
getAgentStatus,
|
|
141
|
+
getAgentStatus, getInboxFiles,
|
|
142
142
|
collectSkillFiles, getSkillIndex, getKnowledgeBaseIndex,
|
|
143
143
|
getPrs, SKILLS_DIR } = queries;
|
|
144
144
|
|
|
@@ -10564,7 +10564,7 @@ module.exports = {
|
|
|
10564
10564
|
|
|
10565
10565
|
// State readers/writers
|
|
10566
10566
|
getConfig, getControl, getDispatch, getRouting, getNotes,
|
|
10567
|
-
getAgentStatus,
|
|
10567
|
+
getAgentStatus, getInboxFiles, getPrs,
|
|
10568
10568
|
validateConfig,
|
|
10569
10569
|
|
|
10570
10570
|
// Dispatch management (re-exported from engine/dispatch.js)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@yemi33/minions",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2306",
|
|
4
4
|
"description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
|
|
5
5
|
"bin": {
|
|
6
6
|
"minions": "bin/minions.js"
|