@yemi33/minions 0.1.2305 → 0.1.2306

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -188,6 +188,65 @@ function _collectAutoCleanablePaths(dirtyFiles) {
188
188
  return cleanable.length > 0 ? cleanable : null;
189
189
  }
190
190
 
191
+ // issue #608 — detects a branch name that the ENGINE created for a live-mode
192
+ // dispatch (never the operator's own branch). Matches ONLY the literal
193
+ // `work/<wi-id>` convention produced by `shared.deriveWorkItemBranchName` —
194
+ // deliberately narrow so an operator's own `user/...`/`feature/...` branch is
195
+ // never mistaken for agent litter. Used by the orphaned-agent-branch self-heal
196
+ // below (prepareLiveCheckout) to recognize a stray branch left behind by an
197
+ // earlier dispatch whose dispatch-end restore did not complete.
198
+ function _looksLikeAgentBranch(name) {
199
+ return typeof name === 'string' && /^work\//.test(name.trim());
200
+ }
201
+
202
+ // issue #608 — pull the current local branch name out of the `## <branch>...`
203
+ // header line that `git status --porcelain=v1 -b` already returns, so the
204
+ // orphaned-agent-branch self-heal below needs ZERO extra git calls on the
205
+ // common (non-stray) dirty path — preserving the existing "no extra git calls
206
+ // when the tree isn't all-artifact-cleanable" contract asserted by
207
+ // test/unit/prepare-live-checkout.test.js. Returns '' for a detached HEAD
208
+ // (`## HEAD (no branch)`) or an unparsable header.
209
+ function _extractBranchFromPorcelainHeader(branchInfo) {
210
+ if (!branchInfo || typeof branchInfo !== 'string') return '';
211
+ const body = branchInfo.replace(/^##\s*/, '');
212
+ const firstToken = body.split(' ')[0] || '';
213
+ const localBranch = firstToken.split('...')[0];
214
+ return (!localBranch || localBranch === 'HEAD') ? '' : localBranch;
215
+ }
216
+
217
+ // issue #608 — commit whatever is staged/unstaged onto the CURRENT branch
218
+ // with bounded retry+backoff, specifically to survive a transient
219
+ // `.git/index.lock` collision against a sibling git process (the root cause
220
+ // of the 2026-06-30 incident: two live-checkout dispatches raced `git add -A`
221
+ // / `git commit` against the same working tree, the commit lost the lock race
222
+ // once, and the prior code gave up on the first failure with no retry). Only
223
+ // RETRIES when the failure looks like transient lock contention; any other
224
+ // failure (e.g. "nothing to commit") returns immediately so callers fall
225
+ // through to their existing best-effort handling. Never throws — callers get
226
+ // `{ ok:false, error }` on exhaustion.
227
+ async function _commitAgentWipWithRetry(git, baseOpts, message, options = {}) {
228
+ const attempts = Number.isFinite(options.attempts) ? options.attempts : 3;
229
+ const baseDelayMs = Number.isFinite(options.baseDelayMs) ? options.baseDelayMs : 300;
230
+ const sleep = typeof options.sleep === 'function'
231
+ ? options.sleep
232
+ : (ms) => new Promise((resolve) => setTimeout(resolve, ms));
233
+ let lastErr = null;
234
+ for (let attempt = 1; attempt <= attempts; attempt++) {
235
+ try {
236
+ await git(['add', '-A'], baseOpts);
237
+ await git(['commit', '--no-verify', '-m', message], baseOpts);
238
+ return { ok: true, attempts: attempt };
239
+ } catch (e) {
240
+ lastErr = e;
241
+ const msg = String((e && e.message) || '');
242
+ const transientLock = /index\.lock/i.test(msg) || /another git process/i.test(msg);
243
+ if (!transientLock || attempt === attempts) break;
244
+ await sleep(baseDelayMs * attempt);
245
+ }
246
+ }
247
+ return { ok: false, error: lastErr, attempts };
248
+ }
249
+
191
250
  // W-mqvejug6000eeb20 — production default for resolving the live-checkout
192
251
  // auto-reset decision when the caller (engine.js spawnAgent) does not pass an
193
252
  // explicit `autoReset` boolean. engine.js is intentionally not threading the
@@ -237,6 +296,7 @@ async function prepareLiveCheckout(opts = {}) {
237
296
  _rm, // private injection for testing — defaults to fs.rmSync (recursive, force)
238
297
  _resolveAutoReset, // private injection for testing — defaults to config-based resolver
239
298
  _writeInboxNote, // private injection for testing — defaults to dispatch.writeInboxAlert
299
+ _sleep, // private injection for testing — defaults to a real setTimeout-based sleep
240
300
  } = opts;
241
301
 
242
302
  // ── Required-arg guards. Throw rather than return {ok:false} so a
@@ -319,6 +379,52 @@ async function prepareLiveCheckout(opts = {}) {
319
379
  const reRaw = await git(['status', '--porcelain=v1', '-b'], baseOpts);
320
380
  ({ branchInfo, dirtyFiles } = parsePorcelain(reRaw));
321
381
  }
382
+ // issue #608 — orphaned-agent-branch self-heal. If the tree is STILL dirty
383
+ // and HEAD is sitting on a DIFFERENT engine-created branch than the one
384
+ // THIS dispatch is about to use, the dirt is provably leftover from an
385
+ // earlier dispatch whose dispatch-end restore never completed (e.g. an
386
+ // `.git/index.lock` race against a sibling dispatch — see
387
+ // restoreLiveCheckoutAtDispatchEnd). Left alone, the ORIGINAL-REF CAPTURE
388
+ // below (Step 3) would adopt that stray branch as `originalRef`, and every
389
+ // future dispatch would inherit it in turn — the tree never stops
390
+ // orbiting agent branches and every dispatch fails LIVE_CHECKOUT_DIRTY
391
+ // forever (321 incidents / 12 days). Since the branch was created BY the
392
+ // engine (matched narrowly via `_looksLikeAgentBranch` — the literal
393
+ // `work/<id>` convention from shared.deriveWorkItemBranchName, never the
394
+ // operator's own branch), committing the leftover WIP there is exactly as
395
+ // safe as the existing dispatch-end self-heal: additive, fully
396
+ // recoverable from that branch's own history, and never touches the
397
+ // operator's real branch. Unconditional (not opt-in) because it is
398
+ // non-destructive, unlike the auto-reset path below. After committing,
399
+ // switch back to `mainRef` — the one ref that is always a real,
400
+ // non-agent baseline — so the tree stops drifting onto agent branches.
401
+ if (dirtyFiles.length > 0) {
402
+ const strayBranch = _extractBranchFromPorcelainHeader(branchInfo);
403
+ if (strayBranch && strayBranch !== branchName && _looksLikeAgentBranch(strayBranch)) {
404
+ const commitRes = await _commitAgentWipWithRetry(
405
+ git, baseOpts,
406
+ `minions: auto-save orphaned WIP from ${strayBranch} (dispatch ${dispatchId || '(unknown)'})`,
407
+ { sleep: _sleep },
408
+ );
409
+ if (commitRes.ok) {
410
+ logFn(`[live-checkout] self-healed orphaned agent branch '${strayBranch}' — committed WIP` +
411
+ (commitRes.attempts > 1 ? ` after ${commitRes.attempts} attempts` : '') +
412
+ `, switching back to '${mainRef}'`, 'info');
413
+ try {
414
+ await git(['checkout', mainRef], baseOpts);
415
+ } catch (backErr) {
416
+ logFn(`[live-checkout] could not switch back to '${mainRef}' after self-healing '${strayBranch}': ${backErr && backErr.message}`, 'warn');
417
+ }
418
+ // Re-run the dirty check. Either the tree is now clean (self-heal
419
+ // succeeded end-to-end) or something new appeared (rare) — either
420
+ // way the normal auto-reset / dirty-bail logic below sees fresh state.
421
+ const healRaw = await git(['status', '--porcelain=v1', '-b'], baseOpts);
422
+ ({ branchInfo, dirtyFiles } = parsePorcelain(healRaw));
423
+ } else {
424
+ logFn(`[live-checkout] could not self-heal orphaned agent branch '${strayBranch}' (commit failed): ${commitRes.error && commitRes.error.message}`, 'warn');
425
+ }
426
+ }
427
+ }
322
428
  if (dirtyFiles.length > 0) {
323
429
  // W-mqvejug6000eeb20 — opt-in auto-reset. Default behavior is to bail with
324
430
  // reason:'dirty' (spawnAgent translates to non-retryable LIVE_CHECKOUT_DIRTY
@@ -641,6 +747,7 @@ async function restoreLiveCheckoutAtDispatchEnd(opts = {}) {
641
747
  log,
642
748
  writeInboxAlert,
643
749
  _git, // private injection for testing — defaults to shared.shellSafeGit
750
+ _sleep, // private injection for testing — defaults to a real setTimeout-based sleep
644
751
  } = opts;
645
752
 
646
753
  const logFn = (typeof log === 'function') ? log : () => {};
@@ -750,16 +857,27 @@ async function restoreLiveCheckoutAtDispatchEnd(opts = {}) {
750
857
  onAgentBranch = cur && cur === branchName;
751
858
  } catch { /* best-effort */ }
752
859
  if (onAgentBranch) {
753
- try {
754
- await git(['add', '-A'], baseOpts);
755
- // --no-verify: skip operator commit hooks that may fail headless.
756
- await git(['commit', '--no-verify', '-m', `minions: auto-save agent WIP (dispatch ${did})`], baseOpts);
860
+ // issue #608 — bounded retry+backoff on the commit. The 2026-06-30
861
+ // incident was caused by exactly this commit losing a transient
862
+ // `.git/index.lock` race against a sibling dispatch's git process;
863
+ // the previous single-attempt code gave up immediately and left the
864
+ // stage half-committed, which then perpetuated as a stray agent
865
+ // branch for every subsequent dispatch (see the orphaned-agent-branch
866
+ // self-heal in prepareLiveCheckout). `--no-verify` skips operator
867
+ // commit hooks that may fail headless.
868
+ const commitRes = await _commitAgentWipWithRetry(
869
+ git, baseOpts,
870
+ `minions: auto-save agent WIP (dispatch ${did})`,
871
+ { sleep: _sleep },
872
+ );
873
+ if (commitRes.ok) {
757
874
  result.autoCommitted = true;
758
- logFn('info', `live-checkout: auto-committed agent WIP onto ${branchName} before restore (dispatch ${did})`);
759
- } catch (commitErr) {
875
+ logFn('info', `live-checkout: auto-committed agent WIP onto ${branchName} before restore (dispatch ${did})` +
876
+ (commitRes.attempts > 1 ? ` after ${commitRes.attempts} attempts` : ''));
877
+ } else {
760
878
  // Leave the tree as-is; the plain checkout below will refuse and the
761
879
  // fallback alert fires. Never force, never discard.
762
- logFn('warn', `live-checkout: could not auto-save agent WIP onto ${branchName} for ${did}: ${commitErr && commitErr.message}`);
880
+ logFn('warn', `live-checkout: could not auto-save agent WIP onto ${branchName} for ${did}: ${commitRes.error && commitRes.error.message}`);
763
881
  }
764
882
  }
765
883
  }
@@ -906,5 +1024,8 @@ module.exports = {
906
1024
  _isPartialCloneBlobError,
907
1025
  _isAutoCleanableArtifact,
908
1026
  _collectAutoCleanablePaths,
1027
+ _looksLikeAgentBranch,
1028
+ _extractBranchFromPorcelainHeader,
1029
+ _commitAgentWipWithRetry,
909
1030
  LIVE_CHECKOUT_AUTO_CLEAN_PATTERNS,
910
1031
  };
@@ -13,7 +13,7 @@ const { wrapUntrusted, buildSource } = require('./untrusted-fence');
13
13
  const { MINIONS_BRAND_URL } = require('./comment-format');
14
14
 
15
15
  const { safeJson, safeRead, getProjects, log, ts, dateStamp, truncateTextBytes, ENGINE_DEFAULTS, WI_STATUS, WORK_TYPE, PR_STATUS, DISPATCH_RESULT, getProjectOrg } = shared;
16
- const { getConfig, getDispatch, getNotes, getAgentCharter, getPrs, getKnowledgeBaseIndex, AGENTS_DIR } = queries;
16
+ const { getConfig, getDispatch, getNotes, getPrs, getKnowledgeBaseIndex, AGENTS_DIR } = queries;
17
17
 
18
18
  const MINIONS_DIR = shared.MINIONS_DIR;
19
19
  const PLAYBOOKS_DIR = path.join(MINIONS_DIR, 'playbooks');
@@ -1052,7 +1052,7 @@ function _checkPlaybookSections(content, context = '') {
1052
1052
  // Lean system prompt: agent identity + rules only (~2-4KB, never grows)
1053
1053
  function buildSystemPrompt(agentId, config, project) {
1054
1054
  const agent = config.agents[agentId] || tempAgents.get(agentId) || { name: agentId, role: 'Temporary Agent', expertise: [] };
1055
- const charter = getAgentCharter(agentId); // returns '' for temp agents (no charter file)
1055
+ const charter = safeRead(path.join(AGENTS_DIR, agentId, 'charter.md')); // returns '' for temp agents (no charter file)
1056
1056
  project = project || getProjects(config)[0] || {};
1057
1057
 
1058
1058
  let prompt = '';
package/engine/queries.js CHANGED
@@ -722,10 +722,6 @@ function getAgentStatus(agentId) {
722
722
  // setAgentStatus removed — agent status is derived from dispatch.json.
723
723
  // Status.json files no longer exist.
724
724
 
725
- function getAgentCharter(agentId) {
726
- return safeRead(path.join(AGENTS_DIR, agentId, 'charter.md'));
727
- }
728
-
729
725
  function getAgents(config) {
730
726
  config = config || getConfig();
731
727
  // Fall back to DEFAULT_AGENTS if config has no agents (uninitialized repo)
@@ -2063,13 +2059,21 @@ function getWorkItems(config, opts) {
2063
2059
  });
2064
2060
 
2065
2061
  const cacheMtimes = _snapshotWorkItemsMtimes(config);
2062
+ // Use Date.now() here (not the `now` captured at function entry) so the
2063
+ // 1 s TTL is measured from when the cache was BUILT, not from when the
2064
+ // call started. On a slow or loaded machine the first cold call can take
2065
+ // > 1 s while DB migrations run; if we used the entry-time `now`, the
2066
+ // very next call would see (entry_time_of_call_2 − entry_time_of_call_1)
2067
+ // >= 1 000 ms and miss the cache, defeating the cache entirely.
2068
+ // (W-mr00aooc000i1d6b — queries-work-items-cache CI failure on slow runners)
2069
+ const cacheSetAt = Date.now();
2066
2070
  if (enrich) {
2067
2071
  _workItemsCache = allItems;
2068
- _workItemsCacheAt = now;
2072
+ _workItemsCacheAt = cacheSetAt;
2069
2073
  _workItemsCacheMtimes = cacheMtimes;
2070
2074
  } else {
2071
2075
  _workItemsLeanCache = allItems;
2072
- _workItemsLeanCacheAt = now;
2076
+ _workItemsLeanCacheAt = cacheSetAt;
2073
2077
  _workItemsLeanCacheMtimes = cacheMtimes;
2074
2078
  }
2075
2079
  return allItems;
@@ -3320,7 +3324,7 @@ module.exports = {
3320
3324
  _buildNotesByWiMap, // exported for testing
3321
3325
 
3322
3326
  // Agents
3323
- getAgentStatus, getAgentCharter, getAgents, getAgentDetail,
3327
+ getAgentStatus, getAgents, getAgentDetail,
3324
3328
 
3325
3329
  // Pull requests
3326
3330
  getPrs, getPullRequests,
package/engine/shared.js CHANGED
@@ -3254,11 +3254,12 @@ const ENGINE_DEFAULTS = {
3254
3254
  // When ON, a dirty/broken live-checkout tree is `git fetch origin` +
3255
3255
  // `git reset --hard origin/<branch>`'d before dispatch instead of failing
3256
3256
  // LIVE_CHECKOUT_DIRTY. Per-project `project.liveCheckoutAutoReset` overrides
3257
- // this. Default OFF — auto-reset is destructive (it discards the operator's
3258
- // uncommitted changes), so it is strictly opt-in. Resolution precedence lives
3259
- // in `resolveLiveCheckoutAutoReset`. Fires ONLY on confirmed dirty-tree
3260
- // detection, never on mid-operation / blob-fetch / tooling failures.
3261
- liveCheckoutAutoReset: false,
3257
+ // this. Default ON — discarding uncommitted dirt via reset-to-remote keeps
3258
+ // live-checkout branches from diverging (the old `false` default accumulated
3259
+ // `minions: auto-save agent WIP` commits on every dirty exit). Resolution
3260
+ // precedence lives in `resolveLiveCheckoutAutoReset`. Fires ONLY on confirmed
3261
+ // dirty-tree detection, never on mid-operation / blob-fetch / tooling failures.
3262
+ liveCheckoutAutoReset: true,
3262
3263
  orphanHolderScanTimeoutMs: 5000, // 5s ceiling for the cross-platform holder scan (PowerShell / /proc walk / lsof)
3263
3264
  ccMaxTurns: 50, // max tool-use turns per CC/doc-chat call before CLI stops (per response, not per session)
3264
3265
  ccWorkerIdleTimeoutMs: 30 * 60 * 1000, // W-mr0qs0vw: idle-reaper window for the persistent `copilot --acp` worker pool (engine/cc-worker-pool.js). After this much inactivity with no in-flight turn the warm ACP process is killed; the next message cold-spawns a fresh session with NO memory of prior turns (CC shows a "context cleared after inactivity" notice). Tradeoff: shorter = less idle memory/process footprint, longer = more context durability across gaps between messages. Wired into the pool via ccWorkerPool.setIdleTimeoutMs() on every reloadConfig(); clamped to [60000, 28800000] (1min–8h) in the settings POST handler.
@@ -5,7 +5,14 @@
5
5
  * Spawned `detached: true` by `minions start`/`restart` after the engine and
6
6
  * dashboard are up. Polls every SUPERVISOR_INTERVAL_MS:
7
7
  * - engine PID from `engine/control.json`
8
- * - dashboard PID via port-listener probe (port 7331 by default)
8
+ * - dashboard via a port-listener probe (port 7331 by default) AND, when the
9
+ * port is bound, an HTTP `/api/health` probe. The HTTP layer catches the
10
+ * "frozen but still listening" dashboard (wedged event loop) that the
11
+ * port-listen probe alone reports as healthy — it respawns only after
12
+ * DASH_HEALTH_MAX_FAILS consecutive failed probes so a dashboard merely
13
+ * busy with a heavy synchronous getStatus() rebuild isn't killed. What
14
+ * counts as "responsive" (and why engine 'degraded'/'stopped' does NOT mean
15
+ * the dashboard is frozen) is documented at _probeDashboardHealth.
9
16
  *
10
17
  * When either is dead AND the stop-intent flag is NOT set, respawns the dead
11
18
  * one in the same way the CLI does (detached, stdio routed to the engine-
@@ -76,6 +83,16 @@ function _resolveDashPort() {
76
83
  const POST_SPAWN_GRACE_MS = Number(process.env.MINIONS_SUPERVISOR_GRACE_MS) || 15000;
77
84
  // #421 — heartbeat age threshold before supervisor considers the engine event loop frozen.
78
85
  const SUPERVISOR_STALE_ENGINE_HEARTBEAT_MS = Number(process.env.MINIONS_SUPERVISOR_STALE_HEARTBEAT_MS) || 180000; // 3 min
86
+ // Dashboard HTTP liveness (frozen-but-listening detection). A dashboard whose
87
+ // event loop is wedged keeps the socket LISTENING, so the port-listen probe
88
+ // alone reports it healthy while the in-browser SPA shows the "unreachable"
89
+ // banner forever. We layer an HTTP /api/health probe on top: only respawn after
90
+ // DASH_HEALTH_MAX_FAILS *consecutive* failed probes so a dashboard that is
91
+ // merely busy with a heavy synchronous getStatus() rebuild (documented to block
92
+ // the loop 15-25s) isn't mistaken for a hang. With the default 30s interval,
93
+ // 3 strikes ≈ 90s of continuous unresponsiveness before we act.
94
+ const DASH_HEALTH_TIMEOUT_MS = Number(process.env.MINIONS_SUPERVISOR_DASH_HEALTH_TIMEOUT_MS) || 5000;
95
+ const DASH_HEALTH_MAX_FAILS = Number(process.env.MINIONS_SUPERVISOR_DASH_HEALTH_FAILS) || 3;
79
96
  const isWin = process.platform === 'win32';
80
97
 
81
98
  function safeReadJson(p) {
@@ -354,6 +371,39 @@ function spawnDashboard() {
354
371
  // the engine.
355
372
  let _lastEngineRespawnAt = 0;
356
373
  let _lastDashboardRespawnAt = 0;
374
+ // Consecutive failed dashboard HTTP health probes. Reset on any healthy probe,
375
+ // after a respawn, and inside the post-spawn grace window.
376
+ let _dashHealthFailStreak = 0;
377
+
378
+ // Lazy, defensive HTTP health probe. Reuses restart-health.js's httpGetJson
379
+ // (rule: reuse before re-implementing) but never lets a load failure (e.g. a
380
+ // mid-upgrade broken shared.js that restart-health requires at module top)
381
+ // break the supervisor. Returns null when the probe helper is unavailable so
382
+ // the caller can FALL BACK to port-only liveness instead of false-respawning a
383
+ // healthy dashboard.
384
+ function _probeDashboardHealth(port, timeoutMs = DASH_HEALTH_TIMEOUT_MS) {
385
+ let getJson = null;
386
+ try {
387
+ const rh = require('./restart-health');
388
+ if (rh && rh._private && typeof rh._private.httpGetJson === 'function') {
389
+ getJson = rh._private.httpGetJson;
390
+ }
391
+ } catch { /* restart-health/shared unavailable — fall through */ }
392
+ if (!getJson) return Promise.resolve(null);
393
+ return getJson(`http://127.0.0.1:${port}/api/health`, timeoutMs)
394
+ // "Responsive" == the dashboard event loop executed handleHealth and returned
395
+ // a well-formed 2xx envelope, REGARDLESS of the engine-derived status value.
396
+ // /api/health reports 'degraded' when the engine is paused/stopping and
397
+ // 'stopped' when the engine is down — those are legitimate ENGINE states
398
+ // (owned by checkEngine/checkEngineHung), not evidence the DASHBOARD process
399
+ // is frozen. Gating on status==='healthy' would respawn a perfectly
400
+ // responsive dashboard every time an operator pauses the engine, tearing
401
+ // down the socket and tripping the in-browser "unreachable" banner for no
402
+ // reason. Only a failed/timed-out request (res.ok false or a rejection)
403
+ // means the dashboard's own event loop is actually wedged.
404
+ .then(res => !!(res && res.ok && res.json && typeof res.json.status === 'string'))
405
+ .catch(() => false);
406
+ }
357
407
 
358
408
  // Window during which a `null` pid + recent `restarted_at` is interpreted as
359
409
  // "another watchdog is currently respawning the engine — don't double-spawn."
@@ -364,12 +414,29 @@ let _lastDashboardRespawnAt = 0;
364
414
  // to enter cli.js:443 and write its own PID on cold Windows boots.
365
415
  const RESPAWN_IN_PROGRESS_WINDOW_MS = Number(process.env.MINIONS_SUPERVISOR_RESPAWN_WINDOW_MS) || 60000;
366
416
 
417
+ // #3758 — 'degraded' is a FAILURE state (set by dashboard.js's frozen-tick
418
+ // watchdog, see dashboard.js#_markEngineAsDegradedIfFrozen), not a deliberate
419
+ // user-intent state like 'paused'/'stopping'/'stopped'. When a frozen tick loop
420
+ // finally unblocks, engine.js#tickInner reads control.state !== 'running' and
421
+ // calls process.exit(0) — self-exiting exactly BECAUSE it saw 'degraded'. The
422
+ // dashboard.js comment documents the expected recovery: "the PID-dead watchdog
423
+ // then auto-restarts it." Excluding 'degraded' here left that promise unmet —
424
+ // the engine PID goes away and the control plane silently stays down until a
425
+ // manual `minions restart`. Respawn eligibility must include both 'running'
426
+ // (still up) and 'degraded' (crashed/crashing out of a real failure); only
427
+ // 'paused', 'stopping', and 'stopped' reflect actual user intent to stay down.
428
+ function _isRespawnEligibleEngineState(state) {
429
+ return state === 'running' || state === 'degraded';
430
+ }
431
+
367
432
  function checkEngine(now) {
368
433
  if (now - _lastEngineRespawnAt < POST_SPAWN_GRACE_MS) return;
369
434
  const control = safeReadJson(CONTROL_PATH_FN());
370
- // Only respawn when control.json says "running" — paused/stopped/stopping
371
- // are legitimate states the supervisor must not override.
372
- if (!control || control.state !== 'running') return;
435
+ // Respawn when control.json says "running" (PID died unexpectedly) or
436
+ // "degraded" (watchdog-flagged failure state — see _isRespawnEligibleEngineState
437
+ // above). "paused"/"stopped"/"stopping" are legitimate user-intent states the
438
+ // supervisor must not override.
439
+ if (!control || !_isRespawnEligibleEngineState(control.state)) return;
373
440
  if (control.pid && isPidAlive(control.pid)) return;
374
441
 
375
442
  // Cross-watchdog race guard: dashboard.js's in-process engine watchdog
@@ -403,7 +470,11 @@ function checkEngine(now) {
403
470
  function checkEngineHung(now) {
404
471
  if (now - _lastEngineRespawnAt < POST_SPAWN_GRACE_MS) return;
405
472
  const control = safeReadJson(CONTROL_PATH_FN());
406
- if (!control || control.state !== 'running') return;
473
+ // Include 'degraded' (see _isRespawnEligibleEngineState) so a permanently
474
+ // frozen tick loop that never gets a chance to self-exit (tickInner never
475
+ // re-enters) still gets force-restarted once its own heartbeat goes stale,
476
+ // instead of being ignored forever the moment the state flips to 'degraded'.
477
+ if (!control || !_isRespawnEligibleEngineState(control.state)) return;
407
478
  // Only acts when the PID is alive — a dead PID is handled by checkEngine().
408
479
  if (!control.pid || !isPidAlive(control.pid)) return;
409
480
 
@@ -424,31 +495,69 @@ function checkEngineHung(now) {
424
495
  console.log(`[supervisor] Engine restarted due to stale heartbeat (new PID: ${newPid})`);
425
496
  }
426
497
 
427
- function checkDashboard(now) {
428
- if (now - _lastDashboardRespawnAt < POST_SPAWN_GRACE_MS) return;
498
+ async function checkDashboard(now) {
499
+ if (now - _lastDashboardRespawnAt < POST_SPAWN_GRACE_MS) { _dashHealthFailStreak = 0; return; }
429
500
  const dashPort = _resolveDashPort();
430
501
  const pids = listeningPidsForPort(dashPort);
431
- if (pids.length > 0) return;
432
-
433
- console.log(`[supervisor] Dashboard not listening on port ${dashPort} — respawning...`);
434
- // Reap stray dashboards first. Besides bounding the count, this collapses the
435
- // port-desync case: orphan dashboards holding other ports are killed, freeing
436
- // the canonical port for the fresh one whose beacon _resolveDashPort() reads.
437
- reapStrayProcesses(path.join(MINIONS_DIR, 'dashboard.js'), 'dashboard');
438
- const newPid = spawnDashboard();
439
- _lastDashboardRespawnAt = now;
440
- console.log(`[supervisor] Dashboard respawned (new PID: ${newPid})`);
502
+
503
+ const respawn = (why) => {
504
+ console.log(`[supervisor] Dashboard ${why} on port ${dashPort} — respawning...`);
505
+ // Reap stray dashboards first. Besides bounding the count, this collapses the
506
+ // port-desync case: orphan dashboards holding other ports are killed, freeing
507
+ // the canonical port for the fresh one whose beacon _resolveDashPort() reads.
508
+ // Critically, for the frozen-but-listening case the reap also KILLS the wedged
509
+ // process still holding the port so spawnDashboard() can bind it.
510
+ reapStrayProcesses(path.join(MINIONS_DIR, 'dashboard.js'), 'dashboard');
511
+ const newPid = spawnDashboard();
512
+ _lastDashboardRespawnAt = now;
513
+ _dashHealthFailStreak = 0;
514
+ console.log(`[supervisor] Dashboard respawned (new PID: ${newPid})`);
515
+ };
516
+
517
+ // Hard-down: nothing bound to the port → respawn immediately.
518
+ if (pids.length === 0) {
519
+ _dashHealthFailStreak = 0;
520
+ respawn('not listening');
521
+ return;
522
+ }
523
+
524
+ // Port is bound. Verify the event loop is actually responsive — a frozen-but-
525
+ // listening dashboard keeps the socket LISTENING yet serves the in-browser
526
+ // "unreachable" banner. Probe /api/health; only respawn after a streak of
527
+ // failures so a legitimately busy getStatus() rebuild isn't mistaken for a hang.
528
+ const healthy = await _probeDashboardHealth(dashPort);
529
+ if (healthy === null) {
530
+ // Probe helper unavailable — fall back to port-only liveness (the pre-existing
531
+ // behavior). Don't count this as a failure; we won't respawn a bound dashboard
532
+ // we can't actually prove is frozen.
533
+ _dashHealthFailStreak = 0;
534
+ return;
535
+ }
536
+ if (healthy) { _dashHealthFailStreak = 0; return; }
537
+
538
+ _dashHealthFailStreak++;
539
+ console.log(`[supervisor] Dashboard bound but unresponsive on port ${dashPort} (${_dashHealthFailStreak}/${DASH_HEALTH_MAX_FAILS} strikes)`);
540
+ if (_dashHealthFailStreak < DASH_HEALTH_MAX_FAILS) return;
541
+ respawn('frozen (health probe failed)');
441
542
  }
442
543
 
443
- function tick() {
544
+ let _tickInFlight = false;
545
+ async function tick() {
546
+ // The dashboard health probe is async; guard against a slow probe overlapping
547
+ // the next interval fire. setInterval invoking an async fn is fire-and-forget,
548
+ // so this flag is the only thing preventing concurrent ticks.
549
+ if (_tickInFlight) return;
550
+ _tickInFlight = true;
444
551
  try {
445
552
  if (isStopIntentSet()) return;
446
553
  const now = Date.now();
447
554
  checkEngine(now);
448
555
  checkEngineHung(now);
449
- checkDashboard(now);
556
+ await checkDashboard(now);
450
557
  } catch (e) {
451
558
  console.error(`[supervisor] tick error: ${e && e.message}`);
559
+ } finally {
560
+ _tickInFlight = false;
452
561
  }
453
562
  }
454
563
 
@@ -517,8 +626,11 @@ module.exports = {
517
626
  openAppendFd,
518
627
  checkEngine,
519
628
  checkEngineHung,
629
+ _isRespawnEligibleEngineState,
520
630
  checkDashboard,
631
+ _probeDashboardHealth,
521
632
  SUPERVISOR_STALE_ENGINE_HEARTBEAT_MS,
633
+ DASH_HEALTH_MAX_FAILS,
522
634
  tick,
523
635
  // Path getters honor MINIONS_TEST_DIR via shared.ENGINE_DIR, so test
524
636
  // isolation correctly redirects writes/reads under createTestMinionsDir.
package/engine.js CHANGED
@@ -138,7 +138,7 @@ const worktreePool = require('./engine/worktree-pool');
138
138
  // ─── State Readers (delegated to engine/queries.js) ─────────────────────────
139
139
 
140
140
  const { getConfig, getControl, getDispatch, getNotes,
141
- getAgentStatus, getAgentCharter, getInboxFiles,
141
+ getAgentStatus, getInboxFiles,
142
142
  collectSkillFiles, getSkillIndex, getKnowledgeBaseIndex,
143
143
  getPrs, SKILLS_DIR } = queries;
144
144
 
@@ -10564,7 +10564,7 @@ module.exports = {
10564
10564
 
10565
10565
  // State readers/writers
10566
10566
  getConfig, getControl, getDispatch, getRouting, getNotes,
10567
- getAgentStatus, getAgentCharter, getInboxFiles, getPrs,
10567
+ getAgentStatus, getInboxFiles, getPrs,
10568
10568
  validateConfig,
10569
10569
 
10570
10570
  // Dispatch management (re-exported from engine/dispatch.js)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@yemi33/minions",
3
- "version": "0.1.2305",
3
+ "version": "0.1.2306",
4
4
  "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
5
5
  "bin": {
6
6
  "minions": "bin/minions.js"