@yemi33/minions 0.1.2006 → 0.1.2008

package/dashboard/js/render-dispatch.js

@@ -10,6 +10,32 @@ function _completedNext() { _completedPage++; refresh(); } // clamped in renderD
 function _logPrev() { if (_logPage > 0) { _logPage--; refresh(); } }
 function _logNext() { _logPage++; refresh(); } // clamped in renderEngineLog
 
+// Engine restart grace state (W-mpfw3hgm001gc594). After the operator clicks
+// "Restart engine" we suppress the STALE indicators for ENGINE_RESTART_GRACE_MS
+// while the new engine spawns and writes its first heartbeat. The state is
+// module-scoped (not on window) so any re-render during the grace window can
+// reconstruct the success indicator — earlier code mutated textContent on the
+// click target, which got blown away on the next /api/status render.
+const _ENGINE_RESTART_GRACE_MS = 30000;
+const _ENGINE_RESTART_MAX_RETRIES = 3;
+let _engineRestartState = null; // { restartedAt: number, pid: number|string, retryCount: number }
+
+function _isEngineRestartGrace(now) {
+  const t = (typeof now === 'number') ? now : Date.now();
+  return !!(_engineRestartState && (t - _engineRestartState.restartedAt < _ENGINE_RESTART_GRACE_MS));
+}
+
+function _shouldSuppressStaleSignals(state, now) {
+  // All three stale signals (badge, alert, version warning) consult this helper
+  // so they share a single source of truth for the post-restart grace window.
+  if (state === 'running') return false; // engine recovered — show real state
+  return _isEngineRestartGrace(now);
+}
+
+function _resetEngineRestartStateForTest() {
+  _engineRestartState = null;
+}
+
 function renderEngineStatus(engine) {
   const badge = document.getElementById('engine-badge');
   let state = engine?.state || 'stopped';
@@ -21,40 +47,48 @@ function renderEngineStatus(engine) {
     if (staleMs > 120000) state = 'stale';
   }
 
-  badge.className = 'engine-badge ' + (state === 'stale' ? 'stopped' : state);
-  badge.textContent = state === 'stale' ? 'STALE' : state.toUpperCase();
-  badge.title = state === 'stale'
+  // Clear restart grace as soon as the engine reports a fresh heartbeat — the
+  // new engine has caught up, so STALE/restart banners should vanish.
+  if (state === 'running' && _engineRestartState) {
+    _engineRestartState = null;
+  }
+
+  const suppressStale = _shouldSuppressStaleSignals(state);
+  let displayState = state;
+  if (state === 'stale' && suppressStale) displayState = 'restarting';
+
+  badge.className = 'engine-badge ' + (displayState === 'stale' ? 'stopped' : (displayState === 'restarting' ? 'paused' : displayState));
+  badge.textContent = displayState === 'stale' ? 'STALE' : displayState.toUpperCase();
+  badge.title = displayState === 'stale'
     ? 'Engine claims running but heartbeat is stale (>2min). It may have crashed. Run: node engine.js start'
-    : state === 'stopped' ? 'Engine is stopped. Run: node engine.js start' : '';
+    : displayState === 'restarting'
+      ? 'Engine restart in progress (PID ' + (_engineRestartState?.pid || '?') + ') — waiting for first heartbeat.'
+      : displayState === 'stopped' ? 'Engine is stopped. Run: node engine.js start' : '';
   renderEngineAlert(state, staleMs);
 }
 
-function renderEngineAlert(state, staleMs) {
-  const el = document.getElementById('engine-alert');
-  if (!el) return;
-  if (state !== 'stale' || (window._engineRestartedAt && Date.now() - window._engineRestartedAt < 30000)) {
-    el.style.display = 'none';
-    el.innerHTML = '';
-    return;
-  }
-  const mins = Math.max(1, Math.round(staleMs / 60000));
-  el.innerHTML =
-    '<span class="engine-alert-msg">&#x26A0;&#xFE0F; Engine heartbeat is stale (' + mins + 'm old). Dispatch may be stuck.</span>' +
-    '<span class="engine-alert-action" id="engine-alert-restart">Restart engine</span>';
-  document.getElementById('engine-alert-restart').onclick = async function() {
+function _wireEngineRestartClick(button) {
+  button.onclick = async function() {
     this.classList.add('clicked');
     this.textContent = 'Restarting...';
+    const priorRetries = _engineRestartState?.retryCount || 0;
     try {
       const res = await fetch('/api/engine/restart', { method: 'POST' });
       const data = await res.json();
       if (data.ok) {
-        this.textContent = '\u2713 Restarted (PID ' + data.pid + ')';
-        this.style.color = 'var(--green)';
-        this.style.borderColor = 'var(--green)';
+        // Persist restart state so re-renders during the grace window can
+        // reconstruct the success message even after this DOM node is gone.
+        _engineRestartState = {
+          restartedAt: Date.now(),
+          pid: data.pid,
+          retryCount: priorRetries + 1,
+        };
         showToast('cmd-toast', 'Engine restarted — PID ' + data.pid, true);
-        // Suppress stale banner for 30s while new engine writes its first heartbeat
-        window._engineRestartedAt = Date.now();
-        setTimeout(() => refresh(), 3000);
+        // Trigger a refresh sooner than the 4s poll cadence so the success
+        // indicator surfaces quickly. The grace window itself (30s) plus
+        // ongoing 4s polling will catch the heartbeat advance whenever it
+        // lands; no fixed timeout assumption needed.
+        setTimeout(() => refresh(), 1500);
       } else {
         this.textContent = 'Failed: ' + (data.error || 'unknown');
         this.classList.remove('clicked');
@@ -64,9 +98,62 @@ function renderEngineAlert(state, staleMs) {
       this.classList.remove('clicked');
     }
   };
+}
+
+function _renderEngineRestartSuccessBanner(el) {
+  const pid = _engineRestartState?.pid || '?';
+  el.innerHTML =
+    '<span class="engine-alert-msg" style="color:var(--green)">&#x2713; Engine restarted (PID ' + pid + ') — waiting for first heartbeat...</span>';
   el.style.display = 'flex';
 }
 
+function _renderEngineRestartRetryBanner(el) {
+  const attempts = _engineRestartState?.retryCount || 0;
+  el.innerHTML =
+    '<span class="engine-alert-msg">&#x26A0;&#xFE0F; Engine restart didn\'t take — heartbeat still stale (attempt ' + attempts + ' of ' + _ENGINE_RESTART_MAX_RETRIES + ').</span>' +
+    '<span class="engine-alert-action" id="engine-alert-restart">Retry restart</span>';
+  _wireEngineRestartClick(document.getElementById('engine-alert-restart'));
+  el.style.display = 'flex';
+}
+
+function _renderEngineStaleBanner(el, staleMs) {
+  const mins = Math.max(1, Math.round(staleMs / 60000));
+  el.innerHTML =
+    '<span class="engine-alert-msg">&#x26A0;&#xFE0F; Engine heartbeat is stale (' + mins + 'm old). Dispatch may be stuck.</span>' +
+    '<span class="engine-alert-action" id="engine-alert-restart">Restart engine</span>';
+  _wireEngineRestartClick(document.getElementById('engine-alert-restart'));
+  el.style.display = 'flex';
+}
+
+function renderEngineAlert(state, staleMs) {
+  const el = document.getElementById('engine-alert');
+  if (!el) return;
+
+  // Engine recovered — clear everything (restart state already nulled in
+  // renderEngineStatus when state flipped to 'running').
+  if (state !== 'stale') {
+    el.style.display = 'none';
+    el.innerHTML = '';
+    return;
+  }
+
+  // Stale + within grace window → persistent success indicator.
+  if (_isEngineRestartGrace()) {
+    _renderEngineRestartSuccessBanner(el);
+    return;
+  }
+
+  // Stale + past grace + we previously restarted but didn't recover.
+  // Offer a retry banner instead of snapping back to the original alert,
+  // up to MAX_RETRIES. After that, fall back to the original alert.
+  if (_engineRestartState && _engineRestartState.retryCount < _ENGINE_RESTART_MAX_RETRIES) {
+    _renderEngineRestartRetryBanner(el);
+    return;
+  }
+
+  _renderEngineStaleBanner(el, staleMs);
+}
+
 function renderAdoThrottleAlert(adoThrottle) {
   const el = document.getElementById('ado-throttle-alert');
   if (!el) return;
@@ -272,11 +359,18 @@ function renderVersionBanner(version) {
   const commitLabel = version.dashboardRunningCommit ? ' (' + version.dashboardRunningCommit + ')' : '';
   const warnStyle = 'font-size:9px;padding:2px 8px;background:rgba(210,153,34,0.15);border:1px solid rgba(210,153,34,0.3);border-radius:4px;color:var(--yellow);cursor:help';
 
-  if (version.engineStale && version.dashboardStale) {
+  // During the post-restart grace window the old engine's reported codeVersion
+  // can still be in the cached payload — silently swallow the engineStale flag
+  // so the user doesn't see "Engine running v… disk has v…" right after a
+  // successful restart. The new engine clears the flag once it writes its
+  // first heartbeat. dashboardStale is unrelated and must still surface.
+  const engineStale = version.engineStale && !_isEngineRestartGrace();
+
+  if (engineStale && version.dashboardStale) {
     el.style.cssText = warnStyle;
     el.textContent = '\u26A0 Engine + Dashboard running old code. Run: minions restart';
     el.title = 'Both processes are running v' + (version.running || '?') + ' but disk has v' + (version.disk || '?');
-  } else if (version.engineStale) {
+  } else if (engineStale) {
     el.style.cssText = warnStyle;
     el.textContent = '\u26A0 Engine running v' + (version.running || '?') + ' — disk has v' + (version.disk || '?') + '. Restart engine.';
     el.title = 'The engine process is running older code. Run: minions restart';
@@ -295,4 +389,4 @@ function renderVersionBanner(version) {
   }
 }
 
-window.MinionsDispatch = { renderEngineStatus, renderEngineAlert, renderAdoThrottleAlert, renderGhThrottleAlert, renderVersionBanner, renderDispatch, renderEngineLog, shortTime, showErrorDetails };
+window.MinionsDispatch = { renderEngineStatus, renderEngineAlert, renderAdoThrottleAlert, renderGhThrottleAlert, renderVersionBanner, renderDispatch, renderEngineLog, shortTime, showErrorDetails, _isEngineRestartGrace, _shouldSuppressStaleSignals, _resetEngineRestartStateForTest };

package/dashboard.js

@@ -1525,22 +1525,24 @@ function _ifNoneMatchHasEtag(headerValue, currentEtag) {
   return false;
 }
 
-// mtime-based cache invalidation — skip full rebuild if no tracked files changed
-const _mtimeTrackedFiles = () => {
-  const files = [
-    path.join(ENGINE_DIR, 'dispatch.json'),
-    path.join(ENGINE_DIR, 'control.json'),
-    path.join(ENGINE_DIR, 'log.json'),
-    path.join(ENGINE_DIR, 'metrics.json'),
-  ];
-  // Add per-project work-items.json
-  for (const p of PROJECTS) {
-    files.push(shared.projectWorkItemsPath(p));
-  }
-  // Central work-items.json
-  files.push(path.join(MINIONS_DIR, 'work-items.json'));
-  return files;
-};
+// mtime-based cache invalidation (W-mpftp7na000td0f4).
+//
+// Engine and dashboard are independent processes; `invalidateStatusCache()`
+// lives in dashboard.js memory and is unreachable from engine code. The
+// fast-state TTL is 10 s, but we want sub-second visibility for engine-side
+// state flips (work-item pending→done, PR status changes, dispatch.json
+// mutations). The fix: every `getStatus()` call statSyncs a small set of
+// tracked files; if any mtime advanced since the last rebuild, fast-state
+// is rebuilt and `_statusCacheVersion` bumps (which busts the ETag, so the
+// next /api/status poll sees a 200 + fresh body instead of a 304).
+//
+// The tracked list lives in `engine/queries.js → getStatusFastStateMtimePaths()`
+// as a single source of truth — colocated with the read-side aggregation
+// layer that owns the rest of the fast-state assembly. Add a new tracked
+// file there (one line), NOT here. The dashboard side stays a thin
+// delegate so any module that contributes to `_buildStatusFastState()` can
+// register its mtime inputs in one place.
+const _mtimeTrackedFiles = () => queries.getStatusFastStateMtimePaths(CONFIG);
 let _lastMtimes = {}; // { filePath: mtimeMs }
 
 function _getMtimes() {

package/engine/cli.js

@@ -448,6 +448,15 @@ const commands = {
       codeVersion,
       codeCommit
     }));
+    // W-mpg3bcp800075d4f — Prime control.heartbeat the moment this process
+    // owns control.json, BEFORE the dispatch-recovery / startupReconcile*
+    // chain. Those reconciles are synchronous and on a slow Windows host with
+    // many projects/worktrees can run for 15-40s; without an early prime the
+    // dashboard's 30s restart-grace window (dashboard/js/render-dispatch.js
+    // → renderEngineAlert) expires against the *previous* engine's heartbeat
+    // and the 'Engine heartbeat is stale' banner snaps back even though the
+    // new engine is healthy. The 15s setInterval below keeps it fresh.
+    writeHeartbeatNow();
     // Keep .minions-version in sync so `minions version` stays accurate after git pulls
     if (codeVersion) {
       try { fs.writeFileSync(path.join(shared.MINIONS_DIR, '.minions-version'), codeVersion); } catch {}
@@ -924,8 +933,11 @@ const commands = {
     // surfaced a healthy engine as crashed. We now write every 15s on a
     // dedicated interval — 8× headroom vs the 120s threshold even under
     // event-loop pressure, and orders of magnitude under TICK_TIMEOUT_MS so
-    // a hung tick still looks distinct from a wedged event loop.
-    writeHeartbeatNow(); // prime control.heartbeat immediately
+    // a hung tick still looks distinct from a wedged event loop. The first
+    // (priming) heartbeat write lives near the top of this handler — see the
+    // W-mpg3bcp800075d4f comment above mutateControl — so that the dashboard's
+    // 30s restart grace window sees a fresh heartbeat regardless of how long
+    // the boot reconcile chain takes.
     const heartbeatTimer = setInterval(writeHeartbeatNow, HEARTBEAT_INTERVAL_MS);
 
     // Fast poll: check steering every 1s (lightweight — just fs.stat per agent)

package/engine/queries.js

@@ -1749,6 +1749,69 @@ function resetProjectGitStatusCache() {
   _projectGitStatusCache.clear();
 }
 
+/**
+ * Files whose mtime must trigger a dashboard `_fastState` rebuild
+ * (W-mpftp7na000td0f4). Single source of truth for the dashboard's
+ * cache-invalidation tracker — adding a new fast-state-surfaced JSON file
+ * is a **one-line change here**, next to the related fast-state getters.
+ *
+ * Contract for additions:
+ *   1. The file must be read (directly or transitively) by something in
+ *      `dashboard._buildStatusFastState` (e.g. `getPullRequests`,
+ *      `getDispatchQueue`, `watchesMod.getWatches`). Files surfaced only
+ *      in `_buildStatusSlowState` (60 s TTL) should NOT be added — fast-
+ *      state mtime invalidation has no effect on slow state, and they
+ *      will silently no-op.
+ *   2. The file must be mutated through a `mutate*` helper (so writes
+ *      actually advance mtime via `safeWrite`'s rename). Append-only logs
+ *      can also be added, but expect rebuilds on every line write.
+ *   3. Per-project files must use `shared.projectPrPath` / `shared.
+ *      projectWorkItemsPath` so newly-added projects are picked up
+ *      automatically.
+ *
+ * Files intentionally NOT tracked:
+ *   - `engine/state.json` — surfaced via `getEngineState()` but changes
+ *     only on engine startup / reconcile. Negligible benefit.
+ *   - `engine/cooldowns.json`, `engine/pr-links.json`, `engine/pending-
+ *     rebases.json`, `agents/<id>/managed-spawn.json` — not in the
+ *     `/api/status` payload.
+ *   - `pinned.md`, `schedules`, `pipeline-runs.json`, `schedule-runs.json`,
+ *     PRD JSON — slow-state only.
+ *   - `meetings/` directory — dir mtime semantics are flaky on Windows
+ *     for file-content changes inside the dir; meeting transitions also
+ *     mutate work items, which already invalidate fast state.
+ *
+ * Performance: `_getMtimes()` in dashboard.js does `fs.statSync` per path
+ * per `getStatus()` call. Roughly N=4 engine paths + 2 per project today,
+ * so a 5-project fleet runs 14 statSync calls per cache miss — bounded
+ * and unmeasurable in benchmarks.
+ */
+function getStatusFastStateMtimePaths(config) {
+  const projects = getProjects(config || getConfig());
+  const files = [
+    // Engine-level state surfaced by getDispatchQueue / inline engine block /
+    // getEngineLog / getMetrics.
+    DISPATCH_PATH,
+    CONTROL_PATH,
+    LOG_PATH,
+    path.join(ENGINE_DIR, 'metrics.json'),
+    // Watches surfaced by watchesMod.getWatches() (W-mpftp7na000td0f4 fix).
+    path.join(ENGINE_DIR, 'watches.json'),
+    // Central work-items.json surfaced by getWorkItems().
+    path.join(MINIONS_DIR, 'work-items.json'),
+  ];
+  // Per-project work-items (surfaced by getWorkItems) and pull-requests
+  // (surfaced by getPullRequests). The PR file was the biggest miss in the
+  // original tracked list — PR status flips (running → passing, waiting →
+  // approved) were waiting on the 10 s SSE backstop instead of the next
+  // 4 s SPA poll.
+  for (const p of projects) {
+    files.push(shared.projectWorkItemsPath(p));
+    files.push(shared.projectPrPath(p));
+  }
+  return files;
+}
+
 // ── Exports ─────────────────────────────────────────────────────────────────
 
 module.exports = {
@@ -1766,6 +1829,8 @@ module.exports = {
   getProjectGitStatus,
   warmProjectGitStatus,
   _awaitPendingProjectGitStatusProbes,
+  // W-mpftp7na000td0f4 — engine→dashboard cache-invalidation registry
+  getStatusFastStateMtimePaths,
 
   // Core state
   getConfig, getControl, getDispatch, getDispatchQueue, getDispatchCompletionReport, invalidateDispatchCache,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2006",
+  "version": "0.1.2008",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"