@yemi33/minions 0.1.2026 → 0.1.2028

package/dashboard/js/refresh.js

@@ -13,6 +13,7 @@ const _pageCounters = {
   pipelines: function(d) { return (d.pipelines || []).length + '|' + (d.pipelines || []).reduce(function(s, p) { return s + (p.runs || []).length; }, 0); },
   schedule:  function(d) { return (d.schedules || []).length; },
   engine:    function(d) { return (d.dispatch?.completed || []).filter(function(c) { return c.result === 'error'; }).length; },
+  qa:        function(d) { return (d.qaRuns?.total || 0) + '|' + (d.qaRuns?.sig || ''); },
 };
 let _prevCounts = {};
 function _detectPageChanges(data) {

package/dashboard.js

@@ -1589,7 +1589,9 @@ function _ifNoneMatchHasEtag(headerValue, currentEtag) {
 // delegate so any module that contributes to `_buildStatusFastState()` can
 // register its mtime inputs in one place.
 const _mtimeTrackedFiles = () => queries.getStatusFastStateMtimePaths(CONFIG);
-let _lastMtimes = {}; // { filePath: mtimeMs }
+const _slowMtimeTrackedFiles = () => queries.getStatusSlowStateMtimePaths(CONFIG);
+let _lastMtimes = {}; // { filePath: mtimeMs } — fast-state baseline
+let _lastSlowMtimes = {}; // { filePath: mtimeMs } — slow-state baseline
 
 function _getMtimes() {
   const result = {};
@@ -1599,6 +1601,14 @@ function _getMtimes() {
   return result;
 }
 
+function _getSlowMtimes() {
+  const result = {};
+  for (const fp of _slowMtimeTrackedFiles()) {
+    try { result[fp] = fs.statSync(fp).mtimeMs; } catch { result[fp] = 0; }
+  }
+  return result;
+}
+
 function _mtimesChanged(prev, curr) {
   for (const fp of Object.keys(curr)) {
     if (prev[fp] !== curr[fp]) return true;
@@ -1668,6 +1678,21 @@ function _buildStatusFastState() {
     workItems: getWorkItems(),
     watches: watchesMod.getWatches(),
     meetings: (() => { try { return require('./engine/meeting').getMeetings(); } catch { return []; } })(),
+    // QA runs — surfaced for the sidebar activity-dot counter and any future
+    // CC/aggregate view. Tab-level rendering keeps its own /api/qa/runs poll
+    // (5 s while the QA page is mounted). qa-runs.json is in the mtime tracker
+    // so a new run lights the dot within one /api/status poll cycle (~4 s).
+    qaRuns: (() => {
+      try {
+        const runs = require('./engine/qa-runs').listRuns({ limit: 50 }) || [];
+        return {
+          total: runs.length,
+          // Signature of (id, status) for the most recent 20 runs so the
+          // sidebar counter advances on status flips AND on new entries.
+          sig: runs.slice(0, 20).map(r => (r && r.id || '') + ':' + (r && r.status || '')).join(','),
+        };
+      } catch { return { total: 0, sig: '' }; }
+    })(),
   };
 }
 
@@ -1772,8 +1797,16 @@ function getStatus() {
     if (_mtimesChanged(_lastMtimes, currMtimes)) fastStale = true;
   }
 
-  // Slow state: 60s TTL, pure TTL (no mtime check — these files change rarely)
-  const slowStale = !_slowState || (now - _slowStateTs) >= SLOW_STATE_TTL;
+  // Slow state: 60s TTL with mtime-based validation for early bust.
+  // The mtime tracker covers engine-driven slow-state writes (PRD updates,
+  // pipeline-runs.json, schedule-runs.json, verify guides, project skills)
+  // so changes surface within one SPA poll (~4 s) instead of waiting up to
+  // 60 s for TTL. Same pre-build snapshot semantics as fast-state below.
+  let slowStale = !_slowState || (now - _slowStateTs) >= SLOW_STATE_TTL;
+  if (!slowStale) {
+    const currSlowMtimes = _getSlowMtimes();
+    if (_mtimesChanged(_lastSlowMtimes, currSlowMtimes)) slowStale = true;
+  }
 
   // If nothing stale, return cached merged result
   if (!fastStale && !slowStale && _statusCache) return _statusCache;
@@ -1795,10 +1828,14 @@ function getStatus() {
     _lastMtimes = preBuildMtimes;
   }
 
-  // Rebuild slow state (rarely-changing data: ~8-15 reads, 60s TTL)
+  // Rebuild slow state (rarely-changing data: ~8-15 reads, 60s TTL).
+  // Same pre-build snapshot pattern as fast-state — capture mtimes BEFORE
+  // disk reads so any write landing mid-build busts the next poll.
   if (slowStale) {
+    const preBuildSlowMtimes = _getSlowMtimes();
     _slowState = _buildStatusSlowState();
     _slowStateTs = now;
+    _lastSlowMtimes = preBuildSlowMtimes;
   }
 
   // Merge both tiers — no API contract change
@@ -1839,7 +1876,11 @@ function refreshStatusAsync() {
         const currMtimes = _getMtimes();
         if (_mtimesChanged(_lastMtimes, currMtimes)) fastStale = true;
       }
-      const slowStale = !_slowState || (now - _slowStateTs) >= SLOW_STATE_TTL;
+      let slowStale = !_slowState || (now - _slowStateTs) >= SLOW_STATE_TTL;
+      if (!slowStale) {
+        const currSlowMtimes = _getSlowMtimes();
+        if (_mtimesChanged(_lastSlowMtimes, currSlowMtimes)) slowStale = true;
+      }
 
       if (!fastStale && !slowStale && _statusCache) return _statusCache;
 
@@ -1866,7 +1907,9 @@ function refreshStatusAsync() {
       }
 
       let slow = _slowState;
+      let preBuildSlowMtimes = null;
       if (slowStale) {
+        preBuildSlowMtimes = _getSlowMtimes();
         slow = _buildStatusSlowState();
       }
 
@@ -1887,6 +1930,7 @@ function refreshStatusAsync() {
       if (slowStale) {
         _slowState = slow;
         _slowStateTs = now;
+        _lastSlowMtimes = preBuildSlowMtimes;
       }
       _statusCache = { ..._fastState, ..._slowState, timestamp: new Date().toISOString() };
       _markStatusCacheBuilt();
@@ -1921,6 +1965,7 @@ function _resetStatusCacheForTesting() {
   _statusInvalidationGeneration = 0;
   _statusRefreshHook = null;
   _lastMtimes = {};
+  _lastSlowMtimes = {};
 }
 
 /** Return cached JSON string of status — single stringify, reused by SSE and /api/status */

package/engine/queries.js

@@ -1918,11 +1918,29 @@ function getStatusFastStateMtimePaths(config) {
     // entry-add/remove signal even on Windows NTFS. Without it, PR-comment
     // notifications, agent-failure summaries, follow-up build alerts, and
     // meeting-transcript dumps all lagged up to 10 s before appearing on
-    // the dashboard's inbox view. The meeting directory is still excluded
-    // (see "Files intentionally NOT tracked") because round transitions
-    // edit existing files in-place, which dir mtime can't detect.
+    // the dashboard's inbox view.
     INBOX_DIR,
+    // engine/qa-runs.json (surfaced by listRuns via fast-state qaRuns slice)
+    // — new QA runs and status flips need to light the sidebar activity dot
+    // within one SPA poll cycle. Single file, mtime advances on each write.
+    path.join(ENGINE_DIR, 'qa-runs.json'),
   ];
+  // meetings/<id>.json (surfaced by meeting.getMeetings) — round transitions
+  // edit each file in-place via mutateMeeting, so the parent dir's mtime
+  // does NOT advance on Windows. Tracking each file individually catches
+  // in-file edits. Bounded by meeting count (typically <50 active); a 50-
+  // meeting fleet adds ~50 statSync calls per cache miss — still cheap.
+  // Filter `.backup` sidecars (from safe-write tempfile pattern) and
+  // non-*.json entries so corrupted state can't pollute the registry.
+  try {
+    const meetingsDir = path.join(MINIONS_DIR, 'meetings');
+    const entries = fs.readdirSync(meetingsDir);
+    for (const f of entries) {
+      if (f.endsWith('.json') && !f.endsWith('.backup.json')) {
+        files.push(path.join(meetingsDir, f));
+      }
+    }
+  } catch { /* meetings dir absent → no meetings to track */ }
   // Per-project work-items (surfaced by getWorkItems) and pull-requests
   // (surfaced by getPullRequests). The PR file was the biggest miss in the
   // original tracked list — PR status flips (running → passing, waiting →
@@ -1947,6 +1965,79 @@ function getStatusFastStateMtimePaths(config) {
   return files;
 }
 
+/**
+ * Slow-state mtime tracker — symmetric with the fast-state registry above.
+ *
+ * Slow-state slices (`prdProgress`, `prd`, `verifyGuides`, `archivedPrds`,
+ * `skills`, `mcpServers`, `schedules`, `pipelines`, `pinned`, `projects`,
+ * `version`, etc.) live behind a 60 s TTL in `dashboard.js`. Without mtime
+ * tracking, engine-driven writes to `prd/*.json`, `engine/schedule-runs.json`,
+ * `engine/pipeline-runs.json`, or `prd/guides/*.md` waited up to the full
+ * 60 s before surfacing in the dashboard — which produced the visible
+ * "plan status doesn't update, pipelines never advance" symptom.
+ *
+ * The same conventions as fast-state apply:
+ *   - Entries must back something `_buildStatusSlowState()` reads; otherwise
+ *     the rebuild silently no-ops on detected change.
+ *   - Files mutated through a `mutate*` helper are reliable (`safeWrite`
+ *     rename advances mtime). Append-only logs reset the cache often but
+ *     are not relevant here.
+ *   - Per-project paths must use `shared.project*` so newly-added projects
+ *     are picked up without registry edits.
+ *
+ * Files intentionally NOT tracked here:
+ *   - `mcpServers`, version, autoMode, installId — change only on human/
+ *     CLI edits, which already pop the slow-state via reloadConfig + the
+ *     60 s TTL.
+ *   - `~/.claude/skills/`, `~/.copilot/skills/` — user-home dirs that
+ *     `extractSkillsFromOutput` writes to from the agent-close path,
+ *     which already calls `invalidateStatusCache()` directly.
+ *   - project git state — already invalidated via the
+ *     `_setOnProjectGitStatusChanged` callback into `invalidateStatusCache`
+ *     (W-mpgrk5cy fix); also tracked in fast-state via `.git/logs/HEAD`.
+ */
+function getStatusSlowStateMtimePaths(config) {
+  const projects = getProjects(config || getConfig());
+  const files = [
+    // prd/*.json (surfaced by getPrdInfo) — engine writes via syncPrdFromPrs,
+    // the materializer, and plan-to-prd outputs. Dir mtime advances on entry
+    // add/remove; in-file edits use getPrdInfo's own per-file mtime cache so
+    // small-content flips still surface via the 10 s TTL backstop, but the
+    // big "new PRD created" event surfaces immediately.
+    PRD_DIR,
+    // prd/archive/*.json — manual archive moves PRDs here; dir mtime catches
+    // the move.
+    path.join(PRD_DIR, 'archive'),
+    // prd/guides/*.md — verify agent writes new files here on E2E completion.
+    path.join(MINIONS_DIR, 'prd', 'guides'),
+    // engine/schedule-runs.json — scheduler rewrites this on every cron fire.
+    path.join(ENGINE_DIR, 'schedule-runs.json'),
+    // engine/pipeline-runs.json — pipeline executor rewrites this on each
+    // stage transition (the most user-visible slow-state lag pre-fix).
+    path.join(ENGINE_DIR, 'pipeline-runs.json'),
+    // pipelines/*.json — pipeline definitions, edited by humans + plan agents.
+    // Dir mtime is fine because pipeline edits are wholesale file replacements
+    // (no in-place tweaks once a pipeline is authored).
+    path.join(MINIONS_DIR, 'pipelines'),
+    // pinned.md — single file, dashboard-side writes already call
+    // invalidateStatusCache({includeSlow:true}); tracker entry catches any
+    // CLI/editor edit that bypasses the API.
+    path.join(MINIONS_DIR, 'pinned.md'),
+    // engine/skill-states/ — engine writes when agents extract new skills.
+    // Dir mtime catches new skill files. Skips per-skill in-place edits which
+    // the agent-close invalidate already covers.
+    SKILLS_DIR,
+  ];
+  // Per-project local skill dirs — agents extract project-scoped skills here.
+  for (const p of projects) {
+    if (p && p.localPath) {
+      files.push(path.join(p.localPath, '.claude', 'skills'));
+      files.push(path.join(p.localPath, '.github', 'skills'));
+    }
+  }
+  return files;
+}
+
 // ── Exports ─────────────────────────────────────────────────────────────────
 
 module.exports = {
@@ -1967,6 +2058,7 @@ module.exports = {
   _setOnProjectGitStatusChanged,
   // W-mpftp7na000td0f4 — engine→dashboard cache-invalidation registry
   getStatusFastStateMtimePaths,
+  getStatusSlowStateMtimePaths,
 
   // Core state
   getConfig, getControl, getDispatch, getDispatchQueue, getDispatchCompletionReport, invalidateDispatchCache,

package/engine.js

@@ -588,7 +588,23 @@ function resolveDependencyBranches(depIds, sourcePlan, project, config) {
  *
  * @returns {Promise<{skipped: boolean, reason?: string}>}
  */
-async function syncReusedWorktree(rootDir, worktreePath, branchName, gitOpts = {}) {
+async function syncReusedWorktree(rootDir, worktreePath, branchName, gitOpts = {}, opts = {}) {
+  // W-mph6n4p00006ce38: Freshen origin/<mainRef> and fast-forward the
+  // worktree to it BEFORE the branch sync. Without this, reused worktrees
+  // inherit whatever stale local master the engine clone happens to be on,
+  // so downstream dep-merges (and zero-dep agent work) get layered onto a
+  // stale base. Shared-branch / useExistingBranch dispatches MUST NOT
+  // auto-pick-up master mid-flight, so the carve-out is explicit.
+  const { mainRef, isSharedBranch } = opts;
+  if (mainRef && !isSharedBranch) {
+    try { await shared.shellSafeGit(['fetch', 'origin', mainRef], { ...gitOpts, cwd: rootDir }); }
+    catch (e) { log('warn', `git: failed to fetch origin/${mainRef} during reuse-sync: ${e.message}`); }
+    try { await shared.shellSafeGit(['merge', `origin/${mainRef}`, '--no-edit', '--no-ff'], { ...gitOpts, cwd: worktreePath }); }
+    catch (e) {
+      log('warn', `git: failed to merge origin/${mainRef} into ${branchName} during reuse-sync: ${e.message}`);
+      try { await shared.shellSafeGit(['merge', '--abort'], { ...gitOpts, cwd: worktreePath }); } catch (_) { /* no merge in progress */ }
+    }
+  }
   // ls-remote --exit-code returns 2 when no matching refs are found on the
   // remote. The probe only lists refs (no object transfer), so it's cheap
   // even on slow links.
@@ -1063,7 +1079,12 @@ async function spawnAgent(dispatchItem, config) {
       // (orphan/timeout retry before first push) would otherwise emit a
       // "couldn't find remote ref" warn pair on every reuse.
       _phaseT.reuseSyncStart = Date.now();
-      await syncReusedWorktree(rootDir, existingWt, branchName, _gitOpts);
+      // W-mph6n4p00006ce38: hand syncReusedWorktree the main ref + shared-branch
+      // flag so it can freshen origin/<mainRef> into the worktree (skipped on
+      // shared-branch dispatches — see syncReusedWorktree's opts contract).
+      const _reuseMainRef = sanitizeBranch(shared.resolveMainBranch(rootDir, project.mainBranch));
+      const _reuseIsShared = meta?.branchStrategy === 'shared-branch' || meta?.useExistingBranch;
+      await syncReusedWorktree(rootDir, existingWt, branchName, _gitOpts, { mainRef: _reuseMainRef, isSharedBranch: _reuseIsShared });
       _phaseT.reuseSyncEnd = Date.now();
     } else {
       _phaseT.createWorktreeStart = Date.now();
@@ -1167,8 +1188,24 @@ async function spawnAgent(dispatchItem, config) {
           } else {
             log('info', `Creating worktree: ${worktreePath} on branch ${branchName}`);
             const mainRef = sanitizeBranch(shared.resolveMainBranch(rootDir, project.mainBranch));
+            // W-mph6n4p00006ce38: mirror the pool-borrow path (~line 1110-1114)
+            // — fetch fresh origin/<mainRef> and start the new branch off it,
+            // not the local ref. Without this, fresh-create dispatches inherit
+            // whatever stale local master the engine clone happens to be on
+            // (most painful: long-lived engine processes between restarts).
+            // Non-fatal: if the fetch fails (network blip, transient auth),
+            // fall back to local mainRef so the dispatch still progresses;
+            // the dep-merge phase's own fetch + the on-failure
+            // `git reset --hard origin/<mainRef>` recovery remain as safety nets.
+            let _freshCreateBase = mainRef;
             try {
-              await runWorktreeAdd(rootDir, worktreePath, ['-b', branchName, mainRef], _worktreeGitOpts, worktreeCreateRetries);
+              await shared.shellSafeGit(['fetch', 'origin', mainRef], { ..._gitOpts, cwd: rootDir, timeout: 30000 });
+              _freshCreateBase = `origin/${mainRef}`;
+            } catch (mainFetchErr) {
+              log('warn', `Failed to fetch origin/${mainRef} before fresh-create worktree for ${branchName}: ${mainFetchErr.message} — falling back to local ${mainRef}`);
+            }
+            try {
+              await runWorktreeAdd(rootDir, worktreePath, ['-b', branchName, _freshCreateBase], _worktreeGitOpts, worktreeCreateRetries);
             } catch (e1) {
               const branchExists = e1.message?.includes('already exists');
               log('warn', `Worktree -b failed for ${branchName}: ${e1.message?.split('\n')[0]}`);
@@ -1180,7 +1217,7 @@ async function spawnAgent(dispatchItem, config) {
                 // Clean up partial worktree directory from failed attempt
                 try { if (fs.existsSync(worktreePath)) fs.rmSync(worktreePath, { recursive: true, force: true }); } catch { /* optional */ }
                 try {
-                  await runWorktreeAdd(rootDir, worktreePath, ['-b', branchName, mainRef], _worktreeGitOpts, 0);
+                  await runWorktreeAdd(rootDir, worktreePath, ['-b', branchName, _freshCreateBase], _worktreeGitOpts, 0);
                 } catch (e1b) {
                   log('error', `Worktree -b retry also failed for ${branchName}: ${e1b.message?.split('\n')[0]}`);
                   throw e1b;
@@ -1303,6 +1340,38 @@ async function spawnAgent(dispatchItem, config) {
           // of FAILURE_CLASS.MERGE_CONFLICT (which retries 3x against the
           // same broken auth path).
           let _depAuthFailed = false;
+          // W-mph6n4p00006ce38: Refresh origin/<mainRef> + freshen the worktree
+          // BEFORE the parallel dep fetches and preflight sim. Without this,
+          // preflight + real dep-merges run against whatever stale local master
+          // the engine clone happens to be on, producing false-positive
+          // conflicts whenever master drifted ahead of the engine's last fetch.
+          // The pool-borrow + fresh-create paths already start at fresh
+          // origin/<mainRef>, but the reuse path can outrun its sync; and even
+          // for the start-fresh paths, master can advance between worktree
+          // creation and dep-merge (long-lived dispatches). Shared-branch
+          // dispatches MUST NOT auto-pick-up master mid-flight — the carve-out
+          // is explicit. Non-fatal: failures log + continue with the stale base,
+          // and the existing on-failure `git reset --hard origin/<mainRef>`
+          // recovery (~line 1452) remains the safety net.
+          const _depMainRef = sanitizeBranch(shared.resolveMainBranch(rootDir, project.mainBranch));
+          const _depIsSharedBranch = meta?.branchStrategy === 'shared-branch' || meta?.useExistingBranch;
+          if (!_failedRefCache.has(_depMainRef)) {
+            try {
+              await adoGitAuth.runAdoGit(project, ['fetch', 'origin', _depMainRef], { ..._gitOpts, cwd: rootDir });
+            } catch (mainFetchErr) {
+              log('warn', `Failed to fetch origin/${_depMainRef} before dep merge: ${mainFetchErr.message} — proceeding with stale base`);
+              if (adoGitAuth.isAdoAuthFailure(mainFetchErr)) _depAuthFailed = true;
+              _failedRefCache.add(_depMainRef);
+            }
+          }
+          if (!_depIsSharedBranch) {
+            try {
+              await shared.shellSafeGit(['merge', `origin/${_depMainRef}`, '--no-edit', '--no-ff'], { ..._gitOpts, cwd: worktreePath });
+            } catch (mainMergeErr) {
+              log('warn', `Failed to merge origin/${_depMainRef} into ${branchName} before dep merge: ${mainMergeErr.message} — proceeding without main refresh`);
+              try { await shared.shellSafeGit(['merge', '--abort'], { ..._gitOpts, cwd: worktreePath }); } catch (_) { /* no merge in progress */ }
+            }
+          }
           // Fetch all dependency branches in parallel (git fetches are independent)
           const fetchable = depBranches.filter(d => !_failedRefCache.has(d.branch));
           const unfetchable = depBranches.filter(d => _failedRefCache.has(d.branch));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2026",
+  "version": "0.1.2028",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"