@yemi33/minions 0.1.2251 → 0.1.2252

package/dashboard.js

@@ -299,6 +299,51 @@ try {
   });
 } catch { /* defensive — wiring is best-effort, queries probes still work without it */ }
 
+// opg-microsoft/minions#381 — watch each project's .git/HEAD with fs.watch so
+// a bare `git checkout` / `git switch` outside the dashboard immediately busts
+// the status cache rather than waiting for the next 4-second SPA poll to detect
+// the mtime change. When HEAD changes the watcher calls invalidateStatusCache()
+// which (a) clears _statusCache and (b) debounce-pushes the updated state to
+// connected SSE clients within ~500ms — zero polling lag for SSE consumers.
+//
+// The mtime-based polling in getStatusSlowStateMtimePaths + _projectGitRefFiles
+// (queries.js) remains as a belt-and-suspenders fallback for environments where
+// fs.watch is unreliable (network drives, some WSL mounts).
+const _projectHeadWatchers = new Map(); // localPath → fs.FSWatcher
+
+function _setupProjectHeadWatchers() {
+  const currentPaths = new Set(PROJECTS.filter(p => p && p.localPath).map(p => p.localPath));
+  // Close watchers for projects that are no longer configured.
+  for (const [localPath, watcher] of _projectHeadWatchers) {
+    if (!currentPaths.has(localPath)) {
+      try { watcher.close(); } catch { /* ignore close errors */ }
+      _projectHeadWatchers.delete(localPath);
+    }
+  }
+  // Add watchers for newly-configured projects.
+  for (const project of PROJECTS) {
+    if (!project || !project.localPath || _projectHeadWatchers.has(project.localPath)) continue;
+    let headPath = null;
+    try {
+      const resolvedGitDir = queries._resolveGitDir(project.localPath);
+      if (resolvedGitDir) headPath = path.join(resolvedGitDir, 'HEAD');
+    } catch { /* _resolveGitDir is best-effort */ }
+    if (!headPath) continue;
+    try {
+      const watcher = fs.watch(headPath, () => {
+        try { invalidateStatusCache(); } catch { /* best-effort */ }
+      });
+      watcher.on('error', () => {
+        // Silently discard watch errors (e.g. path deleted) — polling fallback handles it.
+        try { watcher.close(); } catch { /* ignore */ }
+        _projectHeadWatchers.delete(project.localPath);
+      });
+      _projectHeadWatchers.set(project.localPath, watcher);
+    } catch { /* fs.watch not available on this fs (network drive etc.) — polling covers it */ }
+  }
+}
+try { _setupProjectHeadWatchers(); } catch { /* best-effort */ }
+
 function resolveScheduleProjectValue(project, projects = PROJECTS) {
   if (project === undefined) return { project: undefined };
   const target = shared.resolveConfiguredProject(project, projects);
@@ -9042,6 +9087,7 @@ What would you like to discuss or change? When you're happy, say "approve" and I
 
       reloadConfig(); // Update in-memory project list immediately
       warmProjectGitStatusCache(); // Probe the new project's git status in the background
+      try { _setupProjectHeadWatchers(); } catch { /* best-effort */ } // opg#381: watch new project's HEAD
       // includeSlow: PROJECTS lives in the slow-state cache (60s TTL); without
       // flushing it, /api/status keeps returning the previous project list for
       // up to a minute after the add. Matches handleProjectsRemove's behavior.

package/engine/queries.js

@@ -1658,6 +1658,11 @@ let _kbCache = null;       // last good snapshot — never nulled by invalidate
 let _kbCacheTs = 0;
 let _kbCacheStale = true;  // invalidate marks stale; snapshot kept for sync readers
 let _kbRefreshPromise = null; // in-flight scan dedupe
+// Incremental-scan cache: filePath -> { mtimeMs, byteSize, entry }. Lets a
+// rescan reuse the derived entry for any file whose mtime+size are unchanged,
+// so the expensive readFile+parse only runs for genuinely-changed files. Keyed
+// off the live stat, so in-place edits (agent memory appends) are still caught.
+let _kbFileMetaCache = new Map();
 const KB_CACHE_TTL = 30000; // 30s — KB changes infrequently
 
 function invalidateKnowledgeBaseCache() {
@@ -1684,32 +1689,28 @@ async function _scanKnowledgeBase() {
   // ~30 large KB entries. `_flat()` materialises a fresh flat string via
   // Buffer round-trip so the cached entry no longer pins the parent file.
   const _flat = (s) => Buffer.from(String(s || ''), 'utf8').toString('utf8');
-  // Build the full (category, file) work list first, THEN process it with a
-  // bounded concurrency window. The KB tree holds thousands of files; the old
-  // unbounded `Promise.all(files.map(...))` per category fired thousands of
-  // concurrent readFile+stat ops at once, flooding the (default-4, now 16)
-  // libuv thread pool and starving every other fs-dependent dashboard request
-  // — including the /api/status poll — for tens of seconds, which is what
-  // tripped the "stale / unreachable" banner. Capping in-flight reads keeps
-  // the scan from monopolizing the pool. (Load-reduction audit 2026-06-24.)
+  // Incremental, bounded-concurrency scan. The KB tree holds thousands of files;
+  // re-reading every file's full content on each cache rebuild was the dominant
+  // disk load behind dashboard staleness (the readFile + parse + string-alloc of
+  // ~3.5k files, repeated every TTL expiry, on a disk already contended by live
+  // agents). Two defenses: (1) a per-file metadata cache keyed on mtime+size, so
+  // a rescan only re-reads files that actually changed — steady state is one
+  // light stat() per file and ~zero readFile; (2) a bounded in-flight window so
+  // the scan can't flood the libuv thread pool and starve /api/status (which
+  // trips the "stale" banner). (Load-reduction audit 2026-06-24.)
   const work = [];
   for (const cat of KB_CATEGORIES) {
     const catDir = path.join(KNOWLEDGE_DIR, cat);
     const files = (await fsp.readdir(catDir).catch(() => [])).filter(f => f.endsWith('.md'));
     for (const f of files) work.push({ cat, catDir, f });
   }
-  const scanOne = async ({ cat, catDir, f }) => {
-    const filePath = path.join(catDir, f);
-    const [content, stat] = await Promise.all([
-      fsp.readFile(filePath, 'utf8').catch(() => ''),
-      fsp.stat(filePath).catch(() => null),
-    ]);
+  const nextMetaCache = new Map();
+  const deriveEntry = (cat, f, content, sortTs) => {
     const titleMatch = content.match(/^#\s+(.+)/m);
     const title = _flat(titleMatch ? titleMatch[1].trim() : f.replace(/\.md$/, ''));
     const agentMatch = f.match(/^\d{4}-\d{2}-\d{2}-(\w+)-/);
     const dateMatch = f.match(/^(\d{4}-\d{2}-\d{2})/) || content.match(/^date:\s*(\d{4}-\d{2}-\d{2})$/m);
     const sourceMatch = content.match(/^source:\s*(.+)/m);
-    const sortTs = (stat && stat.mtimeMs) || 0;
     const displayDate = dateMatch ? _flat(dateMatch[1]) : (sortTs ? new Date(sortTs).toISOString().slice(0, 10) : '');
     return {
       cat, file: f, title,
@@ -1721,6 +1722,22 @@ async function _scanKnowledgeBase() {
       size: content.length,
     };
   };
+  const scanOne = async ({ cat, catDir, f }) => {
+    const filePath = path.join(catDir, f);
+    const stat = await fsp.stat(filePath).catch(() => null);
+    const sortTs = (stat && stat.mtimeMs) || 0;
+    // Reuse the cached derived entry when mtime+size are unchanged — skips the
+    // readFile + regex parse + string allocations entirely for stable files.
+    const cached = _kbFileMetaCache.get(filePath);
+    if (cached && stat && cached.mtimeMs === stat.mtimeMs && cached.byteSize === stat.size) {
+      nextMetaCache.set(filePath, cached);
+      return cached.entry;
+    }
+    const content = await fsp.readFile(filePath, 'utf8').catch(() => '');
+    const entry = deriveEntry(cat, f, content, sortTs);
+    nextMetaCache.set(filePath, { mtimeMs: sortTs, byteSize: stat ? stat.size : Buffer.byteLength(content), entry });
+    return entry;
+  };
   const entries = [];
   const KB_SCAN_CONCURRENCY = 16;
   let _next = 0;
@@ -1731,6 +1748,8 @@ async function _scanKnowledgeBase() {
     }
   };
   await Promise.all(Array.from({ length: Math.min(KB_SCAN_CONCURRENCY, work.length) }, worker));
+  // Swap in the rebuilt cache (drops metadata for files that were deleted).
+  _kbFileMetaCache = nextMetaCache;
   entries.sort((a, b) =>
     (b.sortTs || 0) - (a.sortTs || 0) ||
     (b.date || '').localeCompare(a.date || '') ||
@@ -2758,6 +2777,11 @@ function _projectGitRefFiles(localPath, configuredMainBranch) {
   if (!gitDir) return null;
   const commonGitDir = _resolveCommonGitDir(gitDir);
   const files = [
+    // opg-microsoft/minions#381 — track HEAD directly so a bare `git checkout`
+    // (which rewrites .git/HEAD but may not advance logs/HEAD when reflog is
+    // disabled or when the snapshot already reflects the post-checkout mtime)
+    // still triggers refsAdvanced=true and the gitStale marker on the next call.
+    path.join(gitDir, 'HEAD'),
     path.join(gitDir, 'logs', 'HEAD'),
     path.join(commonGitDir, 'FETCH_HEAD'),
   ];
@@ -3149,10 +3173,14 @@ function getStatusSlowStateMtimePaths(config) {
   // not into the per-worktree subdir. For the main worktree both
   // resolvers return the same gitdir, so the behavior collapses to the
   // expected `<localPath>/.git/{logs/HEAD,FETCH_HEAD}` pair.
+  // opg-microsoft/minions#381 — also track HEAD itself so `git checkout`
+  // (which always rewrites HEAD) busts the cache even when logs/HEAD is
+  // absent or was already captured at the post-checkout mtime.
   for (const project of projects) {
     if (!project || !project.localPath) continue;
     const gitDir = _resolveGitDir(project.localPath) || path.join(project.localPath, '.git');
     const commonGitDir = _resolveCommonGitDir(gitDir);
+    files.push(path.join(gitDir, 'HEAD'));
     files.push(path.join(gitDir, 'logs', 'HEAD'));
     files.push(path.join(commonGitDir, 'FETCH_HEAD'));
   }
@@ -3183,6 +3211,7 @@ module.exports = {
   _setProbeWatchdogForTest,
   _getProbeTimeoutsForTest,
   _getProjectGitStatusCacheForTest,
+  _resolveGitDir, // opg-microsoft/minions#381 — dashboard.js uses this to locate HEAD for fs.watch
   // W-mpftp7na000td0f4 — engine→dashboard cache-invalidation registry
   getStatusFastStateMtimePaths,
   getStatusSlowStateMtimePaths,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2251",
+  "version": "0.1.2252",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"