@yemi33/minions 0.1.2003 → 0.1.2004

package/dashboard.js

@@ -38,7 +38,7 @@ const os = require('os');
 const { safeRead, safeReadOrNull, safeReadDir, safeWrite, safeJson, safeJsonObj, safeJsonArr, safeJsonNoRestore, safeUnlink, mutateJsonFileLocked, mutateTextFileLocked, mutateControl, mutateCooldowns, mutateWorkItems, getProjects: _getProjects, DONE_STATUSES, WI_STATUS, WORK_TYPE, WORKTREE_REQUIRING_TYPES, reopenWorkItem } = shared;
 const { getAgents, getAgentDetail, getPrdInfo, getWorkItems, getDispatchQueue,
   getSkills, getInbox, getNotesWithMeta, getPullRequests,
-  getEngineLog, getMetrics, getKnowledgeBaseEntries, getProjectGitStatus, timeSince,
+  getEngineLog, getMetrics, getKnowledgeBaseEntries, getKnowledgeBaseEntriesSnapshot, getProjectGitStatus, timeSince,
   MINIONS_DIR, AGENTS_DIR, ENGINE_DIR, INBOX_DIR, DISPATCH_PATH, PRD_DIR } = queries;
 
 // Dev vs binary differentiation. When two dashboards run side-by-side (npm
@@ -912,6 +912,16 @@ function _steeringDeliveryState(agentId) {
 
 const PLANS_DIR = path.join(MINIONS_DIR, 'plans');
 
+// W-mpetru8a000s123a — /api/plans cache. Dashboard auto-polls every 4s; the
+// pre-cache cold path walked 4 directories (plans/, prd/, archive variants),
+// sync-stat'd every file, and parsed/regex-scanned each .md → 2-3s blocking.
+// 5s TTL means external .md edits surface within 5s. Mutation handlers must
+// call invalidatePlansCache() for immediate visibility.
+let _plansCache = null;
+let _plansCacheTs = 0;
+const PLANS_CACHE_TTL_MS = 5000;
+function invalidatePlansCache() { _plansCache = null; _plansCacheTs = 0; }
+
 // Resolve a plan/PRD file path: .json files live in prd/, .md files in plans/
 // Validates that the file stays within the expected directory to prevent path traversal.
 function resolvePlanPath(file) {
@@ -4279,6 +4289,7 @@ const server = http.createServer(async (req, res) => {
         });
         if (existingVerify) {
           invalidateStatusCache();
+          invalidatePlansCache();
           return jsonReply(res, 200, { ok: true, verifyId: existingVerify.id });
         }
       }
@@ -4298,6 +4309,7 @@ const server = http.createServer(async (req, res) => {
         const verify = items.find(w => w.sourcePlan === body.file && w.itemType === 'verify');
         if (verify) {
           invalidateStatusCache();
+          invalidatePlansCache();
           return jsonReply(res, 200, { ok: true, verifyId: verify.id });
         }
       }
@@ -4894,6 +4906,7 @@ const server = http.createServer(async (req, res) => {
 
       const planFile = 'manual-' + shared.uid() + '.json';
       safeWrite(path.join(PRD_DIR, planFile), manualPrd.plan);
+      invalidatePlansCache();
       return jsonReply(res, 200, { ok: true, id: manualPrd.id, file: planFile });
     } catch (e) { return jsonReply(res, 400, { error: e.message }); }
   }
@@ -4949,6 +4962,7 @@ const server = http.createServer(async (req, res) => {
         } catch (e) { console.error('work item sync:', e.message); }
       }
 
+      invalidatePlansCache();
       return jsonReply(res, 200, { ok: true, item, workItemSynced });
     } catch (e) { return jsonReply(res, 400, { error: e.message }); }
   }
@@ -4995,6 +5009,7 @@ const server = http.createServer(async (req, res) => {
         d.meta?.item?.sourcePlan === body.source && d.meta?.item?.id === body.itemId
       );
 
+      invalidatePlansCache();
       return jsonReply(res, 200, { ok: true, cancelled });
     } catch (e) { return jsonReply(res, 400, { error: e.message }); }
   }
@@ -5339,7 +5354,7 @@ const server = http.createServer(async (req, res) => {
   }
 
   async function handleKnowledgeList(req, res) {
-    const entries = getKnowledgeBaseEntries();
+    const entries = await getKnowledgeBaseEntries();
     const result = {};
     for (const cat of shared.KB_CATEGORIES) result[cat] = [];
     for (const e of entries) {
@@ -5353,7 +5368,11 @@ const server = http.createServer(async (req, res) => {
     // Source of truth: kb-sweep-state.json + PID liveness — the in-process
     // sweep moved to a detached runner so in-memory globals are no longer
     // authoritative (they die with the dashboard).
+    // W-mpetru8a000s123a: yield event loop before readSweepLiveness (sync
+    // process.kill + safeJson) so a single /api/knowledge handler can't
+    // chain three sync blocks back-to-back during a stall window.
     try {
+      await new Promise(r => setImmediate(r));
       const { readSweepLiveness } = require('./engine/kb-sweep');
       const liveness = readSweepLiveness({ entryCount: entries.length });
       if (liveness.inFlight && liveness.alive) {
@@ -5386,7 +5405,7 @@ const server = http.createServer(async (req, res) => {
     const {
       readSweepLiveness, staleGuardMs, KB_SWEEP_STATE_PATH, KB_SWEEP_LOG_PATH, KB_SWEEP_RUNNER_PATH,
     } = require('./engine/kb-sweep');
-    const entryCount = (queries.getKnowledgeBaseEntries() || []).length;
+    const entryCount = ((await queries.getKnowledgeBaseEntries()) || []).length;
     const guardMs = staleGuardMs(entryCount);
 
     // Synchronous pre-claim BEFORE awaiting the body so a concurrent POST
@@ -5482,11 +5501,11 @@ const server = http.createServer(async (req, res) => {
   }
 
 
-  function handleKnowledgeSweepStatus(req, res) {
+  async function handleKnowledgeSweepStatus(req, res) {
     // Source of truth = kb-sweep-state.json + PID liveness. Globals are gone —
     // the runner is detached, so its lifecycle is independent of this process.
     const { readSweepLiveness } = require('./engine/kb-sweep');
-    const entries = queries.getKnowledgeBaseEntries() || [];
+    const entries = (await queries.getKnowledgeBaseEntries()) || [];
     const liveness = readSweepLiveness({ entryCount: entries.length });
     const diskState = safeJson(path.join(ENGINE_DIR, 'kb-sweep-state.json'));
     let inFlight = false;
@@ -5513,13 +5532,19 @@ const server = http.createServer(async (req, res) => {
   }
 
   async function handlePlansList(req, res) {
+    const now = Date.now();
+    if (_plansCache && (now - _plansCacheTs) < PLANS_CACHE_TTL_MS) {
+      return jsonReply(res, 200, _plansCache);
+    }
+    const fsp = fs.promises;
     const dirs = [
       { dir: PLANS_DIR, archived: false },
       { dir: path.join(PLANS_DIR, 'archive'), archived: true },
       { dir: PRD_DIR, archived: false },
       { dir: path.join(PRD_DIR, 'archive'), archived: true },
     ];
-    // Load work items to check for completed plan-to-prd conversions
+    // Load work items to check for completed plan-to-prd conversions.
+    // safeJsonArr is sync but reads a single small file — leave as is.
     const centralWi = safeJsonArr(path.join(MINIONS_DIR, 'work-items.json'));
     const completedPrdFiles = new Set(
       centralWi.filter(w => w.type === 'plan-to-prd' && DONE_STATUSES.has(w.status) && w.planFile)
@@ -5527,18 +5552,21 @@ const server = http.createServer(async (req, res) => {
     );
     const plans = [];
     for (const { dir, archived } of dirs) {
-      const allFiles = safeReadDir(dir).filter(f => f.endsWith('.json') || f.endsWith('.md'));
-      for (const f of allFiles) {
+      const allFiles = (await fsp.readdir(dir).catch(() => []))
+        .filter(f => f.endsWith('.json') || f.endsWith('.md'));
+      const dirResults = await Promise.all(allFiles.map(async f => {
         const filePath = path.join(dir, f);
-        const content = safeRead(filePath) || '';
-        let updatedAt = '';
-        try { updatedAt = new Date(fs.statSync(filePath).mtimeMs).toISOString(); } catch { /* optional */ }
+        const [content, stat] = await Promise.all([
+          fsp.readFile(filePath, 'utf8').catch(() => ''),
+          fsp.stat(filePath).catch(() => null),
+        ]);
+        const updatedAt = stat ? new Date(stat.mtimeMs).toISOString() : '';
         const isJson = f.endsWith('.json');
         if (isJson) {
           try {
             const plan = JSON.parse(content);
             const status = plan.status || 'active';
-            plans.push({
+            return {
               file: f, format: 'prd', archived,
               project: plan.project || '',
               summary: plan.plan_summary || '',
@@ -5556,15 +5584,15 @@ const server = http.createServer(async (req, res) => {
               archiveReady: plan._archiveReady || false,
               archiveReadyAt: plan._archiveReadyAt || null,
               planStale: plan.planStale || false,
-            });
-          } catch { /* JSON parse fallback */ }
+            };
+          } catch { return null; /* JSON parse fallback */ }
         } else {
           const titleMatch = content.match(/^#\s+(?:Plan:\s*)?(.+)/m);
           const projectMatch = content.match(/\*\*Project:\*\*\s*(.+)/m);
           const authorMatch = content.match(/\*\*Author:\*\*\s*(.+)/m);
           const dateMatch = content.match(/\*\*Date:\*\*\s*(.+)/m);
           const versionMatch = f.match(/-v(\d+)/);
-          plans.push({
+          return {
             file: f, format: 'draft', archived,
             project: projectMatch ? projectMatch[1].trim() : '',
             summary: titleMatch ? titleMatch[1].trim() : f.replace('.md', ''),
@@ -5578,11 +5606,14 @@ const server = http.createServer(async (req, res) => {
             requiresApproval: false,
             revisionFeedback: null,
             version: versionMatch ? parseInt(versionMatch[1]) : null,
-          });
+          };
         }
-      }
+      }));
+      for (const r of dirResults) if (r) plans.push(r);
     }
     plans.sort((a, b) => (b.generatedAt || '').localeCompare(a.generatedAt || ''));
+    _plansCache = plans;
+    _plansCacheTs = Date.now();
     return jsonReply(res, 200, plans);
   }
 
@@ -5617,6 +5648,7 @@ const server = http.createServer(async (req, res) => {
       }
 
       invalidateStatusCache();
+      invalidatePlansCache();
       return jsonReply(res, 200, { ok: true, unarchivedSource });
     } catch (e) { return jsonReply(res, 400, { error: e.message }); }
   }
@@ -5747,6 +5779,7 @@ const server = http.createServer(async (req, res) => {
       }
 
       invalidateStatusCache();
+      invalidatePlansCache();
       return jsonReply(res, 200, { ok: true, status: 'approved', resumedWorkItems: resumed, diffAwareUpdate: diffAwareQueued });
     } catch (e) { return jsonReply(res, 400, { error: e.message }); }
   }
@@ -5862,6 +5895,7 @@ const server = http.createServer(async (req, res) => {
       }, { defaultValue: { pending: [], active: [], completed: [] } });
 
       invalidateStatusCache();
+      invalidatePlansCache();
       return jsonReply(res, 200, { ok: true, status: 'paused', resetWorkItems: reset });
     } catch (e) { return jsonReply(res, 400, { error: e.message }); }
   }
@@ -5895,6 +5929,7 @@ const server = http.createServer(async (req, res) => {
       });
       if (!queueResult.queued) return jsonReply(res, 200, { ok: true, alreadyQueued: true, id: queueResult.id });
       invalidateStatusCache();
+      invalidatePlansCache();
       return jsonReply(res, 200, { ok: true, id: queueResult.id });
     } catch (e) { return jsonReply(res, 400, { error: e.message }); }
   }
@@ -5913,6 +5948,7 @@ const server = http.createServer(async (req, res) => {
         return data;
       }, { defaultValue: {} });
 
+      invalidatePlansCache();
       return jsonReply(res, 200, { ok: true, status: 'rejected' });
     } catch (e) { return jsonReply(res, 400, { error: e.message }); }
   }
@@ -6040,6 +6076,7 @@ const server = http.createServer(async (req, res) => {
       }
 
       invalidateStatusCache();
+      invalidatePlansCache();
       return jsonReply(res, 200, { ok: true, cleanedWorkItems: cleaned, cleanedDispatches: dispatchCleaned });
     } catch (e) { return jsonReply(res, 400, { error: e.message }); }
   }
@@ -6115,6 +6152,7 @@ const server = http.createServer(async (req, res) => {
       } catch (e) { console.error('plan worktree cleanup:', e.message); }
 
       invalidateStatusCache();
+      invalidatePlansCache();
       const payload = { ok: true, archived: body.file, archivedSource, cancelledItems };
       if (archiveWarnings.length > 0) payload.warnings = archiveWarnings;
       return jsonReply(res, 200, payload);
@@ -6149,6 +6187,7 @@ const server = http.createServer(async (req, res) => {
       }
 
       invalidateStatusCache();
+      invalidatePlansCache();
       return jsonReply(res, 200, { ok: true, unarchivedSource });
     } catch (e) { return jsonReply(res, 400, { error: e.message }); }
   }
@@ -6180,6 +6219,7 @@ const server = http.createServer(async (req, res) => {
           planFile: body.file,
         });
       });
+      invalidatePlansCache();
       return jsonReply(res, 200, { ok: true, status: 'revision-requested', workItemId: id });
     } catch (e) { return jsonReply(res, 400, { error: e.message }); }
   }
@@ -10071,6 +10111,12 @@ if (require.main === module) {
     console.log(`  Projects:   ${PROJECTS.map(p => `${p.name} (${p.localPath})`).join(', ')}`);
     console.log(`\n  Auto-refreshes every 4s. Ctrl+C to stop.\n`);
 
+    // W-mpetru8a000s123a — warm the async KB cache so synchronous callers
+    // (getWorkItems) see real data on first /api/status instead of an empty
+    // snapshot. Fire-and-forget; tolerant of warming failure.
+    Promise.resolve(queries.getKnowledgeBaseEntries())
+      .catch(err => console.warn(`[dashboard] KB cache warm failed: ${err && err.message}`));
+
     // Auto-open the browser unless suppressed. `minions restart` and the
     // upgrade path set MINIONS_NO_AUTO_OPEN=1 because the CLI orchestrates the
     // open itself after observing whether an existing tab reconnected.

package/engine/cli.js

@@ -895,6 +895,21 @@ const commands = {
       }
     })();
 
+    // W-mpetru8a000s123a — warm the async KB cache so synchronous callers
+    // (getWorkItems, getKnowledgeBaseIndex, playbook render) see real data
+    // on first read instead of the empty snapshot. Fire-and-forget; the cache
+    // updates as soon as the scan resolves and any inflight async caller
+    // shares the same promise.
+    (function warmKnowledgeBaseCache() {
+      try {
+        const queries = require('./queries');
+        Promise.resolve(queries.getKnowledgeBaseEntries())
+          .catch(err => e.log('warn', `KB cache warm failed: ${err && err.message}`));
+      } catch (err) {
+        e.log('warn', `KB cache warm setup failed: ${err.message}`);
+      }
+    })();
+
     // Initial tick
     e.tick();
 

package/engine/kb-sweep.js

@@ -484,7 +484,7 @@ async function _runKbSweepImpl(opts = {}) {
   };
   const t0 = Date.now();
 
-  const entries = queries.getKnowledgeBaseEntries();
+  const entries = await queries.getKnowledgeBaseEntries();
   if (entries.length < 2) { summary.summary = 'nothing to sweep (< 2 entries)'; summary.durationMs = Date.now() - t0; return summary; }
 
   const requestPinned = Array.isArray(opts.pinnedKeys)
@@ -535,7 +535,7 @@ async function _runKbSweepImpl(opts = {}) {
   summary.sweptArchivePruned = _pruneOldSwept();
 
   // Final tallies — re-walk surviving entries for accurate bytesAfter
-  const finalEntries = queries.getKnowledgeBaseEntries();
+  const finalEntries = await queries.getKnowledgeBaseEntries();
   for (const e of finalEntries) {
     if (pinned.has(`knowledge/${e.cat}/${e.file}`)) continue;
     const fp = path.join(KB_DIR, e.cat, e.file);

package/engine/queries.js

@@ -5,6 +5,7 @@
  */
 
 const fs = require('fs');
+const fsp = require('fs').promises;
 const path = require('path');
 const os = require('os');
 const shared = require('./shared');
@@ -1058,35 +1059,52 @@ function getCommandIndex(config) {
 
 // ── Knowledge Base ──────────────────────────────────────────────────────────
 
-let _kbCache = null;
+// W-mpetru8a000s123a — async KB scan + stale-while-revalidate cache.
+// Async fs prevents event-loop stalls on /api/knowledge cold-cache hits
+// (previously ~11s of blocking readFileSync/statSync). Sync callers
+// (getWorkItems, getKnowledgeBaseIndex, playbook render) use the in-memory
+// snapshot via getKnowledgeBaseEntriesSnapshot() to avoid forcing async
+// propagation through ~17 callers of getWorkItems.
+let _kbCache = null;       // last good snapshot — never nulled by invalidate
 let _kbCacheTs = 0;
+let _kbCacheStale = true;  // invalidate marks stale; snapshot kept for sync readers
+let _kbRefreshPromise = null; // in-flight scan dedupe
 const KB_CACHE_TTL = 30000; // 30s — KB changes infrequently
 
 function invalidateKnowledgeBaseCache() {
-  _kbCache = null;
+  _kbCacheStale = true;
   _kbCacheTs = 0;
 }
 
-function getKnowledgeBaseEntries() {
-  const now = Date.now();
-  if (_kbCache && (now - _kbCacheTs) < KB_CACHE_TTL) return _kbCache;
+/**
+ * Synchronous snapshot — returns last-known KB entries from memory, never
+ * touches disk. Returns [] only until the first async getKnowledgeBaseEntries()
+ * resolves. Used by sync callers (getWorkItems, getKnowledgeBaseIndex,
+ * playbook render) that historically called the sync version.
+ */
+function getKnowledgeBaseEntriesSnapshot() {
+  return Array.isArray(_kbCache) ? _kbCache : [];
+}
 
+async function _scanKnowledgeBase() {
   const entries = [];
   for (const cat of KB_CATEGORIES) {
     const catDir = path.join(KNOWLEDGE_DIR, cat);
-    const files = safeReadDir(catDir).filter(f => f.endsWith('.md'));
-    for (const f of files) {
+    const files = (await fsp.readdir(catDir).catch(() => [])).filter(f => f.endsWith('.md'));
+    const fileResults = await Promise.all(files.map(async f => {
       const filePath = path.join(catDir, f);
-      const content = safeRead(filePath) || '';
+      const [content, stat] = await Promise.all([
+        fsp.readFile(filePath, 'utf8').catch(() => ''),
+        fsp.stat(filePath).catch(() => null),
+      ]);
       const titleMatch = content.match(/^#\s+(.+)/m);
       const title = titleMatch ? titleMatch[1].trim() : f.replace(/\.md$/, '');
       const agentMatch = f.match(/^\d{4}-\d{2}-\d{2}-(\w+)-/);
       const dateMatch = f.match(/^(\d{4}-\d{2}-\d{2})/) || content.match(/^date:\s*(\d{4}-\d{2}-\d{2})$/m);
       const sourceMatch = content.match(/^source:\s*(.+)/m);
-      let sortTs = 0;
-      try { sortTs = fs.statSync(filePath).mtimeMs || 0; } catch {}
+      const sortTs = (stat && stat.mtimeMs) || 0;
       const displayDate = dateMatch ? dateMatch[1] : (sortTs ? new Date(sortTs).toISOString().slice(0, 10) : '');
-      entries.push({
+      return {
         cat, file: f, title,
         agent: agentMatch ? agentMatch[1] : '',
         date: displayDate,
@@ -1094,22 +1112,36 @@ function getKnowledgeBaseEntries() {
         source: sourceMatch ? sourceMatch[1].trim() : '',
         preview: content.slice(0, 200),
         size: content.length,
-      });
-    }
+      };
+    }));
+    entries.push(...fileResults);
   }
   entries.sort((a, b) =>
     (b.sortTs || 0) - (a.sortTs || 0) ||
     (b.date || '').localeCompare(a.date || '') ||
     a.title.localeCompare(b.title)
   );
-  _kbCache = entries;
-  _kbCacheTs = now;
   return entries;
 }
 
+async function getKnowledgeBaseEntries() {
+  const now = Date.now();
+  if (!_kbCacheStale && _kbCache && (now - _kbCacheTs) < KB_CACHE_TTL) return _kbCache;
+  if (_kbRefreshPromise) return _kbRefreshPromise;
+  _kbRefreshPromise = _scanKnowledgeBase()
+    .then(entries => {
+      _kbCache = entries;
+      _kbCacheTs = Date.now();
+      _kbCacheStale = false;
+      return _kbCache;
+    })
+    .finally(() => { _kbRefreshPromise = null; });
+  return _kbRefreshPromise;
+}
+
 function getKnowledgeBaseIndex() {
   try {
-    const entries = getKnowledgeBaseEntries();
+    const entries = getKnowledgeBaseEntriesSnapshot();
     if (entries.length === 0) return '';
     let index = '## Knowledge Base Reference\n\n';
     index += 'Deep-reference docs from past work. Read the file if you need detail.\n\n';
@@ -1227,8 +1259,9 @@ function getWorkItems(config) {
   const _agentDirCache = {};
   const _inboxFiles = safeReadDir(INBOX_DIR);
   const _archiveFiles = safeReadDir(ARCHIVE_DIR);
-  // Use cached KB entries (includes source frontmatter field)
-  const _kbEntries = getKnowledgeBaseEntries();
+  // Use snapshot — sync access; cold start before any async warm returns [].
+  // Best-effort enrichment for work item _artifacts.notes, not correctness-critical.
+  const _kbEntries = getKnowledgeBaseEntriesSnapshot();
   for (const item of allItems) {
     const arts = {};
     const agentId = item.dispatched_to || item.agent;
@@ -1754,7 +1787,7 @@ module.exports = {
   collectCommandFiles, getCommandIndex,
 
   // Knowledge base
-  getKnowledgeBaseEntries, getKnowledgeBaseIndex,
+  getKnowledgeBaseEntries, getKnowledgeBaseEntriesSnapshot, getKnowledgeBaseIndex,
 
   // Work items & PRD
   getWorkItems, getPrdInfo,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2003",
+  "version": "0.1.2004",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"