@yemi33/minions 0.1.2449 → 0.1.2451

package/dashboard/js/render-inbox.js

@@ -19,13 +19,44 @@ function _timeSinceMs(mtimeMs) {
   return days + 'd ago';
 }
 
+// Per-file content cache keyed by name → { mtimeMs, content }. The inbox can
+// hold thousands of notes; without a cache every 4 s refresh re-fetched EVERY
+// file's body. Firing that many fetches at once exhausted the browser socket
+// pool and flooded the console with `net::ERR_INSUFFICIENT_RESOURCES` on every
+// page (the refresh loop runs regardless of the active page). Caching by
+// (name, mtimeMs) means steady-state refreshes re-fetch nothing — only new or
+// edited notes are pulled — and the pool below bounds the initial load so the
+// socket pool is never saturated. (W-mscjlch1005qdc11)
+const _inboxContentCache = new Map();
+// Max concurrent /state/notes/inbox/<file> fetches. Chrome caps same-host HTTP/1
+// connections at 6; 8 keeps the pipe full without tripping ERR_INSUFFICIENT_RESOURCES.
+const INBOX_FETCH_CONCURRENCY = 8;
+
+// Run async `worker(item)` over `items` with at most `limit` in flight at once.
+// Preserves input order in the returned results array. Used to bound the inbox
+// per-file content fetches so a large inbox can't exhaust the browser's socket pool.
+async function _mapWithConcurrency(items, limit, worker) {
+  const results = new Array(items.length);
+  let next = 0;
+  async function run() {
+    while (next < items.length) {
+      const i = next++;
+      results[i] = await worker(items[i], i);
+    }
+  }
+  const pool = [];
+  for (let i = 0; i < Math.min(limit, items.length); i++) pool.push(run());
+  await Promise.all(pool);
+  return results;
+}
+
 // Fetch the inbox via the /state/notes/inbox directory listing + per-file
 // content fetch. The listing call returns {entries:[{name, mtimeMs, size,
-// isDir}]} and is cheap (one statSync per entry). The per-file fetches run
-// in parallel with their own mtime+size ETag so unchanged files 304 with
-// no body. Issue #2949 — the staleness fix applies here too: the inbox
-// directory mtime advances on every new note, and per-file ETags surface
-// content edits within one 4 s poll.
+// isDir}]} and is cheap (one statSync per entry). Per-file bodies are fetched
+// through a bounded-concurrency pool and cached by (name, mtimeMs), so unchanged
+// notes are served from memory and only new/edited notes hit the network. Issue
+// #2949 — the staleness fix applies here too: the inbox directory mtime advances
+// on every new note, and the mtime cache key surfaces content edits within one 4 s poll.
 async function fetchInboxFromDisk() {
   try {
     const listResp = await fetch('/state/notes/inbox');
@@ -35,21 +66,27 @@ async function fetchInboxFromDisk() {
     const mdFiles = entries
       .filter((e) => e && !e.isDir && typeof e.name === 'string' && e.name.endsWith('.md'))
       .sort((a, b) => (b.mtimeMs || 0) - (a.mtimeMs || 0));
-    const items = await Promise.all(mdFiles.map(async (e) => {
+    // Drop cache entries for notes that no longer exist so the cache can't grow
+    // unbounded across the lifetime of the page.
+    const liveNames = new Set(mdFiles.map((e) => e.name));
+    for (const key of _inboxContentCache.keys()) {
+      if (!liveNames.has(key)) _inboxContentCache.delete(key);
+    }
+    const items = await _mapWithConcurrency(mdFiles, INBOX_FETCH_CONCURRENCY, async (e) => {
+      const cached = _inboxContentCache.get(e.name);
+      if (cached && cached.mtimeMs === e.mtimeMs) {
+        return { name: e.name, mtime: e.mtimeMs, age: _timeSinceMs(e.mtimeMs), content: cached.content };
+      }
       try {
         const r = await fetch('/state/notes/inbox/' + encodeURIComponent(e.name));
         if (!r.ok) return null;
         const content = await r.text();
-        return {
-          name: e.name,
-          mtime: e.mtimeMs,
-          age: _timeSinceMs(e.mtimeMs),
-          content,
-        };
+        _inboxContentCache.set(e.name, { mtimeMs: e.mtimeMs, content });
+        return { name: e.name, mtime: e.mtimeMs, age: _timeSinceMs(e.mtimeMs), content };
       } catch {
         return null;
       }
-    }));
+    });
     return items.filter(Boolean);
   } catch {
     return null;

package/dashboard/js/render-work-items.js

@@ -1238,10 +1238,14 @@ function openWorkItemDetail(id, source) {
   // — see W-mphejzmj000718bf / W-mq5xg5e9000nec0e. We then hydrate the
   // missing/truncated fields from GET /api/work-items/<id> and re-render in
   // place.
-  const needsHydration = !cached.description ||
-    cached._descriptionTruncated === true ||
-    (cached.acceptanceCriteriaCount > 0 && !Array.isArray(cached.acceptanceCriteria)) ||
-    (cached.referencesCount > 0 && !Array.isArray(cached.references));
+  // The bulk /api/work-items list is now a LEAN payload: it carries badges,
+  // status, and the PR link, but NOT the detail-only enrichment fields
+  // (_artifacts / _notes / _model) — those are computed on demand by
+  // GET /api/work-items/<id> to keep the polled list event-loop-safe
+  // (W-mscir7fv000u6321). The detail modal therefore ALWAYS hydrates from the
+  // full record; we still render the cached slim record first so the click
+  // feels instant, then re-render in place once the full record arrives.
+  const needsHydration = true;
 
   const initial = needsHydration ? Object.assign({}, cached, { _descriptionLoading: true }) : cached;
   document.getElementById('modal-title').textContent = initial.title || initial.id;

package/dashboard.js

@@ -3690,6 +3690,19 @@ function handleStateRead(req, res) {
   let lstat;
   try { lstat = fs.lstatSync(resolved); }
   catch {
+    // A not-yet-created but allowlisted top-level state directory (e.g. the
+    // meetings dir before the first meeting is recorded) is an EMPTY directory,
+    // not a missing resource. Returning 404 here made the dashboard's directory-
+    // listing clients (render-meetings.js) log a console error on every refresh.
+    // List it as empty instead so "no items yet" is a clean 200. Missing FILES
+    // and missing nested paths still 404. (W-mscjlch1005qdc11)
+    if (isSingleSegment && STATE_READ_ALLOWED_DIRS.has(top)) {
+      res.statusCode = 200;
+      res.setHeader('Content-Type', 'application/json; charset=utf-8');
+      res.setHeader('Cache-Control', 'private, max-age=0, must-revalidate');
+      res.end(JSON.stringify({ path: rel.replace(/\\/g, '/'), entries: [] }));
+      return;
+    }
     res.statusCode = 404;
     res.setHeader('Content-Type', 'application/json');
     res.end(JSON.stringify({ error: 'not found' }));
@@ -15245,18 +15258,29 @@ What would you like to discuss or change? When you're happy, say "approve" and I
         builder: () => getAgents(),
       });
     }},
-    { method: 'GET', path: '/api/work-items', desc: 'Fully-enriched SQL-backed work items joined with dispatch/PR state — fresh on every request; description hard-capped + acceptanceCriteria/references replaced with *Count integers (W-mq5xg5e9000nec0e); detail modal lazy-loads the full record via GET /api/work-items/<id>. Optional ?limit=&offset= returns a { items, total, hasMore, offset, limit } page (progressive load, W-ms3bbzry000i6612); with no paging params the full array is byte-identical to legacy.', handler: (req, res) => {
+    { method: 'GET', path: '/api/work-items', desc: 'Lean SQL-backed work items joined with dispatch state + PR links (_pr/_prUrl) — fresh on every request; description hard-capped + acceptanceCriteria/references replaced with *Count integers (W-mq5xg5e9000nec0e). The expensive detail-only enrichment (_artifacts/_notes/_model) is NOT computed for the list — the detail modal lazy-loads the full record via GET /api/work-items/<id> (W-mscir7fv000u6321). Optional ?limit=&offset= returns a { items, total, hasMore, offset, limit } page (progressive load, W-ms3bbzry000i6612); with no paging params the full array is returned. Pagination is applied to the lean list BEFORE PR link-up so a large history cannot monopolize the event loop.', handler: (req, res) => {
       const page = _parsePageParams(req);
       return serveFreshJson(req, res, {
         tag: 'work-items',
         inputs: [CONFIG_PATH],
         variant: page ? ('p' + page.offset + '.' + page.limit) : '',
-        // W-mq5xg5e9000nec0e — Slim every item before stringify so the polled
-        // refresh-loop payload stays < ~300 KB even when description fields
-        // include 100+ KB transcripts. slimWorkItemForList shallow-copies so
-        // queries.getWorkItems()'s in-memory cache is never mutated.
-        builder: () => getWorkItems().map(slimWorkItemForList),
-        transform: page ? (list) => _paginateList(list, page) : null,
+        // W-mscir7fv000u6321 — event-loop-safe list build. getWorkItems(enrich:
+        // false) returns the deterministically-ordered lean list (rows + dispatch
+        // cross-ref + sort) with NO O(items × files) _artifacts/_notes scan.
+        // Paginate FIRST, then slim + attach _pr/_prUrl to just the returned
+        // page (or full list when unpaged) via the cheap map-based helper.
+        // W-mq5xg5e9000nec0e — slimWorkItemForList shallow-copies so the cached
+        // lean list from queries.getWorkItems() is never mutated; attach the PR
+        // links to those copies, never the shared cache.
+        builder: () => {
+          const lean = getWorkItems(null, { enrich: false });
+          if (page) {
+            const pageResult = _paginateList(lean, page);
+            pageResult.items = queries.attachWorkItemPrLinks(pageResult.items.map(slimWorkItemForList));
+            return pageResult;
+          }
+          return queries.attachWorkItemPrLinks(lean.map(slimWorkItemForList));
+        },
       });
     }},
     { method: 'GET', path: '/api/pull-requests', desc: 'Fully-enriched SQL-backed pull requests with URL backfill and project stamps. Optional ?limit=&offset= returns a { items, total, hasMore, offset, limit } page (progressive load, W-ms3bbzry000i6612); with no paging params the full array is byte-identical to legacy.', handler: (req, res) => {

package/engine/core/queries.js

@@ -2106,6 +2106,82 @@ function invalidateWorkItemsCache() {
   _workItemsLeanCacheAt = 0;
 }
 
+// attachWorkItemPrLinks(items, config) — backfill the _pr / _prUrl fields the
+// /api/work-items LIST rows render (dashboard/js/render-work-items.js `wiRow`).
+// This is the ONLY cross-reference the list rows need; the far more expensive
+// _artifacts/_notes/_model enrichment stays detail-only (GET /api/work-items/<id>).
+//
+// Root-cause perf note (W-mscir7fv000u6321): the list endpoint used to run the
+// FULL enrich over the entire work-item history on every 4 s poll, whose
+// _artifacts/_notes step is an O(items × files) synchronous KB/inbox/archive/
+// agent-dir scan that hard-froze the single dashboard event loop (~26-30 s
+// requests at ~2,281 completed items), starving /api/health and Command Center
+// SSE. Extracting the cheap PR link-up lets the endpoint paginate FIRST and then
+// attach links to just the page slice. This helper is deliberately map-based —
+// O(items + prs), never O(items × prs) — and performs NO filesystem access, so
+// it is safe to call on the full list or a single page. Mutates and returns
+// `items` (same semantics the enrich block relied on).
+function attachWorkItemPrLinks(items, config) {
+  if (!Array.isArray(items) || items.length === 0) return items || [];
+  config = config || getConfig();
+  const projects = getProjects(config);
+  const allPrs = getPullRequests(config);
+  // Prebuilt lookups so each item is O(1) instead of scanning allPrs.
+  const prById = new Map();
+  const prsByDisplayId = new Map();
+  const prByProjectPrdItem = new Map();
+  const prByProjectBranch = new Map();
+  for (const p of allPrs) {
+    if (p.id != null && !prById.has(p.id)) prById.set(p.id, p);
+    const disp = shared.getPrDisplayId(p);
+    if (disp != null) {
+      const bucket = prsByDisplayId.get(disp);
+      if (bucket) bucket.push(p); else prsByDisplayId.set(disp, [p]);
+    }
+    if (p._project) {
+      for (const wid of (p.prdItems || [])) {
+        const key = p._project + '\0' + wid;
+        if (!prByProjectPrdItem.has(key)) prByProjectPrdItem.set(key, p);
+      }
+      if (p.branch) {
+        const bkey = p._project + '\0' + p.branch;
+        if (!prByProjectBranch.has(bkey)) prByProjectBranch.set(bkey, p);
+      }
+    }
+  }
+  for (const item of items) {
+    if (item._pr && !item._prUrl) {
+      const project = shared.resolveProjectSource(item.project || item._source, projects, { allowCentral: false }).project || null;
+      const canonicalPrId = shared.getCanonicalPrId(project, item._pr);
+      const displayPrId = shared.getPrDisplayId(item._pr);
+      const exactPr = prById.get(canonicalPrId);
+      const displayMatches = exactPr ? [] : (prsByDisplayId.get(displayPrId) || []);
+      const pr = exactPr || (displayMatches.length === 1 ? displayMatches[0] : null);
+      if (pr) {
+        item._pr = pr.id;
+        item._prUrl = pr.url;
+      }
+    }
+    if (!item._pr) {
+      // Derive from PR.prdItems (single source of truth), then fall back to
+      // branch identity. The branch fallback heals decomposed-child WIs whose
+      // PR-opening agent only stamped the parent id into prdItems — branch
+      // names (`work/<wi-id>`) are unique per WI within a project, so a
+      // branch hit is as authoritative as a prdItems hit. Scoped to the same
+      // project so cross-repo branch collisions don't false-link.
+      // (W-mpn0b76200044eed)
+      const itemProject = item.project || item._source;
+      const linkedPr = prByProjectPrdItem.get(itemProject + '\0' + item.id)
+        || (item.branch ? prByProjectBranch.get(itemProject + '\0' + item.branch) : null);
+      if (linkedPr) {
+        item._pr = linkedPr.id;
+        item._prUrl = linkedPr.url;
+      }
+    }
+  }
+  return items;
+}
+
 // getWorkItems(config, { enrich, onlyId })
 //   enrich (default true): also cross-reference PRs and populate the
 //     detail-modal _artifacts/_notes fields. The /api/work-items list payload
@@ -2203,39 +2279,12 @@ function getWorkItems(config, opts) {
   // Detail-modal enrichment (PR cross-ref + _artifacts/_notes). Skipped for
   // enrich:false callers — see the getWorkItems header.
   if (enrich && !skipEnrichment) {
-  const projects = getProjects(config);
-  // Cross-reference with PRs
-  const allPrs = getPullRequests(config);
-  for (const item of enrichTargets) {
-    if (item._pr && !item._prUrl) {
-      const project = shared.resolveProjectSource(item.project || item._source, projects, { allowCentral: false }).project || null;
-      const canonicalPrId = shared.getCanonicalPrId(project, item._pr);
-      const displayPrId = shared.getPrDisplayId(item._pr);
-      const exactPr = allPrs.find(p => p.id === canonicalPrId);
-      const displayMatches = exactPr ? [] : allPrs.filter(p => shared.getPrDisplayId(p) === displayPrId);
-      const pr = exactPr || (displayMatches.length === 1 ? displayMatches[0] : null);
-      if (pr) {
-        item._pr = pr.id;
-        item._prUrl = pr.url;
-      }
-    }
-    if (!item._pr) {
-      // Derive from PR.prdItems (single source of truth), then fall back to
-      // branch identity. The branch fallback heals decomposed-child WIs whose
-      // PR-opening agent only stamped the parent id into prdItems — branch
-      // names (`work/<wi-id>`) are unique per WI within a project, so a
-      // branch hit is as authoritative as a prdItems hit. Scoped to the same
-      // project so cross-repo branch collisions don't false-link.
-      // (W-mpn0b76200044eed)
-      const itemProject = item.project || item._source;
-      const linkedPr = allPrs.find(p => p._project === itemProject && (p.prdItems || []).includes(item.id))
-        || (item.branch && allPrs.find(p => p.branch === item.branch && p._project === itemProject));
-      if (linkedPr) {
-        item._pr = linkedPr.id;
-        item._prUrl = linkedPr.url;
-      }
-    }
-  }
+  // Cross-reference with PRs (the _pr/_prUrl LIST-row fields). Extracted to a
+  // shared, map-based helper so the /api/work-items list endpoint can attach
+  // the same links to a PAGINATED slice without the expensive _artifacts/_notes
+  // filesystem scan below (W-mscir7fv000u6321). Narrowed to enrichTargets so an
+  // onlyId detail lookup only links its own row (W-mscmqk6b03ha5673).
+  attachWorkItemPrLinks(enrichTargets, config);
 
   // Populate _artifacts for the work item detail modal
   // Build dispatch ID → work item ID lookup from completed dispatches
@@ -3609,7 +3658,7 @@ module.exports = {
   getKnowledgeBaseEntries, getKnowledgeBaseEntriesSnapshot, getKnowledgeBaseIndex,
 
   // Work items & PRD
-  getWorkItems, invalidateWorkItemsCache, getPrdInfo,
+  getWorkItems, attachWorkItemPrLinks, invalidateWorkItemsCache, getPrdInfo,
 
   // W-mq5uzmc6001d708f — test hooks for the defensive PR enrichment cache.
   _resetPrEnrichmentCacheForTest,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2449",
+  "version": "0.1.2451",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"