@yemi33/minions 0.1.550 → 0.1.552

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.1.552 (2026-04-07)
+
+### Features
+-  temporary pin-to-top for inbox and KB items
+
+## 0.1.551 (2026-04-07)
+
+### Fixes
+- mark stale build status on ADO auth failure, bypass 6-tick cadence for recovery (#483)
+
 ## 0.1.550 (2026-04-07)
 
 ### Fixes

package/dashboard/js/command-center.js

@@ -9,6 +9,23 @@ let _ccAbortController = null;
 // Clear stale sending state on page load — SSE streams don't survive refresh
 try { localStorage.removeItem('cc-sending'); } catch {}
 
+function _ccFindPinTarget(query) {
+  for (var i = 0; i < (inboxData || []).length; i++) {
+    if (inboxData[i].name.toLowerCase().includes(query)) {
+      return { key: inboxPinKey(inboxData[i].name), label: inboxData[i].name };
+    }
+  }
+  for (var [cat, items] of Object.entries(_kbData || {})) {
+    if (!Array.isArray(items)) continue;
+    for (var j = 0; j < items.length; j++) {
+      if ((items[j].title || '').toLowerCase().includes(query) || (items[j].file || '').toLowerCase().includes(query)) {
+        return { key: kbPinKey(cat, items[j].file), label: items[j].title || items[j].file };
+      }
+    }
+  }
+  return null;
+}
+
 function ccAbort() {
   if (_ccAbortController) {
     _ccAbortController.abort();
@@ -165,6 +182,26 @@ function _renderQueueIndicator() {
 }
 
 async function _ccDoSend(message, skipUserMsg) {
+  // Client-side /pin and /unpin — no LLM round-trip needed
+  var pinMatch = message.match(/^\/(pin|unpin)\s+(.+)/i);
+  if (pinMatch) {
+    if (!skipUserMsg) ccAddMessage('user', escHtml(message));
+    var pinAction = pinMatch[1].toLowerCase();
+    var pinQuery = pinMatch[2].toLowerCase().trim();
+    var found = _ccFindPinTarget(pinQuery);
+    if (found) {
+      var wasPinned = isPinned(found.key);
+      if (pinAction === 'pin' && !wasPinned) { togglePin(found.key); ccAddMessage('assistant', 'Pinned <strong>' + escHtml(found.label) + '</strong> to top'); }
+      else if (pinAction === 'unpin' && wasPinned) { togglePin(found.key); ccAddMessage('assistant', 'Unpinned <strong>' + escHtml(found.label) + '</strong>'); }
+      else { ccAddMessage('assistant', '<strong>' + escHtml(found.label) + '</strong> is already ' + (wasPinned ? 'pinned' : 'unpinned')); }
+      showToast('cmd-toast', pinAction === 'pin' ? 'Pinned to top' : 'Unpinned', true);
+      renderInbox(inboxData); renderKnowledgeBase();
+    } else {
+      ccAddMessage('assistant', 'No inbox or KB item matching "' + escHtml(pinQuery) + '"');
+    }
+    return;
+  }
+
   _ccSending = true;
   _ccAbortController = new AbortController();
   try { localStorage.setItem('cc-sending', JSON.stringify({ sending: true, startedAt: Date.now() })); } catch {}

package/dashboard/js/render-inbox.js

@@ -7,7 +7,12 @@ function _inboxPrev() { if (_inboxPage > 0) { _inboxPage--; renderInbox(inboxDat
 function _inboxNext() { _inboxPage++; renderInbox(inboxData); }
 
 function renderInbox(inbox) {
+  invalidatePinsCache();
   inbox = inbox.filter(function(item) { return !isDeleted('inbox:' + item.name); });
+  // Stable sort — pinned items float to top
+  inbox.sort(function(a, b) {
+    return (isPinned(inboxPinKey(a.name)) ? 0 : 1) - (isPinned(inboxPinKey(b.name)) ? 0 : 1);
+  });
   inboxData = inbox;
   const list = document.getElementById('inbox-list');
   const count = document.getElementById('inbox-count');
@@ -22,12 +27,15 @@ function renderInbox(inbox) {
 
   list.innerHTML = pageInbox.map((item, i) => {
     const idx = inboxStart + i;
-    return `<div class="inbox-item" data-file="notes/inbox/${escHtml(item.name)}">
+    const pk = inboxPinKey(item.name);
+    const pinned = isPinned(pk);
+    return `<div class="inbox-item${pinned ? ' item-pinned' : ''}" data-file="notes/inbox/${escHtml(item.name)}">
       <div class="inbox-name" onclick="openModal(${idx})" style="cursor:pointer">
         <span>${escHtml(item.name)}</span><span>${escHtml(item.age || '')}</span>
       </div>
       <div class="inbox-preview" onclick="openModal(${idx})" style="cursor:pointer">${escHtml(item.content.slice(0,200))}</div>
       <div style="display:flex;gap:6px;margin-top:6px;align-items:center">
+        <button class="pr-pager-btn pin-btn${pinned ? ' pinned' : ''}" style="font-size:9px;padding:2px 8px" data-pin-key="${escHtml(pk)}" onclick="event.stopPropagation();_togglePinAndRefresh(this.dataset.pinKey,'inbox')">${pinned ? 'Unpin' : 'Pin'}</button>
         <button class="pr-pager-btn" style="font-size:9px;padding:2px 8px" data-inbox-name="${escHtml(item.name)}" onclick="event.stopPropagation();promoteToKB(this.dataset.inboxName)">Add to Knowledge Base</button>
         <button class="pr-pager-btn" style="font-size:9px;padding:2px 8px" data-inbox-name="${escHtml(item.name)}" onclick="event.stopPropagation();openInboxInExplorer(this.dataset.inboxName)">Open in Explorer</button>
         <button class="pr-pager-btn" style="font-size:9px;padding:2px 8px;color:var(--red)" data-inbox-name="${escHtml(item.name)}" onclick="event.stopPropagation();deleteInboxItem(this.dataset.inboxName)">Delete</button>

package/dashboard/js/render-kb.js

@@ -32,40 +32,61 @@ function renderKnowledgeBase() {
   const listEl = document.getElementById('kb-list');
   const countEl = document.getElementById('kb-count');
 
-  // Count total (skip non-array keys like lastSwept)
-  let total = 0;
-  for (const [k, v] of Object.entries(_kbData)) if (Array.isArray(v)) total += v.length;
-  countEl.textContent = total;
+  invalidatePinsCache();
+
+  // Single pass: flatten all KB items and count pinned
+  const allItems = [];
+  let pinnedCount = 0;
+  for (const [cat, catItems] of Object.entries(_kbData)) {
+    if (!Array.isArray(catItems)) continue;
+    for (const item of catItems) {
+      const entry = { ...item, category: cat };
+      allItems.push(entry);
+      if (isPinned(kbPinKey(cat, item.file))) pinnedCount++;
+    }
+  }
+  countEl.textContent = allItems.length;
 
-  // Last swept timestamp
   const sweptEl = document.getElementById('kb-swept-time');
   if (sweptEl) sweptEl.textContent = _kbData.lastSwept ? 'swept ' + timeSinceStr(new Date(_kbData.lastSwept)) : '';
 
-  if (total === 0) {
+  if (allItems.length === 0) {
     tabsEl.innerHTML = '';
     listEl.innerHTML = '<p class="empty">No knowledge entries yet. Notes are classified here after consolidation.</p>';
     return;
   }
 
+  if (_kbActiveTab === 'pinned' && pinnedCount === 0) _kbActiveTab = 'all';
+
   // Render tabs
-  let tabsHtml = '<button class="kb-tab ' + (_kbActiveTab === 'all' ? 'active' : '') + '" onclick="kbSetTab(\'all\')">All <span class="badge">' + total + '</span></button>';
-  for (const [cat, items] of Object.entries(_kbData)) {
-    if (!Array.isArray(items) || items.length === 0) continue;
+  let tabsHtml = '';
+  if (pinnedCount > 0) {
+    tabsHtml += '<button class="kb-tab ' + (_kbActiveTab === 'pinned' ? 'active' : '') + '" style="color:var(--yellow)" onclick="kbSetTab(\'pinned\')">Pinned <span class="badge">' + pinnedCount + '</span></button>';
+  }
+  tabsHtml += '<button class="kb-tab ' + (_kbActiveTab === 'all' ? 'active' : '') + '" onclick="kbSetTab(\'all\')">All <span class="badge">' + allItems.length + '</span></button>';
+  for (const [cat, catArr] of Object.entries(_kbData)) {
+    if (!Array.isArray(catArr) || catArr.length === 0) continue;
     const label = KB_CAT_LABELS[cat] || cat;
-    tabsHtml += '<button class="kb-tab ' + (_kbActiveTab === cat ? 'active' : '') + '" onclick="kbSetTab(\'' + cat + '\')">' + label + ' <span class="badge">' + items.length + '</span></button>';
+    tabsHtml += '<button class="kb-tab ' + (_kbActiveTab === cat ? 'active' : '') + '" onclick="kbSetTab(\'' + cat + '\')">' + label + ' <span class="badge">' + catArr.length + '</span></button>';
   }
   tabsEl.innerHTML = tabsHtml;
 
-  // Collect items for active tab
-  let items = [];
-  if (_kbActiveTab === 'all') {
-    for (const [cat, catItems] of Object.entries(_kbData)) {
-      if (!Array.isArray(catItems)) continue;
-      for (const item of catItems) items.push({ ...item, category: cat });
-    }
+  // Filter items for active tab
+  let items;
+  if (_kbActiveTab === 'pinned') {
+    items = allItems.filter(i => isPinned(kbPinKey(i.category, i.file)));
+  } else if (_kbActiveTab === 'all') {
+    items = allItems.slice();
     items.sort((a, b) => (b.date || '').localeCompare(a.date || ''));
   } else {
-    items = (_kbData[_kbActiveTab] || []).map(i => ({ ...i, category: _kbActiveTab }));
+    items = allItems.filter(i => i.category === _kbActiveTab);
+  }
+
+  // Stable sort — pinned items float to top (skip on pinned tab where all are pinned)
+  if (_kbActiveTab !== 'pinned') {
+    items.sort(function(a, b) {
+      return (isPinned(kbPinKey(a.category, a.file)) ? 0 : 1) - (isPinned(kbPinKey(b.category, b.file)) ? 0 : 1);
+    });
   }
 
   if (items.length === 0) {
@@ -82,9 +103,13 @@ function renderKnowledgeBase() {
   listEl.innerHTML = pageItems.map(item => {
     const icon = KB_CAT_ICONS[item.category] || '\u{1F4C4}';
     const label = KB_CAT_LABELS[item.category] || item.category;
-    return '<div class="kb-item" data-file="knowledge/' + escHtml(item.category) + '/' + escHtml(item.file) + '" onclick="kbOpenItem(\'' + escHtml(item.category) + '\', \'' + escHtml(item.file) + '\')">' +
+    var pinKey = kbPinKey(item.category, item.file);
+    var pinned = isPinned(pinKey);
+    return '<div class="kb-item' + (pinned ? ' item-pinned' : '') + '" data-file="knowledge/' + escHtml(item.category) + '/' + escHtml(item.file) + '" onclick="kbOpenItem(\'' + escHtml(item.category) + '\', \'' + escHtml(item.file) + '\')">' +
       '<div class="kb-item-body">' +
-        '<div class="kb-item-title">' + icon + ' ' + escHtml(item.title) + '</div>' +
+        '<div class="kb-item-title">' + icon + ' ' + escHtml(item.title) +
+          ' <button class="pr-pager-btn pin-btn' + (pinned ? ' pinned' : '') + '" style="font-size:9px;padding:1px 6px;margin-left:6px;vertical-align:middle" data-pin-key="' + escHtml(pinKey) + '" onclick="event.stopPropagation();_togglePinAndRefresh(this.dataset.pinKey,\'kb\')">' + (pinned ? 'Unpin' : 'Pin') + '</button>' +
+        '</div>' +
         '<div class="kb-item-meta">' +
           '<span>' + label + '</span>' +
           (item.agent ? '<span>' + escHtml(item.agent) + '</span>' : '') +

package/dashboard/js/render-prs.js

@@ -12,8 +12,8 @@ function prRow(pr) {
   const reviewSource = sq.status || effectiveReviewStatus || 'pending';
   const reviewClass = reviewSource === 'approved' ? 'approved' : (reviewSource === 'changes-requested' || reviewSource === 'rejected') ? 'rejected' : reviewSource === 'waiting' ? 'building' : 'draft';
   const reviewLabel = sq.status === 'waiting' ? 'reviewing (minions)' : sq.status ? sq.status + ' (minions)' : (effectiveReviewStatus || 'pending');
-  const buildClass = pr.buildStatus === 'passing' ? 'build-pass' : pr.buildStatus === 'failing' ? 'build-fail' : pr.buildStatus === 'running' ? 'building' : 'no-build';
-  const buildLabel = pr.buildStatus || 'none';
+  const buildClass = pr._buildStatusStale ? 'build-stale' : pr.buildStatus === 'passing' ? 'build-pass' : pr.buildStatus === 'failing' ? 'build-fail' : pr.buildStatus === 'running' ? 'building' : 'no-build';
+  const buildLabel = (pr.buildStatus || 'none') + (pr._buildStatusStale ? ' (stale)' : '');
   const statusClass = pr.status === 'merged' ? 'merged' : pr.status === 'abandoned' ? 'rejected' : pr.status === 'active' ? 'active' : 'draft';
   const statusLabel = pr.status || 'active';
   const url = pr.url || '#';

package/dashboard/js/utils.js

@@ -8,6 +8,33 @@ const _deletedIds = new Map(); // key → expiry timestamp
 function markDeleted(key) { _deletedIds.set(key, Date.now() + 10000); } // suppress for 10s
 function isDeleted(key) { const exp = _deletedIds.get(key); if (!exp) return false; if (Date.now() > exp) { _deletedIds.delete(key); return false; } return true; }
 
+// Temporary pin-to-top — UI-only, stored in localStorage, does not affect agents
+const PINS_KEY = 'minions-pinned-items';
+let _pinsCache = null;
+function invalidatePinsCache() { _pinsCache = null; }
+function getPinnedItems() {
+  if (_pinsCache) return _pinsCache;
+  try { _pinsCache = JSON.parse(localStorage.getItem(PINS_KEY) || '[]'); } catch { _pinsCache = []; }
+  return _pinsCache;
+}
+function isPinned(key) { return getPinnedItems().includes(key); }
+function togglePin(key) {
+  const pins = getPinnedItems();
+  const idx = pins.indexOf(key);
+  if (idx >= 0) pins.splice(idx, 1); else pins.unshift(key);
+  localStorage.setItem(PINS_KEY, JSON.stringify(pins));
+  invalidatePinsCache();
+  return idx < 0; // true if now pinned
+}
+function inboxPinKey(name) { return 'notes/inbox/' + name; }
+function kbPinKey(cat, file) { return 'knowledge/' + cat + '/' + file; }
+function _togglePinAndRefresh(key, source) {
+  var pinned = togglePin(key);
+  showToast('cmd-toast', pinned ? 'Pinned to top' : 'Unpinned', true);
+  if (source === 'inbox') renderInbox(inboxData);
+  else if (source === 'kb') renderKnowledgeBase();
+}
+
 function escHtml(s) {
   return String(s).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;').replace(/"/g,'&quot;').replace(/'/g,'&#39;');
 }
@@ -217,12 +244,19 @@ function _renderMdChunked(fullText) {
     var target = pos + MD_CHUNK_SIZE;
     var searchStart = Math.max(pos + Math.floor(MD_CHUNK_SIZE * 0.7), pos);
     var searchEnd = Math.min(target + 500, fullText.length);
-    var slice = fullText.slice(searchStart, searchEnd);
-    var lastBlank = slice.lastIndexOf('\n\n');
-    var best;
-    if (lastBlank !== -1) {
-      best = searchStart + lastBlank + 2;
-    } else {
+    // Find the last \n\n in [searchStart, searchEnd] that isn't inside a code fence
+    var fencesBefore = (fullText.slice(pos, searchStart).match(/```/g) || []).length;
+    var inFence = (fencesBefore % 2) === 1;
+    var best = null;
+    for (var si = searchStart; si < searchEnd - 1; si++) {
+      if (fullText[si] === '`' && fullText[si + 1] === '`' && fullText[si + 2] === '`') {
+        inFence = !inFence;
+        si += 2;
+      } else if (!inFence && fullText[si] === '\n' && fullText[si + 1] === '\n') {
+        best = si + 2;
+      }
+    }
+    if (best === null) {
       var nl = fullText.indexOf('\n', target);
       best = (nl !== -1 && nl - target < 500) ? nl + 1 : target;
     }

package/dashboard/styles.css

@@ -154,6 +154,7 @@
   .kb-item { display: flex; align-items: flex-start; gap: var(--space-5); padding: var(--space-4) 0; border-bottom: 1px solid var(--border); cursor: pointer; transition: background var(--transition-fast); }
   .kb-item:hover { background: var(--surface2); }
   .kb-item:last-child { border-bottom: none; }
+  .kb-item.item-pinned { border-left: 3px solid var(--yellow); padding-left: var(--space-4); background: rgba(210, 153, 34, 0.05); }
   .kb-item-body { flex: 1; min-width: 0; }
   .kb-item-title { font-size: var(--text-md); color: var(--text); font-weight: 500; }
   .kb-item-meta { font-size: var(--text-sm); color: var(--muted); margin-top: var(--space-1); display: flex; gap: var(--space-4); }
@@ -211,6 +212,10 @@
   .notes-preview:hover { border-color: var(--blue); }
   .inbox-item { background: var(--surface2); border: 1px solid var(--border); border-left: 3px solid var(--purple); border-radius: var(--radius-sm); padding: var(--space-5) var(--space-6); cursor: pointer; }
   .inbox-item:hover { border-color: var(--blue); border-left-color: var(--blue); }
+  .item-pinned { border-left-color: var(--yellow); background: rgba(210, 153, 34, 0.05); }
+  .item-pinned:hover { border-left-color: var(--yellow); }
+  .pin-btn { color: var(--muted); border-color: var(--border); }
+  .pin-btn.pinned { color: var(--yellow); border-color: var(--yellow); }
   .inbox-name { font-weight: 500; font-size: var(--text-md); color: var(--purple); margin-bottom: var(--space-2); display: flex; justify-content: space-between; }
   .inbox-preview { font-size: var(--text-base); color: var(--muted); line-height: 1.5; max-height: 60px; overflow: hidden; }
 
@@ -251,6 +256,7 @@
   .pr-badge.build-pass { background: rgba(63,185,80,0.15); color: var(--green); border: 1px solid var(--green); }
   .pr-badge.build-fail { background: rgba(248,81,73,0.15); color: var(--red); border: 1px solid var(--red); }
   .pr-badge.no-build { background: var(--surface); color: var(--muted); border: 1px solid var(--border); }
+  .pr-badge.build-stale { background: rgba(210,153,34,0.15); color: var(--orange); border: 1px dashed var(--orange); }
   .error-details-btn { font-size: var(--text-xs); padding: var(--space-1) var(--space-3); margin-left: var(--space-2); background: rgba(248,81,73,0.15); color: var(--red); border: 1px solid var(--red); border-radius: var(--radius-lg); cursor: pointer; font-weight: 600; text-transform: uppercase; letter-spacing: 0.3px; }
   .error-details-btn:hover { background: rgba(248,81,73,0.3); }
   .pr-empty { color: var(--muted); font-style: italic; font-size: var(--text-md); padding: var(--space-6) 0; }

package/docs/design-state-storage.md

@@ -0,0 +1,431 @@
+# Design: Replacing File-Based State with a Structured Database
+
+> Author: Rebecca (Architect) | Date: 2026-04-07 | Status: Proposal
+
+## Executive Summary
+
+Minions persists all runtime state as flat JSON files guarded by file-lock-based concurrency (`mutateJsonFileLocked`). This analysis evaluates five options for migrating to a structured database, benchmarks each against the current approach, and delivers a phased recommendation.
+
+**Verdict:** Stay with improved file-based state short-term. Adopt `node:sqlite` (`DatabaseSync`) as the medium-term target once it exits experimental status, migrating the highest-pain state files first.
+
+---
+
+## 1. Current State Architecture
+
+### 1.1 State Files Inventory
+
+| File | Size (live) | Records | Access Pattern | Contention | Pain Level |
+|------|-------------|---------|----------------|------------|------------|
+| `engine/dispatch.json` | 380 KB | 102 completed + 2 active | R/W every tick; 10+ reads/tick (2s cache) | **High** — engine + dashboard + lifecycle | High |
+| `engine/log.json` | 292 KB | 2,162 entries | Append-only (buffered flush every 500ms) | Medium — log buffer serializes | Medium |
+| `engine/cooldowns.json` | 511 KB | 125 keys | R/W on dispatch failure + retry | Low — infrequent writes | High (bloated) |
+| `engine/metrics.json` | 5 KB | Per-agent stats | R/W on PR approval/merge | Low | Low |
+| `engine/control.json` | 169 B | Single object | R/W on start/stop/heartbeat | Low | Low |
+| `projects/*/work-items.json` | 370 KB | 180 items | R/W every 1-2 ticks; dashboard reads on-demand | **High** — engine + lifecycle + dashboard | High |
+| `projects/*/pull-requests.json` | 241 KB | 128 PRs | R/W every 6 ticks (polling); lifecycle writes | Medium | Medium |
+| `engine/pipeline-runs.json` | 36 KB | Pipeline state | R/W on pipeline execution | Low | Low |
+| `engine/schedule-runs.json` | 115 B | Last-run times | R every 10 ticks; W on schedule execution | Low | Low |
+
+**Total live state:** ~1.8 MB across 9+ JSON files.
+
+(source: `engine/shared.js:233-252` for locking, `engine/queries.js:57-61` for paths, live file sizes from `ls -la engine/*.json`)
+
+### 1.2 Concurrency Model
+
+All mutations go through `mutateJsonFileLocked()` (source: `engine/shared.js:233-252`):
+
+```
+acquire .lock file (exclusive create via fs.openSync 'wx')
+  → read JSON file (full parse)
+  → apply mutation function
+  → write entire file (atomic rename via safeWrite)
+  → create .backup sidecar
+release .lock file
+```
+
+Key properties:
+- **Synchronous blocking** — `withFileLock` spins with `sleepMs(25)` until lock acquired or 5s timeout (source: `engine/shared.js:175-231`)
+- **Whole-file granularity** — updating one field in one work item rewrites all 180 items (370 KB)
+- **Stale lock recovery** — locks older than 60s are force-removed (source: `engine/shared.js:173`, `LOCK_STALE_MS`)
+- **Read caching** — only `dispatch.json` has a 2s TTL cache (source: `engine/queries.js:82-91`)
+
+### 1.3 Read vs Write Ratio
+
+| Consumer | Reads/tick | Writes/tick | Pattern |
+|----------|-----------|-------------|---------|
+| Engine tick cycle | ~15 | ~3 | Heavy read, selective write |
+| Dashboard (per page load) | ~8 | 0 | Read-only display |
+| Dashboard (user action) | ~2 | ~2 | Read-modify-write |
+| PR polling (every 6 ticks) | ~4 | ~2 | Batch read-modify-write |
+| Consolidation (every 10 ticks) | ~3 | ~2 | Read inbox files, write notes.md |
+
+**Read:write ratio is approximately 8:1.** This strongly favors a system that can serve reads without locking (e.g., WAL mode).
+
+---
+
+## 2. Option Evaluation
+
+### 2.1 `node:sqlite` (DatabaseSync) — Node 22.5+
+
+**Current status:** Available in Node v24.12.0 (this machine). Marked `ExperimentalWarning`. Synchronous API via `DatabaseSync`.
+
+**Benchmark results** (measured on this machine):
+
+| Operation | `node:sqlite` | File-based (current) |
+|-----------|---------------|---------------------|
+| Insert 1,000 rows (transaction) | 4.6 ms | N/A (no equivalent) |
+| Single SELECT by status (200 rows) | 0.13 ms | 2.9 ms (full parse) + 0.06 ms (filter) |
+| 100 individual UPDATEs (no txn) | 48.7 ms | ~270 ms (100x full rewrite) |
+| 100 UPDATEs in transaction | 0.7 ms | N/A |
+| SELECT all 1,000 rows | 0.7 ms | 2.9 ms (parse 370 KB) |
+
+**Dependency story:** Zero npm dependencies. Ships with Node.js. No native addon build step.
+
+**Cross-platform:** SQLite is compiled into Node itself — works identically on Windows, macOS, Linux.
+
+**Concurrency model:**
+- WAL mode allows concurrent readers with one writer (verified working — source: benchmark tests above)
+- `DatabaseSync` is synchronous, matching the current blocking model exactly
+- Transactions replace file locks — `BEGIN EXCLUSIVE` provides the same mutual exclusion
+- Row-level updates eliminate whole-file rewrites
+
+**Migration complexity:** High. ~40+ `safeJson` read sites across `engine.js`, `dashboard.js`, `queries.js`, `lifecycle.js`. ~20+ `mutateJsonFileLocked` write sites. Each needs conversion to prepared statements.
+
+**JSON support:** SQLite JSON1 extension works — `json_extract()`, `json_each()` verified functional for dependency resolution queries.
+
+**Pros:**
+- Zero dependencies (built into Node)
+- WAL mode eliminates read-write contention
+- Row-level operations (no more 370 KB rewrites for one field change)
+- Indexed queries (find pending items by status without scanning all items)
+- Transactions provide stronger atomicity than file locks
+- Single `.db` file replaces 9+ JSON files + 9 `.backup` + 9 `.lock` files
+
+**Cons:**
+- **Experimental API** — could change between Node versions with no migration path
+- State files become opaque (can't `cat dispatch.json` for debugging)
+- Schema migrations needed as data model evolves
+- No async API yet (acceptable — current code is sync anyway)
+- `ExperimentalWarning` printed on first import (suppressible with `--no-warnings=ExperimentalWarning`)
+
+**Risk assessment:** The experimental status is the **only** serious blocker. The API surface is small (`DatabaseSync`, `prepare`, `exec`, `get`, `all`, `run`) and mirrors `better-sqlite3` closely — likely to stabilize without breaking changes. But "likely" is not "guaranteed."
+
+### 2.2 LevelDB / LMDB (Embedded Key-Value)
+
+**Dependency story:** npm packages with native C/C++ addons. `level` (LevelDB wrapper) or `lmdb-js`.
+
+**Cross-platform:** Requires native build toolchain (node-gyp, C++ compiler). Windows support historically fragile.
+
+**Concurrency model:**
+- LMDB: MVCC with zero-copy reads — excellent read performance
+- LevelDB: Single-process lock; read-free but writes serialize
+
+**Migration complexity:** Medium-high. Key-value model doesn't naturally support the relational queries needed (e.g., "find all pending work items for project X with unmet dependencies").
+
+**Pros:**
+- Battle-tested in production
+- LMDB: extremely fast reads with memory-mapped I/O
+- Supports ordered iteration (useful for log entries)
+
+**Cons:**
+- **Breaks zero-dependency principle** (hard stop)
+- Native addon build complexity on Windows
+- Key-value model is a poor fit for relational queries on work items
+- Adds ~15-50 MB to install footprint
+
+**Verdict:** Disqualified by the zero-dependency constraint.
+
+### 2.3 Better-SQLite3
+
+**Dependency story:** npm package with native C addon. Prebuilt binaries available for most platforms via `prebuild-install`.
+
+**Cross-platform:** Good — prebuilt binaries for Windows/macOS/Linux. Fallback to node-gyp if no prebuild.
+
+**Concurrency model:** Identical to `node:sqlite` — synchronous, WAL mode, same SQLite engine underneath.
+
+**Migration complexity:** Same as `node:sqlite` — the API is nearly identical (`db.prepare().run()`, `.get()`, `.all()`).
+
+**Pros:**
+- Most popular SQLite binding for Node.js (14M weekly downloads)
+- Stable, well-maintained, extensive documentation
+- Synchronous API (perfect match)
+- Full SQLite feature set including JSON1
+
+**Cons:**
+- **Breaks zero-dependency principle** (hard stop)
+- Native addon (prebuilt binaries help but don't eliminate all build issues)
+- ~8 MB added to node_modules
+
+**Verdict:** Disqualified by the zero-dependency constraint. However, if `node:sqlite` stabilizes with an API modeled on `better-sqlite3` (which it is), then `better-sqlite3` serves as a proven reference implementation.
+
+### 2.4 Improved File-Based Approach
+
+Keep JSON files but address the worst pain points structurally.
+
+**Specific improvements:**
+
+| Improvement | Targets | Effort | Impact |
+|-------------|---------|--------|--------|
+| **Split dispatch.json** into `dispatch/pending.json`, `dispatch/active.json`, `dispatch/completed/` (per-entry files) | dispatch.json (380 KB) | Medium | Reduces lock contention — pending/active/completed are independently lockable |
+| **Cap and rotate cooldowns.json** — delete entries older than 7 days | cooldowns.json (511 KB) | Low | 511 KB → ~20 KB |
+| **Per-entity files for completed dispatches** — `dispatch/completed/{id}.json` | dispatch.json completed array | Medium | Eliminates growing array; reads become `readdir + filter` |
+| **Add read caches** — extend 2s TTL cache from dispatch.json to work-items.json and pull-requests.json | queries.js | Low | 60-80% fewer disk reads per tick |
+| **JSON Lines for log.json** — append-only `.jsonl` format | log.json (292 KB) | Low | Eliminates parse-entire-file-to-append; rotation becomes `tail -n 2000` |
+| **Structured directory layout** — `state/{entity}/{id}.json` | All state files | High | Per-entity locking, but dramatically increases file count |
+
+**Dependency story:** Zero — pure Node.js.
+
+**Cross-platform:** Identical to current.
+
+**Concurrency model:** Same file locks, but with finer granularity (per-entity instead of per-file).
+
+**Migration complexity:** Low-medium. Read/write APIs stay the same shape; internal storage layout changes.
+
+**Pros:**
+- Zero risk — no new dependencies or experimental APIs
+- Human-readable state (critical for debugging)
+- Incremental migration (one file at a time)
+- Preserves existing backup/restore pattern
+
+**Cons:**
+- Doesn't solve the fundamental problem: whole-file read-modify-write for arrays
+- Per-entity files create thousands of small files (OS inode pressure on large deployments)
+- No indexed queries — filtering still requires scanning all files
+- Read caches add TTL staleness risk
+
+### 2.5 Hybrid: JSON Lines + Indexed Views
+
+A creative zero-dep option: use append-only JSON Lines files as the write log, with periodic compaction into indexed JSON snapshots.
+
+**How it works:**
+1. Writes append to `state/{entity}.jsonl` (no locking needed for appends)
+2. Reads come from a cached in-memory index (rebuilt from JSONL on startup)
+3. Periodic compaction rewrites the JSONL file, discarding superseded entries
+
+**Pros:**
+- Append-only writes are naturally lock-free
+- In-memory index serves reads instantly
+- Human-readable (JSONL is `cat`-able)
+- Zero dependencies
+
+**Cons:**
+- Requires custom index implementation (error-prone)
+- Compaction logic is complex to get right under concurrent access
+- Crash recovery requires replaying the full JSONL file
+- Reinventing a (bad) database
+
+**Verdict:** Too much complexity for marginal gain. If we're going to build database-like infrastructure, use an actual database.
+
+---
+
+## 3. Recommendation
+
+### Phase 1: Quick Wins (Now — 1-2 days effort)
+
+Stay with files. Fix the two highest-pain issues immediately:
+
+1. **Cap `cooldowns.json`** — Add a cleanup sweep that deletes entries older than 7 days. This file is 511 KB with 125 keys, most of which are stale. Implement in `engine/cooldown.js` cleanup function. (source: `engine/cooldown.js`)
+
+2. **Cap `dispatch.json` completed array** more aggressively — Currently capped at 100 entries (source: `engine/dispatch.js:112-114`). Reduce to 50 or archive to `dispatch/completed/` directory. The 380 KB file is mostly completed entries.
+
+3. **Add read caches to `work-items.json` and `pull-requests.json`** — Same 2s TTL pattern as dispatch.json (source: `engine/queries.js:82-91`). These are read 8+ times per tick but only written 1-2 times.
+
+4. **Convert `log.json` to append-only JSONL** — Eliminates the parse-entire-file-to-append pattern in `_flushLogBuffer()` (source: `engine/shared.js:49-59`). Log rotation becomes `readFile → keep last 2000 lines → writeFile` instead of `parse JSON array → splice → stringify → write`.
+
+### Phase 2: `node:sqlite` Migration (When API stabilizes — estimated Node 26 LTS)
+
+Monitor `node:sqlite` stability. When it exits experimental:
+
+**Migration order** (highest pain first):
+
+| Priority | State File | Why First |
+|----------|-----------|-----------|
+| 1 | `engine/log.json` | Append-heavy, benefits most from indexed queries, lowest risk (read-only by dashboard) |
+| 2 | `engine/dispatch.json` | Most contended file, benefits from row-level operations, eliminates completed array growth |
+| 3 | `engine/cooldowns.json` | Bloated key-value store, natural fit for TTL-indexed table |
+| 4 | `projects/*/work-items.json` | Core entity, benefits from indexed status/project/type queries |
+| 5 | `projects/*/pull-requests.json` | Similar to work items, lower contention |
+| 6 | `engine/metrics.json` | Small, low contention — migrate for consistency |
+| 7 | `engine/control.json` | Tiny, single-object — migrate last for consistency |
+
+**Migration architecture:**
+
+```
+┌─────────────────────────────────────────────────┐
+│                  StateStore API                  │
+│  (drop-in replacement for mutateJsonFileLocked)  │
+├─────────────────────────────────────────────────┤
+│  getWorkItems(filter?)  → WorkItem[]             │
+│  mutateWorkItem(id, fn) → WorkItem               │
+│  getDispatch()          → DispatchQueue           │
+│  mutateDispatch(fn)     → DispatchQueue           │
+│  appendLog(entry)       → void                    │
+│  ...                                              │
+├─────────────────────────────────────────────────┤
+│          Backend: SQLite (DatabaseSync)           │
+│  ┌─────────┐ ┌──────────┐ ┌──────────────────┐  │
+│  │ WAL mode│ │ Prepared │ │ JSON data column  │  │
+│  │         │ │statements│ │ + indexed columns │  │
+│  └─────────┘ └──────────┘ └──────────────────┘  │
+└─────────────────────────────────────────────────┘
+```
+
+**Key design decisions for the SQLite schema:**
+
+1. **Hybrid column strategy** — Store frequently-queried fields as indexed columns (`id`, `status`, `type`, `project`), keep the full object in a `data TEXT` column as JSON. This allows SQL `WHERE` on hot fields while preserving schema flexibility.
+
+2. **Single database file** — All state in one `.db` file in `engine/minions.db`. WAL mode enables concurrent reads. Transactions replace file locks.
+
+3. **Prepared statement cache** — Create all prepared statements at startup, reuse throughout process lifetime. This avoids the 0.7ms-per-prepare overhead measured in benchmarks.
+
+4. **Migration layer** — On first startup with SQLite, read existing JSON files, populate tables, rename JSON files to `.json.migrated`. On rollback, the `.migrated` files can be renamed back.
+
+5. **Debug tooling** — Add `minions db` CLI command that opens an interactive SQLite shell on `engine/minions.db` for debugging (replaces `cat dispatch.json`).
+
+**Proposed schema (core tables):**
+
+```sql
+-- Dispatch queue entries
+CREATE TABLE dispatch (
+  id TEXT PRIMARY KEY,
+  queue TEXT NOT NULL CHECK(queue IN ('pending','active','completed')),
+  type TEXT NOT NULL,
+  agent TEXT,
+  task TEXT,
+  meta TEXT,          -- JSON blob
+  created_at TEXT,
+  started_at TEXT,
+  completed_at TEXT,
+  result TEXT,
+  reason TEXT
+);
+CREATE INDEX idx_dispatch_queue ON dispatch(queue);
+CREATE INDEX idx_dispatch_agent ON dispatch(agent);
+
+-- Work items
+CREATE TABLE work_items (
+  id TEXT PRIMARY KEY,
+  project TEXT NOT NULL,
+  status TEXT NOT NULL,
+  type TEXT,
+  title TEXT,
+  priority TEXT,
+  data TEXT,          -- Full JSON blob
+  created TEXT,
+  updated_at TEXT DEFAULT (datetime('now'))
+);
+CREATE INDEX idx_wi_status ON work_items(status);
+CREATE INDEX idx_wi_project ON work_items(project);
+CREATE INDEX idx_wi_project_status ON work_items(project, status);
+
+-- Pull requests
+CREATE TABLE pull_requests (
+  id TEXT PRIMARY KEY,
+  project TEXT NOT NULL,
+  status TEXT,
+  branch TEXT,
+  agent TEXT,
+  data TEXT,          -- Full JSON blob
+  created TEXT,
+  updated_at TEXT DEFAULT (datetime('now'))
+);
+CREATE INDEX idx_pr_status ON pull_requests(status);
+CREATE INDEX idx_pr_project ON pull_requests(project);
+
+-- Engine log (append-only)
+CREATE TABLE engine_log (
+  rowid INTEGER PRIMARY KEY AUTOINCREMENT,
+  timestamp TEXT NOT NULL,
+  level TEXT NOT NULL,
+  message TEXT,
+  meta TEXT            -- JSON blob
+);
+CREATE INDEX idx_log_timestamp ON engine_log(timestamp);
+CREATE INDEX idx_log_level ON engine_log(level);
+
+-- Cooldowns (TTL-based)
+CREATE TABLE cooldowns (
+  key TEXT PRIMARY KEY,
+  failures INTEGER DEFAULT 0,
+  last_failure TEXT,
+  cooldown_until TEXT,
+  data TEXT
+);
+CREATE INDEX idx_cd_until ON cooldowns(cooldown_until);
+
+-- Metrics
+CREATE TABLE metrics (
+  agent TEXT PRIMARY KEY,
+  data TEXT            -- JSON blob with token usage, quality, etc.
+);
+
+-- Key-value store for small state (control, schedule-runs, etc.)
+CREATE TABLE kv (
+  key TEXT PRIMARY KEY,
+  value TEXT
+);
+```
+
+### Phase 3: Advanced Optimizations (Post-migration)
+
+Once on SQLite, unlock capabilities impossible with file-based state:
+
+1. **Dashboard SSE from SQLite triggers** — Use `sqlite3_update_hook` (if exposed) or polling with `WHERE updated_at > ?` instead of file-watching
+2. **Dependency resolution via SQL** — Replace in-memory graph traversal with recursive CTEs
+3. **Metrics aggregation** — `SELECT agent, SUM(tokens) FROM dispatch WHERE completed_at > ? GROUP BY agent`
+4. **Log analysis** — `SELECT level, COUNT(*) FROM engine_log WHERE timestamp > ? GROUP BY level`
+5. **Automatic compaction** — `DELETE FROM engine_log WHERE rowid < (SELECT MAX(rowid) - 2000 FROM engine_log)`
+
+---
+
+## 4. What NOT to Do
+
+1. **Don't migrate to SQLite while the API is experimental.** A Node.js upgrade that changes `DatabaseSync` parameters or removes the module would be catastrophic for a state storage layer.
+
+2. **Don't use an async SQLite API** (if one is added to Node). The current codebase is fundamentally synchronous — `mutateJsonFileLocked` is called from synchronous code paths. Mixing sync and async state access is a recipe for race conditions.
+
+3. **Don't split into per-entity files** at scale. Going from 9 files to potentially thousands (one per work item, per dispatch entry) creates inode pressure, `readdir` performance issues, and makes atomic multi-entity operations harder (need to lock multiple files).
+
+4. **Don't add npm dependencies** for state storage. The zero-dependency principle is a genuine architectural strength — it means `git clone && node engine.js` works everywhere with zero setup.
+
+5. **Don't build a custom database.** The JSON Lines + compaction + in-memory index approach is literally reimplementing SQLite badly. Use the real thing when it's stable.
+
+---
+
+## 5. Risk Analysis
+
+| Risk | Probability | Impact | Mitigation |
+|------|-------------|--------|------------|
+| `node:sqlite` API breaks on Node upgrade | Medium (experimental) | High — state inaccessible | Phase 2 only after API stabilizes; keep JSON export/import |
+| File-based approach hits scaling limit | Low (current data tiny) | Medium — slower ticks | Phase 1 caching + capping buys years of headroom |
+| SQLite `.db` corruption | Very low (SQLite is ACID) | High — state lost | WAL mode + periodic `.db` backup to `.db.backup` |
+| Migration bugs lose state | Medium | High | Dual-write period: write to both JSON and SQLite for 1 week |
+| Dashboard performance degrades during migration | Low | Low | Read API stays the same shape; backend changes only |
+
+---
+
+## 6. Decision Matrix
+
+| Criterion | Files (current) | Files (improved) | `node:sqlite` | `better-sqlite3` | LevelDB/LMDB |
+|-----------|----------------|------------------|---------------|-------------------|---------------|
+| Zero dependencies | **Yes** | **Yes** | **Yes** | No | No |
+| Cross-platform | **Yes** | **Yes** | **Yes** | Mostly | Fragile on Win |
+| Concurrency | File locks | File locks (finer) | WAL + transactions | WAL + transactions | MVCC |
+| Row-level ops | No | Partial | **Yes** | **Yes** | Yes (KV) |
+| Indexed queries | No | No | **Yes** | **Yes** | No |
+| Human-readable | **Yes** | **Yes** | No | No | No |
+| API stability | Stable | Stable | **Experimental** | Stable | Stable |
+| Migration effort | None | Low | High | High | High |
+| Debugging ease | **Excellent** | **Excellent** | Needs tooling | Needs tooling | Poor |
+
+---
+
+## 7. Summary
+
+**Short-term (this week):** Implement Phase 1 quick wins — cap cooldowns, add read caches, reduce dispatch completed cap. Zero risk, immediate improvement.
+
+**Medium-term (Node 26 LTS timeframe):** Adopt `node:sqlite` with the hybrid column schema. Migrate log.json first (lowest risk), then dispatch.json (highest pain), then work-items.json. Use a dual-write period for safety.
+
+**The current file-based system is not broken.** At 180 work items and 128 PRs, we're well within the comfortable range for JSON files. The biggest issues (511 KB cooldowns.json, 380 KB dispatch.json) are capping/rotation problems, not fundamental architecture problems. Fix those first.
+
+SQLite is the right long-term answer, but only when `node:sqlite` is no longer experimental. The API is excellent, the performance is superior, and it maintains the zero-dependency principle. Patience here avoids a painful migration if the API changes.

package/engine/ado.js

@@ -21,6 +21,20 @@ function engine() {
 let _adoTokenCache = { token: null, expiresAt: 0 };
 let _adoTokenFailedUntil = 0; // backoff: skip azureauth calls until this timestamp
 
+// ─── Auth Failure Tracking ──────────────────────────────────────────────────
+// Set when pollPrStatus encounters auth errors mid-loop. The engine checks this
+// to bypass the normal 6-tick cadence and re-poll on the next tick.
+let _adoPollHadAuthFailure = false;
+
+/** Check if auth failure during PR poll means an early re-poll is needed. */
+function needsAdoPollRetry() { return _adoPollHadAuthFailure; }
+
+/** Detect auth-related errors from adoFetch (HTML redirect, 401, 403). */
+function isAdoAuthError(err) {
+  const msg = err?.message || '';
+  return msg.includes('auth redirect') || msg.includes('HTML instead of JSON') || /ADO API (401|403)/.test(msg);
+}
+
 async function getAdoToken() {
   if (_adoTokenCache.token && Date.now() < _adoTokenCache.expiresAt) {
     return _adoTokenCache.token;
@@ -133,16 +147,23 @@ async function forEachActivePr(config, token, callback) {
 // ─── PR Status Polling ───────────────────────────────────────────────────────
 
 async function pollPrStatus(config) {
+  _adoPollHadAuthFailure = false; // reset before polling — set again if errors recur
+
   const token = await getAdoToken();
   if (!token) {
     log('warn', 'Skipping PR status poll — no ADO token available');
+    _adoPollHadAuthFailure = true; // trigger retry on next tick
     return;
   }
 
   const totalUpdated = await forEachActivePr(config, token, async (project, pr, prNum, orgBase) => {
+    try {
     const repoBase = `${orgBase}/${project.adoProject}/_apis/git/repositories/${project.repositoryId}/pullrequests/${prNum}`;
     let updated = false;
 
+    // Clear stale flag — we're attempting a fresh poll
+    if (pr._buildStatusStale) { delete pr._buildStatusStale; updated = true; }
+
     const prData = await adoFetch(`${repoBase}?api-version=7.1`, token);
 
     let newStatus = pr.status;
@@ -278,6 +299,17 @@ async function pollPrStatus(config) {
     }
 
     return updated;
+    } catch (err) {
+      // Auth errors → mark build status stale so dashboard shows uncertainty
+      // and engine re-polls on next tick instead of waiting 6 ticks
+      if (isAdoAuthError(err)) {
+        pr._buildStatusStale = true;
+        _adoPollHadAuthFailure = true;
+        log('warn', `PR ${pr.id}: build status marked stale (auth error: ${err.message})`);
+        return true; // count as updated to persist the stale flag
+      }
+      throw err; // re-throw non-auth errors for forEachActivePr to handle
+    }
   });
 
   if (totalUpdated > 0) {
@@ -520,5 +552,7 @@ module.exports = {
   pollPrHumanComments,
   reconcilePrs,
   checkLiveReviewStatus,
+  needsAdoPollRetry,
+  isAdoAuthError, // exported for testing
 };
 

package/engine.js

@@ -937,7 +937,7 @@ function reconcileItemsWithPrs(items, allPrs, { onlyIds } = {}) {
 // ─── Inbox Consolidation (extracted to engine/consolidation.js) ──────────────
 
 const { consolidateInbox } = require('./engine/consolidation');
-const { pollPrStatus, pollPrHumanComments, reconcilePrs, checkLiveReviewStatus: adoCheckLiveReview } = require('./engine/ado');
+const { pollPrStatus, pollPrHumanComments, reconcilePrs, checkLiveReviewStatus: adoCheckLiveReview, needsAdoPollRetry } = require('./engine/ado');
 const { pollPrStatus: ghPollPrStatus, pollPrHumanComments: ghPollPrHumanComments, reconcilePrs: ghReconcilePrs, checkLiveReviewStatus: ghCheckLiveReview } = require('./engine/github');
 
 // ─── State Snapshot ─────────────────────────────────────────────────────────
@@ -2447,7 +2447,8 @@ async function tickInner() {
 
   // 2.6. Poll PR status: build, review, merge (every 6 ticks = ~3 minutes)
   // Awaited so PR state is consistent before discoverWork reads it
-  if (tickCount % 6 === 0) {
+  // Also re-polls early if previous tick had ADO auth failures (stale build status recovery)
+  if (tickCount % 6 === 0 || needsAdoPollRetry()) {
     try { await pollPrStatus(config); } catch (err) { log('warn', `ADO PR status poll error: ${err?.message || err}${err?.stack ? ' | ' + err.stack.split('\n')[1]?.trim() : ''}`); }
     try { await ghPollPrStatus(config); } catch (err) { log('warn', `GitHub PR status poll error: ${err?.message || err}${err?.stack ? ' | ' + err.stack.split('\n')[1]?.trim() : ''}`); }
     // Sync PR status back to PRD items (missing → done when active PR exists)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.550",
+  "version": "0.1.552",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"