@yemi33/minions 0.1.1577 → 0.1.1579

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.1.1579 (2026-04-28)
+
+### Features
+- hash-dedup, compress+normalize pass, dynamic stale-guard, rich result
+
+### Other
+- Keep CC streams reconnectable
+
 ## 0.1.1577 (2026-04-27)
 
 ### Fixes

package/dashboard/js/command-center.js

@@ -94,6 +94,13 @@ function _ccFindPinTarget(query) {
 function ccAbort() {
   var tab = _ccActiveTab();
   if (tab && tab._abortController) {
+    try {
+      fetch('/api/command-center/abort', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ tabId: tab.id })
+      }).catch(function() {});
+    } catch {}
     tab._abortController.abort();
     tab._abortController = null;
   }
@@ -211,6 +218,13 @@ function ccCloseTab(id) {
   var closingTab = _ccTabs.find(function(t) { return t.id === id; });
   if (closingTab && closingTab._sending) {
     if (!confirm('This tab has an active request. Close anyway?')) return;
+    try {
+      fetch('/api/command-center/abort', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ tabId: id })
+      }).catch(function() {});
+    } catch {}
     if (closingTab._abortController) { closingTab._abortController.abort(); closingTab._abortController = null; }
     closingTab._sending = false;
     closingTab._queue = [];
@@ -445,6 +459,8 @@ async function _ccDoSend(message, skipUserMsg, forceTabId) {
   if (existingQueueEl) existingQueueEl.remove();
 
   var ccStartTime = Date.now();
+  var reconnectAttempts = 0;
+  var streamStatusNote = '';
   var phases = [
     [0, 'Thinking...'],
     [3000, 'Reading minions context...'],
@@ -511,6 +527,9 @@ async function _ccDoSend(message, skipUserMsg, forceTabId) {
       if (streamedText) {
         html += renderMd(streamedText);
       }
+      if (streamStatusNote) {
+        html += '<div style="margin-top:6px;font-size:10px;color:var(--muted)">' + escHtml(streamStatusNote) + '</div>';
+      }
       html += '<div style="margin-top:' + (streamedText ? '6px' : '0') + '">' + _getThinkingHtml() + '</div>';
       streamDiv.innerHTML = html;
       // Re-append queue indicators so they stay below the streaming content
@@ -531,35 +550,70 @@ async function _ccDoSend(message, skipUserMsg, forceTabId) {
     var phaseTimer = setInterval(updateStreamDiv, 1000);
     updateStreamDiv(); // render proper layout immediately (not raw "Thinking..." text)
 
-  try {
-    // Stream response via SSE — shows text as it arrives
+  async function _ccConsumeStream(requestBody, isReconnect) {
     var res = await fetch('/api/command-center/stream', {
       method: 'POST', headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({ message: message, tabId: activeTabId, sessionId: activeTab.sessionId || null }),
+      body: JSON.stringify(requestBody),
       signal: activeTab._abortController ? activeTab._abortController.signal : AbortSignal.timeout(960000)
     });
 
     if (!res.ok) {
-      // 429 = server still releasing previous request (abort race) — retry silently up to 3 times
-      if (res.status === 429 && (!activeTab._429retries || activeTab._429retries < 3)) {
+      if (!isReconnect && res.status === 429 && (!activeTab._429retries || activeTab._429retries < 3)) {
         activeTab._429retries = (activeTab._429retries || 0) + 1;
-        _cleanupStreamDiv(); // remove current thinking div — prevents stacking on each retry
         await new Promise(function(r) { setTimeout(r, 1500); });
-        return await _ccDoSend(message, true, forceTabId || activeTabId); // retry — pass tabId so timer closures don't fight
+        return await _ccConsumeStream({ message: message, tabId: activeTabId, sessionId: activeTab.sessionId || null }, false);
       }
       activeTab._429retries = 0;
-      _cleanupStreamDiv();
       var errText = await res.text();
-      addMsg('assistant', '<span style="color:var(--red)">' + escHtml(errText || 'CC error') + '</span>' +
-        (errText.includes('busy') ? ' <button onclick="ccNewTab()" style="margin-top:4px;padding:3px 10px;background:var(--surface2);border:1px solid var(--border);border-radius:4px;color:var(--blue);cursor:pointer;font-size:10px">Reset CC</button>' : ''));
-      return;
+      if (isReconnect && res.status === 409) return { interrupted: true, reconnectable: false, reason: errText || 'No live stream' };
+      throw new Error(errText || 'CC error');
     }
 
+    activeTab._429retries = 0;
+    streamStatusNote = '';
+    updateStreamDiv();
+
     var reader = res.body.getReader();
     var decoder = new TextDecoder();
     var buf = '';
     var terminalEventSeen = false;
 
+    async function _handleEvent(evt) {
+      if (evt.type === 'chunk') {
+        streamedText = evt.text;
+        if (activeTab) activeTab._streamedText = streamedText;
+        updateStreamDiv();
+      } else if (evt.type === 'heartbeat') {
+        return;
+      } else if (evt.type === 'tool') {
+        toolsUsed.push({ name: evt.name, input: evt.input || {} });
+        if (activeTab) activeTab._toolsUsed = toolsUsed.slice();
+        updateStreamDiv();
+        if (msgs.scrollHeight - msgs.scrollTop - msgs.clientHeight < 150) msgs.scrollTop = msgs.scrollHeight;
+      } else if (evt.type === 'done') {
+        terminalEventSeen = true;
+        _cleanupStreamDiv();
+        if (evt.sessionReset) {
+          addMsg('system', '<div style="text-align:center;padding:6px 12px;font-size:11px;color:var(--muted);background:var(--surface2);border-radius:6px;margin:4px 0">Minions was updated — started a fresh session with latest context.</div>', false, activeTabId);
+        }
+        var rendered = renderMd(evt.text || streamedText || '');
+        addMsg('assistant', rendered + _ccElapsedFooter('{seconds}s'));
+        if (evt.sessionId !== undefined) {
+          var originTab = _ccTabs.find(function(t) { return t.id === activeTabId; });
+          if (originTab) { originTab.sessionId = evt.sessionId || null; }
+          ccSaveState(); ccUpdateSessionIndicator();
+        }
+        if (evt.actions && evt.actions.length > 0) {
+          _tagServerExecuted(evt.actions, evt.actionResults);
+          for (var ai = 0; ai < evt.actions.length; ai++) { await ccExecuteAction(evt.actions[ai], activeTabId); }
+        }
+      } else if (evt.type === 'error') {
+        terminalEventSeen = true;
+        _cleanupStreamDiv();
+        addMsg('assistant', '<span style="color:var(--red)">' + escHtml(evt.error) + '</span>');
+      }
+    }
+
     while (true) {
       var readResult = await reader.read();
       if (readResult.done) break;
@@ -569,93 +623,55 @@ async function _ccDoSend(message, skipUserMsg, forceTabId) {
       for (var li = 0; li < lines.length; li++) {
         var line = lines[li];
         if (!line.startsWith('data: ')) continue;
-        try {
-          var evt = JSON.parse(line.slice(6));
-          if (evt.type === 'chunk') {
-            streamedText = evt.text;
-            if (activeTab) activeTab._streamedText = streamedText;
-            updateStreamDiv();
-          } else if (evt.type === 'heartbeat') {
-            continue;
-          } else if (evt.type === 'tool') {
-            toolsUsed.push({ name: evt.name, input: evt.input || {} });
-            if (activeTab) activeTab._toolsUsed = toolsUsed.slice();
-            updateStreamDiv();
-            if (msgs.scrollHeight - msgs.scrollTop - msgs.clientHeight < 150) msgs.scrollTop = msgs.scrollHeight;
-          } else if (evt.type === 'done') {
-            terminalEventSeen = true;
-            _cleanupStreamDiv();
-            // If system prompt changed, show a notice before the response
-            if (evt.sessionReset) {
-              addMsg('system', '<div style="text-align:center;padding:6px 12px;font-size:11px;color:var(--muted);background:var(--surface2);border-radius:6px;margin:4px 0">Minions was updated — started a fresh session with latest context.</div>', false, activeTabId);
-            }
-            // placeholder was added with skipSave=true — nothing to pop
-            var rendered = renderMd(evt.text || streamedText || '');
-            addMsg('assistant', rendered + _ccElapsedFooter('{seconds}s'));
-            if (evt.sessionId !== undefined) {
-              // Save session to the originating tab, not whichever tab is active now
-              var originTab = _ccTabs.find(function(t) { return t.id === activeTabId; });
-              if (originTab) { originTab.sessionId = evt.sessionId || null; }
-              ccSaveState(); ccUpdateSessionIndicator();
-            }
-            if (evt.actions && evt.actions.length > 0) {
-              _tagServerExecuted(evt.actions, evt.actionResults);
-              for (var ai = 0; ai < evt.actions.length; ai++) { await ccExecuteAction(evt.actions[ai], activeTabId); }
-            }
-          } else if (evt.type === 'error') {
-            terminalEventSeen = true;
-            _cleanupStreamDiv();
-            // placeholder was skipSave — no pop needed
-            addMsg('assistant', '<span style="color:var(--red)">' + escHtml(evt.error) + '</span>');
-          }
-        } catch { /* incomplete JSON */ }
+        try { await _handleEvent(JSON.parse(line.slice(6))); } catch {}
       }
     }
-    // Process any remaining buffered data after stream ends
     if (buf.trim()) {
       var remainingLines = buf.split('\n');
       for (var ri = 0; ri < remainingLines.length; ri++) {
         var rline = remainingLines[ri];
         if (!rline.startsWith('data: ')) continue;
-        try {
-          var revt = JSON.parse(rline.slice(6));
-          if (revt.type === 'done') {
-            terminalEventSeen = true;
-            _cleanupStreamDiv();
-            // placeholder was skipSave — no pop needed
-            var rendered2 = renderMd(revt.text || streamedText || '');
-            addMsg('assistant', rendered2 + _ccElapsedFooter('{seconds}s'));
-            if (revt.sessionId !== undefined) {
-              var originTab2 = _ccTabs.find(function(t) { return t.id === activeTabId; });
-              if (originTab2) { originTab2.sessionId = revt.sessionId || null; }
-              ccSaveState(); ccUpdateSessionIndicator();
-            }
-            if (revt.actions && revt.actions.length > 0) {
-              _tagServerExecuted(revt.actions, revt.actionResults);
-              for (var ai2 = 0; ai2 < revt.actions.length; ai2++) { await ccExecuteAction(revt.actions[ai2], activeTabId); }
-            }
-          } else if (revt.type === 'heartbeat') {
-            continue;
-          } else if (revt.type === 'error') {
-            terminalEventSeen = true;
-            _cleanupStreamDiv();
-            addMsg('assistant', '<span style="color:var(--red)">' + escHtml(revt.error) + '</span>');
-          } else if (revt.type === 'chunk') {
-            streamedText = revt.text;
-            updateStreamDiv();
-          }
-        } catch {}
+        try { await _handleEvent(JSON.parse(rline.slice(6))); } catch {}
       }
     }
-    // If stream ended without a 'done' event, finalize with whatever we have
-    if (!terminalEventSeen && (streamDiv.parentNode || document.getElementById('cc-restore-thinking') || document.querySelector('[data-stream-tab="' + activeTabId + '"]'))) {
-      _cleanupStreamDiv();
-      var streamEndedHint = '<div style="font-size:10px;color:var(--muted);margin-top:4px">The response stream ended before completion. Retry to continue from the last user message.</div>';
-      if (streamedText) {
-        addMsg('assistant', renderMd(streamedText) + _ccElapsedFooter('Stream interrupted after {seconds}s') + _ccRetryControls(streamEndedHint, false));
-      } else {
-        addMsg('assistant', '<span style="color:var(--red)">The response stream ended before completion.</span>' + _ccRetryControls(streamEndedHint, false));
+    return { interrupted: !terminalEventSeen, reconnectable: true };
+  }
+
+  try {
+    while (true) {
+      var consume = await _ccConsumeStream(
+        reconnectAttempts === 0
+          ? { message: message, tabId: activeTabId, sessionId: activeTab.sessionId || null }
+          : { tabId: activeTabId, sessionId: activeTab.sessionId || null, reconnect: true },
+        reconnectAttempts > 0
+      );
+      if (!consume.interrupted) break;
+      if (!consume.reconnectable || reconnectAttempts >= 2) {
+        _cleanupStreamDiv();
+        var streamEndedHint = '<div style="font-size:10px;color:var(--muted);margin-top:4px">The response stream ended before completion. Retry to continue from the last user message.</div>';
+        if (streamedText) {
+          addMsg('assistant', renderMd(streamedText) + _ccElapsedFooter('Stream interrupted after {seconds}s') + _ccRetryControls(streamEndedHint, false));
+        } else {
+          addMsg('assistant', '<span style="color:var(--red)">The response stream ended before completion.</span>' + _ccRetryControls(streamEndedHint, false));
+        }
+        break;
+      }
+      var reconnectHealth = await _ccDashboardHealth();
+      if (!reconnectHealth.reachable || reconnectHealth.restarted) {
+        _cleanupStreamDiv();
+        var reconnectHint = reconnectHealth.restarted
+          ? '<div style="font-size:10px;color:var(--muted);margin-top:4px">Dashboard restarted while this response was streaming. Reload the page to reconnect to the new instance.</div>'
+          : '<div style="font-size:10px;color:var(--muted);margin-top:4px">The request stream was interrupted, but the dashboard is still reachable. Retry or start a new session.</div>';
+        addMsg('assistant', (streamedText ? renderMd(streamedText) + _ccElapsedFooter('Stream interrupted after {seconds}s') : '') +
+          _ccRetryControls(reconnectHint, reconnectHealth.restarted));
+        break;
       }
+      reconnectAttempts++;
+      toolsUsed = [];
+      if (activeTab) activeTab._toolsUsed = [];
+      streamStatusNote = 'Connection interrupted — reattaching to the live response...';
+      updateStreamDiv();
+      await new Promise(function(r) { setTimeout(r, 1000 * reconnectAttempts); });
     }
   } catch (e) {
     _cleanupStreamDiv();

package/dashboard/js/render-kb.js

@@ -1,5 +1,11 @@
 // render-kb.js — Knowledge base rendering functions extracted from dashboard.html
 
+function _formatBytes(n) {
+  if (n < 1024) return n + ' B';
+  if (n < 1024 * 1024) return (n / 1024).toFixed(0) + ' KB';
+  return (n / 1024 / 1024).toFixed(1) + ' MB';
+}
+
 const KB_CAT_LABELS = {
   architecture: 'Architecture', conventions: 'Conventions',
   'project-notes': 'Project Notes', 'build-reports': 'Build Reports',
@@ -175,7 +181,18 @@ async function kbSweep() {
           if (result && result.ok) {
             btn.textContent = 'done';
             btn.style.color = 'var(--green)';
-            showToast('cmd-toast', 'KB sweep complete: ' + (result.summary || 'done'), true);
+            // Rich summary toast — show the key counts inline; full breakdown via console.log for now
+            var bytesSaved = (result.bytesBefore || 0) - (result.bytesAfter || 0);
+            var pieces = [];
+            if (result.entriesBefore != null) pieces.push((result.entriesBefore - (result.entriesAfter || 0)) + ' entries removed');
+            if (result.hashDuplicatesArchived) pieces.push(result.hashDuplicatesArchived + ' hash-dup');
+            if (result.llmDuplicatesArchived) pieces.push(result.llmDuplicatesArchived + ' llm-dup');
+            if (result.staleRemoved) pieces.push(result.staleRemoved + ' stale');
+            if (result.reclassified) pieces.push(result.reclassified + ' reclassified');
+            if (result.rewritten) pieces.push(result.rewritten + ' rewritten');
+            if (bytesSaved > 0) pieces.push(_formatBytes(bytesSaved) + ' saved');
+            var msg = pieces.length ? 'KB sweep: ' + pieces.join(' · ') : 'KB sweep: ' + (result.summary || 'done');
+            showToast('cmd-toast', msg, true);
             refreshKnowledgeBase();
           } else {
             btn.style.color = 'var(--red)';

package/dashboard.js

@@ -544,10 +544,86 @@ const CC_SESSION_TTL_MS = shared.ENGINE_DEFAULTS.ccSessionTtlMs;
 let ccSession = { sessionId: null, createdAt: null, lastActiveAt: null, turnCount: 0 };
 const ccInFlightTabs = new Map(); // tabId → timestamp — per-tab in-flight tracking for parallel CC requests
 const ccInFlightAborts = new Map(); // tabId → abortFn — lets a new request kill the stale LLM
+const ccLiveStreams = new Map(); // tabId → buffered live stream state for reconnect-after-disconnect
 const CC_INFLIGHT_TIMEOUT_MS = 2 * 60 * 1000; // 2 minutes — auto-release if request hangs
 const CC_LOCK_WAIT_MS = 200; // grace period for previous handler's finally to release lock
 const CC_STREAM_HEARTBEAT_MS = 15000; // keep streaming responses alive across proxies/restart races
+const CC_STREAM_REATTACH_GRACE_MS = 60000; // keep CC job alive briefly after disconnect so the UI can reattach
+const CC_STREAM_DONE_RETENTION_MS = 30000; // retain final payload briefly so reconnect can still receive it
 function _releaseCCTab(tabId) { ccInFlightTabs.delete(tabId); ccInFlightAborts.delete(tabId); }
+function _getCcLiveStream(tabId) {
+  return ccLiveStreams.get(tabId) || null;
+}
+function _clearCcLiveTimers(tabId) {
+  const state = _getCcLiveStream(tabId);
+  if (!state) return;
+  if (state.abortTimer) {
+    clearTimeout(state.abortTimer);
+    state.abortTimer = null;
+  }
+  if (state.cleanupTimer) {
+    clearTimeout(state.cleanupTimer);
+    state.cleanupTimer = null;
+  }
+}
+function _clearCcLiveStream(tabId) {
+  const state = _getCcLiveStream(tabId);
+  if (!state) return;
+  _clearCcLiveTimers(tabId);
+  ccLiveStreams.delete(tabId);
+}
+function _ensureCcLiveStream(tabId) {
+  let state = _getCcLiveStream(tabId);
+  if (state) return state;
+  state = {
+    tabId,
+    text: '',
+    tools: [],
+    donePayload: null,
+    writer: null,
+    endResponse: null,
+    abortFn: null,
+    abortTimer: null,
+    cleanupTimer: null,
+  };
+  ccLiveStreams.set(tabId, state);
+  return state;
+}
+function _attachCcLiveStream(tabId, writer, endResponse) {
+  const state = _ensureCcLiveStream(tabId);
+  _clearCcLiveTimers(tabId);
+  state.writer = writer;
+  state.endResponse = endResponse;
+  return state;
+}
+function _detachCcLiveStream(tabId, writer) {
+  const state = _getCcLiveStream(tabId);
+  if (!state) return;
+  if (!writer || state.writer === writer) {
+    state.writer = null;
+    state.endResponse = null;
+  }
+}
+function _scheduleCcLiveAbort(tabId) {
+  const state = _getCcLiveStream(tabId);
+  if (!state || state.donePayload) return;
+  _clearCcLiveTimers(tabId);
+  state.abortTimer = setTimeout(() => {
+    const live = _getCcLiveStream(tabId);
+    if (!live || live.donePayload || live.writer) return;
+    try { if (live.abortFn) live.abortFn(); } catch {}
+  }, CC_STREAM_REATTACH_GRACE_MS);
+}
+function _scheduleCcLiveCleanup(tabId, delayMs = CC_STREAM_DONE_RETENTION_MS) {
+  const state = _getCcLiveStream(tabId);
+  if (!state) return;
+  if (state.cleanupTimer) clearTimeout(state.cleanupTimer);
+  state.cleanupTimer = setTimeout(() => {
+    const live = _getCcLiveStream(tabId);
+    if (!live || live.writer) return;
+    _clearCcLiveStream(tabId);
+  }, delayMs);
+}
 function _ccTabIsInFlight(tabId) {
   if (!ccInFlightTabs.has(tabId)) return false;
   // Auto-release stale locks — if a request has been in-flight longer than CC_INFLIGHT_TIMEOUT_MS,
@@ -2577,183 +2653,31 @@ const server = http.createServer(async (req, res) => {
   }
 
   async function handleKnowledgeSweep(req, res) {
-    // Auto-release stale guard after 5 min (LLM may have hung)
-    if (global._kbSweepInFlight && global._kbSweepStartedAt && Date.now() - global._kbSweepStartedAt > 300000) {
-      console.log('[kb-sweep] Auto-releasing stale guard (>5min)');
+    // Auto-release stale guard — dynamic floor based on KB size (30 min min, +1s per entry)
+    const { staleGuardMs } = require('./engine/kb-sweep');
+    const entryCount = (queries.getKnowledgeBaseEntries() || []).length;
+    const guardMs = staleGuardMs(entryCount);
+    if (global._kbSweepInFlight && global._kbSweepStartedAt && Date.now() - global._kbSweepStartedAt > guardMs) {
+      console.log(`[kb-sweep] Auto-releasing stale guard (>${Math.round(guardMs / 60000)}min for ${entryCount} entries)`);
       global._kbSweepInFlight = false;
     }
     if (global._kbSweepInFlight) {
       return jsonReply(res, 200, { ok: true, alreadyRunning: true, startedAt: global._kbSweepStartedAt });
     }
-    // Generation token prevents stale finally blocks from clearing the flag for a new sweep
     const sweepToken = Date.now() + Math.random();
     global._kbSweepToken = sweepToken;
     global._kbSweepInFlight = true;
     global._kbSweepStartedAt = Date.now();
     const body = await readBody(req).catch(() => ({}));
-    // Run sweep in background — return immediately so agents/UI don't time out
     _runKbSweepBackground(body, sweepToken);
     return jsonReply(res, 202, { ok: true, started: true });
   }
 
   async function _runKbSweepBackground(body, sweepToken) {
     try {
-      const entries = getKnowledgeBaseEntries();
-      if (entries.length < 2) {
-        global._kbSweepLastResult = { ok: true, summary: 'nothing to sweep (< 2 entries)' };
-        global._kbSweepLastCompletedAt = Date.now();
-        return;
-      }
-
-      // Build a manifest of all KB entries with their content (skip pinned — user wants to keep them)
-      const requestPinnedKeys = Array.isArray(body.pinnedKeys)
-        ? body.pinnedKeys.filter(k => typeof k === 'string' && k.startsWith('knowledge/'))
-        : [];
-      const serverPinnedKeys = shared.getPinnedItems().filter(k => k.startsWith('knowledge/'));
-      const pinnedKeys = new Set([...serverPinnedKeys, ...requestPinnedKeys]);
-      const manifest = [];
-      for (const e of entries) {
-        if (pinnedKeys.has('knowledge/' + e.cat + '/' + e.file)) continue;
-        const content = safeRead(path.join(MINIONS_DIR, 'knowledge', e.cat, e.file));
-        if (!content) continue;
-        manifest.push({ category: e.cat, file: e.file, title: e.title, agent: e.agent, date: e.date, content: content.slice(0, 3000) });
-      }
-      if (manifest.length < 2) {
-        global._kbSweepLastResult = { ok: true, summary: 'nothing to sweep (< 2 unpinned entries)' };
-        global._kbSweepLastCompletedAt = Date.now();
-        return;
-      }
-
-      const { callLLM, trackEngineUsage } = require('./engine/llm');
-      const BATCH_SIZE = 30; // ~30 entries per batch to stay within Haiku context
-      const batches = [];
-      for (let i = 0; i < manifest.length; i += BATCH_SIZE) {
-        batches.push(manifest.slice(i, i + BATCH_SIZE));
-      }
-
-      const plan = { duplicates: [], reclassify: [], remove: [] };
-      for (let b = 0; b < batches.length; b++) {
-        const batch = batches[b];
-        const offset = b * BATCH_SIZE;
-        const prompt = `You are a knowledge base curator. Analyze these ${batch.length} entries (batch ${b + 1}/${batches.length}, indices ${offset}-${offset + batch.length - 1}) and produce a cleanup plan.
-
-## Entries
-
-${batch.map((m, i) => `[${offset + i}] ${m.category}/${m.file} | ${m.title} | ${m.date} | ${m.agent || '?'} | ${(m.content || '').slice(0, 200).replace(/\n/g, ' ')}`).join('\n')}
-
-## Instructions
-
-1. **Find duplicates**: entries with substantially the same content (same findings, different agents/runs). List pairs by index. Prefer keeping the more recent entry.
-2. **Find misclassified**: entries in the wrong category.
-3. **Find stale/empty**: entries with no actionable content (boilerplate, bail-out notes, "no changes needed").
-
-Respond with ONLY valid JSON: { "duplicates": [{ "keep": N, "remove": [N], "reason": "..." }], "reclassify": [{ "index": N, "from": "cat", "to": "cat", "reason": "..." }], "remove": [{ "index": N, "reason": "..." }] }
-If nothing to do: { "duplicates": [], "reclassify": [], "remove": [] }`;
-
-        const result = await callLLM(prompt, 'Output only JSON.', {
-          timeout: 120000, label: 'kb-sweep', model: 'haiku', maxTurns: 1, direct: true
-        });
-        trackEngineUsage('kb-sweep', result.usage);
-
-        let batchPlan;
-        try {
-          let jsonStr = (result.text || '').trim();
-          const fenceMatch = jsonStr.match(/```(?:json)?\s*([\s\S]*?)```/);
-          if (fenceMatch) jsonStr = fenceMatch[1].trim();
-          batchPlan = JSON.parse(jsonStr);
-        } catch {
-          console.log(`[kb-sweep] batch ${b + 1}/${batches.length} returned invalid JSON, skipping`);
-          continue;
-        }
-        if (batchPlan.duplicates) plan.duplicates.push(...batchPlan.duplicates);
-        if (batchPlan.reclassify) plan.reclassify.push(...batchPlan.reclassify);
-        if (batchPlan.remove) plan.remove.push(...batchPlan.remove);
-      }
-
-      let removed = 0, reclassified = 0, merged = 0;
-      const kbDir = path.join(MINIONS_DIR, 'knowledge');
-
-      // If nothing to do, store result and return
-      const totalActions = (plan.remove || []).length + (plan.duplicates || []).reduce((n, d) => n + (d.remove || []).length, 0) + (plan.reclassify || []).length;
-      if (totalActions === 0) {
-        global._kbSweepLastResult = { ok: true, summary: 'KB is clean — nothing to sweep', plan };
-        global._kbSweepLastCompletedAt = Date.now();
-        return;
-      }
-
-      // Archive dir for swept files (never delete, always preserve)
-      const kbArchiveDir = path.join(kbDir, '_swept');
-      if (!fs.existsSync(kbArchiveDir)) fs.mkdirSync(kbArchiveDir, { recursive: true });
-
-      function archiveKbFile(filePath, reason) {
-        if (!fs.existsSync(filePath)) return;
-        const basename = path.basename(filePath);
-        const destPath = shared.uniquePath(path.join(kbArchiveDir, basename));
-        try {
-          const content = safeRead(filePath);
-          if (content === null) return; // don't delete if we can't read
-          const meta = `<!-- swept: ${new Date().toISOString()} | reason: ${reason} -->\n`;
-          safeWrite(destPath, meta + content);
-          safeUnlink(filePath);
-        } catch (e) { console.error('kb archive:', e.message); }
-      }
-
-      // Process removals (stale/empty) — archive, not delete
-      for (const r of (plan.remove || [])) {
-        const entry = manifest[r.index];
-        if (!entry) continue;
-        const fp = path.join(kbDir, entry.category, entry.file);
-        archiveKbFile(fp, 'stale: ' + (r.reason || ''));
-        removed++;
-      }
-
-      // Process duplicates — archive the duplicates, keep the primary
-      for (const d of (plan.duplicates || [])) {
-        for (const idx of (d.remove || [])) {
-          const entry = manifest[idx];
-          if (!entry) continue;
-          const fp = path.join(kbDir, entry.category, entry.file);
-          archiveKbFile(fp, 'duplicate of index ' + d.keep + ': ' + (d.reason || ''));
-          merged++;
-        }
-      }
-
-      // Process reclassifications (move between categories)
-      for (const r of (plan.reclassify || [])) {
-        const entry = manifest[r.index];
-        if (!entry || !shared.KB_CATEGORIES.includes(r.to)) continue;
-        const srcPath = path.join(kbDir, entry.category, entry.file);
-        const destDir = path.join(kbDir, r.to);
-        if (!fs.existsSync(srcPath)) continue;
-        if (!fs.existsSync(destDir)) fs.mkdirSync(destDir, { recursive: true });
-        try {
-          const srcStats = fs.statSync(srcPath);
-          const content = safeRead(srcPath);
-          const updated = content.replace(/^(category:\s*).+$/m, `$1${r.to}`);
-          const destPath = path.join(destDir, entry.file);
-          safeWrite(destPath, updated);
-          fs.utimesSync(destPath, srcStats.atime, srcStats.mtime);
-          safeUnlink(srcPath);
-          reclassified++;
-        } catch (e) { console.error('kb reclassify:', e.message); }
-      }
-
-      // Prune swept files older than 30 days
-      let pruned = 0;
-      const SWEPT_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
-      try {
-        for (const f of fs.readdirSync(kbArchiveDir)) {
-          const fp = path.join(kbArchiveDir, f);
-          try {
-            if (Date.now() - fs.statSync(fp).mtimeMs > SWEPT_RETENTION_MS) { safeUnlink(fp); pruned++; }
-          } catch { /* cleanup */ }
-        }
-      } catch { /* optional */ }
-
-      const summary = `${merged} duplicates merged, ${removed} stale removed, ${reclassified} reclassified${pruned ? ', ' + pruned + ' old swept files pruned' : ''}`;
-      safeWrite(path.join(ENGINE_DIR, 'kb-swept.json'), JSON.stringify({ timestamp: new Date().toISOString(), summary }));
-      queries.invalidateKnowledgeBaseCache();
-      global._kbSweepLastResult = { ok: true, summary, plan };
+      const { runKbSweep } = require('./engine/kb-sweep');
+      const result = await runKbSweep({ pinnedKeys: body.pinnedKeys });
+      global._kbSweepLastResult = result;
       global._kbSweepLastCompletedAt = Date.now();
     } catch (e) {
       console.error('[kb-sweep] background error:', e.message);
@@ -2762,6 +2686,7 @@ If nothing to do: { "duplicates": [], "reclassify": [], "remove": [] }`;
     } finally { if (global._kbSweepToken === sweepToken) global._kbSweepInFlight = false; }
   }
 
+
   function handleKnowledgeSweepStatus(req, res) {
     return jsonReply(res, 200, {
       inFlight: !!global._kbSweepInFlight,
@@ -4051,10 +3976,31 @@ What would you like to discuss or change? When you're happy, say "approve" and I
   async function handleCommandCenterNewSession(req, res) {
     ccSession = { sessionId: null, createdAt: null, lastActiveAt: null, turnCount: 0 };
     ccInFlightTabs.clear(); // Reset all in-flight guards
+    for (const [tabId, live] of ccLiveStreams.entries()) {
+      try { if (live.abortFn) live.abortFn(); } catch {}
+      _clearCcLiveStream(tabId);
+    }
     safeWrite(path.join(ENGINE_DIR, 'cc-session.json'), ccSession);
     return jsonReply(res, 200, { ok: true });
   }
 
+  async function handleCommandCenterAbort(req, res) {
+    try {
+      const body = await readBody(req);
+      const tabId = body.tabId || 'default';
+      const live = _getCcLiveStream(tabId);
+      if (live?.abortFn) {
+        try { live.abortFn(); } catch {}
+      } else {
+        const abort = ccInFlightAborts.get(tabId);
+        if (abort) { try { abort(); } catch {} }
+      }
+      _clearCcLiveStream(tabId);
+      _releaseCCTab(tabId);
+      return jsonReply(res, 200, { ok: true });
+    } catch (e) { return jsonReply(res, 400, { error: e.message }); }
+  }
+
   async function handleCCSessionsList(req, res) {
     const sessions = _readCcTabSessions();
     return jsonReply(res, 200, { sessions });
@@ -4188,8 +4134,51 @@ What would you like to discuss or change? When you're happy, say "approve" and I
     };
     try {
       const body = await readBody(req);
-      if (!body.message) { res.statusCode = 400; res.end('message required'); return; }
+      if (!body.message && !body.reconnect) { res.statusCode = 400; res.end('message required'); return; }
       tabId = body.tabId || 'default';
+      if (body.reconnect) {
+        const live = _getCcLiveStream(tabId);
+        if (!live) { res.statusCode = 409; res.end('No live command-center response to reconnect'); return; }
+        res.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', 'Connection': 'keep-alive' });
+        writeCcEvent({ type: 'heartbeat' });
+        _ccHeartbeatTimer = setInterval(() => {
+          if (_ccStreamEnded) {
+            stopCcHeartbeat();
+            return;
+          }
+          if (!writeCcEvent({ type: 'heartbeat' })) stopCcHeartbeat();
+        }, CC_STREAM_HEARTBEAT_MS);
+        let reconnectDone;
+        const reconnectDonePromise = new Promise(resolve => { reconnectDone = resolve; });
+        _attachCcLiveStream(tabId, writeCcEvent, () => {
+          if (_ccStreamEnded) return;
+          _ccStreamEnded = true;
+          stopCcHeartbeat();
+          try { res.end(); } catch {}
+          reconnectDone();
+        });
+        req.on('close', () => {
+          if (_ccStreamEnded) return;
+          stopCcHeartbeat();
+          _detachCcLiveStream(tabId, writeCcEvent);
+          _scheduleCcLiveAbort(tabId);
+          reconnectDone();
+        });
+        for (const tool of live.tools || []) {
+          writeCcEvent({ type: 'tool', name: tool.name, input: _lightToolInput(tool.input) });
+        }
+        if (live.text) writeCcEvent({ type: 'chunk', text: live.text });
+        if (live.donePayload) {
+          writeCcEvent(live.donePayload);
+          _ccStreamEnded = true;
+          stopCcHeartbeat();
+          try { res.end(); } catch {}
+          _scheduleCcLiveCleanup(tabId);
+          return;
+        }
+        await reconnectDonePromise;
+        return;
+      }
       if (_ccTabIsInFlight(tabId)) {
         // Previous request still in-flight — abort its LLM (handles keep-alive abort where close event didn't fire)
         const prevAbort = ccInFlightAborts.get(tabId);
@@ -4200,6 +4189,13 @@ What would you like to discuss or change? When you're happy, say "approve" and I
         }
       }
       ccInFlightTabs.set(tabId, Date.now());
+      _clearCcLiveStream(tabId);
+      const liveState = _attachCcLiveStream(tabId, writeCcEvent, () => {
+        if (_ccStreamEnded) return;
+        _ccStreamEnded = true;
+        stopCcHeartbeat();
+        try { res.end(); } catch {}
+      });
 
       res.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', 'Connection': 'keep-alive' });
       writeCcEvent({ type: 'heartbeat' }); // flush headers quickly and keep intermediaries from idling out
@@ -4210,17 +4206,12 @@ What would you like to discuss or change? When you're happy, say "approve" and I
         }
         if (!writeCcEvent({ type: 'heartbeat' })) stopCcHeartbeat();
       }, CC_STREAM_HEARTBEAT_MS);
-      // Kill LLM process immediately if client disconnects mid-stream.
-      // Guard with !_ccStreamEnded: when the stream ends normally, finally already released the lock;
-      // without the guard, this close event (which fires after res.end) could wipe a new request's lock.
+      // Keep the LLM alive briefly after disconnect so the UI can reattach to the same in-flight turn.
       req.on('close', () => {
         if (!_ccStreamEnded) {
           stopCcHeartbeat();
-          _releaseCCTab(tabId);
-          if (_ccStreamAbort) {
-            console.log(`[CC-stream] Client disconnected for tab ${tabId} — aborting LLM`);
-            _ccStreamAbort();
-          }
+          _detachCcLiveStream(tabId, writeCcEvent);
+          _scheduleCcLiveAbort(tabId);
         }
       });
 
@@ -4257,14 +4248,17 @@ What would you like to discuss or change? When you're happy, say "approve" and I
           onChunk: (text) => {
             const actIdx = findCCActionsDelimiter(text);
             const display = actIdx >= 0 ? text.slice(0, actIdx).trim() : text;
-            writeCcEvent({ type: 'chunk', text: display });
+            liveState.text = display;
+            if (liveState.writer) liveState.writer({ type: 'chunk', text: display });
           },
           onToolUse: (name, input) => {
             toolUses.push({ name, input: input || {} });
-            writeCcEvent({ type: 'tool', name, input: _lightToolInput(input) });
+            liveState.tools.push({ name, input: input || {} });
+            if (liveState.writer) liveState.writer({ type: 'tool', name, input: _lightToolInput(input) });
           }
         });
         _ccStreamAbort = llmPromise.abort;
+        liveState.abortFn = _ccStreamAbort;
         ccInFlightAborts.set(tabId, _ccStreamAbort);
         const result = await llmPromise;
         trackUsage('command-center', result.usage);
@@ -4278,21 +4272,24 @@ What would you like to discuss or change? When you're happy, say "approve" and I
           toolUses = []; // discard stale metadata from the failed resume attempt
           const retryPromise = callLLMStreaming(freshPrompt, CC_STATIC_SYSTEM_PROMPT, {
             timeout: 900000, label: 'command-center', model: streamModel, maxTurns: ccMaxTurns,
-            allowedTools: 'Bash,Read,Write,Edit,Glob,Grep,WebFetch,WebSearch',
-            effort: streamEffort, direct: true,
-            onChunk: (text) => {
-              const actIdx = findCCActionsDelimiter(text);
-              const display = actIdx >= 0 ? text.slice(0, actIdx).trim() : text;
-              writeCcEvent({ type: 'chunk', text: display });
-            },
-            onToolUse: (name, input) => {
-              toolUses.push({ name, input: input || {} });
-              writeCcEvent({ type: 'tool', name, input: _lightToolInput(input) });
-            }
-          });
-          _ccStreamAbort = retryPromise.abort;
-          ccInFlightAborts.set(tabId, _ccStreamAbort);
-          const retryResult = await retryPromise;
+              allowedTools: 'Bash,Read,Write,Edit,Glob,Grep,WebFetch,WebSearch',
+              effort: streamEffort, direct: true,
+              onChunk: (text) => {
+                const actIdx = findCCActionsDelimiter(text);
+                const display = actIdx >= 0 ? text.slice(0, actIdx).trim() : text;
+                liveState.text = display;
+                if (liveState.writer) liveState.writer({ type: 'chunk', text: display });
+              },
+              onToolUse: (name, input) => {
+                toolUses.push({ name, input: input || {} });
+                liveState.tools.push({ name, input: input || {} });
+                if (liveState.writer) liveState.writer({ type: 'tool', name, input: _lightToolInput(input) });
+              }
+            });
+            _ccStreamAbort = retryPromise.abort;
+            liveState.abortFn = _ccStreamAbort;
+            ccInFlightAborts.set(tabId, _ccStreamAbort);
+            const retryResult = await retryPromise;
           trackUsage('command-center', retryResult.usage);
           if (retryResult.text) {
             // Fresh session succeeded — use retryResult from here
@@ -4305,8 +4302,10 @@ What would you like to discuss or change? When you're happy, say "approve" and I
           const stderrTail = (result.stderr || '').trim().split('\n').filter(Boolean).slice(-3).join(' | ');
           console.error(`[CC-stream] Failed: code=${result.code}, stderr=${(result.stderr || '').slice(0, 500)}, stdout_tail=${(result.raw || '').slice(-500)}`);
           const retryHint = 'Send your message again to retry.';
-          writeCcEvent({ type: 'done', text: `I had trouble processing that ${debugInfo}. ${stderrTail ? 'Detail: ' + stderrTail : ''}\n\n${retryHint}`, actions: [], sessionId: null });
-          _ccStreamEnded = true; res.end();
+          liveState.donePayload = { type: 'done', text: `I had trouble processing that ${debugInfo}. ${stderrTail ? 'Detail: ' + stderrTail : ''}\n\n${retryHint}`, actions: [], sessionId: null };
+          if (liveState.writer) liveState.writer(liveState.donePayload);
+          if (liveState.endResponse) liveState.endResponse();
+          _scheduleCcLiveCleanup(tabId);
           return;
         }
 
@@ -4348,7 +4347,8 @@ What would you like to discuss or change? When you're happy, say "approve" and I
         }
         const donePayload = { type: 'done', text: displayText, actions, actionResults, sessionId: responseSessionId, newSession: !wasResume };
         if (sessionReset) donePayload.sessionReset = true;
-        writeCcEvent(donePayload);
+        liveState.donePayload = donePayload;
+        if (liveState.writer) liveState.writer(donePayload);
 
         // Mirror CC response to Teams (non-blocking, skip Teams-originated)
         const _streamTabId = body.tabId || 'default';
@@ -4356,7 +4356,8 @@ What would you like to discuss or change? When you're happy, say "approve" and I
           teams.teamsPostCCResponse(body.message, result.text).catch(() => {});
         }
 
-        _ccStreamEnded = true; res.end();
+        if (liveState.endResponse) liveState.endResponse();
+        _scheduleCcLiveCleanup(tabId);
       } finally {
         stopCcHeartbeat();
         _releaseCCTab(tabId);
@@ -5250,6 +5251,7 @@ What would you like to discuss or change? When you're happy, say "approve" and I
 
     // Command Center
     { method: 'POST', path: '/api/command-center/new-session', desc: 'Clear active CC session', handler: handleCommandCenterNewSession },
+    { method: 'POST', path: '/api/command-center/abort', desc: 'Abort an in-flight CC request for a tab', params: 'tabId?', handler: handleCommandCenterAbort },
     { method: 'POST', path: '/api/command-center', desc: 'Conversational command center with full minions context', params: 'message, sessionId?', handler: handleCommandCenter },
     { method: 'POST', path: '/api/command-center/stream', desc: 'Streaming CC — SSE with text chunks as they arrive', params: 'message, tabId?', handler: handleCommandCenterStream },
     { method: 'GET', path: '/api/cc-sessions', desc: 'List CC session metadata for all tabs', handler: handleCCSessionsList },

package/engine/kb-sweep.js

@@ -0,0 +1,383 @@
+/**
+ * engine/kb-sweep.js — Knowledge base sweep: dedup, compress, normalize.
+ *
+ * Replaces the inline sweep that lived in dashboard.js. Three passes:
+ *   1. Hash-based dedup    — cheap, catches cross-batch duplicates
+ *   2. LLM batch sweep     — finds remaining dupes + reclassify + stale-remove
+ *   3. Compress & normalize — per-entry LLM rewrite, flagged via _swept frontmatter
+ *
+ * Returns a rich summary so the dashboard can show before/after byte counts.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const shared = require('./shared');
+const queries = require('./queries');
+const { safeRead, safeWrite, safeUnlink, log, ts } = shared;
+const { MINIONS_DIR, ENGINE_DIR } = queries;
+
+const KB_DIR = path.join(MINIONS_DIR, 'knowledge');
+const SWEPT_DIR = path.join(KB_DIR, '_swept');
+const SWEPT_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
+const COMPRESS_THRESHOLD_BYTES = 5000;
+const LLM_BATCH_SIZE = 30;
+const NORMALIZE_CONCURRENCY = 5;
+const SWEPT_FLAG_KEY = '_swept'; // frontmatter key — entries with this skip the rewrite pass
+
+function _hashEntry(content) {
+  const normalized = String(content || '').replace(/\s+/g, ' ').trim().slice(0, 500);
+  return crypto.createHash('sha256').update(normalized + ':' + (content?.length || 0)).digest('hex');
+}
+
+/**
+ * Parse YAML-ish frontmatter at the top of a markdown file.
+ * Returns { fm: {key:value}, body: string }.
+ */
+function _parseFrontmatter(content) {
+  const m = String(content || '').match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
+  if (!m) return { fm: {}, body: content || '' };
+  const fm = {};
+  for (const line of m[1].split('\n')) {
+    const lm = line.match(/^([\w-]+):\s*(.*)$/);
+    if (lm) fm[lm[1]] = lm[2].trim();
+  }
+  return { fm, body: m[2].replace(/^\n+/, '') };
+}
+
+function _serializeFrontmatter(fm, body) {
+  const keys = Object.keys(fm);
+  if (keys.length === 0) return body;
+  const lines = keys.map(k => `${k}: ${fm[k]}`);
+  return `---\n${lines.join('\n')}\n---\n\n${body.replace(/^\n+/, '')}`;
+}
+
+function _archiveKbFile(filePath, reason) {
+  if (!fs.existsSync(filePath)) return false;
+  if (!fs.existsSync(SWEPT_DIR)) fs.mkdirSync(SWEPT_DIR, { recursive: true });
+  const destPath = shared.uniquePath(path.join(SWEPT_DIR, path.basename(filePath)));
+  try {
+    const content = safeRead(filePath);
+    if (content === null) return false;
+    safeWrite(destPath, `<!-- swept: ${new Date().toISOString()} | reason: ${reason} -->\n${content}`);
+    safeUnlink(filePath);
+    return true;
+  } catch (e) { log('warn', `[kb-sweep] archive ${path.basename(filePath)}: ${e.message}`); return false; }
+}
+
+function _pruneOldSwept() {
+  if (!fs.existsSync(SWEPT_DIR)) return 0;
+  let pruned = 0;
+  try {
+    for (const f of fs.readdirSync(SWEPT_DIR)) {
+      const fp = path.join(SWEPT_DIR, f);
+      try {
+        if (Date.now() - fs.statSync(fp).mtimeMs > SWEPT_RETENTION_MS) { safeUnlink(fp); pruned++; }
+      } catch { /* ignore */ }
+    }
+  } catch { /* ignore */ }
+  return pruned;
+}
+
+/** Group entries by content hash, keep most-recent per group. Cheap, no LLM. */
+function _hashDedup(manifest, opts = {}) {
+  const groups = new Map(); // hash → entries[]
+  for (const e of manifest) {
+    const h = _hashEntry(e.content);
+    if (!groups.has(h)) groups.set(h, []);
+    groups.get(h).push(e);
+  }
+  let archived = 0;
+  const survivors = [];
+  for (const [, group] of groups) {
+    if (group.length === 1) { survivors.push(group[0]); continue; }
+    // Keep most recent (by date frontmatter, then mtime)
+    group.sort((a, b) => (b.date || '').localeCompare(a.date || '') || b.mtimeMs - a.mtimeMs);
+    survivors.push(group[0]);
+    for (const dup of group.slice(1)) {
+      if (opts.dryRun) { archived++; continue; }
+      const fp = path.join(KB_DIR, dup.category, dup.file);
+      if (_archiveKbFile(fp, `hash-duplicate of ${group[0].category}/${group[0].file}`)) archived++;
+    }
+  }
+  return { survivors, archived };
+}
+
+/** Batched LLM sweep — finds within-batch dupes, reclassifies, removes stale. */
+async function _llmBatchSweep(manifest, callLLM, trackEngineUsage) {
+  const plan = { duplicates: [], reclassify: [], remove: [] };
+  const batches = [];
+  for (let i = 0; i < manifest.length; i += LLM_BATCH_SIZE) {
+    batches.push(manifest.slice(i, i + LLM_BATCH_SIZE));
+  }
+  for (let b = 0; b < batches.length; b++) {
+    const batch = batches[b];
+    const offset = b * LLM_BATCH_SIZE;
+    const prompt = `You are a knowledge base curator. Analyze these ${batch.length} entries (batch ${b + 1}/${batches.length}, indices ${offset}-${offset + batch.length - 1}) and produce a cleanup plan.
+
+## Entries
+
+${batch.map((m, i) => `[${offset + i}] ${m.category}/${m.file} | ${m.title} | ${m.date} | ${m.agent || '?'} | ${(m.content || '').slice(0, 200).replace(/\n/g, ' ')}`).join('\n')}
+
+## Instructions
+
+1. **Find duplicates**: entries with substantially the same content (same findings, different agents/runs). List pairs by index. Prefer keeping the more recent entry.
+2. **Find misclassified**: entries in the wrong category.
+3. **Find stale/empty**: entries with no actionable content (boilerplate, bail-out notes, "no changes needed").
+
+Respond with ONLY valid JSON: { "duplicates": [{ "keep": N, "remove": [N], "reason": "..." }], "reclassify": [{ "index": N, "from": "cat", "to": "cat", "reason": "..." }], "remove": [{ "index": N, "reason": "..." }] }
+If nothing to do: { "duplicates": [], "reclassify": [], "remove": [] }`;
+
+    let result;
+    try {
+      result = await callLLM(prompt, 'Output only JSON.', { timeout: 120000, label: 'kb-sweep', model: 'haiku', maxTurns: 1, direct: true });
+      trackEngineUsage('kb-sweep', result.usage);
+    } catch (e) { log('warn', `[kb-sweep] batch ${b + 1} LLM error: ${e.message}`); continue; }
+
+    let batchPlan;
+    try {
+      let jsonStr = (result.text || '').trim();
+      const fence = jsonStr.match(/```(?:json)?\s*([\s\S]*?)```/);
+      if (fence) jsonStr = fence[1].trim();
+      batchPlan = JSON.parse(jsonStr);
+    } catch { log('warn', `[kb-sweep] batch ${b + 1} returned invalid JSON, skipping`); continue; }
+    if (batchPlan.duplicates) plan.duplicates.push(...batchPlan.duplicates);
+    if (batchPlan.reclassify) plan.reclassify.push(...batchPlan.reclassify);
+    if (batchPlan.remove) plan.remove.push(...batchPlan.remove);
+  }
+  return plan;
+}
+
+/**
+ * Per-entry rewrite pass: compress large entries + normalize structure into
+ * a fixed template. Only runs on entries lacking the `_swept` frontmatter flag.
+ * Concurrency-limited via Promise pool.
+ */
+async function _rewritePass(survivors, callLLM, trackEngineUsage, opts = {}) {
+  const REWRITE_PROMPT = (entry, body) => `You are restructuring a knowledge-base entry so future agents can scan it quickly.
+
+Reshape the content into this exact template, preserving ALL actionable findings, file:line references, and code snippets. Compress to <=800 words by dropping boilerplate (dates, full file paths that aren't actionable, agent IDs in the body, narrative scaffolding).
+
+Template:
+## Summary
+2-3 sentence overview.
+
+## Key Findings
+- Bullet 1 (specific, includes file:line where relevant)
+- Bullet 2
+
+## Action Items
+- Bullet (omit section entirely if none)
+
+## References
+- file:line citations or doc links (omit section if none)
+
+Output ONLY the template body — no frontmatter, no markdown code fence, no preamble.
+
+Original entry (category: ${entry.category}, agent: ${entry.agent || '?'}, date: ${entry.date}):
+
+${body}`;
+
+  const candidates = [];
+  for (const e of survivors) {
+    const fp = path.join(KB_DIR, e.category, e.file);
+    const content = safeRead(fp);
+    if (content == null) continue;
+    const { fm, body } = _parseFrontmatter(content);
+    // Skip already-processed unless the file was modified after the sweep flag was set
+    if (fm[SWEPT_FLAG_KEY]) {
+      try {
+        const mtime = fs.statSync(fp).mtimeMs;
+        const sweptAt = Date.parse(fm[SWEPT_FLAG_KEY]);
+        if (Number.isFinite(sweptAt) && mtime <= sweptAt + 1000) continue;
+      } catch { /* ignore — re-process */ }
+    }
+    candidates.push({ entry: e, fp, fm, body, originalSize: content.length });
+  }
+
+  if (candidates.length === 0) return { processed: 0, bytesBefore: 0, bytesAfter: 0 };
+
+  let processed = 0, bytesBefore = 0, bytesAfter = 0;
+  // Simple promise pool — NORMALIZE_CONCURRENCY at a time
+  let cursor = 0;
+  async function worker() {
+    while (cursor < candidates.length) {
+      const c = candidates[cursor++];
+      try {
+        const result = await callLLM(REWRITE_PROMPT(c.entry, c.body), 'Output ONLY the template body.', {
+          timeout: 120000, label: 'kb-rewrite', model: 'haiku', maxTurns: 1, direct: true,
+        });
+        trackEngineUsage('kb-sweep', result.usage);
+        let newBody = (result.text || '').trim();
+        // Strip accidental code fence
+        const fence = newBody.match(/^```(?:markdown|md)?\s*([\s\S]*?)```$/);
+        if (fence) newBody = fence[1].trim();
+        if (!newBody || newBody.length < 50) continue; // suspicious — skip
+        const newFm = { ...c.fm, [SWEPT_FLAG_KEY]: new Date().toISOString() };
+        const newContent = _serializeFrontmatter(newFm, newBody);
+        if (!opts.dryRun) safeWrite(c.fp, newContent);
+        bytesBefore += c.originalSize;
+        bytesAfter += newContent.length;
+        processed++;
+      } catch (e) { log('warn', `[kb-sweep] rewrite ${c.entry.category}/${c.entry.file}: ${e.message}`); }
+    }
+  }
+  const workers = Array.from({ length: NORMALIZE_CONCURRENCY }, worker);
+  await Promise.all(workers);
+  return { processed, bytesBefore, bytesAfter };
+}
+
+function _applyLlmPlan(plan, manifest, opts = {}) {
+  let removed = 0, merged = 0, reclassified = 0;
+  for (const r of (plan.remove || [])) {
+    const entry = manifest[r.index];
+    if (!entry) continue;
+    if (opts.dryRun) { removed++; continue; }
+    if (_archiveKbFile(path.join(KB_DIR, entry.category, entry.file), `stale: ${r.reason || ''}`)) removed++;
+  }
+  for (const d of (plan.duplicates || [])) {
+    for (const idx of (d.remove || [])) {
+      const entry = manifest[idx];
+      if (!entry) continue;
+      if (opts.dryRun) { merged++; continue; }
+      if (_archiveKbFile(path.join(KB_DIR, entry.category, entry.file), `duplicate of index ${d.keep}: ${d.reason || ''}`)) merged++;
+    }
+  }
+  for (const r of (plan.reclassify || [])) {
+    const entry = manifest[r.index];
+    if (!entry || !shared.KB_CATEGORIES.includes(r.to)) continue;
+    if (opts.dryRun) { reclassified++; continue; }
+    const srcPath = path.join(KB_DIR, entry.category, entry.file);
+    const destDir = path.join(KB_DIR, r.to);
+    if (!fs.existsSync(srcPath)) continue;
+    if (!fs.existsSync(destDir)) fs.mkdirSync(destDir, { recursive: true });
+    try {
+      const stats = fs.statSync(srcPath);
+      const content = safeRead(srcPath);
+      const updated = (content || '').replace(/^(category:\s*).+$/m, `$1${r.to}`);
+      const destPath = path.join(destDir, entry.file);
+      safeWrite(destPath, updated);
+      fs.utimesSync(destPath, stats.atime, stats.mtime);
+      safeUnlink(srcPath);
+      reclassified++;
+    } catch (e) { log('warn', `[kb-sweep] reclassify ${entry.file}: ${e.message}`); }
+  }
+  return { removed, merged, reclassified };
+}
+
+/**
+ * Run the full sweep. Returns a rich summary.
+ *
+ * @param {object} opts
+ * @param {string[]} [opts.pinnedKeys] - extra pinned keys (e.g. from request body)
+ * @param {boolean}  [opts.dryRun]      - count actions but don't mutate files
+ * @returns {Promise<object>} summary
+ */
+async function runKbSweep(opts = {}) {
+  const { callLLM, trackEngineUsage } = require('./llm');
+  const summary = {
+    ok: true,
+    entriesBefore: 0,
+    entriesAfter: 0,
+    bytesBefore: 0,
+    bytesAfter: 0,
+    hashDuplicatesArchived: 0,
+    llmDuplicatesArchived: 0,
+    staleRemoved: 0,
+    reclassified: 0,
+    rewritten: 0,
+    rewriteBytesBefore: 0,
+    rewriteBytesAfter: 0,
+    sweptArchivePruned: 0,
+    durationMs: 0,
+  };
+  const t0 = Date.now();
+
+  const entries = queries.getKnowledgeBaseEntries();
+  if (entries.length < 2) { summary.summary = 'nothing to sweep (< 2 entries)'; summary.durationMs = Date.now() - t0; return summary; }
+
+  const requestPinned = Array.isArray(opts.pinnedKeys)
+    ? opts.pinnedKeys.filter(k => typeof k === 'string' && k.startsWith('knowledge/'))
+    : [];
+  const pinned = new Set([
+    ...shared.getPinnedItems().filter(k => k.startsWith('knowledge/')),
+    ...requestPinned,
+  ]);
+
+  // Build manifest with full content + mtime
+  const manifest = [];
+  for (const e of entries) {
+    if (pinned.has(`knowledge/${e.cat}/${e.file}`)) continue;
+    const fp = path.join(KB_DIR, e.cat, e.file);
+    const content = safeRead(fp);
+    if (!content) continue;
+    let mtimeMs = 0;
+    try { mtimeMs = fs.statSync(fp).mtimeMs; } catch { /* ignore */ }
+    manifest.push({ category: e.cat, file: e.file, title: e.title, agent: e.agent, date: e.date, content: content.slice(0, 3000), mtimeMs });
+    summary.entriesBefore++;
+    summary.bytesBefore += content.length;
+  }
+  if (manifest.length < 2) { summary.summary = 'nothing to sweep (< 2 unpinned entries)'; summary.durationMs = Date.now() - t0; return summary; }
+
+  // 1. Hash-based dedup (cheap, catches cross-batch duplicates)
+  const { survivors: afterHash, archived: hashArchived } = _hashDedup(manifest, opts);
+  summary.hashDuplicatesArchived = hashArchived;
+
+  // 2. LLM batch sweep — within-batch dupes + reclassify + remove stale
+  // Only runs against survivors, but we need indices that match the LIST sent to the LLM
+  const llmManifest = afterHash;
+  const plan = await _llmBatchSweep(llmManifest, callLLM, trackEngineUsage);
+  const llmActions = _applyLlmPlan(plan, llmManifest, opts);
+  summary.llmDuplicatesArchived = llmActions.merged;
+  summary.staleRemoved = llmActions.removed;
+  summary.reclassified = llmActions.reclassified;
+
+  // 3. Per-entry rewrite (compress + normalize)
+  // Filter to entries that survived hash + LLM passes (still on disk)
+  const stillOnDisk = afterHash.filter(e => fs.existsSync(path.join(KB_DIR, e.category, e.file)));
+  const rewriteResult = await _rewritePass(stillOnDisk, callLLM, trackEngineUsage, opts);
+  summary.rewritten = rewriteResult.processed;
+  summary.rewriteBytesBefore = rewriteResult.bytesBefore;
+  summary.rewriteBytesAfter = rewriteResult.bytesAfter;
+
+  // 4. Prune old swept files (>30 days)
+  summary.sweptArchivePruned = _pruneOldSwept();
+
+  // Final tallies — re-walk surviving entries for accurate bytesAfter
+  const finalEntries = queries.getKnowledgeBaseEntries();
+  for (const e of finalEntries) {
+    if (pinned.has(`knowledge/${e.cat}/${e.file}`)) continue;
+    const fp = path.join(KB_DIR, e.cat, e.file);
+    const content = safeRead(fp);
+    if (!content) continue;
+    summary.entriesAfter++;
+    summary.bytesAfter += content.length;
+  }
+
+  summary.durationMs = Date.now() - t0;
+  summary.summary = `${summary.hashDuplicatesArchived} hash-dup, ${summary.llmDuplicatesArchived} llm-dup, ${summary.staleRemoved} stale, ${summary.reclassified} reclassified, ${summary.rewritten} rewritten (${(summary.bytesBefore - summary.bytesAfter).toLocaleString()} bytes saved)`;
+
+  if (!opts.dryRun) {
+    try { safeWrite(path.join(ENGINE_DIR, 'kb-swept.json'), JSON.stringify({ timestamp: ts(), summary: summary.summary, detail: summary })); } catch { /* ignore */ }
+    try { queries.invalidateKnowledgeBaseCache(); } catch { /* ignore */ }
+  }
+  return summary;
+}
+
+/** Compute a dynamic stale-guard timeout based on KB size. */
+function staleGuardMs(entryCount) {
+  // 30 minutes minimum, plus 1 second per entry (for the rewrite pass)
+  return Math.max(30 * 60 * 1000, entryCount * 1000);
+}
+
+module.exports = {
+  runKbSweep,
+  staleGuardMs,
+  // Exported for tests
+  _hashEntry,
+  _parseFrontmatter,
+  _serializeFrontmatter,
+  _hashDedup,
+  COMPRESS_THRESHOLD_BYTES,
+  SWEPT_FLAG_KEY,
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.1577",
+  "version": "0.1.1579",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"