@yemi33/minions 0.1.2347 → 0.1.2349

package/dashboard.js

@@ -9845,8 +9845,11 @@ What would you like to discuss or change? When you're happy, say "approve" and I
       const body = await readBody(req);
       const message = body && typeof body.message === 'string' ? body.message : '';
       if (!message.trim()) return jsonReply(res, 400, { error: 'message required' });
-      const watchId = body.watchId ? String(body.watchId) : '';
-      const label = watchId ? `watch-cc-triage:${watchId}` : 'watch-cc-triage';
+      // W-mrb1a3950003d9ea — flat, stable label (no per-watch-id interpolation).
+      // Every other _engine category (kb-sweep, consolidation, command-center, …)
+      // uses one flat label; a dynamic watchId suffix here used to mint a brand-new
+      // permanent metrics.json._engine key per watch firing (20+ confirmed live).
+      const label = 'watch-cc-triage';
 
       const model = CONFIG.engine?.ccModel || shared.ENGINE_DEFAULTS.ccModel;
       const maxTurns = CONFIG.engine?.ccMaxTurns || shared.ENGINE_DEFAULTS.ccMaxTurns;

package/engine/kb-sweep.js

@@ -29,6 +29,7 @@ const COMPRESS_THRESHOLD_BYTES = 5000;
 const LLM_BATCH_SIZE = 30;
 const NORMALIZE_CONCURRENCY = 5;
 const SWEPT_FLAG_KEY = '_swept'; // frontmatter key — entries with this skip the rewrite pass
+const LLM_SCAN_FLAG_KEY = '_llmScannedAt'; // frontmatter key — entries with this (and an unchanged mtime) skip the LLM batch dedup/reclassify pass
 
 function _hashEntry(content) {
   const normalized = String(content || '').replace(/\s+/g, ' ').trim().slice(0, 500);
@@ -420,27 +421,99 @@ function _structuralConsolidate(manifest, opts = {}) {
   return { survivors, consolidated, groupsCollapsed, digestsWritten };
 }
 
-/** Batched LLM sweep — finds within-batch dupes, reclassifies, removes stale. */
+/**
+ * Split a manifest into entries that need a full-content LLM dedup/reclassify
+ * scan ("changed") vs entries that were already scanned in a prior sweep and
+ * haven't been modified since ("unchanged"). Mirrors the `_rewritePass` /
+ * `SWEPT_FLAG_KEY` skip pattern, but tracks it under a separate frontmatter
+ * key (`LLM_SCAN_FLAG_KEY`) since the two passes run independently and an
+ * entry can be llm-scanned without yet being rewritten (or vice versa).
+ *
+ * @returns {{ changedIdx:number[], unchangedIdx:number[] }} indices into `manifest`
+ */
+function _classifyLlmScanFreshness(manifest) {
+  const changedIdx = [];
+  const unchangedIdx = [];
+  for (let i = 0; i < manifest.length; i++) {
+    const e = manifest[i];
+    const fp = path.join(KB_DIR, e.category, e.file);
+    let scannedAt = null;
+    try {
+      const content = safeRead(fp);
+      if (content != null) {
+        const { fm } = _parseFrontmatter(content);
+        if (fm[LLM_SCAN_FLAG_KEY]) {
+          const parsed = Date.parse(fm[LLM_SCAN_FLAG_KEY]);
+          if (Number.isFinite(parsed)) scannedAt = parsed;
+        }
+      }
+    } catch { /* treat as unscanned */ }
+    let mtime = 0;
+    try { mtime = fs.statSync(fp).mtimeMs; } catch { /* missing/unreadable — treat as changed */ }
+    if (scannedAt != null && mtime > 0 && mtime <= scannedAt + 1000) {
+      unchangedIdx.push(i);
+    } else {
+      changedIdx.push(i);
+    }
+  }
+  return { changedIdx, unchangedIdx };
+}
+
+/**
+ * Batched LLM sweep — finds within-batch dupes, reclassifies, removes stale.
+ *
+ * Incremental-scan optimization (W-mrb1c2r20004883a): previously this re-sent
+ * the ENTIRE current KB manifest through the LLM on every sweep cycle
+ * (AUTO_SWEEP_INTERVAL_MS = 4h), even for entries scanned in the prior cycle
+ * with no changes since — cost scaled with (KB size) x (sweep count),
+ * unbounded. Now only entries that are new or modified since their last scan
+ * (`LLM_SCAN_FLAG_KEY` frontmatter, mtime-gated like `SWEPT_FLAG_KEY`) get
+ * full-content batches sent to the LLM. Previously-scanned entries are still
+ * included as lightweight (title/date/category only, no content) roster
+ * context in every batch so cross-entry duplicate detection against the full
+ * KB — not just the new/changed subset — remains correct.
+ *
+ * @returns {{ plan:object, scannedIdx:number[] }} scannedIdx are manifest
+ *   indices actually analyzed this run (used by the caller to stamp
+ *   LLM_SCAN_FLAG_KEY once the whole sweep — including the rewrite pass — has
+ *   finished, avoiding a same-run mtime race with `_rewritePass`).
+ */
 async function _llmBatchSweep(manifest, callLLM, trackEngineUsage, opts = {}) {
   const plan = { duplicates: [], reclassify: [], remove: [] };
+  const { changedIdx, unchangedIdx } = _classifyLlmScanFreshness(manifest);
+
+  if (changedIdx.length === 0) {
+    // Nothing new or modified since the last scan — no LLM calls needed.
+    return { plan, scannedIdx: [] };
+  }
+
+  const rosterLine = (i) => {
+    const m = manifest[i];
+    return `[${i}] ${m.category}/${m.file} | ${m.title} | ${m.date} | ${m.agent || '?'}`;
+  };
+
   const batches = [];
-  for (let i = 0; i < manifest.length; i += LLM_BATCH_SIZE) {
-    batches.push(manifest.slice(i, i + LLM_BATCH_SIZE));
+  for (let i = 0; i < changedIdx.length; i += LLM_BATCH_SIZE) {
+    batches.push(changedIdx.slice(i, i + LLM_BATCH_SIZE));
   }
-  for (let b = 0; b < batches.length; b++) {
-    const batch = batches[b];
-    const offset = b * LLM_BATCH_SIZE;
-    const prompt = `You are a knowledge base curator. Analyze these ${batch.length} entries (batch ${b + 1}/${batches.length}, indices ${offset}-${offset + batch.length - 1}) and produce a cleanup plan.
 
-## Entries
+  const scannedIdx = [];
+  for (let b = 0; b < batches.length; b++) {
+    const batchIdxs = batches[b];
+    const rosterSection = unchangedIdx.length
+      ? `\n## Existing Entries (context only — already scanned previously, title/date/category only, no content)\n\n${unchangedIdx.map(rosterLine).join('\n')}\n`
+      : '';
+    const prompt = `You are a knowledge base curator. Analyze the ${batchIdxs.length} NEW/CHANGED entries below (batch ${b + 1}/${batches.length}) and produce a cleanup plan. ${unchangedIdx.length} additional EXISTING entries are listed below for context only (already scanned in a prior sweep) — use them ONLY to detect whether a new/changed entry duplicates one of them; do not flag pairs where BOTH sides are existing-only entries.
 
-${batch.map((m, i) => `[${offset + i}] ${m.category}/${m.file} | ${m.title} | ${m.date} | ${m.agent || '?'} | ${(m.content || '').slice(0, 200).replace(/\n/g, ' ')}`).join('\n')}
+## New/Changed Entries (analyze content)
 
+${batchIdxs.map(i => `[${i}] ${manifest[i].category}/${manifest[i].file} | ${manifest[i].title} | ${manifest[i].date} | ${manifest[i].agent || '?'} | ${(manifest[i].content || '').slice(0, 200).replace(/\n/g, ' ')}`).join('\n')}
+${rosterSection}
 ## Instructions
 
-1. **Find duplicates**: entries with substantially the same content (same findings, different agents/runs). List pairs by index. Prefer keeping the more recent entry.
-2. **Find misclassified**: entries in the wrong category.
-3. **Find stale/empty**: entries with no actionable content (boilerplate, bail-out notes, "no changes needed").
+1. **Find duplicates**: entries with substantially the same content (same findings, different agents/runs). At least one side of each pair must be a New/Changed entry. List pairs by index. Prefer keeping the more recent entry.
+2. **Find misclassified**: New/Changed entries in the wrong category.
+3. **Find stale/empty**: New/Changed entries with no actionable content (boilerplate, bail-out notes, "no changes needed").
 
 Respond with ONLY valid JSON: { "duplicates": [{ "keep": N, "remove": [N], "reason": "..." }], "reclassify": [{ "index": N, "from": "cat", "to": "cat", "reason": "..." }], "remove": [{ "index": N, "reason": "..." }] }
 If nothing to do: { "duplicates": [], "reclassify": [], "remove": [] }`;
@@ -468,8 +541,34 @@ If nothing to do: { "duplicates": [], "reclassify": [], "remove": [] }`;
     if (batchPlan.duplicates) plan.duplicates.push(...batchPlan.duplicates);
     if (batchPlan.reclassify) plan.reclassify.push(...batchPlan.reclassify);
     if (batchPlan.remove) plan.remove.push(...batchPlan.remove);
+    scannedIdx.push(...batchIdxs);
+  }
+  return { plan, scannedIdx };
+}
+
+/**
+ * Stamp `LLM_SCAN_FLAG_KEY` on entries that were successfully analyzed by
+ * `_llmBatchSweep` this run. Called at the very end of the sweep (after the
+ * rewrite pass) so the rewrite pass's own content/mtime update doesn't
+ * immediately invalidate the freshness marker we're about to write.
+ * Entries that no longer exist at their original path (removed as
+ * stale/duplicate, or moved by reclassify) are skipped — best-effort only.
+ */
+function _markLlmScanned(manifest, scannedIdx, opts = {}) {
+  if (opts.dryRun || !Array.isArray(scannedIdx) || scannedIdx.length === 0) return;
+  const now = new Date().toISOString();
+  for (const i of scannedIdx) {
+    const e = manifest[i];
+    if (!e) continue;
+    const fp = path.join(KB_DIR, e.category, e.file);
+    try {
+      const content = safeRead(fp);
+      if (content == null) continue; // removed/reclassified elsewhere — skip
+      const { fm, body } = _parseFrontmatter(content);
+      fm[LLM_SCAN_FLAG_KEY] = now;
+      safeWrite(fp, _serializeFrontmatter(fm, body));
+    } catch (err) { log('warn', `[kb-sweep] mark-llm-scanned ${e.category}/${e.file}: ${err.message}`); }
   }
-  return plan;
 }
 
 /**
@@ -535,7 +634,7 @@ ${body}`;
           timeout: 120000, label: 'kb-rewrite', model: 'haiku', maxTurns: 1, direct: true,
           engineConfig: opts.engineConfig,
         });
-        trackEngineUsage('kb-sweep', result.usage);
+        trackEngineUsage('kb-rewrite', result.usage);
         if (result.missingRuntime) {
           log('warn', `[kb-sweep] rewrite ${c.entry.category}/${c.entry.file} skipped: ${result.text || result.stderr || 'runtime unavailable'}`);
           continue;
@@ -854,7 +953,7 @@ async function _runKbSweepImpl(opts = {}) {
   // 2. LLM batch sweep — within-batch dupes + reclassify + remove stale
   // Only runs against survivors, but we need indices that match the LIST sent to the LLM
   const llmManifest = afterConsolidate;
-  const plan = await _llmBatchSweep(llmManifest, callLLM, trackEngineUsage, opts);
+  const { plan, scannedIdx: llmScannedIdx } = await _llmBatchSweep(llmManifest, callLLM, trackEngineUsage, opts);
   const llmActions = _applyLlmPlan(plan, llmManifest, opts);
   summary.llmDuplicatesArchived = llmActions.merged;
   summary.staleRemoved = llmActions.removed;
@@ -877,6 +976,11 @@ async function _runKbSweepImpl(opts = {}) {
   summary.rewriteBytesBefore = rewriteResult.bytesBefore;
   summary.rewriteBytesAfter = rewriteResult.bytesAfter;
 
+  // Stamp LLM_SCAN_FLAG_KEY on entries analyzed by _llmBatchSweep this run —
+  // deferred until after the rewrite pass so its content/mtime update doesn't
+  // race the freshness marker we're about to write (see _markLlmScanned doc).
+  _markLlmScanned(llmManifest, llmScannedIdx, opts);
+
   // 4. Prune old swept files (>30 days)
   summary.sweptArchivePruned = _pruneOldSwept();
 
@@ -1063,6 +1167,10 @@ module.exports = {
   _digestRow,
   _isDigestEntry,
   _extractHeadField,
+  _llmBatchSweep,
+  _classifyLlmScanFreshness,
+  _markLlmScanned,
+  LLM_SCAN_FLAG_KEY,
   CONSOLIDATION_DIGEST_PREFIX,
   COMPRESS_THRESHOLD_BYTES,
   SWEPT_FLAG_KEY,

package/engine/metrics-store.js

@@ -172,6 +172,8 @@ function readMetrics() {
   try { db = getDb(); }
   catch { return _readJsonObjectFallback(); }
 
+  migrateWatchCcTriageKeysOnce();
+
   _resyncIfJsonDiverged(db);
 
   const rows = _readAllRows(db);
@@ -285,9 +287,73 @@ function _resetMetricsTableForTest() {
   _lastMirrorHash = null;
 }
 
+// W-mrb1a3950003d9ea — one-time collapse of legacy `watch-cc-triage:<watchId>`
+// per-watch _engine keys (minted before dashboard.js's handleCommandCenterTriage
+// stopped interpolating watchId into the trackEngineUsage label) into a single
+// flat `watch-cc-triage` bucket, matching every other _engine category. Sums
+// calls/cost/tokens/duration/errorsByCode across all matching keys, then deletes
+// the per-id keys. Idempotent — a no-op (no SQL/JSON write) once no key matches
+// /^watch-cc-triage:/, so it's safe to call unconditionally on every process
+// startup via migrateWatchCcTriageKeysOnce().
+const WATCH_CC_TRIAGE_LEGACY_KEY_RE = /^watch-cc-triage:/;
+const WATCH_CC_TRIAGE_FLAT_KEY = 'watch-cc-triage';
+
+function migrateWatchCcTriageKeys() {
+  return applyMetricsMutation((metrics) => {
+    const engineMetrics = metrics._engine;
+    if (!engineMetrics || typeof engineMetrics !== 'object') return metrics;
+    const staleKeys = Object.keys(engineMetrics).filter((k) => WATCH_CC_TRIAGE_LEGACY_KEY_RE.test(k));
+    if (staleKeys.length === 0) return metrics;
+
+    if (!engineMetrics[WATCH_CC_TRIAGE_FLAT_KEY]) {
+      engineMetrics[WATCH_CC_TRIAGE_FLAT_KEY] = { calls: 0, costUsd: 0, inputTokens: 0, outputTokens: 0, cacheRead: 0, cacheCreation: 0 };
+    }
+    const flat = engineMetrics[WATCH_CC_TRIAGE_FLAT_KEY];
+    for (const key of staleKeys) {
+      const cat = engineMetrics[key] || {};
+      flat.calls = (flat.calls || 0) + (cat.calls || 0);
+      flat.costUsd = (flat.costUsd || 0) + (cat.costUsd || 0);
+      flat.inputTokens = (flat.inputTokens || 0) + (cat.inputTokens || 0);
+      flat.outputTokens = (flat.outputTokens || 0) + (cat.outputTokens || 0);
+      flat.cacheRead = (flat.cacheRead || 0) + (cat.cacheRead || 0);
+      flat.cacheCreation = (flat.cacheCreation || 0) + (cat.cacheCreation || 0);
+      if (cat.totalDurationMs) flat.totalDurationMs = (flat.totalDurationMs || 0) + cat.totalDurationMs;
+      if (cat.timedCalls) flat.timedCalls = (flat.timedCalls || 0) + cat.timedCalls;
+      if (cat.errorsByCode) {
+        if (!flat.errorsByCode) flat.errorsByCode = {};
+        for (const [code, count] of Object.entries(cat.errorsByCode)) {
+          flat.errorsByCode[code] = (flat.errorsByCode[code] || 0) + count;
+        }
+      }
+      delete engineMetrics[key];
+    }
+    return metrics;
+  });
+}
+
+let _watchCcTriageMigratedThisProcess = false;
+
+// Startup-safe wrapper: runs migrateWatchCcTriageKeys() at most once per
+// process (subsequent calls, e.g. on every readMetrics(), are a cheap
+// no-op flag check). Failures (e.g. SQLite unavailable) are swallowed and
+// retried on the next process start rather than surfaced to callers.
+function migrateWatchCcTriageKeysOnce() {
+  if (_watchCcTriageMigratedThisProcess) return;
+  _watchCcTriageMigratedThisProcess = true;
+  try { migrateWatchCcTriageKeys(); }
+  catch { /* best-effort — next process start retries */ }
+}
+
+function _resetWatchCcTriageMigrationFlagForTest() {
+  _watchCcTriageMigratedThisProcess = false;
+}
+
 module.exports = {
   readMetrics,
   applyMetricsMutation,
   _mirrorJsonFromSql,
   _resetMetricsTableForTest,
+  migrateWatchCcTriageKeys,
+  migrateWatchCcTriageKeysOnce,
+  _resetWatchCcTriageMigrationFlagForTest,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2347",
+  "version": "0.1.2349",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"