@yemi33/minions 0.1.2348 → 0.1.2350

package/engine/kb-sweep.js

@@ -29,6 +29,7 @@ const COMPRESS_THRESHOLD_BYTES = 5000;
 const LLM_BATCH_SIZE = 30;
 const NORMALIZE_CONCURRENCY = 5;
 const SWEPT_FLAG_KEY = '_swept'; // frontmatter key — entries with this skip the rewrite pass
+const LLM_SCAN_FLAG_KEY = '_llmScannedAt'; // frontmatter key — entries with this (and an unchanged mtime) skip the LLM batch dedup/reclassify pass
 
 function _hashEntry(content) {
   const normalized = String(content || '').replace(/\s+/g, ' ').trim().slice(0, 500);
@@ -420,27 +421,99 @@ function _structuralConsolidate(manifest, opts = {}) {
   return { survivors, consolidated, groupsCollapsed, digestsWritten };
 }
 
-/** Batched LLM sweep — finds within-batch dupes, reclassifies, removes stale. */
+/**
+ * Split a manifest into entries that need a full-content LLM dedup/reclassify
+ * scan ("changed") vs entries that were already scanned in a prior sweep and
+ * haven't been modified since ("unchanged"). Mirrors the `_rewritePass` /
+ * `SWEPT_FLAG_KEY` skip pattern, but tracks it under a separate frontmatter
+ * key (`LLM_SCAN_FLAG_KEY`) since the two passes run independently and an
+ * entry can be llm-scanned without yet being rewritten (or vice versa).
+ *
+ * @returns {{ changedIdx:number[], unchangedIdx:number[] }} indices into `manifest`
+ */
+function _classifyLlmScanFreshness(manifest) {
+  const changedIdx = [];
+  const unchangedIdx = [];
+  for (let i = 0; i < manifest.length; i++) {
+    const e = manifest[i];
+    const fp = path.join(KB_DIR, e.category, e.file);
+    let scannedAt = null;
+    try {
+      const content = safeRead(fp);
+      if (content != null) {
+        const { fm } = _parseFrontmatter(content);
+        if (fm[LLM_SCAN_FLAG_KEY]) {
+          const parsed = Date.parse(fm[LLM_SCAN_FLAG_KEY]);
+          if (Number.isFinite(parsed)) scannedAt = parsed;
+        }
+      }
+    } catch { /* treat as unscanned */ }
+    let mtime = 0;
+    try { mtime = fs.statSync(fp).mtimeMs; } catch { /* missing/unreadable — treat as changed */ }
+    if (scannedAt != null && mtime > 0 && mtime <= scannedAt + 1000) {
+      unchangedIdx.push(i);
+    } else {
+      changedIdx.push(i);
+    }
+  }
+  return { changedIdx, unchangedIdx };
+}
+
+/**
+ * Batched LLM sweep — finds within-batch dupes, reclassifies, removes stale.
+ *
+ * Incremental-scan optimization (W-mrb1c2r20004883a): previously this re-sent
+ * the ENTIRE current KB manifest through the LLM on every sweep cycle
+ * (AUTO_SWEEP_INTERVAL_MS = 4h), even for entries scanned in the prior cycle
+ * with no changes since — cost scaled with (KB size) x (sweep count),
+ * unbounded. Now only entries that are new or modified since their last scan
+ * (`LLM_SCAN_FLAG_KEY` frontmatter, mtime-gated like `SWEPT_FLAG_KEY`) get
+ * full-content batches sent to the LLM. Previously-scanned entries are still
+ * included as lightweight (title/date/category only, no content) roster
+ * context in every batch so cross-entry duplicate detection against the full
+ * KB — not just the new/changed subset — remains correct.
+ *
+ * @returns {{ plan:object, scannedIdx:number[] }} scannedIdx are manifest
+ *   indices actually analyzed this run (used by the caller to stamp
+ *   LLM_SCAN_FLAG_KEY once the whole sweep — including the rewrite pass — has
+ *   finished, avoiding a same-run mtime race with `_rewritePass`).
+ */
 async function _llmBatchSweep(manifest, callLLM, trackEngineUsage, opts = {}) {
   const plan = { duplicates: [], reclassify: [], remove: [] };
+  const { changedIdx, unchangedIdx } = _classifyLlmScanFreshness(manifest);
+
+  if (changedIdx.length === 0) {
+    // Nothing new or modified since the last scan — no LLM calls needed.
+    return { plan, scannedIdx: [] };
+  }
+
+  const rosterLine = (i) => {
+    const m = manifest[i];
+    return `[${i}] ${m.category}/${m.file} | ${m.title} | ${m.date} | ${m.agent || '?'}`;
+  };
+
   const batches = [];
-  for (let i = 0; i < manifest.length; i += LLM_BATCH_SIZE) {
-    batches.push(manifest.slice(i, i + LLM_BATCH_SIZE));
+  for (let i = 0; i < changedIdx.length; i += LLM_BATCH_SIZE) {
+    batches.push(changedIdx.slice(i, i + LLM_BATCH_SIZE));
   }
-  for (let b = 0; b < batches.length; b++) {
-    const batch = batches[b];
-    const offset = b * LLM_BATCH_SIZE;
-    const prompt = `You are a knowledge base curator. Analyze these ${batch.length} entries (batch ${b + 1}/${batches.length}, indices ${offset}-${offset + batch.length - 1}) and produce a cleanup plan.
 
-## Entries
+  const scannedIdx = [];
+  for (let b = 0; b < batches.length; b++) {
+    const batchIdxs = batches[b];
+    const rosterSection = unchangedIdx.length
+      ? `\n## Existing Entries (context only — already scanned previously, title/date/category only, no content)\n\n${unchangedIdx.map(rosterLine).join('\n')}\n`
+      : '';
+    const prompt = `You are a knowledge base curator. Analyze the ${batchIdxs.length} NEW/CHANGED entries below (batch ${b + 1}/${batches.length}) and produce a cleanup plan. ${unchangedIdx.length} additional EXISTING entries are listed below for context only (already scanned in a prior sweep) — use them ONLY to detect whether a new/changed entry duplicates one of them; do not flag pairs where BOTH sides are existing-only entries.
 
-${batch.map((m, i) => `[${offset + i}] ${m.category}/${m.file} | ${m.title} | ${m.date} | ${m.agent || '?'} | ${(m.content || '').slice(0, 200).replace(/\n/g, ' ')}`).join('\n')}
+## New/Changed Entries (analyze content)
 
+${batchIdxs.map(i => `[${i}] ${manifest[i].category}/${manifest[i].file} | ${manifest[i].title} | ${manifest[i].date} | ${manifest[i].agent || '?'} | ${(manifest[i].content || '').slice(0, 200).replace(/\n/g, ' ')}`).join('\n')}
+${rosterSection}
 ## Instructions
 
-1. **Find duplicates**: entries with substantially the same content (same findings, different agents/runs). List pairs by index. Prefer keeping the more recent entry.
-2. **Find misclassified**: entries in the wrong category.
-3. **Find stale/empty**: entries with no actionable content (boilerplate, bail-out notes, "no changes needed").
+1. **Find duplicates**: entries with substantially the same content (same findings, different agents/runs). At least one side of each pair must be a New/Changed entry. List pairs by index. Prefer keeping the more recent entry.
+2. **Find misclassified**: New/Changed entries in the wrong category.
+3. **Find stale/empty**: New/Changed entries with no actionable content (boilerplate, bail-out notes, "no changes needed").
 
 Respond with ONLY valid JSON: { "duplicates": [{ "keep": N, "remove": [N], "reason": "..." }], "reclassify": [{ "index": N, "from": "cat", "to": "cat", "reason": "..." }], "remove": [{ "index": N, "reason": "..." }] }
 If nothing to do: { "duplicates": [], "reclassify": [], "remove": [] }`;
@@ -468,8 +541,34 @@ If nothing to do: { "duplicates": [], "reclassify": [], "remove": [] }`;
     if (batchPlan.duplicates) plan.duplicates.push(...batchPlan.duplicates);
     if (batchPlan.reclassify) plan.reclassify.push(...batchPlan.reclassify);
     if (batchPlan.remove) plan.remove.push(...batchPlan.remove);
+    scannedIdx.push(...batchIdxs);
+  }
+  return { plan, scannedIdx };
+}
+
+/**
+ * Stamp `LLM_SCAN_FLAG_KEY` on entries that were successfully analyzed by
+ * `_llmBatchSweep` this run. Called at the very end of the sweep (after the
+ * rewrite pass) so the rewrite pass's own content/mtime update doesn't
+ * immediately invalidate the freshness marker we're about to write.
+ * Entries that no longer exist at their original path (removed as
+ * stale/duplicate, or moved by reclassify) are skipped — best-effort only.
+ */
+function _markLlmScanned(manifest, scannedIdx, opts = {}) {
+  if (opts.dryRun || !Array.isArray(scannedIdx) || scannedIdx.length === 0) return;
+  const now = new Date().toISOString();
+  for (const i of scannedIdx) {
+    const e = manifest[i];
+    if (!e) continue;
+    const fp = path.join(KB_DIR, e.category, e.file);
+    try {
+      const content = safeRead(fp);
+      if (content == null) continue; // removed/reclassified elsewhere — skip
+      const { fm, body } = _parseFrontmatter(content);
+      fm[LLM_SCAN_FLAG_KEY] = now;
+      safeWrite(fp, _serializeFrontmatter(fm, body));
+    } catch (err) { log('warn', `[kb-sweep] mark-llm-scanned ${e.category}/${e.file}: ${err.message}`); }
   }
-  return plan;
 }
 
 /**
@@ -535,7 +634,7 @@ ${body}`;
           timeout: 120000, label: 'kb-rewrite', model: 'haiku', maxTurns: 1, direct: true,
           engineConfig: opts.engineConfig,
         });
-        trackEngineUsage('kb-sweep', result.usage);
+        trackEngineUsage('kb-rewrite', result.usage);
         if (result.missingRuntime) {
           log('warn', `[kb-sweep] rewrite ${c.entry.category}/${c.entry.file} skipped: ${result.text || result.stderr || 'runtime unavailable'}`);
           continue;
@@ -559,8 +658,14 @@ ${body}`;
   return { processed, bytesBefore, bytesAfter };
 }
 
-function _applyLlmPlan(plan, manifest, opts = {}) {
+function _applyLlmPlan(plan, manifest, changedIdx, opts = {}) {
   let removed = 0, merged = 0, reclassified = 0;
+  // changedIdx is the set of manifest indices actually content-analyzed by
+  // the LLM this batch (see _llmBatchSweep's scannedIdx). Duplicate pairs
+  // referencing only unanalyzed indices (LLM hallucination / stale index
+  // mis-reference) must never be archived — a pair is only trustworthy if
+  // at least one side was actually analyzed this batch (W-mrb447th00062813).
+  const changedSet = new Set(changedIdx || []);
   for (const r of (plan.remove || [])) {
     const entry = manifest[r.index];
     if (!entry) continue;
@@ -568,6 +673,11 @@ function _applyLlmPlan(plan, manifest, opts = {}) {
     if (_archiveKbFile(path.join(KB_DIR, entry.category, entry.file), `stale: ${r.reason || ''}`)) removed++;
   }
   for (const d of (plan.duplicates || [])) {
+    const pairIdx = [d.keep, ...(d.remove || [])];
+    if (!pairIdx.some((idx) => changedSet.has(idx))) {
+      log('warn', `[kb-sweep] duplicate pair (keep=${d.keep}, remove=${JSON.stringify(d.remove)}) references no analyzed (changedIdx) entry — skipping archive to avoid acting on a hallucinated/stale LLM reference`);
+      continue;
+    }
     for (const idx of (d.remove || [])) {
       const entry = manifest[idx];
       if (!entry) continue;
@@ -854,8 +964,8 @@ async function _runKbSweepImpl(opts = {}) {
   // 2. LLM batch sweep — within-batch dupes + reclassify + remove stale
   // Only runs against survivors, but we need indices that match the LIST sent to the LLM
   const llmManifest = afterConsolidate;
-  const plan = await _llmBatchSweep(llmManifest, callLLM, trackEngineUsage, opts);
-  const llmActions = _applyLlmPlan(plan, llmManifest, opts);
+  const { plan, scannedIdx: llmScannedIdx } = await _llmBatchSweep(llmManifest, callLLM, trackEngineUsage, opts);
+  const llmActions = _applyLlmPlan(plan, llmManifest, llmScannedIdx, opts);
   summary.llmDuplicatesArchived = llmActions.merged;
   summary.staleRemoved = llmActions.removed;
   summary.reclassified = llmActions.reclassified;
@@ -877,6 +987,11 @@ async function _runKbSweepImpl(opts = {}) {
   summary.rewriteBytesBefore = rewriteResult.bytesBefore;
   summary.rewriteBytesAfter = rewriteResult.bytesAfter;
 
+  // Stamp LLM_SCAN_FLAG_KEY on entries analyzed by _llmBatchSweep this run —
+  // deferred until after the rewrite pass so its content/mtime update doesn't
+  // race the freshness marker we're about to write (see _markLlmScanned doc).
+  _markLlmScanned(llmManifest, llmScannedIdx, opts);
+
   // 4. Prune old swept files (>30 days)
   summary.sweptArchivePruned = _pruneOldSwept();
 
@@ -1063,6 +1178,11 @@ module.exports = {
   _digestRow,
   _isDigestEntry,
   _extractHeadField,
+  _llmBatchSweep,
+  _classifyLlmScanFreshness,
+  _markLlmScanned,
+  _applyLlmPlan,
+  LLM_SCAN_FLAG_KEY,
   CONSOLIDATION_DIGEST_PREFIX,
   COMPRESS_THRESHOLD_BYTES,
   SWEPT_FLAG_KEY,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2348",
+  "version": "0.1.2350",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"