onbuzz 4.8.3 → 4.9.0

package/src/services/codebaseKnowledgeService.js

@@ -0,0 +1,430 @@
+/**
+ * @file codebaseKnowledgeService.js
+ * @description Per-agent cache of structural knowledge the agent has
+ *   already gathered about the codebase. Powers two layers:
+ *
+ *   1. The `## CODEBASE KNOWLEDGE` injection in the system prompt
+ *      every turn (agentScheduler) — a salient index of "you already
+ *      mapped these files" that the agent reads BEFORE deciding to
+ *      issue another code-map / filesystem read.
+ *
+ *   2. A transparent cache underneath the code-map tool — when the
+ *      agent does decide to re-issue, the tool can serve from cache
+ *      (no parser run, no disk read) provided the file hasn't been
+ *      written to since the last cache fill.
+ *
+ *   Together these address the production-observed pattern in the
+ *   Monopoly mission (1556 messages, 8.5h, see DESIGN doc): 22
+ *   skeleton calls on the same file, 49 reads of the EXACT same
+ *   (file, line-range) tuple, 80% of redundant calls happening
+ *   while the original tool result was STILL in the active context
+ *   window — i.e. the agent had the answer in front of it but had
+ *   no salient signal to that effect.
+ *
+ *   Freshness model (defense in depth):
+ *     - `markDirty` is the FAST-PATH signal — called from the tools we
+ *       know about (filesystem write/append/delete/copy/move,
+ *       file-content-replace). Cheap; no stat needed when we already
+ *       know we wrote.
+ *     - mtime-on-lookup is the CATCH-ALL — stats the file on every
+ *       cache hit, compares against the mtime captured at record
+ *       time, treats any mismatch as dirty. Catches mutations from
+ *       sources `markDirty` doesn't know about: terminal commands
+ *       (sed -i, prettier --write, git checkout, npm install),
+ *       external editors, build outputs, etc.
+ *
+ *   Design constraints honored:
+ *     - per-agent isolation (no cross-agent leakage)
+ *     - never blocks the agent's turn — every public method is
+ *       defensively try/catched at the call sites
+ *     - bounded growth (MAX_FILES_PER_AGENT)
+ *     - workingDir-relative canonicalization, no /src/-token magic
+ *       (monorepo-safe; the old heuristic collided across apps/* in
+ *        Yarn/pnpm workspaces and across vendor/* sibling paths)
+ *     - asymmetric-lie prevention: a record without retrievable
+ *       content is rejected at the door, never created as a stub
+ */
+
+import { promises as fs } from 'fs';
+
+const MAX_FILES_PER_AGENT = 50;     // LRU cap; prevents unbounded growth
+const MAX_RANGE_KEYS_PER_FILE = 30; // bound per-file growth (multiple ranges)
+
+let _singleton = null;
+
+class CodebaseKnowledgeService {
+  constructor(logger = null) {
+    this.logger = logger;
+    // agentId → Map(canonicalPath → entry)
+    this._byAgent = new Map();
+  }
+
+  /**
+   * Canonicalize a path so:
+   *   1. forward slashes only (Windows backslashes normalized)
+   *   2. relative to workingDir when an absolute path falls under it
+   *   3. lower-cased drive letter on Windows so `C:` and `c:` collide
+   *   4. trailing slash stripped
+   *   5. `./` prefix stripped
+   *
+   * Monorepo-safe: TWO files like `apps/web/src/Button.tsx` and
+   * `apps/api/src/Button.tsx` produce TWO distinct keys. The previous
+   * implementation collapsed both to `src/Button.tsx`, silently
+   * returning the wrong file's content on lookup.
+   *
+   * No regex — String.prototype only — so the path-handling logic is
+   * easy to audit and step through.
+   *
+   * @param {string} rawPath - the path as the tool received it (relative or absolute)
+   * @param {string|null} workingDir - the agent's CWD; absolute paths
+   *   under it become relative. Pass null/undefined if not known —
+   *   we'll still canonicalize separators + casing.
+   */
+  _canonicalize(rawPath, workingDir = null) {
+    if (typeof rawPath !== 'string') return null;
+    let p = rawPath.split('\\').join('/').trim();
+    if (!p) return null;
+
+    // Strip trailing slashes for stable keying ("src/" === "src").
+    while (p.length > 1 && p.endsWith('/')) p = p.slice(0, -1);
+    // Strip leading "./".
+    while (p.startsWith('./')) p = p.slice(2);
+    if (!p) return null;
+
+    // Windows drive-letter case normalization: "C:/foo" === "c:/foo".
+    if (p.length >= 2 && p[1] === ':') {
+      p = p[0].toLowerCase() + p.slice(1);
+    }
+
+    if (workingDir) {
+      let wd = workingDir.split('\\').join('/').trim();
+      while (wd.length > 1 && wd.endsWith('/')) wd = wd.slice(0, -1);
+      if (wd.length >= 2 && wd[1] === ':') wd = wd[0].toLowerCase() + wd.slice(1);
+      // If absolute path falls under workingDir, store as relative.
+      // Add a trailing slash to wd so we don't match a sibling whose
+      // name starts with the same prefix (`/foo` vs `/foobar`).
+      const wdSlash = wd + '/';
+      if (p === wd) return '.';
+      if (p.startsWith(wdSlash)) return p.slice(wdSlash.length);
+    }
+
+    return p;
+  }
+
+  /**
+   * Reconstruct an absolute path from the raw path + workingDir.
+   * String-native (no regex). Used for the mtime stat call.
+   *
+   * Returns null if we can't determine an absolute path (e.g. relative
+   * path given with no workingDir). In that case the caller falls back
+   * to the pre-mtime behaviour (no freshness check; rely on markDirty).
+   */
+  _toAbsolutePath(rawPath, workingDir) {
+    if (typeof rawPath !== 'string' || !rawPath) return null;
+    let p = rawPath.split('\\').join('/').trim();
+    if (!p) return null;
+    // Unix absolute: starts with /
+    if (p.startsWith('/')) return p;
+    // Windows absolute: X: or X:/
+    if (p.length >= 2 && p[1] === ':') {
+      return p[0].toLowerCase() + p.slice(1);
+    }
+    // Relative — need workingDir to anchor.
+    if (!workingDir) return null;
+    let wd = workingDir.split('\\').join('/').trim();
+    while (wd.length > 1 && wd.endsWith('/')) wd = wd.slice(0, -1);
+    if (wd.length >= 2 && wd[1] === ':') wd = wd[0].toLowerCase() + wd.slice(1);
+    while (p.startsWith('./')) p = p.slice(2);
+    return wd + '/' + p;
+  }
+
+  _agentMap(agentId) {
+    if (!this._byAgent.has(agentId)) {
+      this._byAgent.set(agentId, new Map());
+    }
+    return this._byAgent.get(agentId);
+  }
+
+  /**
+   * Touch an entry to keep it fresh under LRU (most-recently-accessed wins).
+   * @private
+   */
+  _touch(agentId, canonical) {
+    const m = this._agentMap(agentId);
+    const entry = m.get(canonical);
+    if (!entry) return;
+    m.delete(canonical);
+    m.set(canonical, entry);
+    // Evict oldest if over cap
+    while (m.size > MAX_FILES_PER_AGENT) {
+      const oldestKey = m.keys().next().value;
+      m.delete(oldestKey);
+      this.logger?.debug?.('codebaseKnowledge: evicted oldest', { agentId, file: oldestKey });
+    }
+  }
+
+  /**
+   * Record that a skeleton was just fetched for a file.
+   *
+   * Asymmetric-lie prevention: if content is empty/null we REJECT the
+   * record outright rather than creating a metadata-only entry. The
+   * old behaviour (drop content over 200KB but keep the metadata)
+   * meant the index injection advertised entries the lookup couldn't
+   * fulfil — actively misleading the agent. The 200KB cap is gone:
+   * RAM is cheap, LRU + per-file/per-agent counts are sufficient
+   * bounds.
+   *
+   * @param {string} agentId
+   * @param {string} rawPath
+   * @param {string} content
+   * @param {string|null} [workingDir] - the agent's CWD; used to
+   *   anchor canonicalization. Optional for backward-compat.
+   */
+  async recordSkeleton(agentId, rawPath, content, workingDir = null) {
+    if (typeof content !== 'string' || content.length === 0) {
+      this.logger?.debug?.('codebaseKnowledge: skipping recordSkeleton (no content)', {
+        agentId, rawPath,
+      });
+      return;
+    }
+    const canonical = this._canonicalize(rawPath, workingDir);
+    if (!canonical) return;
+
+    // Capture mtime for the lookup-time freshness check. Best-effort:
+    // if stat fails (file gone? synthetic test path?), we just skip
+    // the mtime check at lookup — markDirty remains the only signal.
+    const absPath = this._toAbsolutePath(rawPath, workingDir);
+    let recordedMtimeMs = null;
+    if (absPath) {
+      try {
+        const stat = await fs.stat(absPath);
+        recordedMtimeMs = stat.mtimeMs;
+      } catch (e) {
+        this.logger?.debug?.('codebaseKnowledge: stat at record time failed (mtime check disabled for this entry)', {
+          agentId, absPath, error: e.message,
+        });
+      }
+    }
+
+    const m = this._agentMap(agentId);
+    const existing = m.get(canonical) || { ranges: new Map() };
+    existing.skeleton = { content, ts: Date.now(), absPath, recordedMtimeMs };
+    existing.dirty = false;
+    existing.lastUpdated = Date.now();
+    m.set(canonical, existing);
+    this._touch(agentId, canonical);
+  }
+
+  /**
+   * Record that a specific line range was fetched for a file.
+   *
+   * Same asymmetric-lie protection as recordSkeleton: no content →
+   * no entry.
+   */
+  async recordRange(agentId, rawPath, startLine, endLine, content, workingDir = null) {
+    if (typeof content !== 'string' || content.length === 0) {
+      this.logger?.debug?.('codebaseKnowledge: skipping recordRange (no content)', {
+        agentId, rawPath, startLine, endLine,
+      });
+      return;
+    }
+    if (typeof startLine !== 'number' || typeof endLine !== 'number') return;
+    const canonical = this._canonicalize(rawPath, workingDir);
+    if (!canonical) return;
+
+    // Capture mtime + absPath at the file level so range lookups can
+    // freshness-check the same way skeleton lookups do.
+    const absPath = this._toAbsolutePath(rawPath, workingDir);
+    let recordedMtimeMs = null;
+    if (absPath) {
+      try {
+        const stat = await fs.stat(absPath);
+        recordedMtimeMs = stat.mtimeMs;
+      } catch (e) {
+        this.logger?.debug?.('codebaseKnowledge: stat at record-range time failed', {
+          agentId, absPath, error: e.message,
+        });
+      }
+    }
+
+    const m = this._agentMap(agentId);
+    const existing = m.get(canonical) || { ranges: new Map() };
+    if (!(existing.ranges instanceof Map)) existing.ranges = new Map();
+    const key = `${startLine}-${endLine}`;
+    existing.ranges.set(key, { content, ts: Date.now(), absPath, recordedMtimeMs });
+    // Bound per-file ranges (FIFO within a file — file-level eviction is LRU).
+    while (existing.ranges.size > MAX_RANGE_KEYS_PER_FILE) {
+      const oldestKey = existing.ranges.keys().next().value;
+      existing.ranges.delete(oldestKey);
+    }
+    existing.dirty = false;
+    existing.lastUpdated = Date.now();
+    m.set(canonical, existing);
+    this._touch(agentId, canonical);
+  }
+
+  /**
+   * Mark a file as modified — invalidates cached skeleton + ranges.
+   * Called by filesystem write/append/delete and file-content-replace hooks.
+   */
+  markDirty(agentId, rawPath, workingDir = null) {
+    const canonical = this._canonicalize(rawPath, workingDir);
+    if (!canonical) return;
+    const m = this._agentMap(agentId);
+    const e = m.get(canonical);
+    if (e) {
+      e.dirty = true;
+      e.lastUpdated = Date.now();
+    }
+  }
+
+  /**
+   * Best-effort mtime freshness check. Returns true if the file has
+   * been modified since the recorded mtime, OR if the file is gone
+   * or unreadable. Returns false if the file is unchanged. Returns
+   * null when no check is possible (no absPath / no recorded mtime
+   * captured at record time) — caller should treat null as "no
+   * signal, keep going".
+   * @private
+   */
+  async _isStale(entry) {
+    if (!entry || !entry.absPath || typeof entry.recordedMtimeMs !== 'number') {
+      return null;
+    }
+    try {
+      const stat = await fs.stat(entry.absPath);
+      return stat.mtimeMs > entry.recordedMtimeMs;
+    } catch {
+      // File gone, unreadable, or otherwise inaccessible — treat as stale.
+      return true;
+    }
+  }
+
+  /**
+   * Look up cached skeleton content. Returns null if not cached, dirty,
+   * or stale per mtime check. Caller should fall through to a fresh
+   * fetch on null. ASYNC because of the mtime stat.
+   */
+  async lookupSkeleton(agentId, rawPath, workingDir = null) {
+    const canonical = this._canonicalize(rawPath, workingDir);
+    if (!canonical) return null;
+    const e = this._agentMap(agentId).get(canonical);
+    if (!e || e.dirty) return null;
+    if (!e.skeleton || !e.skeleton.content) return null;
+
+    // mtime catch-all. If the file was mutated by something `markDirty`
+    // doesn't know about (terminal, external editor, build output),
+    // this is where we catch it. Flip the entry to dirty so the next
+    // index injection shows STALE; return null so the agent re-fetches.
+    const stale = await this._isStale(e.skeleton);
+    if (stale === true) {
+      e.dirty = true;
+      return null;
+    }
+    return { content: e.skeleton.content, canonical };
+  }
+
+  /**
+   * Look up a cached line range. Returns null if missing, dirty, stale,
+   * or no exact range match. We don't currently slice cached ranges —
+   * exact matches only. ASYNC because of the mtime stat.
+   */
+  async lookupRange(agentId, rawPath, startLine, endLine, workingDir = null) {
+    const canonical = this._canonicalize(rawPath, workingDir);
+    if (!canonical) return null;
+    const e = this._agentMap(agentId).get(canonical);
+    if (!e || e.dirty || !(e.ranges instanceof Map)) return null;
+    const key = `${startLine}-${endLine}`;
+    const hit = e.ranges.get(key);
+    if (!hit || !hit.content) return null;
+
+    // Range entries carry their own absPath/recordedMtimeMs (captured
+    // at record-time). If a write happens between record and lookup,
+    // this catches it even for ranges that were never touched by
+    // markDirty.
+    const stale = await this._isStale(hit);
+    if (stale === true) {
+      // Mark the whole file dirty — any range on a mutated file is suspect.
+      e.dirty = true;
+      return null;
+    }
+    return { content: hit.content, canonical };
+  }
+
+  /**
+   * Build the system-prompt index block for this agent. Returns an empty
+   * string if nothing is cached — keeps the system prompt clean for new
+   * sessions and for agents that don't do code work.
+   */
+  buildIndexBlock(agentId) {
+    const m = this._byAgent.get(agentId);
+    if (!m || m.size === 0) return '';
+
+    // Sorted: dirty (STALE) entries first to grab attention, then by
+    // most-recently-accessed (the Map preserves insertion order, and
+    // _touch re-inserts on each access, so iteration order is LRU).
+    const entries = Array.from(m.entries());
+    const dirty = entries.filter(([_, e]) => e.dirty);
+    const fresh = entries.filter(([_, e]) => !e.dirty).reverse();
+
+    // Format: MINIMAL — just the file list. The explanatory paragraph
+    // ("re-issuing is wasted work, the cache will serve…") that earlier
+    // versions of this code emitted was REMOVED after empirical A/B
+    // testing showed it actively REGRESSED smaller models (gpt-4.1-nano
+    // went 5/5 to redundant `code-map skeleton` calls on the explained
+    // variant; 1/5 on the minimal variant; 0/5 with no index at all).
+    // Hypothesis: the verbose text repeats the tokens "code-map" and
+    // "skeleton" multiple times, which acts as an in-context-learning
+    // cue for the model to emit that exact pattern. The minimal list
+    // is descriptive (what's in cache) without prescriptive boilerplate
+    // (what to do about it) — the prescriptive guidance lives in the
+    // tool descriptions / OPERATING POSTURE, surfaced once, not per-turn.
+    // See tmp-compaction-experiment/behavior-test.mjs for the harness.
+    const lines = [
+      '',
+      '## CODEBASE KNOWLEDGE (cached this session)',
+      '',
+    ];
+    for (const [file, e] of [...dirty, ...fresh]) {
+      const what = [];
+      // Just `skeleton` — no `#msgIdx` citation. The phantom-citation
+      // feature never actually worked end-to-end (messageIndex was
+      // hard-coded to -1 because no caller populated it, and the LLM
+      // had no way to navigate by message number anyway).
+      if (e.skeleton) what.push('skeleton');
+      if (e.ranges instanceof Map && e.ranges.size > 0) {
+        // Show up to 3 most-recent range labels for compactness.
+        const rks = Array.from(e.ranges.keys()).slice(-3);
+        what.push(`ranges=[${rks.join(', ')}]`);
+      }
+      const flag = e.dirty ? ' STALE' : ' current';
+      lines.push(`- \`${file}\`  ${what.join('  ')} ${flag}`);
+    }
+    return lines.join('\n');
+  }
+
+  /**
+   * Drop all knowledge for one agent — called when agent is deleted or reset.
+   */
+  clear(agentId) {
+    this._byAgent.delete(agentId);
+  }
+
+  /** For tests: reset everything. */
+  _resetForTests() {
+    this._byAgent.clear();
+  }
+}
+
+/** Singleton accessor. Mirrors the pattern used by memoryService. */
+export function getCodebaseKnowledgeService(logger = null) {
+  if (!_singleton) _singleton = new CodebaseKnowledgeService(logger);
+  return _singleton;
+}
+
+/** For tests: replace the singleton. */
+export function _resetCodebaseKnowledgeServiceForTests() {
+  _singleton = null;
+}
+
+export default CodebaseKnowledgeService;