@maxgfr/codeindex 2.17.1 → 2.18.0

package/src/mcp.ts

@@ -1,20 +1,27 @@
 // MCP (Model Context Protocol) server over stdio — hand-rolled JSON-RPC 2.0 so
 // the engine stays zero-dependency. Newline-delimited JSON messages, protocol
 // 2024-11-05 (compatible with later revisions' initialize handshake). Exposes
-// the engine's read-only indexing capabilities as MCP tools; every tool takes a
-// `repo` path and returns JSON text content.
+// the engine's indexing capabilities as MCP tools; every tool takes a `repo`
+// path and returns text content — JSON, except repo_map, mermaid and
+// read_memory, which return their own formats.
 //
-// Register in an MCP client as:  node scripts/engine.mjs mcp
-import { readFileSync, statSync } from "node:fs";
-import { join } from "node:path";
+// Register in an MCP client as:  codeindex mcp
+// (NOT `node scripts/engine.mjs mcp`: engine.mjs is a side-effect-free library
+// with no main-module guard — see src/engine.ts — so that command does nothing.
+// The entrypoint is the `codeindex` bin, i.e. scripts/cli.mjs.)
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { isAbsolute, join } from "node:path";
+import { pathToFileURL } from "node:url";
 import { createInterface } from "node:readline";
-import { ENGINE_VERSION, SCHEMA_VERSION, EXTRACTOR_VERSION, type FileRecord, type Graph, type SymbolIndex } from "./types.js";
+import { ENGINE_VERSION } from "./types.js";
 import { ensureGrammars, grammarKeysForExts } from "./ast/loader.js";
 import { buildArtifactsFromScan, type IndexArtifacts } from "./pipeline.js";
 import { renderGraphJson } from "./render/graph-json.js";
-import { scanRepo, type RepoScan, type ScanOptions } from "./scan.js";
-import { walk } from "./walk.js";
+import { scanRepo, scanSummary, type RepoScan, type ScanOptions, type ScanSummary } from "./scan.js";
+import { preloadSession, toCacheMap, INDEX_DIR, type PersistedCacheEntry, type PersistedCacheMap } from "./preload.js";
+import { walk, type WalkResult } from "./walk.js";
 import { buildCallerIndex } from "./callers.js";
+import { callerIndexFor } from "./derived.js";
 import { detectWorkspaces } from "./workspaces.js";
 import { gitChurn } from "./git.js";
 import { grepRepo } from "./grep.js";
@@ -77,7 +84,15 @@ const TOOLS = [
       "Who calls a function? Per-symbol caller index: each defined symbol with the exact (file, line) call sites that bind to it. Omit `name` for the full index.",
     inputSchema: {
       type: "object",
-      properties: { ...repoProp, name: { type: "string", description: "Symbol name to look up" } },
+      properties: {
+        ...repoProp,
+        name: { type: "string", description: "Symbol name to look up" },
+        recall: {
+          type: "boolean",
+          description:
+            "Recall-oriented binding: relax the JS/TS import gate to unique repo-wide names, labelling each site corroborated|unique-name (default false = precision)",
+        },
+      },
       required: ["repo"],
     },
   },
@@ -117,6 +132,7 @@ const TOOLS = [
         namePath: { type: "string", description: "Symbol name or Parent/child path" },
         substring: { type: "boolean" },
         includeBody: { type: "boolean" },
+        maxResults: { type: "number", description: "Cap matches (default 50)" },
       },
       required: ["repo", "namePath"],
     },
@@ -232,8 +248,16 @@ const TOOLS = [
   {
     name: "dead_code",
     description:
-      "Dead-code candidates in two labeled tiers: 'unreferenced' (no call site binds AND nothing references the name) and 'uncalled' (referenced somewhere — re-export, type position — but never called). Exported symbols only; test files and entrypoint-looking files excluded as roots.",
-    inputSchema: { type: "object", properties: { ...repoProp, ...scopeProps }, required: ["repo"] },
+      "Dead-code candidates in two labeled tiers: 'unreferenced' (no call site binds AND nothing references the name) and 'uncalled' (referenced somewhere — re-export, type position — but never called). Exported symbols only; test files and entrypoint-looking files excluded as roots. On a large repo this list runs to thousands of entries — pass `limit`, or `scope` to one subdirectory.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        ...repoProp,
+        ...scopeProps,
+        limit: { type: "number", description: "Cap entries (default: all)" },
+      },
+      required: ["repo"],
+    },
   },
   {
     name: "complexity",
@@ -264,6 +288,7 @@ const TOOLS = [
       properties: {
         ...repoProp,
         pattern: { type: "string", description: "Regular expression to search for" },
+        scope: { type: "string", description: "Restrict to one directory (repo-relative)" },
         globs: { type: "array", items: { type: "string" }, description: "Restrict to matching paths" },
         ignoreCase: { type: "boolean" },
         maxHits: { type: "number" },
@@ -312,8 +337,13 @@ const TOOLS = [
         ...repoProp,
         ...scopeProps,
         rules: { type: "array", description: "Rules array (inline JSON — see description)" },
+        configPath: {
+          type: "string",
+          description:
+            "Read the rules from this JSON file instead (repo-relative or absolute) — the CLI's --config. Ignored when `rules` is given.",
+        },
       },
-      required: ["repo", "rules"],
+      required: ["repo"],
     },
   },
 ] as const;
@@ -324,6 +354,13 @@ function str(v: unknown): string | undefined {
 function strArray(v: unknown): string[] | undefined {
   return Array.isArray(v) && v.every((x) => typeof x === "string") && v.length ? (v as string[]) : undefined;
 }
+// A positive numeric argument. Also accepts the numeric STRING a JSON-Schema-less
+// client may send: `"50"` used to fall through to the default in silence, which
+// reads to the caller as the option being ignored.
+function num(v: unknown): number | undefined {
+  const n = typeof v === "number" ? v : typeof v === "string" && v.trim() !== "" ? Number(v) : NaN;
+  return Number.isFinite(n) && n > 0 ? n : undefined;
+}
 function errMessage(e: unknown): string {
   return e instanceof Error ? e.message : String(e);
 }
@@ -419,13 +456,48 @@ export function memoizedEmbedModel(modelDir: string): StaticEmbedModel | undefin
 // and a caller-supplied stale walk would desynchronize the freshness oracle.
 export type SessionScanOptions = Omit<ScanOptions, "cache" | "precomputedWalk">;
 
-type SessionCacheEntry = { hash: string; record: FileRecord; size?: number; mtimeMs?: number };
-type SessionCacheMap = Map<string, SessionCacheEntry>;
+type SessionCacheEntry = PersistedCacheEntry;
+type SessionCacheMap = PersistedCacheMap;
 
-// A SINGLE entry — never an unbounded map — holding the most recent scan.
-let sessionCache:
-  | { key: string; scan: RepoScan; cacheMap: SessionCacheMap; arts?: IndexArtifacts }
-  | undefined;
+interface SessionEntry {
+  key: string;
+  scan: RepoScan;
+  cacheMap: SessionCacheMap;
+  arts?: IndexArtifacts;
+}
+
+// A SMALL bounded LRU — never an unbounded map.
+//
+// This was a single entry, which made two entirely normal agent behaviours
+// pathological: alternating between two repos, and alternating between two
+// `scope` values on one repo. Either one evicted the other on every call, so
+// every call paid a full cold rebuild. Four entries covers those patterns while
+// keeping the memory story the same order of magnitude as before.
+const SESSION_CACHE_MAX = 4;
+const sessionCaches: SessionEntry[] = []; // most-recently-used first
+
+function sessionGet(key: string): SessionEntry | undefined {
+  const i = sessionCaches.findIndex((e) => e.key === key);
+  if (i < 0) return undefined;
+  const [entry] = sessionCaches.splice(i, 1);
+  sessionCaches.unshift(entry!);
+  return entry;
+}
+
+function sessionPut(entry: SessionEntry): SessionEntry {
+  const i = sessionCaches.findIndex((e) => e.key === entry.key);
+  if (i >= 0) sessionCaches.splice(i, 1);
+  sessionCaches.unshift(entry);
+  sessionCaches.length = Math.min(sessionCaches.length, SESSION_CACHE_MAX);
+  return entry;
+}
+
+// Drop every entry. Used by the symbolic-edit tools: an edit landing in the same
+// mtime tick with the same byte count would pass the (size, mtimeMs) fastpath
+// and serve a stale scan.
+function sessionClear(): void {
+  sessionCaches.length = 0;
+}
 
 // Fixed property order (and JSON.stringify dropping undefined) keeps the key
 // deterministic regardless of how the caller assembled the options object.
@@ -448,149 +520,22 @@ function sessionKey(repo: string, opts: SessionScanOptions): string {
   );
 }
 
-// A scan re-expressed as the `ScanOptions.cache` shape (the exact map the CLI
-// persists as cache.json): rel → (hash, record, size, mtimeMs), so the next
-// scanRepo can take the stat fastpath / hash-match reuse paths against it.
-// Exported for tests.
-export function toCacheMap(scan: RepoScan): SessionCacheMap {
-  const m: SessionCacheMap = new Map();
-  for (const f of scan.files) m.set(f.rel, { hash: f.hash, record: f, size: f.size, mtimeMs: scan.mtimes.get(f.rel) });
-  return m;
-}
-
-// --- persisted-index preload -------------------------------------------------
-// On the FIRST tool call for a repo, a committed .codeindex/ index (written by
-// `codeindex index`) lets the session skip work TWO ways. cache.json seeds the
-// session scan, so every unchanged file takes scan.ts's stat fastpath instead
-// of a read + hash + extraction; and — only when the T4 freshness guard holds —
-// the persisted graph.json/symbols.json are deserialized straight into the
-// session, so the first graph/symbols/mermaid/repo_map/check_rules call skips
-// the whole downstream pipeline (buildArtifactsFromScan). Both are pure
-// optimizations: the seeded scan's reused records are value-identical to a cold
-// scan's (T3/T4 determinism), and the guard is the SAME oracle the CLI's index
-// fastpath uses to prove the on-disk artifacts equal a fresh build here. Absent,
-// stale, corrupt, or any version/commit/sha mismatch → every step falls back to
-// today's cold path EXACTLY (a fresh scanRepo / buildArtifactsFromScan), never a
-// throw.
-
-// ADDITIVE cache.json meta describing the artifacts a prior `index` run wrote
-// (see engine-cli.ts's CacheMeta). Old caches lacking these keys simply never
-// pass the guard below — their per-file records are still reused to seed the
-// scan. Only the graph/symbols shas matter here; the embed sidecar has its own
-// memoization path.
-interface PersistedMeta {
-  engineVersion?: string;
-  commit?: string;
-  graphSha1?: string;
-  symbolsSha1?: string;
-}
-
-// Read <repo>/.codeindex/cache.json into the (cacheMap, meta) the preload needs.
-// Per-file records are reusable ONLY when (schemaVersion, extractorVersion)
-// match this engine — the exact gate the CLI applies before trusting a cache —
-// otherwise the whole cache is discarded (cold scan). Any read/parse failure (no
-// index yet, unreadable, malformed) returns undefined: the cold path.
-function readPersistedIndex(repo: string): { cacheMap: SessionCacheMap; meta: PersistedMeta } | undefined {
-  let parsed:
-    | ({ schemaVersion?: number; extractorVersion?: number; files?: Record<string, SessionCacheEntry> } & PersistedMeta)
-    | undefined;
-  try {
-    parsed = JSON.parse(readFileSync(join(repo, ".codeindex", "cache.json"), "utf8")) as typeof parsed;
-  } catch {
-    return undefined;
-  }
-  if (!parsed || parsed.schemaVersion !== SCHEMA_VERSION || parsed.extractorVersion !== EXTRACTOR_VERSION || !parsed.files) {
-    return undefined;
-  }
-  const cacheMap: SessionCacheMap = new Map(Object.entries(parsed.files));
-  const meta: PersistedMeta = {
-    engineVersion: parsed.engineVersion,
-    commit: parsed.commit,
-    graphSha1: parsed.graphSha1,
-    symbolsSha1: parsed.symbolsSha1,
-  };
-  return { cacheMap, meta };
-}
-
-// The T4 freshness guard, applied to a session scan seeded from cache.json:
-// contentUnchanged proves this scan's records are the ones that built the
-// on-disk artifacts; engineVersion pins the version stamp graph.json embeds and
-// commit the HEAD it embeds; the sha checks prove the on-disk bytes ARE that
-// build's output. All true ⇒ graph.json/symbols.json are byte-equal to
-// buildArtifactsFromScan(scan) run here, so deserialize them instead of
-// rebuilding. Graph/SymbolIndex are pure JSON POJOs (no Map/Set/typed fields),
-// so JSON.parse is a lossless round-trip — a schemaVersion assert is the only
-// reconstruction needed (see the round-trip test). ANY failure — a stale scan,
-// a version/commit/sha mismatch, a missing/corrupt/partial artifact, an
-// unexpected schemaVersion — returns undefined so the caller rebuilds. NEVER
-// throws (a corrupt artifact must degrade, not crash the session).
-function preloadArtifacts(repo: string, scan: RepoScan, meta: PersistedMeta): IndexArtifacts | undefined {
-  if (
-    !scan.contentUnchanged ||
-    meta.engineVersion !== ENGINE_VERSION ||
-    meta.commit !== scan.commit ||
-    meta.graphSha1 === undefined ||
-    meta.symbolsSha1 === undefined
-  ) {
-    return undefined;
-  }
-  const dir = join(repo, ".codeindex");
-  let graphBytes: Buffer;
-  let symbolsBytes: Buffer;
-  try {
-    graphBytes = readFileSync(join(dir, "graph.json"));
-    symbolsBytes = readFileSync(join(dir, "symbols.json"));
-  } catch {
-    return undefined; // a sha'd artifact went missing since cache.json — rebuild
-  }
-  // sha over the raw bytes; sha1(string) hashes the same UTF-8 bytes writeFileSync
-  // put on disk, so this equals the meta sha the CLI computed over the render.
-  if (sha1(graphBytes) !== meta.graphSha1 || sha1(symbolsBytes) !== meta.symbolsSha1) {
-    return undefined; // tampered / partial / corrupt on-disk bytes — rebuild
-  }
-  try {
-    const graph = JSON.parse(graphBytes.toString("utf8")) as Graph;
-    const symbols = JSON.parse(symbolsBytes.toString("utf8")) as SymbolIndex;
-    if (graph.schemaVersion !== SCHEMA_VERSION || symbols.schemaVersion !== SCHEMA_VERSION) return undefined;
-    return { scan, graph, symbols };
-  } catch {
-    // Unreachable once the shas matched (the bytes are valid JSON this engine
-    // wrote), but the contract is "never throw" — degrade to a rebuild.
-    return undefined;
-  }
-}
-
-// First-touch preload: seed the session scan from cache.json and, when the guard
-// holds, the artifacts from graph.json/symbols.json. undefined ⇒ no persisted
-// index ⇒ the caller takes the cold scanRepo path unchanged.
-function preloadSession(
-  repo: string,
-  opts: SessionScanOptions,
-): { scan: RepoScan; cacheMap: SessionCacheMap; arts?: IndexArtifacts } | undefined {
-  const persisted = readPersistedIndex(repo);
-  if (!persisted) return undefined;
-  // Seed the scan from the persisted records — scan.ts's stat fastpath + exact
-  // content-hash reuse make this value-identical to a cold scan (T3/T4), only
-  // cheaper, and it computes the contentUnchanged the artifact guard reads. When
-  // the on-disk content drifted from cache.json, changed files are re-read/
-  // extracted here exactly as a cold scan would, so the scan stays correct and
-  // the guard simply fails (arts undefined → rebuild on demand).
-  const scan = scanRepo(repo, { ...opts, cache: persisted.cacheMap });
-  const arts = preloadArtifacts(repo, scan, persisted.meta);
-  return { scan, cacheMap: toCacheMap(scan), arts };
-}
+// Re-exported for tests and for consumers that imported it from here before the
+// preload machinery moved into src/preload.ts.
+export { toCacheMap };
 
 // The memoizing replacement for scanRepo inside callTool. Exported for tests.
-export function getScan(repo: string, opts: SessionScanOptions = {}): RepoScan {
+export function getScan(repo: string, opts: SessionScanOptions = {}, walked?: WalkResult): RepoScan {
   const key = sessionKey(repo, opts);
-  if (sessionCache && sessionCache.key === key) {
-    const fresh = scanRepo(repo, { ...opts, cache: sessionCache.cacheMap });
+  const hit = sessionGet(key);
+  if (hit) {
+    const fresh = scanRepo(repo, { ...opts, cache: hit.cacheMap, precomputedWalk: walked });
     if (fresh.contentUnchanged) {
       // Content proven identical → return the SAME object (object identity is
       // what keeps derived.ts's WeakMap and the memoized artifacts warm). A
       // stat-only drift (e.g. a bare touch) still refreshes the cache map so
       // the next call's stat fastpath keys on the new (size, mtimeMs).
-      if (fresh.cacheDirty) sessionCache.cacheMap = toCacheMap(fresh);
+      if (fresh.cacheDirty) hit.cacheMap = toCacheMap(fresh);
       // `commit` (headCommit(root)) is NOT part of the stat/hash freshness
       // oracle: a git HEAD move that leaves the worktree untouched — commit /
       // commit --amend / reset --soft / checkout to an identical-tree branch —
@@ -603,35 +548,57 @@ export function getScan(repo: string, opts: SessionScanOptions = {}): RepoScan {
       // derived.ts WeakMap key on. Safe: no artifact carries commit (graph /
       // symbols render byte-identically regardless), so nothing memoized here
       // depends on this field.
-      if (sessionCache.scan.commit !== fresh.commit) sessionCache.scan.commit = fresh.commit;
-      return sessionCache.scan;
+      if (hit.scan.commit !== fresh.commit) hit.scan.commit = fresh.commit;
+      return hit.scan;
     }
-    sessionCache = { key, scan: fresh, cacheMap: toCacheMap(fresh) };
+    sessionPut({ key, scan: fresh, cacheMap: toCacheMap(fresh) });
     return fresh;
   }
   // First touch of this (repo, opts): try the persisted-index preload before a
   // cold scan. A present, version-compatible .codeindex/cache.json seeds the
   // scan (and, when the guard holds, the artifacts); absent it, fall through to
   // the cold path EXACTLY as before.
-  const preloaded = preloadSession(repo, opts);
+  const preloaded = preloadSession(repo, { ...opts, precomputedWalk: walked });
   if (preloaded) {
-    sessionCache = { key, scan: preloaded.scan, cacheMap: preloaded.cacheMap, arts: preloaded.arts };
+    sessionPut({ key, scan: preloaded.scan, cacheMap: preloaded.cacheMap, arts: preloaded.arts });
     return preloaded.scan;
   }
-  const scan = scanRepo(repo, opts);
-  sessionCache = { key, scan, cacheMap: toCacheMap(scan) };
+  const scan = scanRepo(repo, { ...opts, precomputedWalk: walked });
+  sessionPut({ key, scan, cacheMap: toCacheMap(scan) });
   return scan;
 }
 
+// The scan_summary numbers, without paying for a scan.
+//
+// A file count and a language histogram come from the walk plus the path-based
+// classifiers — no read, no hash, no tree-sitter. When this session already
+// holds a scan for the same (repo, opts) we derive from it instead (identical
+// numbers, and it keeps a warm session warm); otherwise scanSummary walks once.
+// The summary is NEVER written into the session cache: it carries no
+// FileRecords, so caching it would starve every record-shaped tool that ran next.
+export function getScanSummary(repo: string, opts: SessionScanOptions = {}, walked?: WalkResult): ScanSummary {
+  if (sessionCaches.some((e) => e.key === sessionKey(repo, opts))) {
+    const scan = getScan(repo, opts, walked);
+    return {
+      root: scan.root,
+      commit: scan.commit,
+      fileCount: scan.files.length,
+      languages: scan.languages,
+      capped: scan.capped,
+      excluded: scan.excluded,
+    };
+  }
+  return scanSummary(repo, { ...opts, precomputedWalk: walked });
+}
+
 // Lazy pipeline memoized on scan OBJECT IDENTITY: graph-shaped tools reuse the
 // artifacts exactly as long as getScan keeps returning the same scan object.
 // Exported for tests.
-export function getArtifacts(repo: string, opts: SessionScanOptions = {}): IndexArtifacts {
-  const scan = getScan(repo, opts);
-  if (sessionCache && sessionCache.scan === scan) {
-    return (sessionCache.arts ??= buildArtifactsFromScan(scan, opts));
-  }
-  // Defensive fallback (getScan always leaves sessionCache holding `scan`).
+export function getArtifacts(repo: string, opts: SessionScanOptions = {}, walked?: WalkResult): IndexArtifacts {
+  const scan = getScan(repo, opts, walked);
+  const entry = sessionCaches.find((e) => e.scan === scan);
+  if (entry) return (entry.arts ??= buildArtifactsFromScan(scan, opts));
+  // Defensive fallback (getScan always leaves an entry holding `scan`).
   return buildArtifactsFromScan(scan, opts);
 }
 
@@ -653,8 +620,19 @@ export function getArtifacts(repo: string, opts: SessionScanOptions = {}): Index
 // calls are independent, so warming fewer grammars cannot alter the parse of a
 // loaded one.
 export async function warmGrammarsForRepo(repo: string): Promise<void> {
-  const { files } = walk(repo, {});
-  await ensureGrammars(grammarKeysForExts(files.map((f) => f.ext)));
+  await warmGrammarsForWalk(walk(repo, {}));
+}
+
+// The same warm, against a walk the caller already has.
+//
+// callTool used to walk TWICE per scan-needing call: once here to derive the
+// present languages, then again inside scanRepo. On a large repo that fixed cost
+// dominated every response (the project's own benchmark shows find-symbol,
+// references and file-overview all landing within a few ms of each other on a
+// 20k-file monorepo — the signature of a per-call constant, not of the query).
+// One walk now feeds both.
+export async function warmGrammarsForWalk(walked: WalkResult): Promise<void> {
+  await ensureGrammars(grammarKeysForExts(walked.files.map((f) => f.ext)));
 }
 
 // Tools that never scan the file tree (git/grep/memory/embed-status only) — they
@@ -665,6 +643,10 @@ const SCANLESS_TOOLS = new Set([
   "workspaces", "churn", "coupling", "grep",
   "write_memory", "read_memory", "list_memories", "delete_memory",
   "embed_status",
+  // scan_summary counts and classifies by path only — it never parses, so the
+  // grammar warm (a whole extra walk) would be pure overhead. When a scan is
+  // already cached getScanSummary reuses it, warm grammars included.
+  "scan_summary",
 ]);
 
 async function callTool(name: string, args: Record<string, unknown>, defaultRepo?: string): Promise<string> {
@@ -676,21 +658,26 @@ async function callTool(name: string, args: Record<string, unknown>, defaultRepo
   const scanOpts = { scope: str(args.scope), include: strArray(args.include), exclude: strArray(args.exclude) };
   // Scan-needing tools warm the present-language grammars (re-derived per call)
   // before any scan so extraction takes the AST tier; scan-less tools skip it.
-  if (!SCANLESS_TOOLS.has(name)) await warmGrammarsForRepo(repo);
+  // ONE walk feeds both the warm and the scan below — see warmGrammarsForWalk.
+  let walked: WalkResult | undefined;
+  if (!SCANLESS_TOOLS.has(name)) {
+    walked = walk(repo, {});
+    await warmGrammarsForWalk(walked);
+  }
 
   if (name === "scan_summary") {
-    const scan = getScan(repo, scanOpts);
+    const s = getScanSummary(repo, scanOpts, walked);
     return JSON.stringify(
-      { engineVersion: ENGINE_VERSION, commit: scan.commit, fileCount: scan.files.length, languages: scan.languages, capped: scan.capped },
+      { engineVersion: ENGINE_VERSION, commit: s.commit, fileCount: s.fileCount, languages: s.languages, capped: s.capped },
       null,
       2,
     );
   }
   if (name === "graph") {
-    return renderGraphJson(getArtifacts(repo, scanOpts).graph);
+    return renderGraphJson(getArtifacts(repo, scanOpts, walked).graph);
   }
   if (name === "symbols") {
-    const { symbols } = getArtifacts(repo, scanOpts);
+    const { symbols } = getArtifacts(repo, scanOpts, walked);
     const lookup = str(args.name);
     if (lookup) {
       return JSON.stringify({ name: lookup, defs: symbols.defs[lookup] ?? [], refs: symbols.refs[lookup] ?? [] }, null, 2);
@@ -698,7 +685,13 @@ async function callTool(name: string, args: Record<string, unknown>, defaultRepo
     return JSON.stringify(symbols, null, 2);
   }
   if (name === "callers") {
-    const index = buildCallerIndex(getScan(repo, scanOpts));
+    // callerIndexFor, not buildCallerIndex: the public builder is unmemoized, so
+    // this rebuilt the whole index on EVERY request (318ms per call on a 20k-file
+    // repo in the project's own benchmark). The memoized one is keyed on scan
+    // object identity, which the session cache preserves across calls.
+    // Recall mode is option-dependent, so it cannot use the memoized index.
+    const scan = getScan(repo, scanOpts, walked);
+    const index = args.recall === true ? buildCallerIndex(scan, undefined, { recall: true }) : callerIndexFor(scan);
     const lookup = str(args.name);
     if (lookup) {
       const entry = index.get(lookup);
@@ -721,27 +714,28 @@ async function callTool(name: string, args: Record<string, unknown>, defaultRepo
   if (name === "symbols_overview") {
     const file = str(args.file);
     if (!file) throw new Error("`file` is required");
-    return JSON.stringify(symbolsOverview(getScan(repo, scanOpts), file), null, 2);
+    return JSON.stringify(symbolsOverview(getScan(repo, scanOpts, walked), file), null, 2);
   }
   if (name === "find_symbol") {
     const namePath = str(args.namePath);
     if (!namePath) throw new Error("`namePath` is required");
-    const matches = findSymbol(getScan(repo, scanOpts), namePath, {
+    const matches = findSymbol(getScan(repo, scanOpts, walked), namePath, {
       substring: args.substring === true,
       includeBody: args.includeBody === true,
+      maxResults: num(args.maxResults),
     });
     return JSON.stringify(matches, null, 2);
   }
   if (name === "find_references") {
     const symName = str(args.name);
     if (!symName) throw new Error("`name` is required");
-    return JSON.stringify(findReferences(getScan(repo, scanOpts), symName), null, 2);
+    return JSON.stringify(findReferences(getScan(repo, scanOpts, walked), symName), null, 2);
   }
   if (name === "replace_symbol_body" || name === "insert_after_symbol" || name === "insert_before_symbol") {
     const namePath = str(args.namePath);
     const body = typeof args.body === "string" ? args.body : undefined;
     if (!namePath || body === undefined) throw new Error("`namePath` and `body` are required");
-    const scan = getScan(repo, scanOpts);
+    const scan = getScan(repo, scanOpts, walked);
     const fn = name === "replace_symbol_body" ? replaceSymbolBody : name === "insert_after_symbol" ? insertAfterSymbol : insertBeforeSymbol;
     const result = fn(scan, namePath, body, str(args.file));
     // A write WE just performed must not be trusted to the stat oracle: an
@@ -750,7 +744,7 @@ async function callTool(name: string, args: Record<string, unknown>, defaultRepo
     // session entry unconditionally — the next call rescans from scratch.
     // (write_memory needs no invalidation: .codeindex/ is excluded from the
     // walk, so memories never enter a scan.)
-    sessionCache = undefined;
+    sessionClear();
     return JSON.stringify(result, null, 2);
   }
   if (name === "write_memory") {
@@ -775,26 +769,31 @@ async function callTool(name: string, args: Record<string, unknown>, defaultRepo
     return JSON.stringify({ deleted: deleteMemory(repo, memName) }, null, 2);
   }
   if (name === "dead_code") {
-    return JSON.stringify(findDeadCode(getScan(repo, scanOpts)), null, 2);
+    const all = findDeadCode(getScan(repo, scanOpts, walked));
+    const limit = num(args.limit);
+    // Additive: without `limit` the payload is exactly what it always was.
+    if (limit === undefined || all.length <= limit) return JSON.stringify(all, null, 2);
+    return JSON.stringify({ total: all.length, shown: limit, truncated: true, candidates: all.slice(0, limit) }, null, 2);
   }
   if (name === "complexity") {
-    const scan = getScan(repo, scanOpts);
+    const scan = getScan(repo, scanOpts, walked);
     if (args.risk === true) {
-      const { churn, ok } = gitChurn(repo);
-      return JSON.stringify({ churnOk: ok, risks: riskHotspots(scan, churn) }, null, 2);
+      // `since` was accepted by the CLI's `risk` but silently dropped here.
+      const { churn, ok } = gitChurn(repo, { since: str(args.since) });
+      return JSON.stringify({ churnOk: ok, risks: riskHotspots(scan, churn, num(args.top)) }, null, 2);
     }
-    return JSON.stringify(symbolComplexity(scan, str(args.file)), null, 2);
+    return JSON.stringify(symbolComplexity(scan, str(args.file), num(args.top)), null, 2);
   }
   if (name === "mermaid") {
-    const { graph } = getArtifacts(repo, scanOpts);
-    return renderMermaid(graph, { module: str(args.module) });
+    const { graph } = getArtifacts(repo, scanOpts, walked);
+    return renderMermaid(graph, { module: str(args.module), maxEdges: num(args.maxEdges) });
   }
   if (name === "repo_map") {
-    const { scan, graph } = getArtifacts(repo, scanOpts);
+    const { scan, graph } = getArtifacts(repo, scanOpts, walked);
     return renderRepoMap(scan, graph, { budgetTokens: typeof args.budgetTokens === "number" ? args.budgetTokens : undefined });
   }
   if (name === "hotspots") {
-    const scan = getScan(repo, scanOpts);
+    const scan = getScan(repo, scanOpts, walked);
     const { churn, ok } = gitChurn(repo, { since: str(args.since) });
     return JSON.stringify({ churnOk: ok, hotspots: rankHotspots(scan, churn) }, null, 2);
   }
@@ -805,8 +804,12 @@ async function callTool(name: string, args: Record<string, unknown>, defaultRepo
   if (name === "grep") {
     const pattern = str(args.pattern);
     if (!pattern) throw new Error("`pattern` is required");
+    // `scope` was CLI-only: the a205c34 fix folded it into the CLI's glob list
+    // but this handler ignored scanOpts entirely.
+    const scope = str(args.scope);
+    const globs = strArray(args.globs);
     const hits = grepRepo(repo, pattern, {
-      globs: strArray(args.globs),
+      globs: scope ? [...(globs ?? []), `${scope.replace(/\/+$/, "")}/**`] : globs,
       ignoreCase: args.ignoreCase === true,
       maxHits: typeof args.maxHits === "number" ? args.maxHits : undefined,
     });
@@ -815,7 +818,7 @@ async function callTool(name: string, args: Record<string, unknown>, defaultRepo
   if (name === "search") {
     const query = str(args.query);
     if (!query) throw new Error("`query` is required");
-    const scan = getScan(repo, scanOpts);
+    const scan = getScan(repo, scanOpts, walked);
     const limit = typeof args.limit === "number" ? args.limit : undefined;
     const fuzzy = typeof args.fuzzy === "boolean" ? args.fuzzy : undefined;
     if (args.semantic === true) {
@@ -882,33 +885,258 @@ async function callTool(name: string, args: Record<string, unknown>, defaultRepo
     return JSON.stringify(status, null, 2);
   }
   if (name === "check_rules") {
-    const rules = parseRules(args.rules); // throws a descriptive error on a malformed payload
-    const { graph } = getArtifacts(repo, scanOpts);
+    // Inline `rules` stays the primary form; `configPath` is the CLI's --config,
+    // which had no MCP equivalent, so a repo with a committed rules file had to
+    // have it re-pasted into every call.
+    const configPath = str(args.configPath);
+    let payload: unknown = args.rules;
+    if (payload === undefined && configPath) {
+      const abs = isAbsolute(configPath) ? configPath : join(repo, configPath);
+      try {
+        payload = JSON.parse(readFileSync(abs, "utf8"));
+      } catch (e) {
+        throw new Error(`cannot read rules from ${abs}: ${errMessage(e)}`);
+      }
+    }
+    if (payload === undefined) throw new Error("`rules` (or `configPath`) is required");
+    const rules = parseRules(payload); // throws a descriptive error on a malformed payload
+    const { graph } = getArtifacts(repo, scanOpts, walked);
     return JSON.stringify(checkRules(graph, rules), null, 2);
   }
   throw new Error(`unknown tool: ${name}`);
 }
 
-// The advertised tool list. Without a server-level repo pin this is TOOLS
-// verbatim (byte-compat for existing consumers). With one, every tool drops
-// `repo` from its `required` set and documents the default — so a client that
-// omits it is spec-correct rather than relying on the server being lenient.
-// The pin is reflected in the schema, not just honoured at call time.
-function toolsFor(defaultRepo?: string): readonly unknown[] {
-  if (!defaultRepo) return TOOLS;
+// --- protocol versions -------------------------------------------------------
+// The server announced "2024-11-05" hard-coded and never even read the version
+// the client asked for. Three revisions have shipped since.
+//
+// Negotiation is what makes moving forward non-breaking: a client that asks for
+// an old revision gets that revision, and every field introduced later is
+// withheld — so its responses are exactly the bytes it received before. Newer
+// clients opt themselves in simply by asking.
+//
+// Dates sort lexicographically, so `>=` on the strings is a version comparison.
+const PROTOCOL_VERSIONS = ["2024-11-05", "2025-03-26", "2025-06-18", "2025-11-25"] as const;
+const LATEST_PROTOCOL = PROTOCOL_VERSIONS[PROTOCOL_VERSIONS.length - 1]!;
+
+// Feature floors, by the revision that introduced them.
+const ANNOTATIONS_SINCE = "2025-03-26"; // tool behaviour hints
+const RICH_TOOLS_SINCE = "2025-06-18"; // Tool.title, resource_link content
+
+// Validate `arguments` against the tool's declared inputSchema.
+//
+// There was no validation at all beyond presence checks, and the readers failed
+// silently in both directions: str() returns undefined for a non-string, so a
+// number where a path belongs became "missing", and every boolean was `=== true`,
+// so `"false"` and `1` alike read as false. The caller saw its option ignored
+// with no way to tell why.
+//
+// Only the shapes these schemas actually use are checked (string / number /
+// boolean / array-of-string) — this is a guard against silent misreads, not a
+// JSON Schema implementation. The spec (2025-11-25) is explicit that input
+// validation failures belong in a Tool Execution Error, not a protocol error,
+// precisely so the model can read the message and retry.
+// Required-ness stays with callTool, which raises tool-specific messages
+// ("`rules` (or `configPath`) is required"); duplicating it here would only let
+// the two drift.
+export function validateArgs(
+  schema: { properties?: Record<string, unknown> },
+  args: Record<string, unknown>,
+): string | undefined {
+  const props = (schema.properties ?? {}) as Record<string, { type?: string; items?: { type?: string } }>;
+  for (const [key, value] of Object.entries(args)) {
+    if (value === undefined || value === null) continue;
+    const spec = props[key];
+    if (!spec?.type) continue; // undeclared extras stay tolerated
+    const actual = Array.isArray(value) ? "array" : typeof value;
+    if (spec.type === "number") {
+      // A numeric string is accepted (num() coerces it); anything else is not.
+      if (actual === "number") continue;
+      if (actual === "string" && Number.isFinite(Number(value as string)) && (value as string).trim() !== "") continue;
+      return `\`${key}\` must be a number, got ${actual === "string" ? JSON.stringify(value) : actual}`;
+    }
+    if (spec.type === "array") {
+      if (actual !== "array") return `\`${key}\` must be an array of strings, got ${actual}`;
+      if (spec.items?.type === "string" && !(value as unknown[]).every((x) => typeof x === "string")) {
+        return `\`${key}\` must be an array of strings`;
+      }
+      continue;
+    }
+    if (actual !== spec.type) return `\`${key}\` must be a ${spec.type}, got ${actual}`;
+  }
+  return undefined;
+}
+
+export function negotiateProtocol(requested: unknown): string {
+  return typeof requested === "string" && (PROTOCOL_VERSIONS as readonly string[]).includes(requested)
+    ? requested
+    : LATEST_PROTOCOL;
+}
+
+// Per-tool display title and behaviour hints.
+//
+// The hints matter operationally: they are what lets a host auto-approve the 23
+// read-only tools and hold a confirmation for the 5 that write. Without them a
+// client must treat `scan_summary` and `replace_symbol_body` alike.
+//
+// openWorldHint is true only where a call can leave this machine — `search`
+// with semantic:true and `embed_status` may contact CODEINDEX_EMBED_ENDPOINT.
+// Everything else reads the repo and nothing but the repo.
+interface ToolMeta {
+  title: string;
+  write?: boolean;
+  destructive?: boolean;
+  idempotent?: boolean;
+  openWorld?: boolean;
+}
+
+const TOOL_META: Record<string, ToolMeta> = {
+  scan_summary: { title: "Scan summary" },
+  graph: { title: "Link graph" },
+  symbols: { title: "Symbol index" },
+  callers: { title: "Caller index" },
+  workspaces: { title: "Monorepo workspaces" },
+  churn: { title: "Git churn" },
+  symbols_overview: { title: "File symbol overview" },
+  find_symbol: { title: "Find symbol" },
+  find_references: { title: "Find references" },
+  repo_map: { title: "Repository map" },
+  hotspots: { title: "Hotspots" },
+  coupling: { title: "Change coupling" },
+  replace_symbol_body: { title: "Replace symbol body", write: true, destructive: true, idempotent: true },
+  insert_after_symbol: { title: "Insert after symbol", write: true, destructive: false, idempotent: false },
+  insert_before_symbol: { title: "Insert before symbol", write: true, destructive: false, idempotent: false },
+  write_memory: { title: "Write memory", write: true, destructive: false, idempotent: true },
+  read_memory: { title: "Read memory" },
+  list_memories: { title: "List memories" },
+  delete_memory: { title: "Delete memory", write: true, destructive: true, idempotent: true },
+  dead_code: { title: "Dead-code candidates" },
+  complexity: { title: "Complexity" },
+  mermaid: { title: "Mermaid module diagram" },
+  grep: { title: "Grep file contents" },
+  search: { title: "Lexical search", openWorld: true },
+  embed_status: { title: "Embedding tier status", openWorld: true },
+  check_rules: { title: "Check architecture rules" },
+};
+
+function annotationsFor(name: string): Record<string, boolean> | undefined {
+  const meta = TOOL_META[name];
+  if (!meta) return undefined;
+  return {
+    readOnlyHint: !meta.write,
+    ...(meta.write ? { destructiveHint: meta.destructive === true, idempotentHint: meta.idempotent === true } : {}),
+    openWorldHint: meta.openWorld === true,
+  };
+}
+
+// The advertised tool list, for one negotiated protocol version.
+//
+// Without a server-level repo pin and on 2024-11-05 this is TOOLS verbatim —
+// byte-compat for every existing consumer. A pin drops `repo` from each
+// `required` set and documents the default, so a client that omits it is
+// spec-correct rather than relying on the server being lenient. Newer protocol
+// revisions additionally get `title` and `annotations`.
+function toolsFor(defaultRepo?: string, protocolVersion: string = PROTOCOL_VERSIONS[0]): readonly unknown[] {
+  const withAnnotations = protocolVersion >= ANNOTATIONS_SINCE;
+  const withTitle = protocolVersion >= RICH_TOOLS_SINCE;
+  if (!defaultRepo && !withAnnotations && !withTitle) return TOOLS;
   return TOOLS.map((t) => ({
     ...t,
-    inputSchema: {
-      ...t.inputSchema,
-      properties: {
-        ...t.inputSchema.properties,
-        repo: { type: "string", description: `Absolute path to the repository root (optional — defaults to ${defaultRepo})` },
-      },
-      required: (t.inputSchema.required as readonly string[]).filter((r) => r !== "repo"),
-    },
+    ...(withTitle && TOOL_META[t.name] ? { title: TOOL_META[t.name]!.title } : {}),
+    ...(withAnnotations ? { annotations: annotationsFor(t.name) } : {}),
+    inputSchema: !defaultRepo
+      ? t.inputSchema
+      : {
+          ...t.inputSchema,
+          properties: {
+            ...t.inputSchema.properties,
+            repo: {
+              type: "string",
+              description: `Absolute path to the repository root (optional — defaults to ${defaultRepo})`,
+            },
+          },
+          required: (t.inputSchema.required as readonly string[]).filter((r) => r !== "repo"),
+        },
   }));
 }
 
+// --- response size guard -----------------------------------------------------
+// Several tools returned unbounded payloads. On facebook/react (7091 files):
+// graph 9.4 MB, symbols 6.3 MB, callers 6.0 MB, dead_code 771 KB — roughly
+// 2.35M, 1.57M, 1.51M and 193k tokens. A single `graph` call does not merely
+// bloat an agent's context, it exceeds what any MCP client can accept, so the
+// call fails and the turn is wasted.
+//
+// The guard is deliberately NOT a default page size: below the limit a response
+// is byte-identical to what it always was. Above it, the response could not be
+// consumed by any client anyway, so replacing it with something actionable
+// cannot regress a working call — it converts a hard failure into a usable
+// answer that says how big the payload is, where the artifact already sits on
+// disk, and which narrower tool answers the question.
+export const DEFAULT_MAX_RESPONSE_BYTES = 1_000_000;
+
+// What to steer a caller toward when their whole-repo request is too large.
+const NARROWER: Record<string, string> = {
+  graph: "pass `scope` to a subdirectory, or use repo_map / mermaid for an overview",
+  symbols: "pass `name` to look up one symbol, or use find_symbol / symbols_overview",
+  callers: "pass `name` to look up one symbol's call sites",
+  dead_code: "pass `scope` to a subdirectory",
+  find_references: "the symbol is referenced very widely — narrow with `scope` on a graph query",
+  check_rules: "narrow the rule set, or pass `scope` to a subdirectory",
+};
+
+// The persisted artifact backing a tool, when a `codeindex index` already wrote
+// one — far more useful to hand back than a truncated blob.
+const ARTIFACT_FOR: Record<string, string> = { graph: "graph.json", symbols: "symbols.json" };
+
+export function capResponse(text: string, tool: string, repo: string, maxBytes: number): string {
+  const bytes = Buffer.byteLength(text, "utf8");
+  if (bytes <= maxBytes) return text;
+  const artifact = ARTIFACT_FOR[tool] ? join(repo, INDEX_DIR, ARTIFACT_FOR[tool]!) : undefined;
+  return (
+    JSON.stringify(
+      {
+        truncated: true,
+        tool,
+        bytes,
+        maxBytes,
+        reason:
+          "This response exceeds the configured limit and was withheld rather than sent as an unusable partial payload.",
+        narrower: NARROWER[tool] ?? "narrow the request with `scope`, `include`/`exclude`, or a `limit`",
+        ...(artifact && existsSync(artifact)
+          ? { artifact, artifactNote: "The full result is already on disk here — read it directly if you need all of it." }
+          : artifact
+            ? { artifactNote: `Run \`codeindex index --repo ${repo} --out ${join(repo, INDEX_DIR)}\` to get this as a file.` }
+            : {}),
+      },
+      null,
+      2,
+    ) + "\n"
+  );
+}
+
+// When capResponse withheld a payload AND the artifact is on disk, hand the
+// client a resource_link to it. Returns undefined for every normal response —
+// this only ever adds a second content block to a capped one.
+export function resourceLinkFor(text: string, tool: string): Record<string, unknown> | undefined {
+  const artifactName = ARTIFACT_FOR[tool];
+  if (!artifactName) return undefined;
+  let parsed: { truncated?: boolean; artifact?: string };
+  try {
+    parsed = JSON.parse(text) as typeof parsed;
+  } catch {
+    return undefined; // a normal (non-JSON, or non-capped) response
+  }
+  if (parsed.truncated !== true || typeof parsed.artifact !== "string") return undefined;
+  return {
+    type: "resource_link",
+    uri: pathToFileURL(parsed.artifact).href,
+    name: artifactName,
+    description: `The full ${tool} result this call was too large to inline.`,
+    mimeType: "application/json",
+  };
+}
+
 export interface McpServerOptions {
   // Override the serverInfo announced in the initialize response — for
   // downstream consumers embedding this server under their own identity.
@@ -919,6 +1147,9 @@ export interface McpServerOptions {
   // spawn one server per workspace — `codeindex mcp --repo <dir>` — instead of
   // requiring the agent to thread an absolute path through every single call.
   defaultRepo?: string;
+  // Cap on a single tool response, in bytes (default DEFAULT_MAX_RESPONSE_BYTES).
+  // Responses under it are untouched; see capResponse for what happens above it.
+  maxResponseBytes?: number;
 }
 
 export async function runMcpServer(opts: McpServerOptions = {}): Promise<void> {
@@ -926,8 +1157,13 @@ export async function runMcpServer(opts: McpServerOptions = {}): Promise<void> {
     name: opts.serverInfo?.name ?? "codeindex",
     version: opts.serverInfo?.version ?? ENGINE_VERSION,
   };
-  // Derived ONCE: the pin cannot change mid-session, so neither can the list.
-  const tools = toolsFor(opts.defaultRepo);
+  // The negotiated protocol version, settled by `initialize` and fixed for the
+  // session. Until a client says otherwise we assume the oldest revision, so a
+  // client that skips the handshake sees exactly the pre-negotiation server.
+  let protocolVersion: string = PROTOCOL_VERSIONS[0];
+  // Rebuilt when negotiation lands: the pin cannot change mid-session, but the
+  // fields we are allowed to advertise depend on the version.
+  let tools = toolsFor(opts.defaultRepo, protocolVersion);
   // No startup warm: each scan-needing tool warms the present-language grammars
   // for its repo before it runs (warmGrammarsForRepo re-derives them per call),
   // so a session that never scans — or only touches one language — loads no
@@ -959,10 +1195,12 @@ export async function runMcpServer(opts: McpServerOptions = {}): Promise<void> {
 
     try {
       if (req.method === "initialize") {
+        protocolVersion = negotiateProtocol(req.params?.protocolVersion);
+        tools = toolsFor(opts.defaultRepo, protocolVersion);
         send({
           id: req.id,
           result: {
-            protocolVersion: "2024-11-05",
+            protocolVersion,
             capabilities: { tools: {} },
             serverInfo,
           },
@@ -976,8 +1214,27 @@ export async function runMcpServer(opts: McpServerOptions = {}): Promise<void> {
         const name = str(params.name) ?? "";
         const args = (params.arguments ?? {}) as Record<string, unknown>;
         try {
-          const text = await callTool(name, args, opts.defaultRepo);
-          send({ id: req.id, result: { content: [{ type: "text", text }] } });
+          const decl = (tools as { name: string; inputSchema: { properties?: Record<string, unknown> } }[]).find(
+            (t) => t.name === name,
+          );
+          const invalid = decl ? validateArgs(decl.inputSchema, args) : undefined;
+          if (invalid) throw new Error(invalid);
+          const raw = await callTool(name, args, opts.defaultRepo);
+          const repo = str(args.repo) ?? opts.defaultRepo ?? "";
+          const text = capResponse(raw, name, repo, opts.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES);
+          // A capped whole-repo response points at an artifact already on disk.
+          // From 2025-06-18 the protocol has a content type that says exactly
+          // that, so the client can fetch the bytes instead of re-asking.
+          //
+          // Gated on `text !== raw` — i.e. capResponse actually replaced the
+          // payload. Otherwise a normal 900 KB graph would be JSON.parsed on
+          // every single call just to discover it was not truncated.
+          const link =
+            text !== raw && protocolVersion >= RICH_TOOLS_SINCE ? resourceLinkFor(text, name) : undefined;
+          send({
+            id: req.id,
+            result: { content: link ? [{ type: "text", text }, link] : [{ type: "text", text }] },
+          });
         } catch (e) {
           send({
             id: req.id,