@maxgfr/codeindex 2.14.0 → 2.15.0

package/docs/MIGRATION.md

@@ -220,6 +220,58 @@ surface is additive (semver-minor).
   `resolveGrammarsPullTarget`, `fetchGrammarsTarball`, `fetchExpectedSha256`,
   `extractGrammarsTarball`, `GrammarsPullTarget`.
 
+## v2.15.0 — MCP serves from a persisted index (additive, no re-pin required)
+
+Pure fastpath, MCP-only: on the **first** tool call for a repo the long-lived
+server seeds its session from a committed `.codeindex/` index (written by
+`codeindex index`) instead of doing every step cold. Served tool responses stay
+**byte-identical** to a cold-process build on the same repo state, so **no
+re-pin is required** and no consumer needs to act. `SCHEMA_VERSION` /
+`EMBED_VERSION` / `EXTRACTOR_VERSION` are untouched, the public API is unchanged
+(**no new exports** — the preload is entirely internal to `mcp.ts`), and a repo
+with no `.codeindex/` behaves exactly as before.
+
+- **Scan seed from `cache.json`.** `getScan`'s first touch reads
+  `.codeindex/cache.json` and re-expresses its per-file records as the session
+  `cache`, so scan.ts's stat fastpath / exact content-hash reuse rebuilds the
+  `RepoScan` value-identically to a cold scan (the T3/T4 determinism the CLI's
+  `index` fastpath already relies on) without a read + hash + extraction per
+  unchanged file. Records are only trusted when the cache's
+  `(schemaVersion, extractorVersion)` match this engine — the same gate the CLI
+  applies — otherwise the whole cache is discarded and the scan runs cold. A
+  file whose content drifted since `cache.json` is re-read/extracted here exactly
+  as a cold scan would, so the scan stays correct and only the derived freshness
+  flags differ (they never feed artifacts).
+- **Artifact preload from `graph.json` / `symbols.json` — gated by the T4
+  oracle.** Only when the **exact** T4 freshness guard holds does the session
+  deserialize the on-disk artifacts straight in, skipping the whole downstream
+  pipeline (`buildArtifactsFromScan`) for the first
+  `graph`/`symbols`/`mermaid`/`repo_map`/`check_rules` call. The guard is the one
+  the CLI `index` fastpath uses to prove the on-disk bytes equal a fresh build:
+  `scan.contentUnchanged` **and** `cache.json`'s `engineVersion === ENGINE_VERSION`
+  **and** its `commit === scan.commit` **and**
+  `sha1(graph.json) === meta.graphSha1` **and**
+  `sha1(symbols.json) === meta.symbolsSha1`. When it holds, the on-disk
+  `graph.json`/`symbols.json` are byte-equal to `buildArtifactsFromScan(scan)` run
+  here, so deserializing them equals rebuilding.
+- **Deserialize is `JSON.parse`, not a new codec — no new exports.** `Graph` and
+  `SymbolIndex` are pure JSON POJOs (no `Map`/`Set`/typed fields), so
+  `JSON.parse` is a lossless round-trip and a `schemaVersion` assert is the only
+  reconstruction needed. `renderGraphJson` re-sorts `graph.languages` anyway, so
+  render→parse→render reproduces the same bytes (numbers reproduce their
+  shortest-round-trip form, absent optionals stay absent, V8's integer-key
+  hoisting is identical). No (de)serializer was added to the barrel; the preload
+  helpers are private to `mcp.ts`.
+- **Fallback is today's build-on-demand path, EXACTLY — never a throw.** No
+  `.codeindex/`, a `(schemaVersion, extractorVersion)` mismatch, a stale scan, a
+  version/commit/sha mismatch, or a missing/corrupt/partial/tampered
+  `graph.json`/`symbols.json` each returns `undefined` and falls through to a
+  fresh `scanRepo` / `buildArtifactsFromScan`. So the preload self-heals on
+  corruption and a deleted or truncated artifact never crashes the session — the
+  worst case is the cold cost the server paid before this release. The embed
+  sidecar keeps its own memoization path (the graph/symbols shas are all the
+  guard checks).
+
 ## Typical mapping (what to replace with what)
 
 What a consumer usually deletes from its own codebase, and the engine export

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@maxgfr/codeindex",
-  "version": "2.14.0",
+  "version": "2.15.0",
   "description": "Self-contained, deterministic repo-indexing engine: walk + language detection + symbol/import extraction (tree-sitter AST with regex fallback) + import resolution + typed cross-file link-graph + analytics. Ships as a single zero-dependency engine.mjs that consumer tools vendor.",
   "type": "module",
   "packageManager": "pnpm@10.33.0",

package/scripts/engine.d.mts

@@ -1,4 +1,4 @@
-declare const ENGINE_VERSION = "2.14.0";
+declare const ENGINE_VERSION = "2.15.0";
 declare const SCHEMA_VERSION = 4;
 declare const EXTRACTOR_VERSION = 10;
 type FileKind = "code" | "doc" | "config" | "asset" | "other";
@@ -255,7 +255,7 @@ declare function extractAst(rel: string, ext: string, content: string, opts?: {
     maxCalls?: number;
 }): AstResult | undefined;
 
-declare const DEFAULT_GRAMMARS_URL = "https://github.com/maxgfr/codeindex/releases/download/v2.14.0/grammars-2.14.0.tar.gz";
+declare const DEFAULT_GRAMMARS_URL = "https://github.com/maxgfr/codeindex/releases/download/v2.15.0/grammars-2.15.0.tar.gz";
 interface GrammarsPullTarget {
     url: string;
     sha256Url?: string;

package/scripts/engine.mjs

@@ -14,7 +14,7 @@ var ENGINE_VERSION, SCHEMA_VERSION, EXTRACTOR_VERSION;
 var init_types = __esm({
   "src/types.ts"() {
     "use strict";
-    ENGINE_VERSION = "2.14.0";
+    ENGINE_VERSION = "2.15.0";
     SCHEMA_VERSION = 4;
     EXTRACTOR_VERSION = 10;
   }
@@ -10303,7 +10303,7 @@ __export(mcp_exports, {
   toCacheMap: () => toCacheMap,
   warmGrammarsForRepo: () => warmGrammarsForRepo
 });
-import { statSync as statSync4 } from "fs";
+import { readFileSync as readFileSync6, statSync as statSync4 } from "fs";
 import { join as join14 } from "path";
 import { createInterface } from "readline";
 function str(v) {
@@ -10357,6 +10357,57 @@ function toCacheMap(scan2) {
   for (const f of scan2.files) m.set(f.rel, { hash: f.hash, record: f, size: f.size, mtimeMs: scan2.mtimes.get(f.rel) });
   return m;
 }
+function readPersistedIndex(repo) {
+  let parsed;
+  try {
+    parsed = JSON.parse(readFileSync6(join14(repo, ".codeindex", "cache.json"), "utf8"));
+  } catch {
+    return void 0;
+  }
+  if (!parsed || parsed.schemaVersion !== SCHEMA_VERSION || parsed.extractorVersion !== EXTRACTOR_VERSION || !parsed.files) {
+    return void 0;
+  }
+  const cacheMap = new Map(Object.entries(parsed.files));
+  const meta = {
+    engineVersion: parsed.engineVersion,
+    commit: parsed.commit,
+    graphSha1: parsed.graphSha1,
+    symbolsSha1: parsed.symbolsSha1
+  };
+  return { cacheMap, meta };
+}
+function preloadArtifacts(repo, scan2, meta) {
+  if (!scan2.contentUnchanged || meta.engineVersion !== ENGINE_VERSION || meta.commit !== scan2.commit || meta.graphSha1 === void 0 || meta.symbolsSha1 === void 0) {
+    return void 0;
+  }
+  const dir = join14(repo, ".codeindex");
+  let graphBytes;
+  let symbolsBytes;
+  try {
+    graphBytes = readFileSync6(join14(dir, "graph.json"));
+    symbolsBytes = readFileSync6(join14(dir, "symbols.json"));
+  } catch {
+    return void 0;
+  }
+  if (sha1(graphBytes) !== meta.graphSha1 || sha1(symbolsBytes) !== meta.symbolsSha1) {
+    return void 0;
+  }
+  try {
+    const graph = JSON.parse(graphBytes.toString("utf8"));
+    const symbols = JSON.parse(symbolsBytes.toString("utf8"));
+    if (graph.schemaVersion !== SCHEMA_VERSION || symbols.schemaVersion !== SCHEMA_VERSION) return void 0;
+    return { scan: scan2, graph, symbols };
+  } catch {
+    return void 0;
+  }
+}
+function preloadSession(repo, opts) {
+  const persisted = readPersistedIndex(repo);
+  if (!persisted) return void 0;
+  const scan2 = scanRepo(repo, { ...opts, cache: persisted.cacheMap });
+  const arts = preloadArtifacts(repo, scan2, persisted.meta);
+  return { scan: scan2, cacheMap: toCacheMap(scan2), arts };
+}
 function getScan(repo, opts = {}) {
   const key = sessionKey(repo, opts);
   if (sessionCache && sessionCache.key === key) {
@@ -10369,6 +10420,11 @@ function getScan(repo, opts = {}) {
     sessionCache = { key, scan: fresh, cacheMap: toCacheMap(fresh) };
     return fresh;
   }
+  const preloaded = preloadSession(repo, opts);
+  if (preloaded) {
+    sessionCache = { key, scan: preloaded.scan, cacheMap: preloaded.cacheMap, arts: preloaded.arts };
+    return preloaded.scan;
+  }
   const scan2 = scanRepo(repo, opts);
   sessionCache = { key, scan: scan2, cacheMap: toCacheMap(scan2) };
   return scan2;
@@ -11450,7 +11506,7 @@ init_util();
 init_types();
 init_types();
 init_loader();
-import { existsSync as existsSync4, mkdirSync as mkdirSync3, mkdtempSync, readFileSync as readFileSync6, renameSync, rmSync as rmSync2, writeFileSync as writeFileSync4 } from "fs";
+import { existsSync as existsSync4, mkdirSync as mkdirSync3, mkdtempSync, readFileSync as readFileSync7, renameSync, rmSync as rmSync2, writeFileSync as writeFileSync4 } from "fs";
 import { dirname as dirname4, join as join15, resolve as resolve2 } from "path";
 init_pipeline();
 init_hash();
@@ -11651,7 +11707,7 @@ async function runCli(argv) {
     let cache;
     let meta = {};
     try {
-      const parsed = JSON.parse(readFileSync6(cachePath, "utf8"));
+      const parsed = JSON.parse(readFileSync7(cachePath, "utf8"));
       if (parsed.schemaVersion === SCHEMA_VERSION && parsed.extractorVersion === EXTRACTOR_VERSION) {
         cache = new Map(Object.entries(parsed.files));
         meta = {
@@ -11672,7 +11728,7 @@ async function runCli(argv) {
     const embedPath = join15(outDir, "embeddings.bin");
     const artifactSha = (path) => {
       try {
-        return sha1(readFileSync6(path));
+        return sha1(readFileSync7(path));
       } catch {
         return void 0;
       }
@@ -11922,7 +11978,7 @@ async function runCli(argv) {
       if (existsSync4(runtime) && expected && existsSync4(markerPath)) {
         let marker = "";
         try {
-          marker = readFileSync6(markerPath, "utf8").trim();
+          marker = readFileSync7(markerPath, "utf8").trim();
         } catch {
         }
         if (marker === expected) {
@@ -11975,7 +12031,7 @@ async function runCli(argv) {
     }
   } else if (cmd === "rules") {
     if (!flags2.config) throw new Error("rules needs --config <codeindex.rules.json>");
-    const rules = parseRules(JSON.parse(readFileSync6(flags2.config, "utf8")));
+    const rules = parseRules(JSON.parse(readFileSync7(flags2.config, "utf8")));
     const { graph } = buildIndexArtifacts(flags2.repo, scanOptions(flags2, precomputedWalk));
     const violations = checkRules(graph, rules);
     const errors = violations.filter((v) => v.severity === "error").length;

package/src/mcp.ts

@@ -5,10 +5,10 @@
 // `repo` path and returns JSON text content.
 //
 // Register in an MCP client as:  node scripts/engine.mjs mcp
-import { statSync } from "node:fs";
+import { readFileSync, statSync } from "node:fs";
 import { join } from "node:path";
 import { createInterface } from "node:readline";
-import { ENGINE_VERSION, type FileRecord } from "./types.js";
+import { ENGINE_VERSION, SCHEMA_VERSION, EXTRACTOR_VERSION, type FileRecord, type Graph, type SymbolIndex } from "./types.js";
 import { ensureGrammars, grammarKeysForExts } from "./ast/loader.js";
 import { buildArtifactsFromScan, type IndexArtifacts } from "./pipeline.js";
 import { renderGraphJson } from "./render/graph-json.js";
@@ -419,7 +419,8 @@ export function memoizedEmbedModel(modelDir: string): StaticEmbedModel | undefin
 // and a caller-supplied stale walk would desynchronize the freshness oracle.
 export type SessionScanOptions = Omit<ScanOptions, "cache" | "precomputedWalk">;
 
-type SessionCacheMap = Map<string, { hash: string; record: FileRecord; size?: number; mtimeMs?: number }>;
+type SessionCacheEntry = { hash: string; record: FileRecord; size?: number; mtimeMs?: number };
+type SessionCacheMap = Map<string, SessionCacheEntry>;
 
 // A SINGLE entry — never an unbounded map — holding the most recent scan.
 let sessionCache:
@@ -457,6 +458,128 @@ export function toCacheMap(scan: RepoScan): SessionCacheMap {
   return m;
 }
 
+// --- persisted-index preload -------------------------------------------------
+// On the FIRST tool call for a repo, a committed .codeindex/ index (written by
+// `codeindex index`) lets the session skip work TWO ways. cache.json seeds the
+// session scan, so every unchanged file takes scan.ts's stat fastpath instead
+// of a read + hash + extraction; and — only when the T4 freshness guard holds —
+// the persisted graph.json/symbols.json are deserialized straight into the
+// session, so the first graph/symbols/mermaid/repo_map/check_rules call skips
+// the whole downstream pipeline (buildArtifactsFromScan). Both are pure
+// optimizations: the seeded scan's reused records are value-identical to a cold
+// scan's (T3/T4 determinism), and the guard is the SAME oracle the CLI's index
+// fastpath uses to prove the on-disk artifacts equal a fresh build here. Absent,
+// stale, corrupt, or any version/commit/sha mismatch → every step falls back to
+// today's cold path EXACTLY (a fresh scanRepo / buildArtifactsFromScan), never a
+// throw.
+
+// ADDITIVE cache.json meta describing the artifacts a prior `index` run wrote
+// (see engine-cli.ts's CacheMeta). Old caches lacking these keys simply never
+// pass the guard below — their per-file records are still reused to seed the
+// scan. Only the graph/symbols shas matter here; the embed sidecar has its own
+// memoization path.
+interface PersistedMeta {
+  engineVersion?: string;
+  commit?: string;
+  graphSha1?: string;
+  symbolsSha1?: string;
+}
+
+// Read <repo>/.codeindex/cache.json into the (cacheMap, meta) the preload needs.
+// Per-file records are reusable ONLY when (schemaVersion, extractorVersion)
+// match this engine — the exact gate the CLI applies before trusting a cache —
+// otherwise the whole cache is discarded (cold scan). Any read/parse failure (no
+// index yet, unreadable, malformed) returns undefined: the cold path.
+function readPersistedIndex(repo: string): { cacheMap: SessionCacheMap; meta: PersistedMeta } | undefined {
+  let parsed:
+    | ({ schemaVersion?: number; extractorVersion?: number; files?: Record<string, SessionCacheEntry> } & PersistedMeta)
+    | undefined;
+  try {
+    parsed = JSON.parse(readFileSync(join(repo, ".codeindex", "cache.json"), "utf8")) as typeof parsed;
+  } catch {
+    return undefined;
+  }
+  if (!parsed || parsed.schemaVersion !== SCHEMA_VERSION || parsed.extractorVersion !== EXTRACTOR_VERSION || !parsed.files) {
+    return undefined;
+  }
+  const cacheMap: SessionCacheMap = new Map(Object.entries(parsed.files));
+  const meta: PersistedMeta = {
+    engineVersion: parsed.engineVersion,
+    commit: parsed.commit,
+    graphSha1: parsed.graphSha1,
+    symbolsSha1: parsed.symbolsSha1,
+  };
+  return { cacheMap, meta };
+}
+
+// The T4 freshness guard, applied to a session scan seeded from cache.json:
+// contentUnchanged proves this scan's records are the ones that built the
+// on-disk artifacts; engineVersion pins the version stamp graph.json embeds and
+// commit the HEAD it embeds; the sha checks prove the on-disk bytes ARE that
+// build's output. All true ⇒ graph.json/symbols.json are byte-equal to
+// buildArtifactsFromScan(scan) run here, so deserialize them instead of
+// rebuilding. Graph/SymbolIndex are pure JSON POJOs (no Map/Set/typed fields),
+// so JSON.parse is a lossless round-trip — a schemaVersion assert is the only
+// reconstruction needed (see the round-trip test). ANY failure — a stale scan,
+// a version/commit/sha mismatch, a missing/corrupt/partial artifact, an
+// unexpected schemaVersion — returns undefined so the caller rebuilds. NEVER
+// throws (a corrupt artifact must degrade, not crash the session).
+function preloadArtifacts(repo: string, scan: RepoScan, meta: PersistedMeta): IndexArtifacts | undefined {
+  if (
+    !scan.contentUnchanged ||
+    meta.engineVersion !== ENGINE_VERSION ||
+    meta.commit !== scan.commit ||
+    meta.graphSha1 === undefined ||
+    meta.symbolsSha1 === undefined
+  ) {
+    return undefined;
+  }
+  const dir = join(repo, ".codeindex");
+  let graphBytes: Buffer;
+  let symbolsBytes: Buffer;
+  try {
+    graphBytes = readFileSync(join(dir, "graph.json"));
+    symbolsBytes = readFileSync(join(dir, "symbols.json"));
+  } catch {
+    return undefined; // a sha'd artifact went missing since cache.json — rebuild
+  }
+  // sha over the raw bytes; sha1(string) hashes the same UTF-8 bytes writeFileSync
+  // put on disk, so this equals the meta sha the CLI computed over the render.
+  if (sha1(graphBytes) !== meta.graphSha1 || sha1(symbolsBytes) !== meta.symbolsSha1) {
+    return undefined; // tampered / partial / corrupt on-disk bytes — rebuild
+  }
+  try {
+    const graph = JSON.parse(graphBytes.toString("utf8")) as Graph;
+    const symbols = JSON.parse(symbolsBytes.toString("utf8")) as SymbolIndex;
+    if (graph.schemaVersion !== SCHEMA_VERSION || symbols.schemaVersion !== SCHEMA_VERSION) return undefined;
+    return { scan, graph, symbols };
+  } catch {
+    // Unreachable once the shas matched (the bytes are valid JSON this engine
+    // wrote), but the contract is "never throw" — degrade to a rebuild.
+    return undefined;
+  }
+}
+
+// First-touch preload: seed the session scan from cache.json and, when the guard
+// holds, the artifacts from graph.json/symbols.json. undefined ⇒ no persisted
+// index ⇒ the caller takes the cold scanRepo path unchanged.
+function preloadSession(
+  repo: string,
+  opts: SessionScanOptions,
+): { scan: RepoScan; cacheMap: SessionCacheMap; arts?: IndexArtifacts } | undefined {
+  const persisted = readPersistedIndex(repo);
+  if (!persisted) return undefined;
+  // Seed the scan from the persisted records — scan.ts's stat fastpath + exact
+  // content-hash reuse make this value-identical to a cold scan (T3/T4), only
+  // cheaper, and it computes the contentUnchanged the artifact guard reads. When
+  // the on-disk content drifted from cache.json, changed files are re-read/
+  // extracted here exactly as a cold scan would, so the scan stays correct and
+  // the guard simply fails (arts undefined → rebuild on demand).
+  const scan = scanRepo(repo, { ...opts, cache: persisted.cacheMap });
+  const arts = preloadArtifacts(repo, scan, persisted.meta);
+  return { scan, cacheMap: toCacheMap(scan), arts };
+}
+
 // The memoizing replacement for scanRepo inside callTool. Exported for tests.
 export function getScan(repo: string, opts: SessionScanOptions = {}): RepoScan {
   const key = sessionKey(repo, opts);
@@ -486,6 +609,15 @@ export function getScan(repo: string, opts: SessionScanOptions = {}): RepoScan {
     sessionCache = { key, scan: fresh, cacheMap: toCacheMap(fresh) };
     return fresh;
   }
+  // First touch of this (repo, opts): try the persisted-index preload before a
+  // cold scan. A present, version-compatible .codeindex/cache.json seeds the
+  // scan (and, when the guard holds, the artifacts); absent it, fall through to
+  // the cold path EXACTLY as before.
+  const preloaded = preloadSession(repo, opts);
+  if (preloaded) {
+    sessionCache = { key, scan: preloaded.scan, cacheMap: preloaded.cacheMap, arts: preloaded.arts };
+    return preloaded.scan;
+  }
   const scan = scanRepo(repo, opts);
   sessionCache = { key, scan, cacheMap: toCacheMap(scan) };
   return scan;

package/src/types.ts

@@ -1,7 +1,7 @@
 // Single source of truth for the engine version the bundle reports. Kept in
 // lockstep with package.json by the release pipeline. Do not edit by hand
 // outside a release.
-export const ENGINE_VERSION = "2.14.0";
+export const ENGINE_VERSION = "2.15.0";
 
 // Bumped whenever the on-disk artifact shape changes, so a consumer can reject
 // an index written by an incompatible engine instead of misreading it. The