@maxgfr/codeindex 2.17.1 → 2.18.0

package/src/scan.ts

@@ -1,6 +1,6 @@
 import { basename } from "node:path";
-import type { FileRecord } from "./types.js";
-import { walk, readText, type WalkResult } from "./walk.js";
+import type { FileRecord, FileKind } from "./types.js";
+import { walk, readText, type WalkResult, type WalkedFile } from "./walk.js";
 import { headCommit } from "./git.js";
 import { sha1 } from "./hash.js";
 import { classify, MARKDOWN_EXT } from "./classify.js";
@@ -73,6 +73,18 @@ export interface ScanOptions {
   // from the tree. Exists for callers that already walked (e.g. a freshness
   // probe) so scanRepo does not pay a second full directory traversal.
   precomputedWalk?: WalkResult;
+  // Records a worker pool already extracted (rel → record + the stat the worker
+  // itself observed). Consulted after the cache fastpaths and before the read,
+  // and only when the worker's stat matches the walk's. Records are built by the
+  // shared buildCodeRecord, so a hit is byte-identical to extracting here.
+  // Populated by scanRepoParallel — never set this by hand.
+  extracted?: Map<string, ExtractedRecord>;
+}
+
+export interface ExtractedRecord {
+  size: number;
+  mtimeMs: number;
+  record: FileRecord;
 }
 
 function countLines(s: string): number {
@@ -82,9 +94,54 @@ function countLines(s: string): number {
   return n;
 }
 
-// Walk the repo once and turn every in-scope file into a FileRecord. Pure file
-// I/O + deterministic extraction — never reads the repo into the model.
-export function scanRepo(root: string, opts: ScanOptions = {}): RepoScan {
+// The FileRecord for ONE code file. Factored out because the worker pool builds
+// records off the main thread and they must be identical to the sequential
+// ones, byte for byte — sharing this function is what guarantees that rather
+// than two copies of the same logic drifting apart.
+export function buildCodeRecord(
+  rel: string,
+  ext: string,
+  size: number,
+  content: string,
+  hash: string,
+  lang: string,
+  opts: { maxCallsPerFile?: number } = {},
+): FileRecord {
+  const record: FileRecord = {
+    rel,
+    ext,
+    size,
+    lines: countLines(content),
+    hash,
+    kind: "code",
+    lang,
+    headings: [],
+    symbols: [],
+    refs: [],
+  };
+  if (content) {
+    const code = extractCode(rel, ext, content, { maxCallsPerFile: opts.maxCallsPerFile });
+    record.title = basename(rel);
+    record.summary = code.summary;
+    record.symbols = code.symbols;
+    record.refs = code.refs;
+    record.pkg = code.pkg;
+    record.idents = code.idents;
+    record.calls = code.calls;
+    record.importedNames = code.importedNames;
+  } else {
+    record.title = basename(rel);
+  }
+  return record;
+}
+
+// Which walked files this scan keeps, and how each is labelled. Shared by
+// scanRepo and scanSummary so the two can never disagree on a file count or a
+// language histogram — the summary path is exactly this loop, stopped early.
+function* keptFiles(
+  root: string,
+  opts: ScanOptions,
+): Generator<{ f: WalkedFile; kind: FileKind; lang: string }, WalkTotals, void> {
   const scoped = opts.scope ? [...(opts.include ?? []), `${opts.scope.replace(/\/+$/, "")}/**`] : opts.include;
   const include = compileGlobs(scoped);
   const exclude = compileGlobs(opts.exclude);
@@ -100,6 +157,61 @@ export function scanRepo(root: string, opts: ScanOptions = {}): RepoScan {
   // would describe the encyclopedia instead of the code.
   const outPrefix = opts.out ? opts.out.replace(/\/+$/, "") + "/" : null;
 
+  for (const f of walked) {
+    if (outPrefix && (f.abs === opts.out || f.abs.startsWith(outPrefix))) continue;
+    if (include && !include(f.rel)) continue;
+    if (exclude && exclude(f.rel)) continue;
+    yield { f, kind: classify(f.rel, f.ext), lang: extToLang(f.ext) };
+  }
+  return { capped, excluded };
+}
+
+interface WalkTotals {
+  capped: boolean;
+  excluded: number;
+}
+
+// The code files this scan would extract, in the same order scanRepo sees them.
+// The worker pool builds its job list from here so the set it extracts is
+// exactly the set scanRepo would have extracted — no file gets a worker record
+// that the sequential loop would have skipped, and vice versa.
+export function keptCodeFiles(root: string, opts: ScanOptions = {}): { f: WalkedFile; lang: string }[] {
+  const out: { f: WalkedFile; lang: string }[] = [];
+  const it = keptFiles(root, opts);
+  for (let step = it.next(); !step.done; step = it.next()) {
+    if (step.value.kind === "code") out.push({ f: step.value.f, lang: step.value.lang });
+  }
+  return out;
+}
+
+// File count + language histogram WITHOUT reading or parsing a single file.
+// `codeindex scan` and the MCP `scan_summary` tool only ever report these, but
+// used to pay a full scanRepo — i.e. tree-sitter over the whole repo — to get
+// them (6.5s on a 7k-file repo, versus ~0.1s here).
+export interface ScanSummary {
+  root: string;
+  commit?: string;
+  fileCount: number;
+  languages: Record<string, number>;
+  capped: boolean;
+  excluded: number;
+}
+
+export function scanSummary(root: string, opts: ScanOptions = {}): ScanSummary {
+  const languages: Record<string, number> = {};
+  let fileCount = 0;
+  const it = keptFiles(root, opts);
+  let step = it.next();
+  for (; !step.done; step = it.next()) {
+    languages[step.value.lang] = (languages[step.value.lang] ?? 0) + 1;
+    fileCount++;
+  }
+  return { root, commit: headCommit(root), fileCount, languages, capped: step.value.capped, excluded: step.value.excluded };
+}
+
+// Walk the repo once and turn every in-scope file into a FileRecord. Pure file
+// I/O + deterministic extraction — never reads the repo into the model.
+export function scanRepo(root: string, opts: ScanOptions = {}): RepoScan {
   const files: FileRecord[] = [];
   const languages: Record<string, number> = {};
   const docText = new Map<string, string>();
@@ -114,13 +226,10 @@ export function scanRepo(root: string, opts: ScanOptions = {}): RepoScan {
   // With no cache, persisting one is all new bytes — dirty by definition.
   let cacheDirty = cache === undefined;
 
-  for (const f of walked) {
-    if (outPrefix && (f.abs === opts.out || f.abs.startsWith(outPrefix))) continue;
-    if (include && !include(f.rel)) continue;
-    if (exclude && exclude(f.rel)) continue;
-
-    const kind = classify(f.rel, f.ext);
-    const lang = extToLang(f.ext);
+  const it = keptFiles(root, opts);
+  let step = it.next();
+  for (; !step.done; step = it.next()) {
+    const { f, kind, lang } = step.value;
     languages[lang] = (languages[lang] ?? 0) + 1;
     mtimes.set(f.rel, f.mtimeMs); // persist the fastpath key for the next build
 
@@ -147,12 +256,23 @@ export function scanRepo(root: string, opts: ScanOptions = {}): RepoScan {
       continue;
     }
 
+    // A record a worker already read, hashed and extracted for this exact file
+    // (see pool.ts). Accepted only when the worker's own stat agrees with the
+    // walk's, so a file rewritten between the two is re-read here instead.
+    //
+    // It supplies the HASH as well as the record, which is what lets the cache
+    // comparison below stay in its original position. Consulting it any earlier
+    // would skip the hash-hit branch, and a bare `touch` — same content, new
+    // mtime — would then report the scan as changed and rewrite every artifact.
+    const pre = opts.extracted?.get(f.rel);
+    const preUsable = pre && pre.size === f.size && pre.mtimeMs === f.mtimeMs ? pre : undefined;
+
     // Read + hash (the staleness oracle stays exact); only EXTRACTION is cached. A
     // hash hit reuses the previous record — content is byte-identical, so every
     // derived field is too. classify()/extToLang() depend only on the path, so
     // kind/lang are stable across the hit.
-    const content = readText(f.abs);
-    const hash = sha1(content);
+    const content = preUsable ? undefined : readText(f.abs);
+    const hash = preUsable ? preUsable.record.hash : sha1(content!);
     if (cached && cached.hash === hash) {
       files.push(cached.record);
       if (kind === "doc" && content) docText.set(f.rel, content);
@@ -167,11 +287,16 @@ export function scanRepo(root: string, opts: ScanOptions = {}): RepoScan {
     allReused = false;
     cacheDirty = true;
 
+    if (preUsable) {
+      files.push(preUsable.record);
+      continue;
+    }
+
     const record: FileRecord = {
       rel: f.rel,
       ext: f.ext,
       size: f.size,
-      lines: countLines(content),
+      lines: countLines(content!),
       hash,
       kind,
       lang,
@@ -230,8 +355,8 @@ export function scanRepo(root: string, opts: ScanOptions = {}): RepoScan {
     languages,
     docText,
     mtimes,
-    capped,
-    excluded,
+    capped: step.value.capped,
+    excluded: step.value.excluded,
     contentUnchanged: allReused,
     cacheDirty,
   };

package/src/traverse.ts

@@ -0,0 +1,164 @@
+// Graph traversal over a built link-graph: "what depends on this" (impact) and
+// "what sits next to this" (neighbors). Both are pure functions of a `Graph` —
+// no scan, no filesystem — so a consumer holding a persisted graph.json can
+// answer them without re-walking the repo.
+//
+// Both share one hub gate. A hyper-connected node (a barrel, a types module)
+// otherwise drags the entire graph into any depth-≥2 neighbourhood, which is
+// the difference between an answer and a dump.
+import type { Edge, Graph } from "./types.js";
+import { byStr } from "./sort.js";
+
+// Only these edge kinds carry a real "depends on" relation. A doc-link or a
+// mention says something references the name, not that it would break.
+const DEPENDS_KINDS = new Set(["import", "use", "call"]);
+
+// The hub-gating threshold over a degree distribution: max(50, p99). p99 = the
+// degree at index min(n-1, floor(0.99n)) of the ASCENDING degree array (numeric
+// sort — deterministic without byStr).
+//
+// Deliberately DEGREE-based even though the graph carries pagerank/betweenness:
+// the gate exists to bound traversal fan-out, and degree IS the fan-out cost —
+// a high-pagerank/low-degree node is safe to expand through. It is also
+// computed per call, so a kind-filtered view gates on that same subgraph rather
+// than on a whole-graph metric that does not describe it. The 50 floor makes it
+// a no-op on small graphs (never worse than ungated).
+export function hubThreshold(degrees: number[]): number {
+  const sorted = degrees.slice().sort((a, b) => a - b);
+  const n = sorted.length;
+  const p99 = n === 0 ? 0 : sorted[Math.min(n - 1, Math.floor(0.99 * n))]!;
+  return Math.max(50, p99);
+}
+
+export interface ImpactedFile {
+  rel: string;
+  module: string;
+  depth: number; // hops from the target (1 = direct dependent)
+}
+
+export interface ImpactResult {
+  target: string;
+  scope: "module" | "file";
+  seeds: string[]; // the files whose dependents we traced
+  files: ImpactedFile[]; // transitive dependents, nearest first
+  modules: string[]; // distinct modules touched
+}
+
+// Reverse dependency closure: every file that transitively IMPORTS, USES, or
+// CALLS one of `seeds`, out to `depth` hops (default: the full closure).
+export function reverseClosure(edges: Edge[], seeds: string[], depth = Infinity): Map<string, number> {
+  const dependents = new Map<string, Edge[]>(); // target file → incoming depends-on edges
+  for (const e of edges) {
+    if (e.dangling || !DEPENDS_KINDS.has(e.kind)) continue;
+    let arr = dependents.get(e.to);
+    if (!arr) dependents.set(e.to, (arr = []));
+    arr.push(e);
+  }
+  const depthOf = new Map<string, number>();
+  const seen = new Set<string>(seeds);
+  let frontier = [...seeds];
+  for (let d = 1; d <= depth && frontier.length; d++) {
+    const next: string[] = [];
+    for (const node of frontier) {
+      for (const e of (dependents.get(node) ?? []).slice().sort((a, b) => byStr(a.from, b.from))) {
+        if (seen.has(e.from)) continue;
+        seen.add(e.from);
+        depthOf.set(e.from, d);
+        next.push(e.from);
+      }
+    }
+    frontier = next;
+  }
+  return depthOf;
+}
+
+// "What breaks if I change this." Accepts a module slug or a file rel.
+export function impactOf(graph: Graph, target: string, depth = Infinity): ImpactResult | undefined {
+  const moduleOf = new Map(graph.files.map((f) => [f.rel, f.module]));
+  const mod = graph.modules.find((m) => m.slug === target);
+  const file = mod ? undefined : graph.files.find((f) => f.rel === target);
+  if (!mod && !file) return undefined;
+
+  const seeds = mod ? mod.members : [file!.rel];
+  const depthOf = reverseClosure(graph.fileEdges, seeds, depth);
+  const files: ImpactedFile[] = [...depthOf.entries()]
+    .map(([rel, d]) => ({ rel, module: moduleOf.get(rel) ?? "root", depth: d }))
+    .sort((a, b) => a.depth - b.depth || byStr(a.rel, b.rel));
+  const modules = [...new Set(files.map((f) => f.module).filter((m) => m !== target))].sort(byStr);
+
+  return { target, scope: mod ? "module" : "file", seeds, files, modules };
+}
+
+export interface NeighborLink {
+  node: string;
+  direction: "out" | "in";
+  kind: string;
+  weight: number;
+  depth: number;
+  confidence?: "extracted" | "inferred"; // only on a `call` edge — see Edge.confidence
+}
+
+export interface NeighborResult {
+  target: string;
+  scope: "module" | "file";
+  links: NeighborLink[];
+  members?: string[]; // for a module target
+}
+
+// Breadth-first walk from `start`, out to `depth` hops, in BOTH directions.
+// With `kinds` set, only those edge kinds are traversed — and the degree
+// distribution feeding the hub gate is measured over that same filtered
+// subgraph, so the gate reflects the view the caller asked for.
+function bfs(edges: Edge[], start: string, depth: number, kinds?: Set<string>): NeighborLink[] {
+  const out = new Map<string, Edge[]>();
+  const inn = new Map<string, Edge[]>();
+  const degree = new Map<string, number>();
+  for (const e of edges) {
+    if (e.dangling) continue;
+    if (kinds && !kinds.has(e.kind)) continue;
+    (out.get(e.from) ?? out.set(e.from, []).get(e.from)!).push(e);
+    (inn.get(e.to) ?? inn.set(e.to, []).get(e.to)!).push(e);
+    degree.set(e.from, (degree.get(e.from) ?? 0) + 1);
+    degree.set(e.to, (degree.get(e.to) ?? 0) + 1);
+  }
+  // A non-start node at or above the threshold is EMITTED as a link but never
+  // expanded THROUGH. Only bites at depth ≥ 2 — depth-1 links all come from
+  // `start`, which always expands.
+  const threshold = hubThreshold([...degree.values()]);
+  const seen = new Set<string>([start]);
+  const links: NeighborLink[] = [];
+  let frontier = [start];
+  for (let d = 1; d <= depth; d++) {
+    const next: string[] = [];
+    for (const node of frontier) {
+      if (node !== start && (degree.get(node) ?? 0) >= threshold) continue;
+      for (const e of (out.get(node) ?? []).slice().sort((a, b) => byStr(a.to, b.to))) {
+        if (seen.has(e.to)) continue;
+        links.push({ node: e.to, direction: "out", kind: e.kind, weight: e.weight, depth: d, confidence: e.confidence });
+        seen.add(e.to);
+        next.push(e.to);
+      }
+      for (const e of (inn.get(node) ?? []).slice().sort((a, b) => byStr(a.from, b.from))) {
+        if (seen.has(e.from)) continue;
+        links.push({ node: e.from, direction: "in", kind: e.kind, weight: e.weight, depth: d, confidence: e.confidence });
+        seen.add(e.from);
+        next.push(e.from);
+      }
+    }
+    frontier = next;
+  }
+  return links;
+}
+
+// What links to / from a module slug or a file rel, out to `depth` hops.
+export function neighborsOf(graph: Graph, target: string, depth = 1, kinds?: Set<string>): NeighborResult | undefined {
+  const mod = graph.modules.find((m) => m.slug === target);
+  if (mod) {
+    return { target, scope: "module", links: bfs(graph.moduleEdges, target, depth, kinds), members: mod.members };
+  }
+  const file = graph.files.find((f) => f.rel === target);
+  if (file) {
+    return { target, scope: "file", links: bfs(graph.fileEdges, target, depth, kinds) };
+  }
+  return undefined;
+}

package/src/types.ts

@@ -1,7 +1,7 @@
 // Single source of truth for the engine version the bundle reports. Kept in
 // lockstep with package.json by the release pipeline. Do not edit by hand
 // outside a release.
-export const ENGINE_VERSION = "2.17.1";
+export const ENGINE_VERSION = "2.18.0";
 
 // Bumped whenever the on-disk artifact shape changes, so a consumer can reject
 // an index written by an incompatible engine instead of misreading it. The

package/src/viz.ts

@@ -1,7 +1,7 @@
 // Mermaid rendering of the module graph — inline-renderable in Claude, GitHub
 // and most markdown viewers with zero infrastructure (the counter to graph-DB
 // browsers). Deterministic: sorted nodes/edges, stable ids.
-import type { Graph } from "./types.js";
+import type { Graph, ModuleNode, Tier } from "./types.js";
 import { byStr } from "./sort.js";
 
 export interface MermaidOptions {
@@ -42,3 +42,93 @@ export function renderMermaid(graph: Graph, opts: MermaidOptions = {}): string {
   if (dropped) lines.push(`  %% ${dropped} lighter edges omitted (maxEdges=${maxEdges})`);
   return lines.join("\n") + "\n";
 }
+
+// ---------------------------------------------------------------------------
+// Clustered whole-graph view. `renderMermaid` above answers "what sits around
+// THIS module" and returns raw mermaid; this one answers "what does the whole
+// repo look like" — grouped into tier subgraphs, capped by MODULE count as well
+// as edge count, fenced, and reporting exactly what it dropped.
+//
+// Two functions rather than one flag because the truncation contract differs:
+// the neighborhood view caps edges only and notes the drop in a comment, while
+// a whole-graph view has to bound nodes first (a 400-module repo is unreadable
+// long before it hits 80 edges) and hand the caller the counts so it can say so
+// in prose. Returning counts from the existing function would have been a
+// breaking change for its CLI and MCP callers.
+// ---------------------------------------------------------------------------
+
+export interface ClusteredMermaidResult {
+  content: string; // a fenced ```mermaid block, ready to embed in markdown
+  shownModules: number;
+  totalModules: number;
+  shownEdges: number;
+  totalEdges: number;
+}
+
+export interface ClusteredMermaidOptions {
+  maxModules?: number; // default 40 — most-connected first
+  maxEdges?: number; // default 80 — heaviest first
+  title?: string; // leading %% comment; defaults to "module graph"
+}
+
+const TIER_LABEL: Record<Tier, string> = { 0: "Foundations", 1: "Features", 2: "Tail" };
+const CLUSTER_MAX_MODULES = 40;
+const CLUSTER_MAX_EDGES = 80;
+
+const degreeOf = (m: ModuleNode): number => m.degIn + m.degOut;
+
+// Mermaid node ids must be identifier-safe; slugs may contain dashes. Prefixed
+// so a slug starting with a digit still yields a valid id.
+function clusterNodeId(slug: string): string {
+  return "m_" + slug.replace(/[^A-Za-z0-9_]/g, "_");
+}
+
+export function renderMermaidClustered(
+  graph: Graph,
+  opts: ClusteredMermaidOptions = {},
+): ClusteredMermaidResult {
+  const maxModules = opts.maxModules ?? CLUSTER_MAX_MODULES;
+  const maxEdges = opts.maxEdges ?? CLUSTER_MAX_EDGES;
+
+  const ranked = graph.modules.slice().sort((a, b) => degreeOf(b) - degreeOf(a) || byStr(a.slug, b.slug));
+  const shown = ranked.slice(0, maxModules);
+  const shownSet = new Set(shown.map((m) => m.slug));
+
+  // Only edges whose BOTH ends survived the module cap — a dangling half-edge
+  // would render as a node the caller was told was dropped.
+  const edges = graph.moduleEdges
+    .filter((e) => shownSet.has(e.from) && shownSet.has(e.to))
+    .sort((a, b) => b.weight - a.weight || byStr(a.from, b.from) || byStr(a.to, b.to))
+    .slice(0, maxEdges);
+
+  const lines: string[] = [];
+  lines.push(
+    `%% ${opts.title ?? "module graph"} — ${shown.length} of ${graph.modules.length} modules, ` +
+      `${edges.length} of ${graph.moduleEdges.length} edges`,
+  );
+  if (shown.length < graph.modules.length || edges.length < graph.moduleEdges.length) {
+    lines.push("%% truncated to the most-connected modules/edges; see graph.json for the full graph");
+  }
+  lines.push("flowchart LR");
+
+  for (const tier of [0, 1, 2] as Tier[]) {
+    const inTier = shown.filter((m) => m.tier === tier);
+    if (!inTier.length) continue;
+    lines.push(`  subgraph ${TIER_LABEL[tier]}`);
+    for (const m of inTier) lines.push(`    ${clusterNodeId(m.slug)}["${m.path.replace(/"/g, "'")}"]`);
+    lines.push("  end");
+  }
+
+  for (const e of edges) {
+    const label = e.weight > 1 ? `|${e.weight}| ` : "";
+    lines.push(`  ${clusterNodeId(e.from)} -->${label ? " " + label : " "}${clusterNodeId(e.to)}`);
+  }
+
+  return {
+    content: "```mermaid\n" + lines.join("\n") + "\n```\n",
+    shownModules: shown.length,
+    totalModules: graph.modules.length,
+    shownEdges: edges.length,
+    totalEdges: graph.moduleEdges.length,
+  };
+}