@maxgfr/codeindex 2.7.0

package/src/ast/loader.ts

@@ -0,0 +1,104 @@
+import { readFileSync, existsSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { Parser, Language } from "web-tree-sitter";
+
+// Extension → committed grammar wasm key (scripts/grammars/<key>.wasm). Only the
+// languages we ship a grammar for appear here; everything else falls back to the
+// regex extractors (still fully searchable, just no AST-exact symbols/imports).
+export const EXT_GRAMMAR: Record<string, string> = {
+  ".ts": "typescript", ".mts": "typescript", ".cts": "typescript",
+  ".tsx": "tsx",
+  ".js": "javascript", ".jsx": "javascript", ".mjs": "javascript", ".cjs": "javascript",
+  ".py": "python", ".pyi": "python",
+  ".go": "go",
+  ".rs": "rust",
+  ".java": "java",
+  ".rb": "ruby", ".rake": "ruby",
+  ".c": "c", ".h": "c",
+  ".cc": "cpp", ".cpp": "cpp", ".cxx": "cpp", ".hpp": "cpp", ".hh": "cpp",
+  ".cs": "c_sharp",
+  ".php": "php",
+  ".scala": "scala", ".sc": "scala",
+  ".sh": "bash", ".bash": "bash",
+  ".lua": "lua",
+};
+
+export function grammarKeyForExt(ext: string): string | undefined {
+  return EXT_GRAMMAR[ext];
+}
+
+// Where the committed wasms live. Resolved relative to this module so the
+// SAME logic works whether we run from the tsup bundle (scripts/engine.mjs →
+// scripts/grammars), from a consumer's vendored copy (src/vendor →
+// ../../scripts/grammars), or from source under vitest (src/ast →
+// ../../scripts/grammars). CODEINDEX_GRAMMAR_DIR overrides for tests/tooling
+// (ULTRAINDEX_GRAMMAR_DIR kept as a legacy alias).
+function resolveGrammarDir(): string {
+  const env = process.env.CODEINDEX_GRAMMAR_DIR ?? process.env.ULTRAINDEX_GRAMMAR_DIR;
+  if (env && existsSync(env)) return env;
+  const here = dirname(fileURLToPath(import.meta.url));
+  const candidates = [
+    join(here, "grammars"), // bundle: <...>/scripts/grammars
+    join(here, "..", "..", "scripts", "grammars"), // dev: src/ast → <repo>/scripts/grammars
+    join(here, "..", "scripts", "grammars"),
+  ];
+  for (const c of candidates) if (existsSync(c)) return c;
+  return join(here, "grammars");
+}
+
+// tree-sitter's runtime + grammars must be initialised asynchronously (wasm
+// instantiation). We do that ONCE at the CLI/test boundary, then extraction is
+// fully synchronous (parser.parse is sync) — so the scan pipeline itself never
+// becomes async. No worker_threads: parsing is CPU-bound but per-file cheap, and
+// the incremental cache removes the repeated cost; a single deterministic thread
+// keeps byte-identical rebuilds trivially guaranteed.
+let runtimeReady = false;
+let parser: Parser | null = null;
+const loaded = new Map<string, Language>();
+const failed = new Set<string>();
+
+// Load the runtime (once) and the requested grammar keys (each once). Idempotent
+// and safe to call repeatedly. A missing/broken wasm is remembered as failed so
+// the caller silently falls back to regex rather than retrying every file.
+export async function ensureGrammars(keys: Iterable<string>): Promise<void> {
+  const dir = resolveGrammarDir();
+  if (!runtimeReady) {
+    const runtime = join(dir, "web-tree-sitter.wasm");
+    if (!existsSync(runtime)) return; // no committed grammars → regex fallback everywhere
+    await Parser.init({ wasmBinary: readFileSync(runtime) as unknown as Uint8Array });
+    runtimeReady = true;
+    parser = new Parser();
+  }
+  for (const key of new Set(keys)) {
+    if (loaded.has(key) || failed.has(key)) continue;
+    const wasm = join(dir, `${key}.wasm`);
+    if (!existsSync(wasm)) {
+      failed.add(key);
+      continue;
+    }
+    try {
+      loaded.set(key, await Language.load(new Uint8Array(readFileSync(wasm))));
+    } catch {
+      failed.add(key);
+    }
+  }
+}
+
+// All grammar keys we ship — used by the CLI/tests to warm every grammar upfront.
+export function allGrammarKeys(): string[] {
+  return [...new Set(Object.values(EXT_GRAMMAR))];
+}
+
+export function grammarReady(key: string): boolean {
+  return loaded.has(key);
+}
+
+// The shared parser, with `key`'s grammar selected. Returns null when the grammar
+// is not loaded (caller uses the regex extractor). Sync — parse happens after.
+export function parserFor(key: string): Parser | null {
+  const lang = loaded.get(key);
+  if (!parser || !lang) return null;
+  parser.setLanguage(lang);
+  return parser;
+}

package/src/bm25.ts

@@ -0,0 +1,156 @@
+// Keyless lexical search (issue #4): BM25 ranking over deterministic per-file
+// documents, so "where is auth handled?"-style questions get a ranked answer
+// with ZERO API keys — no embeddings, no network, no model. Each file's
+// document is built from its symbol names (split into camelCase/snake_case
+// subtokens, originals kept), its path segments, its markdown headings, and its
+// one-line summary/doc-comment. Scoring is textbook BM25 (k1=1.2, b=0.75) with
+// a non-negative Robertson idf; both sides fold diacritics (util foldText) and
+// the query reuses util keywords, so query and haystack always tokenize alike.
+// Deterministic: files are scored in scan order (sorted by rel), scores are
+// fixed to 4 decimal places, and ties break by path.
+import type { RepoScan } from "./scan.js";
+import { foldText, keywords } from "./util.js";
+import { byStr } from "./sort.js";
+
+const K1 = 1.2;
+const B = 0.75;
+const DEFAULT_LIMIT = 20;
+const TOP_SYMBOLS = 5;
+
+export interface SearchOptions {
+  // Maximum results returned (default 20).
+  limit?: number;
+}
+
+export interface SearchResult {
+  file: string; // repo-relative path
+  score: number; // BM25 score, fixed to 4 decimal places
+  matchedTerms: string[]; // query tokens present in this file's document, sorted
+  topSymbols: string[]; // symbols whose name matches the most query tokens (cap 5)
+}
+
+// Split an identifier/phrase into lowercase, diacritic-folded subtokens:
+// camelCase and ACRONYMWord boundaries become spaces, then any non-alphanumeric
+// run splits (snake_case, kebab-case, dots, prose whitespace). The ORIGINAL
+// token (lowercased) is kept alongside its parts so an exact identifier query
+// ("HttpClient") still matches a compound definition. 1-char fragments are
+// dropped as noise; letter↔digit runs stay together ("sha1", "bm25").
+export function subtokens(raw: string): string[] {
+  const folded = foldText(raw)
+    .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+    .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2");
+  const out: string[] = [];
+  const seen = new Set<string>();
+  const push = (t: string): void => {
+    if (t.length < 2 || seen.has(t)) return;
+    seen.add(t);
+    out.push(t);
+  };
+  // The whole identifier (lowercased) is a term of its own — but only for
+  // single tokens: gluing a prose heading into one mega-token would index junk.
+  if (!/\s/.test(raw.trim())) push(foldText(raw).toLowerCase().replace(/[^a-z0-9_]+/g, ""));
+  for (const part of folded.split(/[^A-Za-z0-9]+/)) push(part.toLowerCase());
+  return out;
+}
+
+interface Doc {
+  file: string;
+  tf: Map<string, number>;
+  len: number; // total token occurrences (the BM25 length normalizer)
+  symbols: string[]; // deduped symbol names, declaration order
+}
+
+function addTerms(doc: Doc, text: string): void {
+  for (const t of subtokens(text)) {
+    doc.tf.set(t, (doc.tf.get(t) ?? 0) + 1);
+    doc.len++;
+  }
+}
+
+function buildDocs(scan: RepoScan): Doc[] {
+  const docs: Doc[] = [];
+  for (const f of scan.files) {
+    const doc: Doc = { file: f.rel, tf: new Map(), len: 0, symbols: [] };
+    const seenSym = new Set<string>();
+    for (const s of f.symbols) {
+      addTerms(doc, s.name);
+      if (!seenSym.has(s.name)) {
+        seenSym.add(s.name);
+        doc.symbols.push(s.name);
+      }
+    }
+    for (const seg of f.rel.split("/")) addTerms(doc, seg);
+    for (const h of f.headings) addTerms(doc, h);
+    if (f.summary) addTerms(doc, f.summary);
+    docs.push(doc);
+  }
+  return docs;
+}
+
+// Rank the scanned files against a natural-language (or identifier) query.
+// Pure and deterministic: same scan + query → the same results, byte-for-byte.
+export function searchIndex(scan: RepoScan, query: string, opts: SearchOptions = {}): SearchResult[] {
+  // Query tokens: util keywords (stopwords dropped, identifiers kept) expanded
+  // through the SAME subtoken splitter the documents use.
+  const terms: string[] = [];
+  const seen = new Set<string>();
+  for (const kw of keywords(query)) {
+    for (const t of subtokens(kw)) {
+      if (seen.has(t)) continue;
+      seen.add(t);
+      terms.push(t);
+    }
+  }
+  if (!terms.length) return [];
+
+  const docs = buildDocs(scan);
+  const n = docs.length;
+  if (!n) return [];
+  let totalLen = 0;
+  for (const d of docs) totalLen += d.len;
+  const avgLen = totalLen / n || 1;
+
+  // Document frequency per query term, for idf.
+  const df = new Map<string, number>();
+  for (const t of terms) {
+    let count = 0;
+    for (const d of docs) if (d.tf.has(t)) count++;
+    df.set(t, count);
+  }
+
+  const results: SearchResult[] = [];
+  for (const d of docs) {
+    let score = 0;
+    const matched: string[] = [];
+    for (const t of terms) {
+      const tf = d.tf.get(t);
+      if (!tf) continue;
+      matched.push(t);
+      const idf = Math.log(1 + (n - df.get(t)! + 0.5) / (df.get(t)! + 0.5));
+      score += (idf * (tf * (K1 + 1))) / (tf + K1 * (1 - B + (B * d.len) / avgLen));
+    }
+    if (!matched.length) continue;
+
+    // Symbols ranked by how many query tokens their name carries, then by name.
+    const scored = d.symbols
+      .map((name) => {
+        const toks = new Set(subtokens(name));
+        let hits = 0;
+        for (const t of matched) if (toks.has(t)) hits++;
+        return { name, hits };
+      })
+      .filter((s) => s.hits > 0)
+      .sort((a, b) => b.hits - a.hits || byStr(a.name, b.name));
+
+    results.push({
+      file: d.file,
+      score: Number(score.toFixed(4)),
+      matchedTerms: matched.sort(byStr),
+      topSymbols: scored.slice(0, TOP_SYMBOLS).map((s) => s.name),
+    });
+  }
+
+  // Rounded score first (so 4-dp ties resolve stably), then path.
+  results.sort((a, b) => b.score - a.score || byStr(a.file, b.file));
+  return results.slice(0, opts.limit ?? DEFAULT_LIMIT);
+}

package/src/calls.ts

@@ -0,0 +1,131 @@
+import type { Edge } from "./types.js";
+import type { RepoScan } from "./scan.js";
+import { byStr } from "./sort.js";
+
+// Symbol kinds that are references to a definition elsewhere (a barrel re-export
+// or `export default Foo`) — they must NOT count as a call target, which should
+// resolve to where a symbol is actually declared. Re-declared here (canonical
+// copy lives in graph.ts) so this module has no import cycle with the graph
+// builder, which imports resolveCallEdges.
+const REFERENCE_KINDS = new Set(["reexport", "reexport-all", "default"]);
+
+// Collapse TypeScript/JavaScript to one family so a call in a `.ts` file can bind
+// to a def in a `.js` file (and vice versa) but never crosses into an unrelated
+// language. Every other language is its own family. Exported for callers.ts,
+// which mirrors this binding logic at call-site granularity.
+export function familyOf(lang: string): string {
+  if (lang === "typescript" || lang === "javascript") return "js";
+  // C and C++ interoperate through headers (.h files classify as "c" while
+  // their consumers are often .cpp) — one family, like the JS/TS pair.
+  if (lang === "c" || lang === "cpp") return "c";
+  return lang;
+}
+
+// Leading path segments two repo-relative paths share (the filename never counts,
+// as it always differs between distinct files). Higher = closer in the tree.
+function sharedSegments(a: string, b: string): number {
+  const as = a.split("/");
+  const bs = b.split("/");
+  let n = 0;
+  while (n < as.length && n < bs.length && as[n] === bs[n]) n++;
+  return n;
+}
+
+export interface Cand {
+  file: string;
+  lang: string;
+}
+
+// Pick a single candidate for a call: the sole candidate, else the one sharing
+// the strictly-most leading path segments with the caller. A tie at the maximum
+// (or an empty list) is unresolvable — return undefined so the caller skips it.
+export function pickCandidate(callerRel: string, cands: Cand[]): Cand | undefined {
+  if (cands.length === 1) return cands[0];
+  if (cands.length === 0) return undefined;
+  let best: Cand | undefined;
+  let bestScore = -1;
+  let tied = false;
+  for (const c of cands) {
+    const s = sharedSegments(callerRel, c.file);
+    if (s > bestScore) {
+      bestScore = s;
+      best = c;
+      tied = false;
+    } else if (s === bestScore) {
+      tied = true;
+    }
+  }
+  return tied ? undefined : best;
+}
+
+// Resolve every collected call site to a cross-file `call` edge in a global second
+// pass. An import between the two files promotes the edge to `extracted`; a unique
+// repo-wide name match with no import yields `inferred`. JS/TS is import-gated (no
+// import ⇒ no edge) because its bare identifiers are too ambiguous to infer safely;
+// other languages fall back to a unique-name inference. Deterministic: the emitted
+// array is sorted and never depends on Map iteration order.
+export function resolveCallEdges(scan: RepoScan, importPairs: Set<string>): Edge[] {
+  // name → distinct def sites (deduped per file; overloads collapse to one file).
+  const defs = new Map<string, Cand[]>();
+  const seen = new Set<string>();
+  for (const f of scan.files) {
+    for (const s of f.symbols) {
+      if (!s.exported || REFERENCE_KINDS.has(s.kind)) continue;
+      const dedup = `${s.name} ${s.file}`;
+      if (seen.has(dedup)) continue;
+      seen.add(dedup);
+      let arr = defs.get(s.name);
+      if (!arr) defs.set(s.name, (arr = []));
+      arr.push({ file: s.file, lang: s.lang });
+    }
+  }
+
+  // (from|to) → aggregated edge. Strongest confidence wins; counts sum.
+  const agg = new Map<string, { from: string; to: string; weight: number; confidence: "extracted" | "inferred" }>();
+  for (const f of scan.files) {
+    if (!f.calls?.length) continue;
+    const family = familyOf(f.lang);
+    const ownNames = new Set(f.symbols.map((s) => s.name));
+    const counts = new Map<string, number>();
+    for (const c of f.calls) counts.set(c.name, (counts.get(c.name) ?? 0) + 1);
+
+    for (const [name, count] of counts) {
+      if (ownNames.has(name)) continue; // same-file call — not a cross-file edge
+      const cands = (defs.get(name) ?? []).filter((d) => familyOf(d.lang) === family && d.file !== f.rel);
+      if (!cands.length) continue;
+      const imported = cands.filter((d) => importPairs.has(`${f.rel}|${d.file}`));
+
+      let chosen: Cand | undefined;
+      let confidence: "extracted" | "inferred";
+      if (family === "js") {
+        // JS/TS gate: without an import corroborating the call, drop it entirely.
+        // A named-import binding (f.importedNames) corroborates a name but not the
+        // file it came from, so it can't narrow `imported` further — pick among
+        // the imported candidates by proximity.
+        if (!imported.length) continue;
+        chosen = pickCandidate(f.rel, imported);
+        confidence = "extracted";
+      } else if (imported.length) {
+        chosen = pickCandidate(f.rel, imported);
+        confidence = "extracted";
+      } else {
+        chosen = pickCandidate(f.rel, cands);
+        confidence = "inferred";
+      }
+      if (!chosen) continue;
+
+      const key = `${f.rel}|${chosen.file}`;
+      const prev = agg.get(key);
+      if (prev) {
+        prev.weight += count;
+        if (confidence === "extracted") prev.confidence = "extracted";
+      } else {
+        agg.set(key, { from: f.rel, to: chosen.file, weight: count, confidence });
+      }
+    }
+  }
+
+  return [...agg.values()]
+    .map((e) => ({ from: e.from, to: e.to, kind: "call" as const, weight: Math.min(e.weight, 5), confidence: e.confidence }))
+    .sort((a, b) => byStr(a.from, b.from) || byStr(a.to, b.to));
+}

package/src/categorize.ts

@@ -0,0 +1,80 @@
+// Nine-way file categorization (ported from reconstruct): a coarser, purpose-
+// oriented view than classify()'s FileKind — "what role does this file play in
+// the project" rather than "how should the indexer parse it". Pure path/ext
+// logic; consumers call it themselves (it is not stamped on FileRecord).
+import { basename } from "node:path";
+
+export type FileCategory =
+  | "code"
+  | "test"
+  | "config"
+  | "schema"
+  | "i18n"
+  | "doc"
+  | "style"
+  | "asset"
+  | "data"
+  | "other";
+
+const CODE_EXTS = new Set([
+  ".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs", ".vue", ".svelte", ".astro",
+  ".py", ".rb", ".go", ".rs", ".java", ".kt", ".kts", ".php", ".c", ".cc", ".cpp", ".h", ".hpp",
+  ".cs", ".swift", ".scala", ".clj", ".ex", ".exs", ".dart", ".lua", ".sh", ".bash", ".zig", ".elm",
+]);
+const STYLE_EXTS = new Set([".css", ".scss", ".sass", ".less", ".styl", ".pcss"]);
+const DOC_EXTS = new Set([".md", ".mdx", ".rst", ".adoc", ".txt"]);
+const DATA_EXTS = new Set([".json", ".yaml", ".yml", ".toml", ".csv", ".xml", ".env"]);
+// `.svg` is deliberately "asset" here: in this purpose-oriented view it plays
+// the same image role as .png/.ico. reconstruct's own bundle categorizes it
+// as "other" (SVG is text, and its asset set is exactly its binary-extension
+// set) and keeps that as a local override on top of the engine — a documented
+// divergence, not an accident.
+const ASSET_EXTS = new Set([
+  ".png", ".jpg", ".jpeg", ".gif", ".webp", ".avif", ".ico", ".bmp", ".tiff", ".svg", ".pdf",
+  ".woff", ".woff2", ".ttf", ".otf", ".eot", ".mp3", ".mp4", ".mov", ".avi", ".webm",
+  // Archives / compiled binaries: reconstruct's bundle files these under
+  // "asset" (opaque blob shipped with the repo, not code/data) — the engine
+  // matches instead of letting them fall through to "other".
+  ".zip", ".gz", ".tar", ".rar", ".7z", ".wasm", ".so", ".dylib", ".dll", ".exe", ".bin",
+  ".class", ".jar", ".pyc", ".node",
+]);
+
+const I18N_DIRS = ["locales", "locale", "i18n", "lang", "langs", "translations", "messages"];
+const I18N_EXTS = new Set([".json", ".yaml", ".yml", ".po", ".properties"]);
+const TEST_DIRS = ["__tests__", "test", "tests", "spec", "e2e", "__mocks__"];
+const SCHEMA_DIRS = ["migrations", "entities", "models"];
+
+const CONFIG_BASES = new Set([
+  "package.json", "tsconfig.json", "dockerfile", "makefile", "pyproject.toml", "cargo.toml",
+  "go.mod", "requirements.txt", "gemfile", "composer.json", "pubspec.yaml",
+]);
+
+export function categorize(rel: string, ext: string): FileCategory {
+  const lower = rel.toLowerCase();
+  const base = basename(lower);
+  const segments = lower.split("/");
+  const inDir = (names: readonly string[]): boolean => names.some((n) => segments.includes(n));
+
+  if (inDir(I18N_DIRS) && I18N_EXTS.has(ext)) return "i18n";
+  if (
+    ext === ".prisma" || ext === ".sql" || ext === ".graphql" || ext === ".gql" ||
+    base.startsWith("schema.") || base === "models.py" || inDir(SCHEMA_DIRS)
+  ) {
+    return "schema";
+  }
+  if (lower.includes(".test.") || lower.includes(".spec.") || inDir(TEST_DIRS)) return "test";
+  if (
+    CONFIG_BASES.has(base) ||
+    base.endsWith(".config.js") || base.endsWith(".config.ts") || base.endsWith(".config.mjs") ||
+    base.startsWith(".eslintrc") || base.startsWith(".prettierrc") || base.startsWith(".env") ||
+    base.startsWith("docker-compose")
+  ) {
+    return "config";
+  }
+  if (DOC_EXTS.has(ext)) return "doc";
+  if (STYLE_EXTS.has(ext)) return "style";
+  if (CODE_EXTS.has(ext)) return "code";
+  if (ASSET_EXTS.has(ext)) return "asset";
+  if (DATA_EXTS.has(ext)) return "data";
+  return "other";
+}

package/src/centrality.ts

@@ -0,0 +1,148 @@
+import type { Edge, Graph } from "./types.js";
+
+// Deterministic centrality over the link-graph, feeding hub/bridge ranking and
+// the delta risk panel. Like the community layer, determinism is load-bearing:
+// scores land in graph.json (byte-identical rebuild guarantee), so node arrays
+// are consumed in their stored (sorted) order, adjacency is built in edge order,
+// and every float is rounded once at assignment — a fixed op order plus
+// ECMA-262's specified number→string conversion keeps bytes stable across
+// platforms. No Math.random anywhere.
+
+// PageRank: standard damping, L1-convergence threshold, iteration safety cap.
+const DAMPING = 0.85;
+const MAX_ITERS = 100;
+const CONVERGENCE = 1e-10;
+
+// Brandes betweenness is O(V·E); module graphs are small, but a pathological
+// repo (thousands of single-file modules) could stall a build. Past this cap
+// the field is left absent and a build note says so.
+export const BETWEENNESS_MAX_NODES = 3000;
+
+// Weighted, directed PageRank over the given node ids. Edges are used exactly
+// as stored (from → to), so a heavily-imported node ranks high. Dangling edges,
+// self-loops, and edges whose endpoints are not in `ids` are skipped — the same
+// graph view the degree fields and the Louvain adjacency use. Nodes with no
+// outgoing weight redistribute their mass uniformly each iteration (the classic
+// dangling-node fix), so the returned values always sum to 1.
+export function pagerankOf(ids: string[], edges: Edge[], damping = DAMPING): Map<string, number> {
+  const out = new Map<string, number>();
+  const n = ids.length;
+  if (n === 0) return out;
+  const idx = new Map(ids.map((s, i) => [s, i]));
+  const adj: [number, number][][] = Array.from({ length: n }, () => []);
+  const outW = new Array<number>(n).fill(0);
+  for (const e of edges) {
+    if (e.dangling) continue;
+    const a = idx.get(e.from);
+    const b = idx.get(e.to);
+    if (a === undefined || b === undefined || a === b) continue;
+    adj[a]!.push([b, e.weight]);
+    outW[a]! += e.weight;
+  }
+
+  let pr = new Array<number>(n).fill(1 / n);
+  for (let iter = 0; iter < MAX_ITERS; iter++) {
+    let dangling = 0;
+    for (let i = 0; i < n; i++) if (outW[i] === 0) dangling += pr[i]!;
+    const base = (1 - damping) / n + (damping * dangling) / n;
+    const next = new Array<number>(n).fill(base);
+    for (let i = 0; i < n; i++) {
+      if (outW[i] === 0) continue;
+      const share = (damping * pr[i]!) / outW[i]!;
+      for (const [j, w] of adj[i]!) next[j]! += share * w;
+    }
+    let delta = 0;
+    for (let i = 0; i < n; i++) delta += Math.abs(next[i]! - pr[i]!);
+    pr = next;
+    if (delta < CONVERGENCE) break;
+  }
+  ids.forEach((s, i) => out.set(s, pr[i]!));
+  return out;
+}
+
+// Brandes betweenness over the UNDIRECTED, UNWEIGHTED view of the edges —
+// hop-count BFS keeps the path counts σ integral, so there are no float
+// shortest-path ties to break. Neighbour lists are deduped and iterated in
+// ascending index order; the source loop runs in index order — the accumulation
+// order is therefore a pure function of the (sorted) input. Normalized by the
+// pair count (n-1)(n-2)/2 into [0,1]; below 3 nodes every value is 0.
+export function betweennessOf(ids: string[], edges: Edge[]): Map<string, number> {
+  const out = new Map<string, number>();
+  for (const s of ids) out.set(s, 0);
+  const n = ids.length;
+  if (n < 3) return out;
+  const idx = new Map(ids.map((s, i) => [s, i]));
+  const nbSets: Set<number>[] = Array.from({ length: n }, () => new Set<number>());
+  for (const e of edges) {
+    if (e.dangling) continue;
+    const a = idx.get(e.from);
+    const b = idx.get(e.to);
+    if (a === undefined || b === undefined || a === b) continue;
+    nbSets[a]!.add(b);
+    nbSets[b]!.add(a);
+  }
+  const adj = nbSets.map((s) => [...s].sort((x, y) => x - y));
+
+  const cb = new Array<number>(n).fill(0);
+  for (let s = 0; s < n; s++) {
+    const stack: number[] = [];
+    const pred: number[][] = Array.from({ length: n }, () => []);
+    const sigma = new Array<number>(n).fill(0);
+    const dist = new Array<number>(n).fill(-1);
+    sigma[s] = 1;
+    dist[s] = 0;
+    const queue: number[] = [s];
+    for (let qi = 0; qi < queue.length; qi++) {
+      const v = queue[qi]!;
+      stack.push(v);
+      for (const w of adj[v]!) {
+        if (dist[w]! < 0) {
+          dist[w] = dist[v]! + 1;
+          queue.push(w);
+        }
+        if (dist[w] === dist[v]! + 1) {
+          sigma[w]! += sigma[v]!;
+          pred[w]!.push(v);
+        }
+      }
+    }
+    const delta = new Array<number>(n).fill(0);
+    for (let si = stack.length - 1; si >= 0; si--) {
+      const w = stack[si]!;
+      for (const v of pred[w]!) delta[v]! += (sigma[v]! / sigma[w]!) * (1 + delta[w]!);
+      if (w !== s) cb[w]! += delta[w]!;
+    }
+  }
+  // Each undirected pair was counted from both endpoints → halve, then normalize.
+  const norm = ((n - 1) * (n - 2)) / 2;
+  ids.forEach((id, i) => out.set(id, cb[i]! / 2 / norm));
+  return out;
+}
+
+// Stamp centrality onto the graph's nodes in place (the `m.community` pattern:
+// fields assigned in fixed code order so graph.json key order stays stable).
+// Pagerank is stored SCALED by the node count — the average node is 1.0, hubs
+// read as "3.2× the average" — and survives 4-dp rounding on graphs where the
+// raw probability would round to zero. Returns build notes (skipped passes).
+export function applyCentrality(graph: Graph): string[] {
+  const notes: string[] = [];
+  const nM = graph.modules.length;
+  if (nM > 0) {
+    const mIds = graph.modules.map((m) => m.id);
+    const mPr = pagerankOf(mIds, graph.moduleEdges);
+    for (const m of graph.modules) m.pagerank = Number(((mPr.get(m.id) ?? 0) * nM).toFixed(4));
+    if (nM > BETWEENNESS_MAX_NODES) {
+      notes.push(`betweenness skipped (${nM} modules > ${BETWEENNESS_MAX_NODES})`);
+    } else {
+      const bt = betweennessOf(mIds, graph.moduleEdges);
+      for (const m of graph.modules) m.betweenness = Number((bt.get(m.id) ?? 0).toFixed(6));
+    }
+  }
+  const nF = graph.files.length;
+  if (nF > 0) {
+    const fIds = graph.files.map((f) => f.id);
+    const fPr = pagerankOf(fIds, graph.fileEdges);
+    for (const f of graph.files) f.pagerank = Number(((fPr.get(f.id) ?? 0) * nF).toFixed(4));
+  }
+  return notes;
+}

package/src/classify.ts

@@ -0,0 +1,53 @@
+import type { FileKind } from "./types.js";
+import { languageOf } from "./lang/registry.js";
+
+// Documentation: conventional top-level docs, anything under a docs tree, and
+// prose extensions. (Same heuristics ultradoc uses to feed its `docs` source.)
+const DOC_BASENAME =
+  /^(readme|changelog|contributing|history|news|authors|notice|security|code_of_conduct|faq|getting[-_]?started|usage|guide|tutorial)\b/i;
+const DOC_EXT = new Set([".md", ".mdx", ".rst", ".adoc", ".txt"]);
+const DOC_DIR = /^(docs?|documentation|wiki|guides?|website|site|book)\//i;
+
+// Manifests / config that reveal the stack, deps, scripts and entry points.
+const CONFIG_BASENAME = new Set([
+  "package.json", "pnpm-workspace.yaml", "tsconfig.json", "jsconfig.json", "pyproject.toml",
+  "setup.py", "setup.cfg", "requirements.txt", "pipfile", "go.mod", "cargo.toml", "gemfile",
+  "pom.xml", "build.gradle", "build.gradle.kts", "composer.json", "mix.exs", "pubspec.yaml",
+  "build.sbt", "dockerfile", "docker-compose.yml", "docker-compose.yaml", "makefile",
+  ".env.example", "manifest.json",
+]);
+const CONFIG_EXT = new Set([".json", ".yaml", ".yml", ".toml", ".ini", ".cfg"]);
+
+// Markdown extraction only makes sense for actual markdown.
+export const MARKDOWN_EXT = new Set([".md", ".mdx"]);
+
+export function isDoc(rel: string, ext: string): boolean {
+  const base = rel.split("/").pop()!.toLowerCase();
+  return DOC_EXT.has(ext) || DOC_BASENAME.test(base) || DOC_DIR.test(rel);
+}
+
+function isConfig(rel: string, ext: string): boolean {
+  const base = rel.split("/").pop()!.toLowerCase();
+  return CONFIG_BASENAME.has(base) || CONFIG_EXT.has(ext);
+}
+
+// A code file is one an extractor (or the language table) recognizes as a
+// programming language — not prose, not data/config.
+const NON_CODE_LANGS = new Set([
+  "markdown", "restructuredtext", "text", "json", "yaml", "toml", "ini", "other",
+  "html", "css", "scss",
+]);
+
+export function isCode(ext: string): boolean {
+  return !NON_CODE_LANGS.has(languageOf(ext));
+}
+
+// Classify a walked file. A real code extension ALWAYS wins — a `.ts`/`.tsx`
+// file named News.tsx or under docs/ is code, not documentation (otherwise its
+// imports are never extracted). After that: docs, then config, then other.
+export function classify(rel: string, ext: string): FileKind {
+  if (isCode(ext)) return "code";
+  if (isDoc(rel, ext)) return "doc";
+  if (isConfig(rel, ext)) return "config";
+  return "other";
+}