@maxgfr/codeindex 2.7.0

package/src/types.ts

@@ -0,0 +1,201 @@
+// Single source of truth for the engine version the bundle reports. Kept in
+// lockstep with package.json by the release pipeline. Do not edit by hand
+// outside a release.
+export const ENGINE_VERSION = "2.7.0";
+
+// Bumped whenever the on-disk artifact shape changes, so a consumer can reject
+// an index written by an incompatible engine instead of misreading it. The
+// numbering continues ultraindex's lineage (this core was extracted from
+// ultraindex 5.1.0 at schema v4): v2 added symbols.json, the `use` edge kind,
+// per-symbol parent/endLine, and the extraction cache; v3 added the `call` edge
+// kind, `Edge.confidence`, and `ModuleNode.community`; v4 added node centrality
+// (`pagerank`/`betweenness`), the tests→code fields (`FileNode.testFile`/
+// `ModuleNode.testedBy`), and symbols.json `endLine`.
+export const SCHEMA_VERSION = 4;
+
+// Identifies the extraction engine's output shape independently of the artifact
+// schema. Incremental caches key reused FileRecords on (content hash,
+// EXTRACTOR_VERSION); bump this whenever symbol/import extraction changes so a
+// stale cache is discarded wholesale rather than mixing old and new records. v3
+// added FileRecord.calls (call-site callee names) and importedNames; v4 added
+// CommonJS assignment-style JS/TS definitions (`x.y = function () {}`); v5
+// populates `calls` on the regex tier too (files without an AST grammar), so
+// caller indexes work without the wasm sidecar; v6 added call-site `receiver`
+// (the immediate receiver of a qualified call, both tiers) and JS/TS export
+// parity with ultradoc (CJS `exports.foo =` / `module.exports = {…}` named
+// exports, `export { a, b as c }` local marking, anonymous `export default`
+// named after the file stem, `export default Foo` marking the declaration).
+export const EXTRACTOR_VERSION = 6;
+
+// How a file is classified. `code` gets symbol/import extraction; `doc` gets
+// link/heading extraction; the rest are catalogued but not deeply parsed.
+export type FileKind = "code" | "doc" | "config" | "asset" | "other";
+
+// Edge kinds in the link-graph. `contains` is the module→member hierarchy;
+// `doc-link` a markdown link; `import` a resolved local code import; `call` a
+// resolved cross-file function/method/constructor call (a global second pass over
+// collected call sites); `use` a code file referencing another file's unique
+// exported symbol (AST-derived, suppressed when an `import` or `call` edge already
+// covers the same pair); `mention` a doc naming an exported symbol.
+export type EdgeKind = "contains" | "doc-link" | "import" | "call" | "use" | "mention";
+
+// Dependency tier: 0 = foundations (types, utils, config), 1 = features,
+// 2 = tail (tests, docs, examples, scripts).
+export type Tier = 0 | 1 | 2;
+
+// A symbol extracted deterministically from source (no LLM). Shape matches the
+// lifted lang/* extractors.
+export interface CodeSymbol {
+  name: string;
+  kind: string; // function | class | method | const | type | interface | enum | struct | trait | def
+  file: string; // relative to repo root
+  line: number; // 1-based
+  endLine?: number; // 1-based end of the declaration node (AST extractor only)
+  parent?: string; // enclosing symbol name for a nested member (AST extractor only)
+  signature?: string;
+  exported: boolean;
+  lang: string;
+}
+
+// A raw, UNRESOLVED outbound reference found in a file: a markdown link target,
+// or an import specifier as written. Resolution to a real file happens later in
+// the graph builder, which is where language/path context lives.
+export interface RawRef {
+  kind: "doc-link" | "import";
+  spec: string; // the target/specifier exactly as written
+}
+
+// Everything extracted from one file in a single pass. The unit the graph and
+// renderers consume; nothing here requires the model.
+export interface FileRecord {
+  rel: string; // posix path relative to repo root
+  ext: string;
+  size: number;
+  lines: number;
+  hash: string; // sha1 of content — the staleness oracle
+  kind: FileKind;
+  lang: string;
+  title?: string; // markdown H1, or basename for code
+  summary?: string; // one-line: first doc paragraph / top doc-comment
+  headings: string[]; // markdown section headings
+  symbols: CodeSymbol[]; // declared symbols (capped per file)
+  refs: RawRef[]; // unresolved outbound links/imports
+  pkg?: string; // Java: the file's `package` declaration — anchors source roots
+  idents?: string[]; // distinctive identifiers referenced (transient — feeds `use` edges, not persisted)
+  // Unresolved call-site callee names (cap 512, deduped by name+line, sorted by
+  // name then line). Transient-ish: consumed by the graph builder's global call
+  // resolution pass, not surfaced in the graph itself. `receiver` is the simple
+  // name of the IMMEDIATE receiver of a qualified call — `axios.get(...)` →
+  // {name: "get", receiver: "axios"}, `a.b.c(...)` → {name: "c", receiver: "b"}
+  // — absent for a bare call (`get()`) or a computed/complex receiver
+  // (`fetch().then(...)`). Receiver-gated sink catalogs (ultrasec) key on it.
+  calls?: { name: string; line: number; receiver?: string }[];
+  // JS/TS named-import bindings (cap 256, deduped, sorted) — feeds the JS/TS
+  // import-evidence gate in call resolution.
+  importedNames?: string[];
+}
+
+// A node in the link-graph. Files and modules are both nodes.
+export interface FileNode {
+  id: string; // == rel
+  kind: "file";
+  rel: string;
+  fileKind: FileKind;
+  lang: string;
+  module: string; // owning module slug
+  title?: string;
+  summary?: string;
+  symbols: number;
+  lines: number;
+  degIn: number;
+  degOut: number;
+  // File-graph PageRank scaled by the file count (average file ≈ 1.0), 4 dp.
+  // Absent only on graphs built before centrality existed.
+  pagerank?: number;
+  // Present (true) only when the path classifies as a test file (tests-map.ts).
+  testFile?: true;
+}
+
+export interface ModuleNode {
+  id: string; // == slug
+  kind: "module";
+  slug: string;
+  path: string; // directory path (or "(root)")
+  title: string;
+  summary: string;
+  tier: Tier;
+  members: string[]; // member file rels, sorted
+  symbols: number; // total declared symbols across members
+  degIn: number;
+  degOut: number;
+  // Navigation community (a Louvain cluster of related modules), 0-based; id 0 is
+  // the largest cluster. OPTIONAL/additive: never affects slugs or lexical
+  // ranking. Absent only on graphs built before communities existed.
+  community?: number;
+  // Module-graph PageRank scaled by the module count (average ≈ 1.0), 4 dp.
+  pagerank?: number;
+  // Normalized undirected Brandes betweenness, [0,1], 6 dp. Absent when the
+  // BETWEENNESS_MAX_NODES guard skipped the pass.
+  betweenness?: number;
+  // Sorted test-file rels with a resolved import/call/use edge into a member —
+  // "which tests cover this module". Absent when none do.
+  testedBy?: string[];
+}
+
+// A directed edge. For a resolved edge `to` is a node id; for a dangling edge
+// `to` is the unresolved spec and `dangling` is set with a `reason`.
+export interface Edge {
+  from: string;
+  to: string;
+  kind: EdgeKind;
+  weight: number;
+  dangling?: boolean;
+  reason?: string;
+  // Only `call` edges set this. extracted = an import between the files
+  // corroborates the call; inferred = resolved by a unique name match without
+  // import evidence.
+  confidence?: "extracted" | "inferred";
+}
+
+// The full machine graph, persisted as graph.json by consumers. Holds BOTH
+// file-level and module-level nodes/edges. Deliberately carries NO wall-clock
+// timestamp so two builds of an unchanged repo are byte-identical.
+export interface Graph {
+  schemaVersion: number;
+  version: string;
+  commit?: string; // stable for a given HEAD
+  fileCount: number;
+  languages: Record<string, number>;
+  files: FileNode[];
+  modules: ModuleNode[];
+  fileEdges: Edge[];
+  moduleEdges: Edge[];
+  // Surprising cross-community couplings (see surprise.ts), capped and sorted
+  // (pairEdges asc, from, to). Absent when none were found.
+  surprises?: SurpriseEdge[];
+}
+
+// A dependency edge that is one of at most 2 links between two otherwise-
+// separate communities — an architectural leak worth extra review attention.
+export interface SurpriseEdge {
+  from: string;
+  to: string;
+  kind: EdgeKind;
+  weight: number;
+  communities: [number, number]; // [community(from), community(to)]
+  pairEdges: number; // total module edges between the two communities
+}
+
+// A persisted symbol table (symbols.json), emitted so `symbols <name>` queries
+// can answer "where is X defined?" without re-scanning the repo. `defs` maps a
+// symbol name to its definition sites; `refs` maps a name to the files that
+// reference it (populated by the use/mention pass). Deterministically ordered.
+export interface SymbolIndex {
+  schemaVersion: number;
+  // `endLine` mirrors CodeSymbol.endLine (AST extractor only).
+  defs: Record<
+    string,
+    { file: string; line: number; endLine?: number; kind: string; exported: boolean; lang: string; parent?: string }[]
+  >;
+  refs: Record<string, string[]>;
+}

package/src/util.ts

@@ -0,0 +1,169 @@
+import { spawnSync } from "node:child_process";
+
+// Result of a subprocess call. `ok` is true on exit code 0 with the binary
+// found; `missing` is true when the binary isn't on PATH (so callers can fall
+// back gracefully instead of crashing — e.g. no ripgrep, no gh, no docker).
+export interface ShResult {
+  ok: boolean;
+  status: number | null;
+  stdout: string;
+  stderr: string;
+  missing: boolean;
+}
+
+// Run a command synchronously. Sync keeps the CLI simple and deterministic
+// (mirrors how the engine is structured); the work is I/O-bound git/rg/gh calls
+// where parallelism buys little. `input` feeds stdin; `maxBuffer` is generous
+// for large `rg --json` / `git log` output.
+export function sh(
+  cmd: string,
+  args: string[],
+  opts: { cwd?: string; input?: string; timeoutMs?: number; env?: Record<string, string | undefined> } = {},
+): ShResult {
+  const res = spawnSync(cmd, args, {
+    cwd: opts.cwd,
+    input: opts.input,
+    encoding: "utf8",
+    timeout: opts.timeoutMs ?? 120_000,
+    maxBuffer: 64 * 1024 * 1024,
+    env: opts.env ?? process.env,
+  });
+  const missing = !!res.error && (res.error as NodeJS.ErrnoException).code === "ENOENT";
+  return {
+    ok: !res.error && res.status === 0,
+    status: res.status,
+    stdout: res.stdout ?? "",
+    stderr: res.stderr ?? (res.error ? String(res.error.message) : ""),
+    missing,
+  };
+}
+
+// Is a binary available on PATH? Cached because we probe the same few tools
+// (rg, gh, git, docker) repeatedly within a run.
+const whichCache = new Map<string, boolean>();
+export function have(cmd: string): boolean {
+  const cached = whichCache.get(cmd);
+  if (cached !== undefined) return cached;
+  const probe = sh(process.platform === "win32" ? "where" : "which", [cmd]);
+  const found = probe.ok && probe.stdout.trim().length > 0;
+  whichCache.set(cmd, found);
+  return found;
+}
+
+// Turn an arbitrary repo identifier into a filesystem-safe cache slug, e.g.
+// "github.com/expressjs/express" -> "github.com-expressjs-express".
+export function slugify(input: string): string {
+  return input
+    .toLowerCase()
+    .replace(/^https?:\/\//, "")
+    .replace(/^git@/, "")
+    .replace(/\.git$/, "")
+    .replace(/[^a-z0-9._-]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .slice(0, 120);
+}
+
+// Truncate a string to a max length with an ellipsis marker, for snippets.
+export function clip(s: string, max: number): string {
+  if (s.length <= max) return s;
+  return s.slice(0, max) + `\n… [truncated ${s.length - max} chars]`;
+}
+
+// Truncate a string for INLINE display (a single line): collapse whitespace, cut
+// at a word boundary, and never leave a dangling inline-code backtick — so a
+// clipped module summary like "… in `path/` (typescript)" stays valid markdown
+// instead of the broken "… in `path/` (types" a raw slice produces.
+export function clipInline(s: string, max: number): string {
+  const flat = s.replace(/\s+/g, " ").trim();
+  if (flat.length <= max) return flat;
+  let cut = flat.slice(0, max).replace(/\s+\S*$/, ""); // back off to a word boundary
+  if (!cut) cut = flat.slice(0, max); // a single token longer than max — hard cut
+  // A half-open inline-code span: drop the severed backtick (and its partial
+  // contents) rather than appending one, which would emit an empty `` span.
+  if ((cut.match(/`/g)?.length ?? 0) % 2 === 1) cut = cut.replace(/`[^`]*$/, "");
+  // A half-open markdown link `[text…`: drop from the unmatched `[`.
+  if (cut.lastIndexOf("[") > cut.lastIndexOf("]")) cut = cut.slice(0, cut.lastIndexOf("["));
+  return cut.replace(/\s+$/, "") + "…";
+}
+
+// Escape a string for safe inclusion as a literal inside a RegExp.
+export function escapeRegExp(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+// Pull the meaningful keywords out of a natural-language question: lowercase,
+// split on non-word chars, drop stopwords and very short tokens, dedupe. Used
+// to drive lexical search and symbol ranking deterministically (no LLM).
+const STOPWORDS = new Set([
+  "the","a","an","is","are","was","were","be","been","being","do","does","did","how","what",
+  "why","when","where","which","who","whom","this","that","these","those","of","in","on","to",
+  "for","with","and","or","but","if","then","else","than","as","at","by","from","into","about",
+  "it","its","i","you","we","they","he","she","there","here","can","could","should","would",
+  "will","shall","may","might","must","have","has","had","not","no","yes","so","such","only",
+  "any","some","all","get","set","use","used","using","work","works","working","handle","handled",
+  "happen","happens","default","value","values","please","explain","tell","me","my","our",
+]);
+
+// Fold diacritics to their base letters (NFKD decomposition, then drop the
+// combining marks in the U+0300–U+036F block) so "café" and "cafe" tokenize
+// alike. Query and haystack must both pass through this so the two sides agree.
+export function foldText(s: string): string {
+  return s.normalize("NFKD").replace(/[̀-ͯ]/g, "");
+}
+
+export function keywords(question: string): string[] {
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const raw of foldText(question).split(/[^A-Za-z0-9_]+/)) {
+    if (!raw) continue;
+    const lower = raw.toLowerCase();
+    // Keep identifiers as-is (camelCase/snake_case often carry the real signal),
+    // but filter generic English stopwords and 1-char noise.
+    if (raw.length < 2) continue;
+    if (STOPWORDS.has(lower)) continue;
+    if (seen.has(lower)) continue;
+    seen.add(lower);
+    out.push(raw);
+  }
+  return out;
+}
+
+// Keywords ordered by how *distinctive* they are, most-specific first. Numbers
+// (status codes like 429), camelCase/snake_case identifiers, and long tokens
+// carry more signal than short generic words. Narrow search APIs (GitHub/GitLab
+// issue search, StackOverflow) AND their terms, so feeding them the few most
+// specific keywords — rather than the first N — dramatically improves recall.
+export function rankedKeywords(question: string): string[] {
+  const base = keywords(question);
+  const score = (raw: string): number => {
+    let s = 0;
+    if (/\d/.test(raw)) s += 3;
+    if (/[A-Z]/.test(raw) && !/^[A-Z0-9]+$/.test(raw)) s += 2; // camelCase/PascalCase
+    if (/_/.test(raw)) s += 2;
+    if (raw.length >= 8) s += 1.5;
+    else if (raw.length >= 5) s += 0.5;
+    return s;
+  };
+  return base
+    .map((k, i) => ({ k, s: score(k), i }))
+    .sort((a, b) => b.s - a.s || a.i - b.i)
+    .map((x) => x.k);
+}
+
+// Reciprocal Rank Fusion: merge several ranked lists into one robust ranking
+// without needing comparable scores across lists. `k` damps the contribution of
+// low ranks. Returns keys ordered best-first with a fused score.
+export function rrf<T>(
+  lists: T[][],
+  keyOf: (item: T) => string,
+  k = 60,
+): Map<string, number> {
+  const score = new Map<string, number>();
+  for (const list of lists) {
+    list.forEach((item, idx) => {
+      const key = keyOf(item);
+      score.set(key, (score.get(key) ?? 0) + 1 / (k + idx + 1));
+    });
+  }
+  return score;
+}

package/src/viz.ts

@@ -0,0 +1,44 @@
+// Mermaid rendering of the module graph — inline-renderable in Claude, GitHub
+// and most markdown viewers with zero infrastructure (the counter to graph-DB
+// browsers). Deterministic: sorted nodes/edges, stable ids.
+import type { Graph } from "./types.js";
+import { byStr } from "./sort.js";
+
+export interface MermaidOptions {
+  // Restrict to one module's neighborhood (the module plus every module it
+  // touches, either direction).
+  module?: string;
+  maxEdges?: number; // default 80 — keeps diagrams renderable
+}
+
+const sanitizeId = (slug: string): string => slug.replace(/[^\w]/g, "_");
+
+export function renderMermaid(graph: Graph, opts: MermaidOptions = {}): string {
+  const maxEdges = opts.maxEdges ?? 80;
+  let edges = [...graph.moduleEdges].filter((e) => !e.dangling);
+  if (opts.module) {
+    edges = edges.filter((e) => e.from === opts.module || e.to === opts.module);
+  }
+  edges.sort((a, b) => b.weight - a.weight || byStr(a.from, b.from) || byStr(a.to, b.to));
+  const dropped = Math.max(0, edges.length - maxEdges);
+  edges = edges.slice(0, maxEdges);
+
+  const shown = new Set<string>();
+  for (const e of edges) {
+    shown.add(e.from);
+    shown.add(e.to);
+  }
+  if (opts.module) shown.add(opts.module);
+
+  const lines: string[] = ["graph LR"];
+  for (const m of [...graph.modules].sort((a, b) => byStr(a.slug, b.slug))) {
+    if (!shown.has(m.slug)) continue;
+    lines.push(`  ${sanitizeId(m.slug)}["${m.slug}${m.tier === 0 ? " (core)" : ""}"]`);
+  }
+  for (const e of edges) {
+    const label = e.kind === "import" ? "" : `|${e.kind}|`;
+    lines.push(`  ${sanitizeId(e.from)} -->${label} ${sanitizeId(e.to)}`);
+  }
+  if (dropped) lines.push(`  %% ${dropped} lighter edges omitted (maxEdges=${maxEdges})`);
+  return lines.join("\n") + "\n";
+}

package/src/walk.ts

@@ -0,0 +1,216 @@
+import { readdirSync, statSync, lstatSync, readFileSync, realpathSync } from "node:fs";
+import { join, relative, sep, extname } from "node:path";
+import { parseGitignore, isIgnored, type IgnoreRule } from "./ignore.js";
+
+// Directories that never carry signal for a documentation/code question and
+// would bloat the index (dependencies, build output, VCS internals, caches).
+// Exported so grep.ts can align ripgrep's universe with the walker's.
+export const IGNORE_DIRS = new Set([
+  ".git", "node_modules", ".pnpm", "bower_components", "vendor", "dist", "build", "out",
+  "target", ".next", ".nuxt", ".svelte-kit", ".turbo", "coverage", "__pycache__", ".venv",
+  "venv", ".tox", ".mypy_cache", ".pytest_cache", ".gradle", ".idea", ".vscode", ".cache",
+  "tmp", ".ultraindex", "Pods", "DerivedData", ".terraform", "elm-stuff", ".dart_tool",
+]);
+
+// Lockfiles: huge, machine-generated, and pure noise for a code/docs question —
+// they'd otherwise rank as keyword-dense "code" hits (e.g. package-lock.json
+// matching a dependency name). Skipped entirely.
+export const LOCKFILES = new Set([
+  "package-lock.json", "npm-shrinkwrap.json", "yarn.lock", "pnpm-lock.yaml", "bun.lockb",
+  "composer.lock", "cargo.lock", "poetry.lock", "pipfile.lock", "gemfile.lock", "go.sum",
+  "flake.lock", "packages.lock.json", "podfile.lock", "mix.lock",
+]);
+
+// Binary / non-source extensions to skip when reading file contents.
+export const BINARY_EXT = new Set([
+  ".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".ico", ".icns", ".svg", ".pdf", ".zip",
+  ".gz", ".tar", ".tgz", ".bz2", ".xz", ".7z", ".rar", ".jar", ".war", ".class", ".so", ".dylib",
+  ".dll", ".exe", ".bin", ".o", ".a", ".wasm", ".woff", ".woff2", ".ttf", ".otf", ".eot", ".mp3",
+  ".mp4", ".mov", ".avi", ".webm", ".wav", ".flac", ".ogg", ".lock", ".min.js", ".map",
+]);
+
+export interface WalkOptions {
+  maxFileBytes?: number; // skip files larger than this (default 1 MiB)
+  maxFiles?: number; // hard cap on indexed files (default 20000)
+  // Honor .gitignore files (root and nested, with negation/anchoring/dir-only
+  // semantics — see ignore.ts). Default TRUE: an ignored file is noise for
+  // every consumer; pass false to index generated/ignored trees deliberately.
+  gitignore?: boolean;
+}
+
+export interface WalkedFile {
+  rel: string; // path relative to root, posix-style
+  abs: string;
+  size: number;
+  ext: string;
+  mtimeMs: number; // last-modified ms — the stat fastpath's freshness key with size
+}
+
+export interface WalkResult {
+  files: WalkedFile[];
+  capped: boolean; // true when the maxFiles cap was hit and the walk stopped early
+  // Files that were SEEN and rejected by the size/lockfile/binary/minified/
+  // gitignore rules. Ignored DIRECTORIES (node_modules, gitignored trees…)
+  // are not counted — their contents were never even listed.
+  excluded: number;
+}
+
+export const DEFAULT_MAX_FILES = 20_000;
+
+// Recursively list source-like files under `root`, applying ignore rules. Pure
+// filesystem walk — no git dependency, so it works on any directory. Returns a
+// `capped` flag (never a silent truncation) so the caller can warn when the
+// maxFiles cap stopped the walk with files still unindexed.
+export function walk(root: string, opts: WalkOptions = {}): WalkResult {
+  const maxFileBytes = opts.maxFileBytes ?? 1024 * 1024;
+  const maxFiles = opts.maxFiles ?? DEFAULT_MAX_FILES;
+  const useGitignore = opts.gitignore !== false;
+  const out: WalkedFile[] = [];
+  let capped = false;
+  let excluded = 0;
+
+  // Containment root for the symlink-escape guard: a symlinked file or
+  // directory whose real path leaves the repo must not be indexed (it would
+  // read foreign content and emit citations no one can open).
+  let rootReal: string;
+  try {
+    rootReal = realpathSync(root);
+  } catch {
+    return { files: out, capped, excluded };
+  }
+  const contained = (real: string): boolean => real === rootReal || real.startsWith(rootReal + sep);
+
+  // Each frame carries the ignore-rule chain inherited from its ancestors;
+  // rules from deeper .gitignore files are appended after (later rules win).
+  const stack: { dir: string; rel: string; rules: readonly IgnoreRule[] }[] = [
+    { dir: root, rel: "", rules: [] },
+  ];
+  const seenDirs = new Set<string>(); // resolved real dirs already walked
+  walking: while (stack.length) {
+    const frame = stack.pop()!;
+    // Cycle guard: a directory symlink pointing at an ancestor would otherwise
+    // make walk() loop, flooding the index with phantom duplicate files. Resolve
+    // the real path and skip any directory we've already descended into.
+    let real: string;
+    try {
+      real = realpathSync(frame.dir);
+    } catch {
+      continue;
+    }
+    if (seenDirs.has(real)) continue;
+    seenDirs.add(real);
+    if (!contained(real)) continue; // dir symlink escaping the repo
+    let entries: string[];
+    try {
+      // Sorted so the walk order — and therefore WHICH files survive a
+      // maxFiles cap — is identical across filesystems and machines.
+      entries = readdirSync(frame.dir).sort();
+    } catch {
+      continue;
+    }
+    let rules = frame.rules;
+    if (useGitignore && entries.includes(".gitignore")) {
+      const parsed = parseGitignore(readText(join(frame.dir, ".gitignore")), frame.rel);
+      if (parsed.length) rules = [...rules, ...parsed];
+    }
+    for (const name of entries) {
+      const abs = join(frame.dir, name);
+      const rel = frame.rel ? `${frame.rel}/${name}` : name;
+      let st;
+      let isLink: boolean;
+      try {
+        st = statSync(abs);
+        isLink = lstatSync(abs).isSymbolicLink();
+      } catch {
+        continue;
+      }
+      if (st.isDirectory()) {
+        if (IGNORE_DIRS.has(name)) continue;
+        // An in-repo DIRECTORY symlink is skipped entirely: its target is (or
+        // will be) walked under its canonical name, and letting both paths race
+        // through the cycle guard would keep whichever readdir served first —
+        // aliased, filesystem-order-dependent indexes. Out-of-repo links are
+        // covered by the containment guard above.
+        if (isLink) continue;
+        if (useGitignore && rules.length && isIgnored(rules, rel, true)) continue;
+        stack.push({ dir: abs, rel, rules });
+        continue;
+      }
+      if (!st.isFile()) continue;
+      // Each rejection below is a file the walk SAW and dropped — counted in
+      // `excluded` so consumers can report how much was filtered, not capped.
+      if (st.size > maxFileBytes) {
+        excluded++;
+        continue;
+      }
+      if (LOCKFILES.has(name.toLowerCase())) {
+        excluded++;
+        continue;
+      }
+      const ext = extname(name).toLowerCase();
+      if (BINARY_EXT.has(ext)) {
+        excluded++;
+        continue;
+      }
+      if (name.endsWith(".min.js") || name.endsWith(".min.css")) {
+        excluded++;
+        continue;
+      }
+      if (useGitignore && rules.length && isIgnored(rules, rel, false)) {
+        excluded++;
+        continue;
+      }
+      // Symlink-escape guard for files (statSync above follows links).
+      if (isLink) {
+        try {
+          if (!contained(realpathSync(abs))) continue;
+        } catch {
+          continue;
+        }
+      }
+      // The cap is enforced HERE, on kept files, so a flat directory cannot
+      // silently overshoot it and `capped` is set exactly when a file was
+      // actually dropped (never a silent truncation).
+      if (out.length >= maxFiles) {
+        capped = true;
+        break walking;
+      }
+      out.push({ rel: rel.split(sep).join("/"), abs, size: st.size, ext, mtimeMs: st.mtimeMs });
+    }
+  }
+  return { files: out, capped, excluded };
+}
+
+// Read a file as text, returning "" on any error (unreadable, vanished). Honours
+// a Unicode BOM before the binary sniff — a UTF-16 source file is full of NUL
+// bytes and would otherwise be misread as binary and dropped, and a UTF-8 BOM
+// would otherwise glue "" onto the first token (breaking line-1 extraction
+// and a `[file:1]` citation). Otherwise UTF-8, with a Latin-1 fallback and a
+// whole-buffer NUL sniff for genuinely-binary content.
+export function readText(abs: string): string {
+  try {
+    const buf = readFileSync(abs);
+    // UTF-16LE/BE BOM. Truncate to an even byte length first so an odd trailing
+    // byte can't make swap16() throw (toString already tolerates it; mirror that).
+    if (buf.length >= 2 && buf[0] === 0xff && buf[1] === 0xfe) {
+      return buf.subarray(2, 2 + ((buf.length - 2) & ~1)).toString("utf16le");
+    }
+    if (buf.length >= 2 && buf[0] === 0xfe && buf[1] === 0xff) {
+      const swapped = Buffer.from(buf.subarray(2, 2 + ((buf.length - 2) & ~1)));
+      swapped.swap16(); // UTF-16BE → LE so Node can decode it
+      return swapped.toString("utf16le");
+    }
+    if (buf.length >= 3 && buf[0] === 0xef && buf[1] === 0xbb && buf[2] === 0xbf) return buf.subarray(3).toString("utf8");
+    // Binary sniff over the WHOLE buffer, not just the first 4 KiB — a NUL after
+    // 4 KiB still means binary (else the symbol right after it is dropped and the
+    // content hash is poisoned).
+    if (buf.includes(0)) return "";
+    const text = buf.toString("utf8");
+    // Invalid UTF-8 surfaces as U+FFFD; a Latin-1/Windows-1252 source decodes
+    // cleanly there (every byte maps to a code point), so prefer that over baking
+    // mojibake into symbols, signatures, and the content hash.
+    return text.includes("�") ? buf.toString("latin1") : text;
+  } catch {
+    return "";
+  }
+}