@maxgfr/codeindex 2.7.0

package/src/modules.ts

@@ -0,0 +1,128 @@
+import { posix } from "node:path";
+import type { FileRecord, Tier } from "./types.js";
+import type { RepoScan } from "./scan.js";
+import { slugify } from "./util.js";
+import { sha1 } from "./hash.js";
+import { byStr } from "./sort.js";
+
+// A module is a directory's worth of files — the unit a developer thinks in
+// ("src/lang is the language extractors"). Grouping by immediate parent dir is
+// deterministic and maps cleanly onto how repos are organized.
+export interface ModuleInfo {
+  slug: string;
+  path: string; // directory path, or "(root)" for top-level files
+  title: string;
+  tier: Tier;
+  members: string[]; // member file rels, sorted
+  summary: string;
+}
+
+const ROOT_PATH = "(root)";
+
+// Foundations: shared building blocks features depend on (matched on the leaf).
+const TIER0 = /(^|\/)(types?|util|utils|lib|libs|common|core|config|configs|constants|shared|helpers|internal)$/i;
+// Tail: supporting material, not the product itself. Matched on ANY path
+// segment — `a/__tests__/b` is still test material, not a feature. Covers both
+// singular/plural jest dunder dirs and e2e suites.
+const TIER2_ANY = /(^|\/)(tests?|__tests?__|__mocks?__|__snapshots?__|spec|specs|e2e|examples?|example|benchmark|benchmarks|fixtures?|docs?|documentation|\.github)(\/|$)/i;
+// Scripts/CI only count as tail when they're the leaf (a `scripts/` dir), not
+// when "scripts" merely appears mid-path.
+const TIER2_LEAF = /(^|\/)(scripts?|bin|\.storybook)$/i;
+// A file that is itself a test/spec/e2e, by basename — so a directory of only
+// these is tail even when its own name is unconventional.
+const TEST_FILE = /\.(test|spec|e2e|stories|story)\.[cm]?[jt]sx?$/i;
+
+export function isTestFile(rel: string): boolean {
+  return TEST_FILE.test(rel.split("/").pop()!);
+}
+
+function dirOf(rel: string): string {
+  return rel.includes("/") ? posix.dirname(rel) : ROOT_PATH;
+}
+
+// Path-only tier decision; returns null when the path alone is undecided.
+export function tierForPath(path: string): Tier | null {
+  if (path === ROOT_PATH) return 0; // root manifests/READMEs are foundational context
+  if (TIER2_ANY.test(path) || TIER2_LEAF.test(path)) return 2;
+  if (TIER0.test(path)) return 0;
+  return null;
+}
+
+function tierOf(path: string, members: FileRecord[]): Tier {
+  const byPath = tierForPath(path);
+  if (byPath !== null) return byPath;
+  // A directory that is entirely docs/config/tests is tail material regardless
+  // of its (unconventional) name.
+  if (members.every((m) => m.kind === "doc" || m.kind === "config" || isTestFile(m.rel))) return 2;
+  return 1;
+}
+
+// Deterministic one-line summary for a module, with no model involvement:
+// prefer a directory README/index's own summary, else the richest member's
+// doc-comment, else a structural fallback.
+function summaryOf(path: string, members: FileRecord[]): string {
+  const readme = members.find((m) => /^(readme|index)\.(md|mdx)$/i.test(m.rel.split("/").pop()!));
+  if (readme?.summary) return readme.summary;
+  if (readme?.title) return readme.title; // a README with only a title is still a better name than a file count
+  const withSummary = members
+    .filter((m) => m.summary)
+    .sort((a, b) => (b.summary?.length ?? 0) - (a.summary?.length ?? 0));
+  if (withSummary[0]?.summary) return withSummary[0].summary;
+  const langs = [...new Set(members.map((m) => m.lang))].filter((l) => l !== "other");
+  const where = path === ROOT_PATH ? "the repository root" : `\`${path}/\``;
+  return `${members.length} file(s) in ${where}${langs.length ? ` (${langs.slice(0, 3).join(", ")})` : ""}.`;
+}
+
+// Group all scanned files into modules. Returns the modules (sorted by slug) and
+// a rel → slug lookup the graph builder uses to lift file edges to module edges.
+export function buildModules(scan: RepoScan): {
+  modules: ModuleInfo[];
+  moduleOf: Map<string, string>;
+} {
+  const byDir = new Map<string, FileRecord[]>();
+  for (const f of scan.files) {
+    const dir = dirOf(f.rel);
+    let list = byDir.get(dir);
+    if (!list) byDir.set(dir, (list = []));
+    list.push(f);
+  }
+
+  const dirs = [...byDir.keys()].sort(byStr);
+  // Slug per directory: slugify the path, but when that is empty (an all-non-ASCII
+  // dir) or collides with another dir (slugify truncates at 120 chars, and two
+  // packages can share a leaf dir name), append a short STABLE hash of the full
+  // path. This keeps slugs injective AND order-independent — the slug is the
+  // persistence key for human prose, and the old positional "module"/"-2" scheme
+  // shuffled (re-gluing prose onto the wrong module) when a sibling changed.
+  const baseOf = new Map<string, string>();
+  const baseCount = new Map<string, number>();
+  for (const dir of dirs) {
+    const b = dir === ROOT_PATH ? "root" : slugify(dir);
+    baseOf.set(dir, b);
+    baseCount.set(b, (baseCount.get(b) ?? 0) + 1);
+  }
+  const slugForDir = (dir: string): string => {
+    const b = baseOf.get(dir)!;
+    return b && baseCount.get(b) === 1 ? b : `${b || "module"}-${sha1(dir).slice(0, 8)}`;
+  };
+
+  const modules: ModuleInfo[] = [];
+  const moduleOf = new Map<string, string>();
+  for (const dir of dirs) {
+    const members = byDir.get(dir)!.slice().sort((a, b) => byStr(a.rel, b.rel));
+    const slug = slugForDir(dir);
+    const info: ModuleInfo = {
+      slug,
+      path: dir,
+      title: dir,
+      tier: tierOf(dir, members),
+      members: members.map((m) => m.rel),
+      summary: summaryOf(dir, members),
+    };
+    modules.push(info);
+    for (const m of members) moduleOf.set(m.rel, slug);
+  }
+
+  modules.sort((a, b) => byStr(a.slug, b.slug));
+  return { modules, moduleOf };
+}

package/src/pipeline.ts

@@ -0,0 +1,59 @@
+import type { Graph, SymbolIndex } from "./types.js";
+import { scanRepo, type RepoScan, type ScanOptions } from "./scan.js";
+import { buildResolveContext } from "./resolve.js";
+import { buildModules } from "./modules.js";
+import { buildGraph } from "./graph.js";
+import { detectCommunities } from "./community.js";
+import { applyCentrality } from "./centrality.js";
+import { computeTestMap } from "./tests-map.js";
+import { computeSurprises } from "./surprise.js";
+import { buildSymbolIndex, computeSymbolRefs } from "./render/symbols-json.js";
+
+export interface BuildIndexOptions extends ScanOptions {
+  // Stamped into the graph a consumer persists — lets it carry its own
+  // version/schema instead of the engine's (see buildGraph).
+  meta?: { version?: string; schemaVersion?: number };
+  // Community ids from a previous build (manifest `communities` shape), so an
+  // unchanged partition keeps stable ids across rebuilds.
+  previousCommunities?: Record<string, string[]>;
+}
+
+export interface IndexArtifacts {
+  scan: RepoScan;
+  graph: Graph;
+  symbols: SymbolIndex;
+}
+
+// The full deterministic pipeline in one call: scan → resolve → group → graph →
+// communities → centrality → tests-map → surprises → symbol index. Mirrors the
+// exact composition (and mutation order — it matters for byte-stable output)
+// that ultraindex's build performs before its prose rendering.
+export function buildIndexArtifacts(repo: string, opts: BuildIndexOptions = {}): IndexArtifacts {
+  const scan = scanRepo(repo, opts);
+  const ctx = buildResolveContext(scan);
+  const { modules, moduleOf } = buildModules(scan);
+  const graph = buildGraph(scan, ctx, modules, moduleOf, opts.meta);
+
+  const communities = detectCommunities(graph.modules, graph.moduleEdges, opts.previousCommunities);
+  for (const m of graph.modules) {
+    const id = communities.get(m.slug);
+    if (id !== undefined) m.community = id;
+  }
+
+  applyCentrality(graph);
+
+  const testMap = computeTestMap(graph);
+  for (const f of graph.files) {
+    if (testMap.testFiles.has(f.rel)) f.testFile = true;
+  }
+  for (const m of graph.modules) {
+    const t = testMap.testedByModule.get(m.slug);
+    if (t?.length) m.testedBy = t;
+  }
+
+  const surprises = computeSurprises(graph);
+  if (surprises.length) graph.surprises = surprises;
+
+  const symbols = buildSymbolIndex(scan, computeSymbolRefs(scan));
+  return { scan, graph, symbols };
+}

package/src/query.ts

@@ -0,0 +1,118 @@
+// Symbol-level query API (Serena-parity tier, static edition): overview of a
+// file's symbols, name-path symbol lookup with optional bodies, and reference
+// finding that merges line-precise caller sites with file-level identifier
+// references. Everything is computed from the deterministic scan — no language
+// server, no daemon; precision is honestly labeled per result so an agent
+// knows when it is looking at compiler-grade truth vs a name-based match.
+import { join } from "node:path";
+import type { CodeSymbol } from "./types.js";
+import type { RepoScan } from "./scan.js";
+import { readText } from "./walk.js";
+import { buildCallerIndex, type CallerSite } from "./callers.js";
+import { uniqueSymbolDefs } from "./graph.js";
+import { byStr } from "./sort.js";
+
+const REFERENCE_KINDS = new Set(["reexport", "reexport-all", "default"]);
+
+// All symbols declared in one file, in declaration order — the fastest way to
+// understand a file without reading it.
+export function symbolsOverview(scan: RepoScan, rel: string): CodeSymbol[] {
+  const f = scan.files.find((x) => x.rel === rel);
+  if (!f) return [];
+  return [...f.symbols].filter((s) => !REFERENCE_KINDS.has(s.kind)).sort((a, b) => a.line - b.line || byStr(a.name, b.name));
+}
+
+export interface SymbolMatch extends CodeSymbol {
+  body?: string; // the declaration's source lines (AST spans; absent without endLine unless the decl is one line)
+}
+
+export interface FindSymbolOptions {
+  substring?: boolean; // match name segments by inclusion instead of equality
+  includeBody?: boolean;
+  maxResults?: number; // default 50
+}
+
+// Look up symbols by name or name path ("Class/method" matches a `method`
+// whose enclosing symbol is `Class`). Deterministic ordering: exact name
+// matches first, then (file, line).
+export function findSymbol(scan: RepoScan, namePath: string, opts: FindSymbolOptions = {}): SymbolMatch[] {
+  const segments = namePath.split("/").filter(Boolean);
+  if (!segments.length) return [];
+  const leaf = segments[segments.length - 1]!;
+  const parents = segments.slice(0, -1);
+  const matchName = (name: string, wanted: string): boolean =>
+    opts.substring ? name.toLowerCase().includes(wanted.toLowerCase()) : name === wanted;
+
+  const out: SymbolMatch[] = [];
+  for (const f of scan.files) {
+    for (const s of f.symbols) {
+      if (REFERENCE_KINDS.has(s.kind)) continue;
+      if (!matchName(s.name, leaf)) continue;
+      // Walk the parent chain (single level in practice — extractor records the
+      // enclosing symbol name) against the requested path suffix. Substring
+      // matching applies to the LAST segment only (Serena's contract): parent
+      // segments always match exactly.
+      if (parents.length) {
+        const parent = parents[parents.length - 1]!;
+        if (!s.parent || s.parent !== parent) continue;
+      }
+      out.push({ ...s });
+    }
+  }
+  out.sort(
+    (a, b) => Number(b.name === leaf) - Number(a.name === leaf) || byStr(a.file, b.file) || a.line - b.line,
+  );
+  const capped = out.slice(0, opts.maxResults ?? 50);
+  if (opts.includeBody) {
+    for (const m of capped) {
+      const end = m.endLine ?? m.line;
+      const content = readText(join(scan.root, m.file));
+      if (!content) continue;
+      m.body = content.split("\n").slice(m.line - 1, end).join("\n");
+    }
+  }
+  return capped;
+}
+
+export interface SymbolReferences {
+  defs: CodeSymbol[]; // where the name is declared
+  // Line-precise call sites bound by the caller index (family-gated, import-
+  // corroborated for JS/TS) — the highest-confidence reference tier.
+  callSites: CallerSite[];
+  // Files whose collected identifiers reference the name (AST idents / doc
+  // mentions) — file-level, name-based: may include homonym false positives.
+  referencingFiles: string[];
+}
+
+// Who references this symbol? Merges the caller index (line-precise) with the
+// identifier/mention pass (file-level), each tier labeled by its field.
+export function findReferences(scan: RepoScan, name: string): SymbolReferences {
+  const defs: CodeSymbol[] = [];
+  for (const f of scan.files) {
+    for (const s of f.symbols) {
+      if (s.name === name && !REFERENCE_KINDS.has(s.kind)) defs.push(s);
+    }
+  }
+  defs.sort((a, b) => byStr(a.file, b.file) || a.line - b.line);
+
+  const index = buildCallerIndex(scan);
+  const entry = index.get(name);
+  const callSites = entry ? entry.callers : [];
+
+  const referencingFiles = new Set<string>();
+  const unique = uniqueSymbolDefs(scan);
+  const defFile = unique.get(name);
+  for (const f of scan.files) {
+    if (f.rel === defFile) continue;
+    if (f.kind === "code" && f.idents?.includes(name)) referencingFiles.add(f.rel);
+    else if (f.kind === "doc") {
+      const content = scan.docText.get(f.rel);
+      if (content && new RegExp(`\\b${name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`).test(content)) {
+        referencingFiles.add(f.rel);
+      }
+    }
+  }
+  for (const site of callSites) referencingFiles.add(site.file);
+
+  return { defs, callSites, referencingFiles: [...referencingFiles].sort(byStr) };
+}

package/src/render/graph-json.ts

@@ -0,0 +1,16 @@
+import type { Graph } from "../types.js";
+import { byStr } from "../sort.js";
+
+function sortObject(obj: Record<string, number>): Record<string, number> {
+  const out: Record<string, number> = {};
+  for (const k of Object.keys(obj).sort(byStr)) out[k] = obj[k]!;
+  return out;
+}
+
+// Serialize the graph to stable, pretty JSON. Every array is already sorted
+// deterministically by the graph builder; we only normalize the language map's
+// key order here so two builds of an unchanged repo are byte-identical.
+export function renderGraphJson(graph: Graph): string {
+  const ordered: Graph = { ...graph, languages: sortObject(graph.languages) };
+  return JSON.stringify(ordered, null, 2) + "\n";
+}

package/src/render/symbols-json.ts

@@ -0,0 +1,79 @@
+import { SCHEMA_VERSION } from "../types.js";
+import type { SymbolIndex } from "../types.js";
+import type { RepoScan } from "../scan.js";
+import { byStr } from "../sort.js";
+import { uniqueSymbolDefs } from "../graph.js";
+
+// Files that REFERENCE each unique exported symbol — code files via their AST
+// identifiers, doc files via naming the symbol. Feeds symbols.json `refs` so
+// `symbols <name>` can answer "where is X used?". Mirrors the graph's use/mention
+// eligibility (unique + distinctive), so refs and edges stay consistent.
+export function computeSymbolRefs(scan: RepoScan): Map<string, Set<string>> {
+  const unique = uniqueSymbolDefs(scan);
+  const refs = new Map<string, Set<string>>();
+  if (!unique.size) return refs;
+  const add = (name: string, file: string): void => {
+    let set = refs.get(name);
+    if (!set) refs.set(name, (set = new Set()));
+    set.add(file);
+  };
+  for (const f of scan.files) {
+    if (f.kind === "code" && f.idents) {
+      for (const id of f.idents) {
+        const target = unique.get(id);
+        if (target && target !== f.rel) add(id, f.rel);
+      }
+    } else if (f.kind === "doc") {
+      const content = scan.docText.get(f.rel);
+      if (!content) continue;
+      for (const tok of content.split(/[^A-Za-z0-9_]+/)) {
+        const target = unique.get(tok);
+        if (target && target !== f.rel) add(tok, f.rel);
+      }
+    }
+  }
+  return refs;
+}
+
+// Build the persisted symbol table from the scan. `defs` collects every declared
+// symbol by name; `refs` (files that USE a name) is filled by the graph's
+// use/mention pass and merged in here. Ordering is fully deterministic so
+// symbols.json is byte-stable across rebuilds.
+export function buildSymbolIndex(scan: RepoScan, refs: Map<string, Set<string>> = new Map()): SymbolIndex {
+  const defsByName = new Map<string, SymbolIndex["defs"][string]>();
+  for (const f of scan.files) {
+    for (const s of f.symbols) {
+      let arr = defsByName.get(s.name);
+      if (!arr) defsByName.set(s.name, (arr = []));
+      arr.push({
+        file: s.file,
+        line: s.line,
+        ...(s.endLine !== undefined ? { endLine: s.endLine } : {}),
+        kind: s.kind,
+        exported: s.exported,
+        lang: s.lang,
+        ...(s.parent ? { parent: s.parent } : {}),
+      });
+    }
+  }
+
+  const defs: SymbolIndex["defs"] = {};
+  for (const name of [...defsByName.keys()].sort(byStr)) {
+    defs[name] = defsByName
+      .get(name)!
+      .slice()
+      .sort((a, b) => byStr(a.file, b.file) || a.line - b.line || byStr(a.kind, b.kind));
+  }
+
+  const refsOut: SymbolIndex["refs"] = {};
+  for (const name of [...refs.keys()].sort(byStr)) {
+    const files = [...refs.get(name)!].sort(byStr);
+    if (files.length) refsOut[name] = files;
+  }
+
+  return { schemaVersion: SCHEMA_VERSION, defs, refs: refsOut };
+}
+
+export function renderSymbolsJson(index: SymbolIndex): string {
+  return JSON.stringify(index, null, 2) + "\n";
+}

package/src/repomap.ts

@@ -0,0 +1,52 @@
+// Token-budgeted repo map (Aider-lineage, but deterministic and served from
+// the persisted index): the highest-PageRank files with their key exported
+// signatures, rendered to fit a token budget — the densest possible "what is
+// this codebase" context an agent can load in one read. Unlike Aider's map it
+// never varies with conversation state: same repo → same bytes.
+import type { Graph } from "./types.js";
+import type { RepoScan } from "./scan.js";
+import { byStr } from "./sort.js";
+
+export interface RepoMapOptions {
+  // Approximate token budget (chars/4 heuristic — deterministic, tokenizer-
+  // free). Default 1024 tokens, Aider's default map size.
+  budgetTokens?: number;
+  maxSymbolsPerFile?: number; // default 8
+}
+
+const CHARS_PER_TOKEN = 4;
+
+export function renderRepoMap(scan: RepoScan, graph: Graph, opts: RepoMapOptions = {}): string {
+  const budgetChars = (opts.budgetTokens ?? 1024) * CHARS_PER_TOKEN;
+  const maxSymbols = opts.maxSymbolsPerFile ?? 8;
+
+  // Rank: PageRank first (stamped by the pipeline's centrality pass), then
+  // symbol count, then path — all deterministic tie-breaks.
+  const ranked = [...graph.files]
+    .filter((f) => f.fileKind === "code")
+    .sort((a, b) => (b.pagerank ?? 0) - (a.pagerank ?? 0) || b.symbols - a.symbols || byStr(a.rel, b.rel));
+  const records = new Map(scan.files.map((f) => [f.rel, f]));
+
+  const header = `# repo map — ${graph.fileCount} files\n`;
+  let out = header;
+  let files = 0;
+  for (const node of ranked) {
+    const rec = records.get(node.rel);
+    if (!rec) continue;
+    // Exported symbols first, declaration order within each group; reference
+    // kinds (re-exports) carry no signature worth budget.
+    const symbols = [...rec.symbols]
+      .filter((s) => s.kind !== "reexport" && s.kind !== "reexport-all")
+      .sort((a, b) => Number(b.exported) - Number(a.exported) || a.line - b.line)
+      .slice(0, maxSymbols);
+    let block = `\n${node.rel}:\n`;
+    for (const s of symbols) {
+      const sig = (s.signature ?? `${s.kind} ${s.name}`).replace(/\s+/g, " ").trim().slice(0, 120);
+      block += `  ${s.line}: ${sig}\n`;
+    }
+    if (out.length + block.length > budgetChars) break;
+    out += block;
+    files++;
+  }
+  return `${out}\n(${files} of ${ranked.length} code files shown, ~${Math.ceil(out.length / CHARS_PER_TOKEN)} tokens)\n`;
+}