@polycode-projects/seonix 0.1.0 → 0.2.1

package/src/extract.mjs

@@ -23,6 +23,9 @@ import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import * as codegraph from "./codegraph.mjs"; // optional renderToolsCatalog (other agent owns this file)
 import { ingestRepo, LANG_EXTS } from "./extract_lang.mjs";
+import { loadIgnores } from "./walk.mjs";
+import { attachProseTokens, buildProseIndex } from "./prose.mjs";
+import { ingestSchemaDocs } from "./schema-docs.mjs";
 
 const here = dirname(fileURLToPath(import.meta.url));
 const AST_SCRIPT = join(here, "extract_ast.py");
@@ -92,6 +95,18 @@ async function runAst(repoPath, python) {
   return Array.isArray(parsed?.modules) ? parsed.modules : [];
 }
 
+// Paths git-log itself must never diff, mirroring .seonixignore's top-level excludes.
+// loadIgnores()/the `ignore` predicate (indexRepository, below) only filters the
+// FINAL module list — by then git log -p has already run and Node has already
+// buffered its full output. For a directory carrying many large files (results/:
+// 9,315 files / 529MB as of 2026-07-02), that historical patch content alone is
+// gigabytes, which OOM'd a memory-constrained CI runner even though .seonixignore
+// correctly kept those paths out of the resulting graph. Pathspec excludes stop git
+// from generating that content at all. Keep in sync with .seonixignore (candidate
+// for the seonix.toml config consolidation to share one list instead of two).
+export const GIT_LOG_EXCLUDE = ["corpus", "target", "vendor", "infra/cdk.out", "results"];
+const gitPathspecExcludes = () => GIT_LOG_EXCLUDE.map((p) => `:(exclude)${p}`);
+
 /** git log → [{sha, author, date, subject, files[]}] over the last N commits
  *  (best-effort; [] if no repo). Header record fields are \x1e-separated; commits
  *  are \x1f-separated; the subject (%s) is single-line so it never collides with
@@ -99,7 +114,7 @@ async function runAst(repoPath, python) {
 async function runGitLog(repoPath, depth = gitDepth()) {
   const { code, stdout } = await exec("git",
     ["log", `-n`, String(depth), "--no-renames", "--name-only",
-      "--pretty=format:%x1f%H%x1e%an%x1e%aI%x1e%s", "--", "."],
+      "--pretty=format:%x1f%H%x1e%an%x1e%aI%x1e%s", "--", ".", ...gitPathspecExcludes()],
     { cwd: repoPath });
   if (code !== 0) return [];
   const out = [];
@@ -125,7 +140,7 @@ async function runGitLogHunks(repoPath, depth = historySymbolDepth()) {
   if (!depth) return [];
   const { code, stdout } = await exec("git",
     ["log", `-n`, String(depth), "--no-renames", "--no-color", "--unified=0",
-      "--pretty=format:%x1f%H", "--", "."],
+      "--pretty=format:%x1f%H", "--", ".", ...gitPathspecExcludes()],
     { cwd: repoPath });
   if (code !== 0) return [];
   const out = [];
@@ -160,7 +175,7 @@ async function runGitLogHunks(repoPath, depth = historySymbolDepth()) {
 /** Build the `entities` payload from the parsed modules + git history.
  *  `symbolHistory` (optional) is the runGitLogHunks() output — per-commit changed
  *  line ranges, intersected with symbol spans to emit mgx:touchesSymbol edges. */
-export function buildEntities(modules, commits, { generatedAt = "", symbolHistory = [] } = {}) {
+export function buildEntities(modules, commits, { generatedAt = "", symbolHistory = [], prose = true } = {}) {
   const modById = new Map();         // path -> module record
   const dottedToPath = new Map();    // dotted module name -> path
   const nameToPaths = new Map();     // top-level symbol name -> Set<path> (for call resolution)
@@ -459,6 +474,12 @@ export function buildEntities(modules, commits, { generatedAt = "", symbolHistor
   const countClass = (c) => fnIndividuals.filter((i) => i.class === c).length;
   const sampleClass = (c) => fnIndividuals.filter((i) => i.class === c).slice(0, 3).map((i) => i.label);
 
+  // Second pass (PLAN_PROSE_INDEX.md) — see the returned `proseIndex` field's comment below.
+  const allIndividuals = attachProseTokens(
+    [...moduleIndividuals, ...fnIndividuals, ...commitIndividuals], { enabled: prose },
+  );
+  const proseIndex = prose ? buildProseIndex(allIndividuals) : {};
+
   return {
     generated_at: generatedAt,
     // SEON (se-on.org, FAMIX-derived) vocabulary + our `mgx:` extension, documented
@@ -518,7 +539,12 @@ export function buildEntities(modules, commits, { generatedAt = "", symbolHistor
       rel("cochange", "mgx:changeCoupledWith", cochangeEdges),
       rel("reexports", "mgx:reExports", reExportEdges),
     ],
-    individuals: [...moduleIndividuals, ...fnIndividuals, ...commitIndividuals],
+    individuals: allIndividuals,
+    // Second pass (PLAN_PROSE_INDEX.md): word -> [individual ids], inverted from the
+    // `prose_tokens` attribute attachProseTokens just attached. Disable via
+    // SEONIX_PROSE_INDEX=0 (indexRepository, below) — {} when off. The typed graph above
+    // (individuals' core fields, all edges) is byte-identical either way.
+    proseIndex,
   };
 }
 
@@ -571,26 +597,41 @@ function localToolsCatalog(cliPath) {
  * symbol-history (line-range) pass so the latter can be kept within the ≤10% budget.
  * @returns {Promise<{graphFile, counts}>}
  */
-export async function indexRepository(repoPath, { python = process.env.SEONIX_PYTHON || process.env.SEON_PYTHON || "python3", generatedAt = "" } = {}) {
+export async function indexRepository(repoPath, { python = process.env.SEONIX_PYTHON || process.env.SEON_PYTHON || "python3", generatedAt = "", ignores = true } = {}) {
   const t0 = Date.now();
+  // Second pass (PLAN_PROSE_INDEX.md): on by default; SEONIX_PROSE_INDEX=0 disables it
+  // (mirrors SEONIX_HISTORY_SYMBOL_DEPTH=0's disable convention above).
+  const proseEnabled = process.env.SEONIX_PROSE_INDEX !== "0";
+  // `.seonixignore` (opt-out: `ignores:false`, the benchmark rig's flag). The matcher
+  // prunes the in-process walkers (extract_lang) at parse time; the module filter below
+  // is the AUTHORITATIVE cut — it also covers extractors that walk on their own
+  // (extract_ast.py, the Java fat-jar, Roslyn), and buildEntities drops history edges
+  // for any module not in the list, so the whole graph honours the same exclusions.
+  const ignore = ignores ? await loadIgnores(repoPath) : null;
   // base extraction: ast (Python) + multi-language front-end (TS/JS, C#) + module-level
   // git history (the cheap, full-depth pass). The Python and language passes are
   // independent parsers; their `{modules:[…]}` outputs share one contract and are merged
   // (dedupe by path — Python from runAst wins) before buildEntities.
   const [pyModules, langResult, commits] = await Promise.all([
     runAst(repoPath, python),
-    ingestRepo(repoPath),
+    ingestRepo(repoPath, { ignore }),
     runGitLog(repoPath, gitDepth()),
   ]);
   const seenPaths = new Set(pyModules.map((m) => m.path));
-  const modules = [...pyModules, ...langResult.modules.filter((m) => !seenPaths.has(m.path))];
+  const merged = [...pyModules, ...langResult.modules.filter((m) => !seenPaths.has(m.path))];
+  const modules = ignore ? merged.filter((m) => !ignore(m.path)) : merged;
   const baseMs = Date.now() - t0;
   // added symbol-history pass (the costly per-commit hunk diff) — budgeted via depth
   const tHist = Date.now();
   const symbolHistory = await runGitLogHunks(repoPath, historySymbolDepth());
   const historyMs = Date.now() - tHist;
 
-  const entities = buildEntities(modules, commits, { generatedAt, symbolHistory });
+  const entities = buildEntities(modules, commits, { generatedAt, symbolHistory, prose: proseEnabled });
+  // Schema self-documentation (schema-docs.mjs): static, repo-independent — merges in
+  // SchemaClass/SchemaPredicate individuals + backfills entities.classes[].description
+  // and entities.vocabulary[].note, so "what does cochange mean" is answerable by the
+  // same graph traversal as any other question. Fixed-size, ~0ms marginal cost.
+  ingestSchemaDocs(entities);
   const graphFile = join(repoPath, ".seonix", "graph.json");
   await mkdir(dirname(graphFile), { recursive: true });
   await writeFile(graphFile, JSON.stringify(entities));
@@ -616,6 +657,8 @@ export async function indexRepository(repoPath, { python = process.env.SEONIX_PY
       callsSymbol: propCount("mgx:callsSymbol"),
       touchesSymbol: propCount("mgx:touchesSymbol"),
       historySymbolDepth: historySymbolDepth(),
+      proseEnabled,
+      proseWords: Object.keys(entities.proseIndex || {}).length,
       baseMs,
       historyMs,
       historyPct,

package/src/extract_lang.mjs

@@ -10,6 +10,8 @@
 import * as jstsTsc from "./jsts_tsc.mjs";
 import * as csRoslyn from "./cs_roslyn.mjs";
 import * as csTree from "./cs_treesitter.mjs";
+import * as javaJavaparser from "./java_javaparser.mjs";
+import * as javaTree from "./java_treesitter.mjs";
 
 // THE EXTRACTOR REGISTRY — the single place a new language plugs in. Keyed by
 // language; `exts` is the file-extension surface it owns, `pick` the default
@@ -17,6 +19,7 @@ import * as csTree from "./cs_treesitter.mjs";
 export const REGISTRY = {
   "js/ts": { exts: [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"], pick: jstsTsc, alt: null },
   "c#": { exts: [".cs"], pick: csRoslyn, alt: csTree },
+  "java": { exts: [".java"], pick: javaJavaparser, alt: javaTree },
 };
 
 /** The full set of file extensions this front-end indexes (sorted, lower-cased).
@@ -32,15 +35,27 @@ async function chooseCs(prefer) {
   return csTree;
 }
 
+/** Choose the Java extractor: JavaParser+SymbolSolver (JDK helper) when a `java`
+ *  runtime and the built jar are present, else the tree-sitter fallback. */
+async function chooseJava(prefer) {
+  if (prefer === "treesitter") return javaTree;
+  if (prefer === "javaparser" || prefer === undefined) {
+    if (await javaJavaparser.toolAvailable()) return javaJavaparser;
+  }
+  return javaTree;
+}
+
 /**
  * Parse every supported non-Python language under `root` and merge the results.
  * @returns {Promise<{modules:Array, perLang:Object, totalFiles:number, failures:Array}>}
  */
-export async function ingestRepo(root, { cs } = {}) {
+export async function ingestRepo(root, { cs, java, ignore = null } = {}) {
   const csExtractor = await chooseCs(cs);
+  const javaExtractor = await chooseJava(java);
   const plan = [
     { lang: "js/ts", extractor: jstsTsc },
     { lang: "c#", extractor: csExtractor },
+    { lang: "java", extractor: javaExtractor },
   ];
   const allModules = [];
   const perLang = {};
@@ -48,7 +63,7 @@ export async function ingestRepo(root, { cs } = {}) {
   const allFailures = [];
   for (const { lang, extractor } of plan) {
     const t0 = Date.now();
-    const { modules, failures, fileCount } = await extractor.ingest(root);
+    const { modules, failures, fileCount } = await extractor.ingest(root, { ignore });
     const ms = Date.now() - t0;
     if (fileCount === 0) continue; // language not present
     allModules.push(...modules);

package/src/java_javaparser.mjs

@@ -0,0 +1,54 @@
+// Java extractor — JavaParser + JavaSymbolSolver (resolved calls) via the bundled
+// JVM helper. PICKED Java candidate when a JDK/`java` runtime is present. Spawns
+// `java -jar java/dist/seonix-java-extract.jar <root>` which prints the
+// {modules:[…]} contract; this wrapper just shells out and parses it. Falling back
+// (java_treesitter) is the caller's job when `java`/the jar is absent.
+import { spawn } from "node:child_process";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { access } from "node:fs/promises";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const TOOL_JAR = join(here, "..", "java", "dist", "seonix-java-extract.jar");
+
+function run(cmd, args) {
+  return new Promise((resolve) => {
+    const child = spawn(cmd, args);
+    let out = ""; let err = "";
+    child.stdout.on("data", (d) => (out += d));
+    child.stderr.on("data", (d) => (err += d));
+    child.on("close", (code) => resolve({ code: code ?? -1, out, err }));
+    child.on("error", (e) => resolve({ code: -1, out, err: String(e) }));
+  });
+}
+
+let _javaOk = null;
+async function javaOnPath() {
+  if (_javaOk !== null) return _javaOk;
+  const { code } = await run("java", ["-version"]);
+  _javaOk = code === 0;
+  return _javaOk;
+}
+
+async function jarExists() {
+  try { await access(TOOL_JAR); return true; } catch { return false; }
+}
+
+/** True iff a `java` runtime is on PATH AND the shaded fat-jar is present. */
+export async function toolAvailable() {
+  if (!(await jarExists())) return false;
+  return javaOnPath();
+}
+
+export async function ingest(root) {
+  if (!(await jarExists())) throw new Error("seonix-java-extract.jar not built (run `mvn package` in java/)");
+  if (!(await javaOnPath())) throw new Error("`java` runtime not on PATH (install a JDK, or use the tree-sitter fallback)");
+  const { code, out, err } = await run("java", ["-jar", TOOL_JAR, root]);
+  if (code !== 0) throw new Error(`seonix-java-extract failed (exit ${code}): ${err.slice(-300)}`);
+  let parsed;
+  try { parsed = JSON.parse(out); }
+  catch { throw new Error(`seonix-java-extract produced non-JSON (${out.slice(0, 120)})`); }
+  return { modules: parsed.modules || [], failures: parsed.failures || [], fileCount: parsed.fileCount || 0 };
+}
+
+export const meta = { id: "javaparser", language: "java", lib: "JavaParser + JavaSymbolSolver (resolved calls)", exts: [".java"] };

package/src/java_treesitter.mjs

@@ -0,0 +1,216 @@
+// Java extractor — tree-sitter-java (CST-only) fallback for the bake-off. The
+// no-JDK path: a uniform CST, no symbol solver (so call targets are syntax-level
+// names, NOT resolved to in-repo/JDK qualified names — that promotion is the
+// JavaParser backend's job). Emits the same {modules:[{path,dotted,imports,
+// defines,calls,exports}]} contract so codegraph.mjs/digest consume it unchanged.
+// Module identity is per-file; `dotted` is the file's package + primary type.
+import { readFile } from "node:fs/promises";
+import { createRequire } from "node:module";
+import { walk, relPath } from "./walk.mjs";
+
+// tree-sitter is the OPTIONAL fallback backend (used only when a JDK/`java` is
+// absent). It is a native module and NOT a declared seonix dependency, so the
+// require is LAZY — importing this module (e.g. via extract_lang's REGISTRY) must
+// not force the native build. Resolved on first parse; a clear error if missing.
+const require = createRequire(import.meta.url);
+let _Parser = null;
+let _Java = null;
+function loadTreeSitter() {
+  if (_Parser) return { Parser: _Parser, Java: _Java };
+  try {
+    _Parser = require("tree-sitter");
+    _Java = require("tree-sitter-java");
+  } catch (e) {
+    throw new Error(
+      `tree-sitter Java fallback unavailable (install tree-sitter + tree-sitter-java, or use the JavaParser backend): ${e?.message || e}`,
+    );
+  }
+  return { Parser: _Parser, Java: _Java };
+}
+
+const EXTS = [".java"];
+const line = (n) => n.startPosition.row + 1;
+const endLine = (n) => n.endPosition.row + 1;
+const oneLine = (s) => (s || "").replace(/\s+/g, " ").trim();
+const clip = (s, n) => (s.length <= n ? s : s.slice(0, n));
+const fieldText = (n, f) => n.childForFieldName(f)?.text || "";
+
+const TYPE_DECLS = new Set(["class_declaration", "interface_declaration", "enum_declaration", "record_declaration"]);
+const BASE_WRAPPERS = new Set(["superclass", "super_interfaces", "extends_interfaces", "interfaces"]);
+const TYPE_NODES = new Set(["type_identifier", "generic_type", "scoped_type_identifier"]);
+
+/** Callee names within a node (syntax-level; unresolved). */
+function collectCalls(node) {
+  const out = new Set();
+  const stack = [node];
+  while (stack.length) {
+    const n = stack.pop();
+    if (n.type === "method_invocation") {
+      const nm = fieldText(n, "name");
+      if (nm) out.add(clip(nm, 80));
+    }
+    for (let i = 0; i < n.namedChildCount; i++) stack.push(n.namedChild(i));
+  }
+  return [...out].sort();
+}
+
+/** Base types (extends + implements) declared on a type node. */
+function basesOf(node) {
+  const bases = [];
+  for (let i = 0; i < node.namedChildCount; i++) {
+    const c = node.namedChild(i);
+    if (!BASE_WRAPPERS.has(c.type)) continue;
+    const stack = [c];
+    while (stack.length) {
+      const w = stack.pop();
+      if (TYPE_NODES.has(w.type)) { bases.push(clip(oneLine(w.text), 80)); continue; }
+      for (let j = 0; j < w.namedChildCount; j++) stack.push(w.namedChild(j));
+    }
+  }
+  return bases;
+}
+
+function modifiersNode(node) {
+  for (let i = 0; i < node.namedChildCount; i++) {
+    if (node.namedChild(i).type === "modifiers") return node.namedChild(i);
+  }
+  return null;
+}
+function decoratorsFrom(mods) {
+  if (!mods) return [];
+  const out = [];
+  for (let i = 0; i < mods.namedChildCount; i++) {
+    const c = mods.namedChild(i);
+    if (c.type.endsWith("annotation")) out.push(clip(oneLine(c.text), 80));
+  }
+  return out;
+}
+function visFrom(mods) {
+  const t = mods?.text || "";
+  return /\bprivate\b/.test(t) ? "private" : /\bprotected\b/.test(t) ? "protected" : "";
+}
+const isPublic = (mods) => /\bpublic\b/.test(mods?.text || "");
+const isStatic = (mods) => /\bstatic\b/.test(mods?.text || "");
+
+function paramsText(node) {
+  const fp = node.childForFieldName("parameters");
+  if (!fp) return "";
+  return clip(oneLine(fp.text.replace(/^\(|\)$/g, "")), 160);
+}
+
+function bodyContainer(typeNode) {
+  return typeNode.childForFieldName("body")
+    || typeNode.namedChildren?.find((c) => c.type === "class_body" || c.type === "enum_body" || c.type === "interface_body")
+    || null;
+}
+
+export async function extractFile(absPath, root) {
+  const text = await readFile(absPath, "utf8").catch(() => null);
+  const path = relPath(root, absPath);
+  if (text == null) return { failed: true, path };
+  let tree;
+  try {
+    const { Parser, Java } = loadTreeSitter();
+    const p = new Parser(); p.setLanguage(Java);
+    tree = p.parse(text);
+  } catch { return { failed: true, path }; }
+
+  const imports = new Set();
+  const defines = [];
+  const exports = new Set();
+  let pkg = "";
+
+  const stack = [tree.rootNode];
+  while (stack.length) {
+    const n = stack.pop();
+    if (n.type === "package_declaration" && !pkg) {
+      pkg = oneLine(n.namedChildren.map((c) => c.text).join(""));
+    } else if (n.type === "import_declaration") {
+      if (/\bstatic\b/.test(n.text)) { /* skip static imports (mirror JavaParser) */ }
+      else {
+        const nm = n.namedChildren.find((c) => /identifier/.test(c.type))?.text;
+        if (nm) imports.add(oneLine(nm));
+      }
+    } else if (TYPE_DECLS.has(n.type)) {
+      const cname = fieldText(n, "name");
+      if (cname) {
+        const mods = modifiersNode(n);
+        defines.push({
+          name: cname, kind: "class", lineno: line(n), end_lineno: endLine(n),
+          bases: basesOf(n), decorators: decoratorsFrom(mods),
+          ...(visFrom(mods) ? { visibility: visFrom(mods) } : {}),
+        });
+        if (isPublic(mods)) exports.add(cname);
+
+        // record components (formal params) surface as attributes (match JavaParser)
+        if (n.type === "record_declaration") {
+          const fp = n.childForFieldName("parameters");
+          if (fp) for (let i = 0; i < fp.namedChildCount; i++) {
+            const fpc = fp.namedChild(i);
+            if (fpc.type !== "formal_parameter") continue;
+            const pn = fieldText(fpc, "name");
+            if (pn) defines.push({ name: `${cname}.${pn}`, kind: "attribute", lineno: line(fpc), end_lineno: endLine(fpc), decorators: [] });
+          }
+        }
+
+        const body = bodyContainer(n);
+        if (body) {
+          for (let i = 0; i < body.namedChildCount; i++) {
+            const mem = body.namedChild(i);
+            if (mem.type === "method_declaration" || mem.type === "constructor_declaration") {
+              const mn = fieldText(mem, "name") || cname;
+              const mmods = modifiersNode(mem);
+              const returns = mem.type === "method_declaration"
+                ? clip(oneLine(mem.childForFieldName("type")?.text || ""), 80) : "";
+              defines.push({
+                name: `${cname}.${mn}`, kind: "method", lineno: line(mem), end_lineno: endLine(mem),
+                decorators: decoratorsFrom(mmods), params: paramsText(mem),
+                ...(returns ? { returns } : {}), calls: collectCalls(mem),
+                ...(isStatic(mmods) ? { is_static: true } : {}),
+                ...(visFrom(mmods) ? { visibility: visFrom(mmods) } : {}),
+              });
+            } else if (mem.type === "field_declaration") {
+              const fmods = modifiersNode(mem);
+              for (let j = 0; j < mem.namedChildCount; j++) {
+                const vd = mem.namedChild(j);
+                if (vd.type !== "variable_declarator") continue;
+                const fn = fieldText(vd, "name");
+                if (fn) defines.push({
+                  name: `${cname}.${fn}`, kind: "attribute", lineno: line(mem), end_lineno: endLine(mem),
+                  decorators: decoratorsFrom(fmods),
+                  ...(visFrom(fmods) ? { visibility: visFrom(fmods) } : {}),
+                });
+              }
+            }
+          }
+        }
+      }
+    }
+    // descend, but not back into a type body (members handled above)
+    if (!TYPE_DECLS.has(n.type)) for (let i = 0; i < n.namedChildCount; i++) stack.push(n.namedChild(i));
+  }
+
+  const base = path.replace(/\.java$/i, "").split("/").pop();
+  const dotted = pkg ? `${pkg}.${base}` : path.replace(/\.java$/i, "").split("/").join(".");
+  return {
+    path, dotted, imports: [...imports].sort(), defines,
+    calls: collectCalls(tree.rootNode), exports: [...exports].sort(),
+  };
+}
+
+export async function ingest(root, { ignore = null } = {}) {
+  const files = await walk(root, EXTS, ignore);
+  const modules = [];
+  const failures = [];
+  for (const f of files) {
+    const mod = await extractFile(f, root);
+    if (mod.failed) failures.push(mod.path); else modules.push(mod);
+  }
+  return { modules, failures, fileCount: files.length };
+}
+
+export function toolAvailable() {
+  try { loadTreeSitter(); return true; } catch { return false; }
+}
+
+export const meta = { id: "treesitter-java", language: "java", lib: "tree-sitter-java (CST, syntax-level)", exts: EXTS };

package/src/jsts_tsc.mjs

@@ -196,8 +196,8 @@ export async function extractFile(absPath, root) {
 }
 
 /** Ingest a whole tree → {modules:[…]} contract + parse-failure list. */
-export async function ingest(root) {
-  const files = await walk(root, EXTS);
+export async function ingest(root, { ignore = null } = {}) {
+  const files = await walk(root, EXTS, ignore);
   const modules = [];
   const failures = [];
   for (const f of files) {

package/src/prose.mjs

@@ -0,0 +1,145 @@
+// prose.mjs — the second-pass prose extraction + cross-reference index (PLAN_PROSE_INDEX.md).
+//
+// Deterministic, zero-model-calls, zero dependencies: pure string transforms + a plain-object
+// inverted index over data extract.mjs's typed pass has already produced (identifier names,
+// captured doc text). Two token sources, one combined set per individual:
+//   1. identifier decomposition — names are often literal sentence fragments
+//      ("calculateTotalPriceIncludingTax" -> calculate/total/price/including/tax); splitting
+//      them turns every symbol/module name into free-text search surface for zero extra cost.
+//   2. prose literals — docstrings/doc-comments already captured as the `doc` attribute.
+//      Python (extract_ast.py) and JS/TS (jsts_tsc.mjs's firstDocLine) already populate this;
+//      C#/Java's PRIMARY extractors (cs_roslyn.mjs, java_javaparser.mjs) shell out to compiled
+//      Roslyn/JavaParser binaries — adding doc capture there means modifying and rebuilding
+//      external .NET/JVM tooling, not a JS-side change. Deliberately deferred (PLAN_PROSE_INDEX.md
+//      backlog); this pass consumes whatever `doc` is already present, regardless of source
+//      language, so C#/Java modules still get identifier-decomposition tokens today.
+//
+// Tokenizer pattern (stopwords, length bounds, per-doc token cap) mirrors marginalia's
+// app/lib/text-index.mjs — a proven lexical-inverted-index tokenizer for the same "search by
+// keyword, no embeddings, no stemmer dependency" problem, adapted for code identifiers.
+//
+// Storage: (a) a `prose_tokens` attribute (space-joined, deduped, sorted) on each individual —
+// the graph stays self-describing, works if a consumer only has one individual in hand; AND
+// (b) a real inverted index (word -> [individual ids]) built from those same tokens and
+// attached as `entities.proseIndex` — an O(1) word lookup for consumers (resolveObject-style
+// fuzzy object-term resolution, scoreModules-style lexical boosting) instead of scanning every
+// individual's attributes. Both are derived from the identical token set, so they can never
+// disagree; (b) is just (a) inverted once, cheaply, at build time.
+
+const STOPWORDS = new Set(
+  ("a an and or but the of to in on at for with from by as is are was were be been being " +
+    "it its this that these those i you he she they we me my your our do does did not no " +
+    "yes if then else than so such can will would should could may might about into over " +
+    "under out up down off again more most some any all what which who whom whose when " +
+    "where why how").split(/\s+/),
+);
+
+const MAX_TOKEN_LEN = 40;    // drops hash-like/garbage tokens (marginalia text-index.mjs)
+const MAX_TOKENS_PER_DOC = 120; // bounds cost on a pathologically long docstring/name
+
+/** Split an identifier or a path-like name into lowercase word tokens.
+ *  Handles camelCase, PascalCase, snake_case, kebab-case, dotted names, path
+ *  separators, and acronym runs ("HTTPSConnection" -> https/connection,
+ *  "parseXML" -> parse/xml). Filters single-character tokens (loop-variable noise). */
+export function splitIdentifierWords(raw) {
+  if (!raw) return [];
+  let s = String(raw).replace(/\.[A-Za-z0-9]+$/, ""); // strip a trailing file extension only
+  s = s
+    .replace(/[/\\]/g, " ")            // path separators
+    .replace(/([a-z0-9])([A-Z])/g, "$1 $2")     // camelCase / word|Digit boundary
+    .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")  // acronym run -> TitleCase (HTTPSConnection)
+    .replace(/([A-Za-z])([0-9])/g, "$1 $2")
+    .replace(/([0-9])([A-Za-z])/g, "$1 $2")
+    .replace(/[_\-.]+/g, " ");
+  return s.split(/\s+/).map((w) => w.toLowerCase()).filter((w) => w.length > 1 && w.length <= MAX_TOKEN_LEN);
+}
+
+/** Tokenize free prose (a docstring/doc-comment) — lowercase words, punctuation stripped,
+ *  common stopwords and single-/over-length tokens dropped, capped at MAX_TOKENS_PER_DOC. */
+export function tokenizeProse(text) {
+  if (!text) return [];
+  const out = [];
+  const seen = new Set();
+  for (const raw of String(text).toLowerCase().split(/[^a-z0-9]+/)) {
+    if (raw.length < 2 || raw.length > MAX_TOKEN_LEN || STOPWORDS.has(raw)) continue;
+    if (seen.has(raw)) continue;
+    seen.add(raw);
+    out.push(raw);
+    if (out.length >= MAX_TOKENS_PER_DOC) break;
+  }
+  return out;
+}
+
+/** The combined, deduped, sorted token set for one individual: its (decomposed) name
+ *  plus any captured doc text. Returns [] if there's nothing to index (never null). */
+export function proseTokensFor({ name, doc } = {}) {
+  const set = new Set([...splitIdentifierWords(name), ...tokenizeProse(doc)]);
+  return [...set].sort();
+}
+
+/** Second pass (PLAN_PROSE_INDEX.md): attach a `prose_tokens` attribute to every
+ *  individual in `individuals`, deriving tokens from its name and captured prose text.
+ *  Class-aware source selection: Module/Function/Method/Class/Attribute/GlobalVariable use
+ *  `label` (a real identifier/path — decomposable, e.g. "calculateTotalPrice") + a `doc`
+ *  attribute if present (docstring/doc-comment). Commit is different: its `label` is a
+ *  truncated SHA (hex noise if decomposed, e.g. "e6a9419567f7" -> "9419567" garbage) — skip
+ *  decomposing it and tokenize its `message` attribute instead, which is the real prose
+ *  (commit messages are often the richest free text in the whole graph). Mutates and
+ *  returns the same array — safe to call once after the typed individuals are built.
+ *  `enabled=false` is a no-op (the disable path `SEONIX_PROSE_INDEX=0` in extract.mjs), so
+ *  the core typed graph is never affected by turning this pass off. */
+export function attachProseTokens(individuals, { enabled = true } = {}) {
+  if (!enabled) return individuals;
+  for (const ind of individuals) {
+    const attrs = ind.attributes || [];
+    const isCommit = ind.class === "Commit";
+    const name = isCommit ? null : ind.label;
+    const doc = isCommit
+      ? attrs.find((a) => a.key === "message")?.value
+      : attrs.find((a) => a.key === "doc")?.value;
+    const tokens = proseTokensFor({ name, doc });
+    if (tokens.length) {
+      ind.attributes = [...(ind.attributes || []), { prop: "mgx:hasProseTokens", key: "prose_tokens", value: tokens.join(" ") }];
+    }
+  }
+  return individuals;
+}
+
+/** Build the inverted index (word -> sorted, deduped [individual ids]) from individuals
+ *  that already carry a `prose_tokens` attribute (i.e. after attachProseTokens ran).
+ *  Plain object, JSON-serializable — this is what lands as `entities.proseIndex`. */
+export function buildProseIndex(individuals) {
+  const index = Object.create(null);
+  for (const ind of individuals) {
+    const tokAttr = (ind.attributes || []).find((a) => a.key === "prose_tokens");
+    if (!tokAttr?.value) continue;
+    for (const word of tokAttr.value.split(" ")) {
+      if (!index[word]) index[word] = [];
+      index[word].push(ind.id);
+    }
+  }
+  for (const word of Object.keys(index)) index[word].sort();
+  return index;
+}
+
+/** Consumer-facing lookup: individual ids whose prose tokens overlap `query` (free text,
+ *  tokenized the same way as a docstring), ranked by overlap count (most shared words
+ *  first). This is the integration point for ask.mjs's resolveObject (fuzzy object-term
+ *  resolution beyond exact/substring match) and codegraph.mjs's scoreModules (a lexical
+ *  boost source) — see PLAN_PROSE_INDEX.md for the exact call-site recommendation; not
+ *  wired into either file here to avoid colliding with concurrent work on them.
+ *  `proseIndex` is `entities.proseIndex` (buildProseIndex's output). */
+export function lookupByProseTokens(proseIndex, query, { limit = 10 } = {}) {
+  const queryTokens = [...new Set([...splitIdentifierWords(query), ...tokenizeProse(query)])];
+  if (!queryTokens.length) return [];
+  const scoreById = new Map();
+  for (const word of queryTokens) {
+    for (const id of proseIndex?.[word] || []) {
+      scoreById.set(id, (scoreById.get(id) || 0) + 1);
+    }
+  }
+  return [...scoreById.entries()]
+    .sort((a, b) => b[1] - a[1] || String(a[0]).localeCompare(b[0]))
+    .slice(0, limit)
+    .map(([id, score]) => ({ id, score }));
+}