@staix/agent-hub 0.20.0 → 0.21.0

package/src/hub/control-client.ts

@@ -1,7 +1,7 @@
 import { readFileSync } from "node:fs";
 import { join, resolve } from "node:path";
 import { projectContext } from "./project.ts";
-import { externalAdapterMetadata, type ExternalAdapterMetadata } from "../adapters/external-metadata.ts";
+import { externalAdapterMetadata, externalInstanceCatalog, type ExternalInstanceCatalog, type ExternalAdapterMetadata } from "../adapters/external-metadata.ts";
 
 export function stateDirFor(cwd: string): string {
   return projectContext(cwd).stateDir;
@@ -9,7 +9,7 @@ export function stateDirFor(cwd: string): string {
 
 /** Control WS wire version. 2 = `deliver` carries `envs` (digests); 3 = `tools` role and task messages; 4 = budget messages and `hub_checkpoint`; 5 = `ask`; 6 = console-only `ui` session bootstrap; 8 = controlled recovery; 9 = Pi bridge metadata; 10 = durable delivery receipts; 11 = queue hold diagnostics; 12 = generation-bound channel settlement and execution budgets; 13 = turn-free facts (`facts` with tool, session and transcript binding, `silenced`) and per-recipient send results (issue #108). 14 = native context-window status and session-bound context checkpoint request ids (issue #185). 15 = conductor tools and permission deadlines/closed events (issues #190, #194). 16 = a Claude session without channel evidence: hello `channel: false` and `inbox` (issue #205). The plugin is installed apart from the daemon, so they can drift. */
 export const PROTOCOL = 16;
-export const DAEMON_FEATURES = ["external_adapter_modules", "configured_acp", "native_hook_launch", "task_cancel", "recovery_deferred_hub", "peer_instances", "role_stages", "task_planners", "release_trains", "reviewer_pool", "native_tui_launch", "terminal_owner", "cli_audit_delegated"] as const;
+export const DAEMON_FEATURES = ["conductor_recovery", "encrypted_rooms", "external_adapter_instances", "external_adapter_modules", "configured_acp", "native_hook_launch", "task_cancel", "recovery_deferred_hub", "peer_instances", "role_stages", "task_planners", "release_trains", "reviewer_pool", "native_tui_launch", "terminal_owner", "cli_audit_delegated"] as const;
 /** Protocols a current coordinator may authenticate while upgrading a running source. */
 export const RECOVERY_SOURCE_PROTOCOLS = [9, 10, 11, 12, 13, 14, 15, PROTOCOL] as const;
 
@@ -166,6 +166,18 @@ export class ControlClient {
     });
   }
 
+  /** Read inert instance policy only from this authenticated, still-current daemon connection. */
+  async readExternalInstanceCatalog(projectRoot: string, timeoutMs = 3000): Promise<ExternalInstanceCatalog> {
+    const epoch = this.instanceId;
+    if (!epoch || !this.isOpen || !["external_adapter_instances", "external_adapter_modules", "peer_instances"].every(feature => this.features.includes(feature))) throw new Error("running hub lacks a live external instance catalog");
+    const reply = await this.request({ t: "status" }, timeoutMs);
+    const status = reply.status;
+    if (!this.isOpen || this.instanceId !== epoch || reply.ok === false || !status || status.instanceId !== epoch || status.cwd !== projectRoot) throw new Error("external instance catalog observation changed");
+    const catalog = externalInstanceCatalog(status.externalInstanceCatalog);
+    if (catalog.epoch !== epoch || catalog.projectRoot !== projectRoot) throw new Error("external instance catalog binding changed");
+    return catalog;
+  }
+
   /** `timeoutMs`: for calls a person waits on. The hub has its own budget; this one only makes sure the CLI never hangs. */
   request(msg: Record<string, unknown>, timeoutMs = 30_000): Promise<any> {
     if (this.ws.readyState !== WebSocket.OPEN) return Promise.resolve({ ok: false, error: "hub connection is not open" });

package/src/hub/corpus-config.ts

@@ -0,0 +1,34 @@
+// agent-hub #510 (hub task #331): the `corpus` config block. Pure on purpose: it is read by the config schema, which the Claude channel bundle reaches, so it
+// imports nothing that touches the file system. The index and the vault refusal are in `corpus.ts`.
+
+export const MAX_CORPUS_ROOTS = 16;
+export const MAX_CORPUS_PATH_CHARS = 1024;
+
+export interface CorpusConfig { paths: string[] }
+
+/** Why a path is not accepted, or undefined. A path is absolute or project-relative; `~` is never expanded, so what the person typed is what is read. */
+export function corpusPathProblem(value: unknown): string | undefined {
+  if (typeof value !== "string" || !value.trim()) return "a corpus path is a non-empty string";
+  if (value.length > MAX_CORPUS_PATH_CHARS) return `a corpus path is at most ${MAX_CORPUS_PATH_CHARS} characters`;
+  if (/[\x00-\x1f\x7f]/.test(value)) return "a corpus path has no control characters";
+  if (value.startsWith("~")) return "write the absolute path: ~ is not expanded";
+  if (value !== value.trim()) return "a corpus path has no leading or trailing spaces";
+  return undefined;
+}
+
+/** The checked block. `corpus.paths` names read-only folders of plain notes; nothing here reads the disk (the vault check does, per index refresh). */
+export function corpusConfig(input: unknown): CorpusConfig {
+  if (input === undefined) return { paths: [] };
+  if (!input || typeof input !== "object" || Array.isArray(input)) throw new Error("corpus must be an object");
+  const block = input as Record<string, unknown>;
+  for (const key of Object.keys(block)) if (key !== "paths") throw new Error(`corpus.${key}: unknown key`);
+  if (block.paths === undefined) return { paths: [] };
+  if (!Array.isArray(block.paths) || block.paths.length > MAX_CORPUS_ROOTS) throw new Error(`corpus.paths takes a list of at most ${MAX_CORPUS_ROOTS} folders`);
+  const out: string[] = [];
+  block.paths.forEach((path, index) => {
+    const problem = corpusPathProblem(path);
+    if (problem) throw new Error(`corpus.paths[${index}]: ${problem}`);
+    if (!out.includes(path as string)) out.push(path as string);
+  });
+  return { paths: out };
+}

package/src/hub/corpus-evidence.ts

@@ -0,0 +1,184 @@
+import { sanitize } from "./envelope.ts";
+import { CorpusIndex, MAX_CORPUS_READS, MAX_FRONTMATTER_BYTES, MAX_NOTE_BYTES, MAX_HEADINGS, parseNote, type NoteRecord } from "./corpus.ts";
+
+export const MAX_CORPUS_EVIDENCE = 6;
+export const MAX_CORPUS_EVIDENCE_ITEM_BYTES = 768;
+export const MAX_CORPUS_EVIDENCE_BYTES = 4096;
+
+export interface CorpusEvidencePolicy {
+  allowed(): boolean;
+  version(): string;
+  /** Screen the complete, fresh note plus its raw metadata before any excerpt is produced. */
+  safe(text: string, paths: readonly string[]): boolean;
+}
+
+export interface CorpusEvidence { id: string; kind: "note"; text: string }
+
+const normalize = (text: string): string => text.normalize("NFKC").toLowerCase();
+const words = (text: string): string[] => [...new Set(normalize(text).match(/[\p{L}\p{N}]{2,}/gu) ?? [])];
+const score = (query: readonly string[], text: string): number => {
+  const haystack = normalize(text);
+  return query.reduce((total, word) => total + (haystack.includes(word) ? 1 : 0), 0);
+};
+
+/** Slugs retain Unicode letters and numbers, fold case, and use hyphens for punctuation or whitespace. */
+export function headingSlug(heading: string): string {
+  return normalize(heading).replace(/[^\p{L}\p{N}]+/gu, "-").replace(/^-+|-+$/g, "") || "section";
+}
+
+/** Escape only the delimiters in the citation grammar; Unicode and its spelling remain literal. */
+function escapeCitationPart(value: string): string {
+  return value.replace(/[\\,#\[\]]/g, char => `\\${char}`);
+}
+
+export function noteCitationId(baseId: string, slug: string): string {
+  const id = `${escapeCitationPart(baseId)}#${escapeCitationPart(slug)}`;
+  if (!isNoteCitation(id)) throw new Error("unsupported note citation id");
+  return id;
+}
+
+/** A syntactic guard for note citations. Callers still need to match the full ID against evidence they actually supplied. */
+export function isNoteCitation(id: string): boolean {
+  if (!id.startsWith("note:") || Buffer.byteLength(id) > 1024 || Array.from(id).some(char => { const point = char.codePointAt(0)!; return point <= 0x1f || point === 0x7f; })) return false;
+  let separator = -1, escaped = false;
+  for (let i = 5; i < id.length; i++) {
+    const char = id[i]!;
+    if (escaped) { if (!["\\", ",", "[", "]", "#"].includes(char)) return false; escaped = false; continue; }
+    if (char === "\\") { escaped = true; continue; }
+    if (char === "#") { if (separator >= 0) return false; separator = i; continue; }
+    if (char === "," || char === "[" || char === "]") return false;
+  }
+  return !escaped && separator > 5 && separator < id.length - 1 && id.slice(5, separator).includes("/");
+}
+
+interface Section { slug: string; heading?: string; body: string }
+
+function contentStart(lines: readonly string[]): number {
+  if (lines[0]?.trimEnd() !== "---") return 0;
+  let bytes = 0;
+  for (let i = 1; i < lines.length; i++) {
+    bytes += Buffer.byteLength(lines[i]!) + 1;
+    if (bytes > MAX_FRONTMATTER_BYTES) return lines.length;
+    if (/^(---|\.\.\.)\s*$/.test(lines[i]!)) return i + 1;
+  }
+  return lines.length;
+}
+
+/** Split prose using the same Markdown heading expression and fenced-code rules as parseNote. */
+function sections(text: string): Section[] {
+  const lines = text.split(/\r?\n/), start = contentStart(lines), bodyLines = lines.slice(start);
+  let endOfSections = bodyLines.length;
+  const boundaries: { line: number; text: string }[] = [];
+  let fence: string | undefined;
+  for (let i = 0; i < bodyLines.length; i++) {
+    const line = bodyLines[i]!;
+    const marker = /^\s{0,3}(`{3,}|~{3,})/.exec(line);
+    if (marker) { const mark = marker[1]![0]!; if (!fence) fence = mark; else if (fence === mark) fence = undefined; continue; }
+    if (fence) continue;
+    const heading = /^ {0,3}(#{1,6})\s+(.+?)(?:\s+#+)?\s*$/.exec(line);
+    const cleanHeading = heading ? parseNote(line).headings[0]?.text : undefined;
+    if (cleanHeading) {
+      // A heading beyond the metadata cap is still a section boundary, never content of the preceding citation.
+      if (boundaries.length >= MAX_HEADINGS) { endOfSections = i; break; }
+      boundaries.push({ line: i, text: cleanHeading });
+    }
+  }
+
+  const occupied = new Set<string>();
+  const allocate = (raw: string): string => {
+    const base = headingSlug(raw);
+    let candidate = base, suffix = 2;
+    while (occupied.has(candidate)) candidate = `${base}-${suffix++}`;
+    occupied.add(candidate);
+    return candidate;
+  };
+  const hasIntro = boundaries.length === 0 || bodyLines.slice(0, boundaries[0]!.line).some(line => line.trim());
+  const result: Section[] = [];
+  if (hasIntro) {
+    occupied.add("top");
+    const end = boundaries[0]?.line ?? bodyLines.length;
+    result.push({ slug: "top", body: bodyLines.slice(0, end).join("\n").trim() });
+  }
+  for (let i = 0; i < boundaries.length; i++) {
+    const heading = boundaries[i]!;
+    const end = boundaries[i + 1]?.line ?? endOfSections;
+    result.push({ slug: allocate(heading.text), heading: heading.text, body: bodyLines.slice(heading.line + 1, end).join("\n").trim() });
+  }
+  if (!result.length) result.push({ slug: "top", body: bodyLines.join("\n").trim() });
+  return result;
+}
+
+function metadata(record: NoteRecord): string {
+  return JSON.stringify({ id: record.id, root: record.root, path: record.path, title: record.title, frontmatter: record.frontmatter, tags: record.tags, aliases: record.aliases, headings: record.headings });
+}
+
+function truncateUtf8(text: string, maxBytes: number): string {
+  let out = "", bytes = 0;
+  for (const char of text) {
+    const size = Buffer.byteLength(char);
+    if (bytes + size > maxBytes) break;
+    out += char; bytes += size;
+  }
+  return out;
+}
+
+function truncateExcerpt(text: string, maxBytes: number): string {
+  if (Buffer.byteLength(text) <= maxBytes) return text;
+  const marker = " [cut]", markerBytes = Buffer.byteLength(marker);
+  if (maxBytes < markerBytes) return "";
+  return `${truncateUtf8(text, maxBytes - markerBytes)}${marker}`;
+}
+
+function permitted(policy: CorpusEvidencePolicy): boolean {
+  try { return policy.allowed(); } catch { return false; }
+}
+
+/** Metadata-first retrieval with a fresh, whole-note privacy screen before any excerpt leaves this module. */
+export function corpusEvidence(query: string, index: CorpusIndex, policy: CorpusEvidencePolicy): CorpusEvidence[] {
+  if (!query.trim() || !permitted(policy)) return [];
+  let version: string;
+  try { version = policy.version(); } catch { return []; }
+  index.refresh();
+  if (!permitted(policy)) return [];
+  const terms = words(query);
+  if (!terms.length) return [];
+  // ponytail: ceiling is 32 metadata-ranked candidates; upgrade by adding a query index over approved note text, never by opening every note body.
+  const candidates = index.notes().map(record => ({ record, score: score(terms, [record.title, record.path, ...record.tags, ...record.aliases, ...record.headings.map(h => h.text)].filter(Boolean).join(" ")) }))
+    .filter(item => item.score > 0).sort((a, b) => b.score - a.score || a.record.id.localeCompare(b.record.id)).slice(0, MAX_CORPUS_READS);
+  const reads = index.readMany(candidates.map(item => item.record.id));
+  if (!permitted(policy)) return [];
+  let currentVersion: string;
+  try { currentVersion = policy.version(); } catch { return []; }
+  if (currentVersion !== version) return [];
+
+  const out: CorpusEvidence[] = [];
+  let aggregate = 0;
+  for (const read of reads) {
+    const record = read.record;
+    const currentMetadata = metadata(record);
+    if (Buffer.byteLength(read.text) > MAX_NOTE_BYTES) continue;
+    try { if (!policy.safe(`${read.text}\n${currentMetadata}`, [read.path, record.path, record.id])) continue; }
+    catch { continue; }
+    const currentScore = score(terms, [record.title, record.path, ...record.tags, ...record.aliases, ...record.headings.map(h => h.text)].filter(Boolean).join(" "));
+    if (!currentScore) continue;
+    const noteSections = sections(read.text);
+    const candidatesForNote = noteSections.map(section => ({ section, score: score(terms, `${section.heading ?? ""} ${section.body}`) }))
+      .sort((a, b) => b.score - a.score);
+    const best = candidatesForNote[0]?.section;
+    if (!best) continue;
+    const rawExcerpt = [best.heading ? `# ${best.heading}` : "", best.body].filter(Boolean).join("\n");
+    const safeText = sanitize(rawExcerpt);
+    let id: string;
+    try { id = noteCitationId(record.id, best.slug); } catch { continue; }
+    const remainingText = MAX_CORPUS_EVIDENCE_BYTES - aggregate - Buffer.byteLength(id);
+    if (!id || remainingText <= 0) continue;
+    const text = truncateExcerpt(safeText, Math.min(MAX_CORPUS_EVIDENCE_ITEM_BYTES, remainingText));
+    if (!text) continue;
+    const cost = Buffer.byteLength(id) + Buffer.byteLength(text);
+    out.push({ id, kind: "note", text });
+    aggregate += cost;
+    if (out.length >= MAX_CORPUS_EVIDENCE) break;
+  }
+  try { if (policy.version() !== version || !permitted(policy)) return []; } catch { return []; }
+  return out;
+}

package/src/hub/corpus.ts

@@ -0,0 +1,373 @@
+import { closeSync, constants, fstatSync, lstatSync, openSync, readSync, readdirSync, type Dirent, type Stats } from "node:fs";
+import { createHash } from "node:crypto";
+import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { isDenied } from "../local/deny.ts";
+import { foldSegment } from "./permission-mode.ts";
+import { matchesPolicy, matchesPolicyLexically, policyRealPath } from "./path-policy.ts";
+import { realPath } from "./project.ts";
+
+// agent-hub #510 (hub task #331): the read-only plain-notes index. Spec: "Knowledge corpus (#510), owner decision 2026-10-22".
+// The hub reads plain notes folders. A folder that is, lies in or contains an Obsidian vault is refused: a vault is reached only through the Obsidian MCP tools.
+
+export const MAX_NOTES = 5000;
+export const MAX_DIRS = 4096;
+export const MAX_ENTRIES = 100_000;
+export const MAX_DEPTH = 24;
+export const MAX_NOTE_BYTES = 256 * 1024;
+export const MAX_HEADINGS = 256;
+export const MAX_LINKS = 512;
+export const MAX_TAGS = 64;
+export const MAX_FRONTMATTER_BYTES = 8 * 1024;
+/** A single evidence request may open at most this many freshly checked notes. */
+export const MAX_CORPUS_READS = 32;
+const NOTE_EXT = /\.(md|markdown)$/i;
+/** The one source of the sentence that names where a vault is read, so the refusal and its tests cannot drift. */
+export const VAULT_REASON = "is, lies in or contains an Obsidian vault (a .obsidian/ folder); the hub never reads a vault from disk. Reach it through the Obsidian MCP tools (mcp__obsidian__*) instead";
+
+export interface CorpusContext {
+  cwd: string;
+  paths: readonly string[];
+  /** `pii.paths`: project-relative exact paths, subtrees or globs. A match is never read. */
+  piiPaths?: readonly string[];
+  /** `local.deny` substrings, on top of the built-in secrets denylist. */
+  deny?: readonly string[];
+  /** `signals.pii_patterns`: a note whose text matches (or whose pattern cannot be evaluated) is not indexed. */
+  piiPatterns?: readonly string[];
+}
+
+export type CorpusRootCheck = { ok: true; real: string } | { ok: false; reason: string };
+
+const errno = (error: unknown): string | undefined => (error as NodeJS.ErrnoException | undefined)?.code;
+const clean = (text: string): string => text.replace(/[\x00-\x1f\x7f]/g, " ").replace(/\s+/g, " ").trim();
+
+/**
+ * Whether an entry named `.obsidian` exists directly in `dir`, in any folded spelling (the walk below a root folds names the same way, so a root and its ancestors are
+ * judged alike on a case-sensitive disk, where `.OBSIDIAN` is a different name than `.obsidian`). A literal `lstat` answers first (it also covers a case-insensitive disk
+ * without a listing); otherwise the folder is listed, bounded. A thrown error for anything but absence: unreadable evidence is never "no vault".
+ */
+function hasObsidian(dir: string): boolean {
+  try { lstatSync(join(dir, ".obsidian")); return true; }
+  catch (error) { if (errno(error) !== "ENOENT" && errno(error) !== "ENOTDIR") throw error; }
+  let names: string[];
+  try { names = readdirSync(dir); }
+  catch (error) { if (errno(error) === "ENOTDIR") return false; throw error; }
+  if (names.length > MAX_ENTRIES) throw Object.assign(new Error("too many entries"), { code: "E2BIG" });
+  return names.some(name => foldSegment(name) === ".obsidian");
+}
+
+/**
+ * Whether a configured corpus root may be read. The check runs on the `realPath` of the root (a symlink into a vault is caught), looks for `.obsidian/` at the
+ * root and in every ancestor of the real path (a root inside a vault), and refuses on evidence it cannot read. `pii.paths` and the deny rules are checked here too:
+ * a root under either is never a corpus, and the walk refuses a root that holds a protected folder, because that folder is never listed and cannot be shown free of a vault. The walk below the root (refusing any `.obsidian` it meets) is `CorpusIndex.refresh`.
+ */
+export function checkCorpusRoot(configured: string, ctx: CorpusContext): CorpusRootCheck {
+  let real: string;
+  try {
+    const cwd = realPath(ctx.cwd);
+    real = policyRealPath(resolve(cwd, configured));
+    if (!lstatSync(real).isDirectory()) return { ok: false, reason: `${configured} is not a folder` };
+    real = realPath(real);
+  } catch (error) { return { ok: false, reason: `${configured} cannot be resolved (${errno(error) ?? "unreadable"}); a path that cannot be checked is never read` }; }
+  try {
+    for (let dir = real; ; dir = dirname(dir)) {
+      if (hasObsidian(dir)) return { ok: false, reason: `${configured} ${VAULT_REASON}${dir === real ? "" : ` (the vault folder is ${dir})`}` };
+      if (dirname(dir) === dir) break;
+    }
+  } catch (error) { return { ok: false, reason: `${configured}: a folder above it cannot be read (${errno(error) ?? "unreadable"}), so it cannot be shown to be free of a vault` }; }
+  if (isDenied(real, [...(ctx.deny ?? [])])) return { ok: false, reason: `${configured} is on the secrets denylist or local.deny` };
+  if (ctx.piiPaths?.length && matchesPolicy(ctx.cwd, real, [...ctx.piiPaths])) return { ok: false, reason: `${configured} is, or lies in, a path under pii.paths` };
+  return { ok: true, real };
+}
+
+export interface NoteLink { target: string; heading?: string; embed: boolean }
+export interface ParsedNote {
+  title?: string;
+  frontmatter: Record<string, string | string[]>;
+  tags: string[];
+  aliases: string[];
+  headings: { level: number; text: string }[];
+  links: NoteLink[];
+  /** A cap cut headings, links or frontmatter short. */
+  truncated: boolean;
+}
+
+const scalar = (raw: string): string => {
+  const text = raw.trim();
+  if ((text.startsWith('"') && text.endsWith('"') && text.length >= 2) || (text.startsWith("'") && text.endsWith("'") && text.length >= 2)) return clean(text.slice(1, -1));
+  return clean(text);
+};
+const listOf = (raw: string): string[] => raw.trim().replace(/^\[/, "").replace(/\]$/, "").split(",").map(scalar).filter(Boolean);
+
+/**
+ * A bounded frontmatter reader for the shapes notes use: `key: value`, `key: [a, b]` and a block list under `key:`. It is not a YAML parser (zero runtime
+ * dependencies): nested maps, anchors and multi-line scalars are kept as their first line or ignored, never executed or evaluated.
+ * ponytail: ceiling is flat keys; upgrade path is a vetted YAML subset parser if notes need nested metadata.
+ */
+function frontmatterOf(text: string): { values: Record<string, string | string[]>; body: string; truncated: boolean } {
+  const lines = text.split(/\r?\n/);
+  if (lines[0]?.trimEnd() !== "---") return { values: {}, body: text, truncated: false };
+  let end = -1, bytes = 0;
+  for (let i = 1; i < lines.length; i++) {
+    bytes += lines[i]!.length + 1;
+    if (bytes > MAX_FRONTMATTER_BYTES) return { values: {}, body: lines.slice(i).join("\n"), truncated: true };
+    if (/^(---|\.\.\.)\s*$/.test(lines[i]!)) { end = i; break; }
+  }
+  if (end < 0) return { values: {}, body: text, truncated: false };
+  const values: Record<string, string | string[]> = {};
+  let key: string | undefined;
+  for (const line of lines.slice(1, end)) {
+    const item = /^\s+-\s+(.*)$/.exec(line);
+    if (item && key) { const current = values[key]; values[key] = [...(Array.isArray(current) ? current : current ? [current] : []), scalar(item[1]!)].filter(Boolean).slice(0, MAX_TAGS); continue; }
+    const pair = /^([A-Za-z0-9_][A-Za-z0-9_ -]{0,63}):\s*(.*)$/.exec(line);
+    if (!pair || /^\s/.test(line)) { if (!/^\s/.test(line)) key = undefined; continue; }
+    key = pair[1]!.trim().toLowerCase();
+    if (Object.hasOwn(values, key) || ["__proto__", "constructor", "prototype"].includes(key)) { key = undefined; continue; }
+    const value = pair[2]!;
+    if (value.startsWith("[")) values[key] = listOf(value).slice(0, MAX_TAGS);
+    else if (value.trim() === "" ) values[key] = [];
+    else values[key] = scalar(value).slice(0, 500);
+  }
+  return { values, body: lines.slice(end + 1).join("\n"), truncated: false };
+}
+
+/** The body without fenced and inline code, so a `[[link]]` or a `# heading` inside code is not read as one. */
+function prose(body: string): { text: string; headings: { level: number; text: string }[]; truncated: boolean } {
+  const kept: string[] = [], headings: { level: number; text: string }[] = [];
+  let fence: string | undefined, truncated = false;
+  for (const line of body.split(/\r?\n/)) {
+    const marker = /^\s{0,3}(`{3,}|~{3,})/.exec(line);
+    if (marker) { const mark = marker[1]![0]!; if (!fence) fence = mark; else if (fence === mark) fence = undefined; continue; }
+    if (fence) continue;
+    const heading = /^ {0,3}(#{1,6})\s+(.+?)(?:\s+#+)?\s*$/.exec(line); // a closing run of # is a closer only after whitespace: `# C#` is "C#"
+    if (heading) { if (headings.length < MAX_HEADINGS) headings.push({ level: heading[1]!.length, text: clean(heading[2]!).slice(0, 200) }); else truncated = true; }
+    kept.push(line.replace(/`[^`\n]*`/g, " "));
+  }
+  return { text: kept.join("\n"), headings: headings.filter(item => item.text), truncated };
+}
+
+/** Frontmatter, headings and wikilinks of one note. Never evaluates anything and never throws on malformed input. */
+export function parseNote(text: string): ParsedNote {
+  const { values, body, truncated: frontTruncated } = frontmatterOf(text);
+  const { text: scan, headings, truncated: headTruncated } = prose(body);
+  const links: NoteLink[] = [];
+  let linkTruncated = false;
+  for (const match of scan.matchAll(/(!?)\[\[([^\[\]\n|#]{1,200})(?:#([^\[\]\n|]{1,200}))?(?:\|[^\[\]\n]{0,200})?\]\]/g)) {
+    const target = clean(match[2]!);
+    if (!target) continue;
+    if (links.length >= MAX_LINKS) { linkTruncated = true; break; }
+    links.push({ target, ...(match[3] ? { heading: clean(match[3]) } : {}), embed: match[1] === "!" });
+  }
+  const list = (key: string): string[] => { const value = values[key]; return (Array.isArray(value) ? value : value ? value.split(",").map(scalar) : []).map(item => item.replace(/^#/, "")).filter(Boolean).slice(0, MAX_TAGS); };
+  const title = typeof values.title === "string" && values.title ? values.title : headings.find(item => item.level === 1)?.text;
+  return { ...(title ? { title } : {}), frontmatter: values, tags: [...new Set([...list("tags"), ...list("tag")])], aliases: [...new Set([...list("aliases"), ...list("alias")])], headings, links, truncated: frontTruncated || headTruncated || linkTruncated };
+}
+
+export interface NoteRecord extends ParsedNote {
+  /** `note:<root label>/<path>`; a heading is cited as `<id>#<heading>`. */
+  id: string;
+  root: string;
+  path: string;
+  bytes: number;
+  mtimeMs: number;
+  sha256: string;
+}
+export interface CorpusRootReport {
+  configured: string;
+  label?: string;
+  real?: string;
+  refused?: string;
+  notes: number;
+  /** Counts of what was left out, by reason: never silently dropped. */
+  skipped: { protected: number; links: number; large: number; unreadable: number; pii: number; overCap: number };
+}
+export interface CorpusRefresh { roots: CorpusRootReport[]; notes: number; parsed: number; reused: number; removed: number }
+export interface CorpusRead { record: NoteRecord; text: string; path: string }
+
+const sizeKey = (st: { size: number; mtimeMs: number; ctimeMs: number; ino: number; dev: number }) => `${st.dev}:${st.ino}:${st.size}:${st.mtimeMs}:${st.ctimeMs}`;
+
+function readNote(path: string, expected: { dev: number; ino: number }): Buffer {
+  const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+  try {
+    const st = fstatSync(fd);
+    if (!st.isFile() || st.nlink !== 1 || st.dev !== expected.dev || st.ino !== expected.ino || st.size > MAX_NOTE_BYTES) throw new Error("not a bounded regular note");
+    const bytes = Buffer.alloc(st.size); let got = 0;
+    while (got < bytes.length) { const n = readSync(fd, bytes, got, bytes.length - got, got); if (!n) break; got += n; }
+    if (got !== st.size) throw new Error("note changed while read");
+    return bytes;
+  } finally { closeSync(fd); }
+}
+
+/**
+ * The read-only index of the configured plain-notes folders. `refresh` walks each root (bounded in folders, entries and notes), skips what `pii.paths`, the deny
+ * rules and symbolic links protect, refuses a root that holds an Obsidian vault anywhere, and re-reads only the notes whose size, times or identity changed.
+ * It stores structure (title, tags, headings, links, a hash), never the note's text, and it writes nothing. The index is in memory: a restart rebuilds it.
+ * ponytail: the cache is per process; persist it in hub.db if a rebuild becomes slow for big folders.
+ */
+export class CorpusIndex {
+  private cache = new Map<string, { key: string; record: NoteRecord }>();
+  private byId = new Map<string, NoteRecord>();
+  constructor(private readonly ctx: CorpusContext) {}
+
+  notes(): NoteRecord[] { return [...this.byId.values()].sort((a, b) => a.id.localeCompare(b.id)); }
+  get(id: string): NoteRecord | undefined { return this.byId.get(id); }
+
+  /** Read one indexed note from disk after rebuilding the bounded index and rechecking current policy. Text is never cached. */
+  read(id: string): CorpusRead | undefined { return this.readMany([id])[0]; }
+
+  /**
+   * Freshly read a bounded batch. One full refresh revalidates every configured root and its vault-safe walk; each selected file then gets current lexical policy,
+   * path, inode, size, content-hash and PII checks immediately before any text is returned. A cache hit is metadata only and never supplies old text.
+   */
+  readMany(ids: readonly string[]): CorpusRead[] {
+    if (ids.length === 0) return [];
+    this.refresh();
+    const selected = [...new Set(ids.slice(0, MAX_CORPUS_READS))];
+    const roots = new Map(this.refreshReports().filter((item): item is CorpusRootReport & { label: string; real: string } => !!item.label && !!item.real).map(item => [item.label, item]));
+    const checkedRoots = new Map<string, string>();
+    for (const root of new Set(selected.map(id => this.byId.get(id)?.root).filter((label): label is string => !!label))) {
+      const report = roots.get(root);
+      if (!report) continue;
+      const checked = checkCorpusRoot(report.configured, this.ctx);
+      if (checked.ok && checked.real === report.real) checkedRoots.set(root, checked.real);
+    }
+    const output: CorpusRead[] = [];
+    for (const id of selected) {
+      const record = this.byId.get(id);
+      if (!record) continue;
+      const root = roots.get(record.root);
+      const checkedReal = checkedRoots.get(record.root);
+      if (!root || !checkedReal) continue;
+      const abs = resolve(checkedReal, record.path);
+      const rel = relative(checkedReal, abs);
+      if (!rel || rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel) || !NOTE_EXT.test(abs)) continue;
+      const pii = [...(this.ctx.piiPaths ?? [])], deny = [...(this.ctx.deny ?? [])];
+      if (isDenied(abs, deny) || (pii.length > 0 && (matchesPolicyLexically(this.ctx.cwd, abs, pii) || matchesPolicy(this.ctx.cwd, abs, pii)))) continue;
+      try {
+        let parent = checkedReal;
+        const segments = record.path.split("/");
+        if (!segments.length || segments.some(part => !part || part === "." || part === "..")) continue;
+        for (const segment of segments.slice(0, -1)) {
+          parent = join(parent, segment);
+          const parentStat = lstatSync(parent);
+          if (parentStat.isSymbolicLink() || !parentStat.isDirectory()) { parent = ""; break; }
+        }
+        if (!parent) continue;
+        const st = lstatSync(abs);
+        if (!st.isFile() || st.nlink !== 1 || st.size > MAX_NOTE_BYTES || st.size !== record.bytes) continue;
+        // Each root-relative parent is checked with lstat; an in-root symlink is not an approved notes path either.
+        const bytes = readNote(abs, st);
+        if (bytes.includes(0)) continue;
+        const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+        if (createHash("sha256").update(bytes).digest("hex") !== record.sha256 || this.piiFlagged(text)) continue;
+        output.push({ record, text, path: abs });
+      } catch { /* A disappeared, replaced, unreadable or invalid note is not evidence. */ }
+    }
+    return output;
+  }
+
+  private refreshReports(): CorpusRootReport[] { return this.lastReports; }
+
+  private lastReports: CorpusRootReport[] = [];
+
+  /** The notes a wikilink target names, in the same root as `fromId`: by path (with or without the extension) or by file name, case and spelling folded. */
+  resolveLink(fromId: string, target: string): NoteRecord[] {
+    const from = this.byId.get(fromId);
+    if (!from) return [];
+    const want = foldSegment(target.replace(/\\/g, "/").replace(NOTE_EXT, "").replace(/^\/+/, ""));
+    if (!want) return [];
+    return this.notes().filter(note => {
+      if (note.root !== from.root) return false;
+      const path = foldSegment(note.path.replace(NOTE_EXT, ""));
+      return want.includes("/") ? path === want || path.endsWith(`/${want}`) : path === want || path.endsWith(`/${want}`);
+    });
+  }
+
+  refresh(): CorpusRefresh {
+    const seen = new Set<string>(), labels = new Set<string>(), reports: CorpusRootReport[] = [];
+    let parsed = 0, reused = 0;
+    const nextById = new Map<string, NoteRecord>();
+    for (const configured of this.ctx.paths) {
+      const report: CorpusRootReport = { configured, notes: 0, skipped: { protected: 0, links: 0, large: 0, unreadable: 0, pii: 0, overCap: 0 } };
+      reports.push(report);
+      const checked = checkCorpusRoot(configured, this.ctx);
+      if (!checked.ok) { report.refused = checked.reason; continue; }
+      report.real = checked.real;
+      let label = clean(basename(checked.real)).replace(/[^\w.-]+/g, "-").replace(/^-+|-+$/g, "") || "notes", n = 1;
+      while (labels.has(label)) label = `${clean(basename(checked.real)).replace(/[^\w.-]+/g, "-") || "notes"}-${++n}`;
+      labels.add(label); report.label = label;
+      const found = this.walk(checked.real, report);
+      if (typeof found === "string") { report.refused = `${configured} ${found}`; continue; }
+      for (const file of found) {
+        const abs = join(checked.real, file.path), key = sizeKey(file.st), id = `note:${label}/${file.path}`;
+        const cached = this.cache.get(abs);
+        let record: NoteRecord;
+        if (cached && cached.key === key && cached.record.id === id) { record = cached.record; reused++; }
+        else {
+          try {
+            const bytes = readNote(abs, file.st), text = new TextDecoder("utf-8", { fatal: false }).decode(bytes);
+            if (bytes.includes(0) || text.includes("\uFFFD")) { report.skipped.unreadable++; continue; }
+            if (this.piiFlagged(text)) { report.skipped.pii++; continue; }
+            record = { ...parseNote(text), id, root: label, path: file.path, bytes: bytes.length, mtimeMs: file.st.mtimeMs, sha256: createHash("sha256").update(bytes).digest("hex") };
+            parsed++;
+          } catch { report.skipped.unreadable++; continue; }
+        }
+        this.cache.set(abs, { key, record }); seen.add(abs);
+        nextById.set(record.id, record); report.notes++;
+      }
+    }
+    let removed = 0;
+    for (const path of [...this.cache.keys()]) if (!seen.has(path)) { this.cache.delete(path); removed++; }
+    this.byId = nextById;
+    this.lastReports = reports;
+    return { roots: reports, notes: nextById.size, parsed, reused, removed };
+  }
+
+  private piiFlagged(text: string): boolean {
+    if (!this.ctx.piiPatterns?.length) return false;
+    try { return this.ctx.piiPatterns.some(pattern => new RegExp(pattern).test(text)); } catch { return true; } // a pattern that cannot be evaluated is not a clear verdict
+  }
+
+  /** The notes under `root`, or a refusal string (a vault below it, or evidence the walk could not read). Directories are visited in name order so a cap cuts deterministically. */
+  private walk(root: string, report: CorpusRootReport): { path: string; st: Stats }[] | string {
+    const out: { path: string; st: Stats }[] = [];
+    let dirs = 0, entries = 0;
+    const pii = [...(this.ctx.piiPaths ?? [])], deny = [...(this.ctx.deny ?? [])];
+    const stack: { dir: string; depth: number }[] = [{ dir: root, depth: 0 }];
+    while (stack.length) {
+      const { dir, depth } = stack.pop()!;
+      if (++dirs > MAX_DIRS || depth > MAX_DEPTH) return `has more than ${MAX_DIRS} folders or nests deeper than ${MAX_DEPTH}, so it cannot be shown to be free of a vault; point corpus.paths at a smaller folder`;
+      let list: Dirent[];
+      try { list = readdirSync(dir, { withFileTypes: true }); }
+      catch (error) { return `has a folder that cannot be read (${errno(error) ?? "unreadable"}), so it cannot be shown to be free of a vault`; }
+      list.sort((a, b) => a.name.localeCompare(b.name));
+      const subdirs: string[] = [];
+      for (const entry of list) {
+        if (++entries > MAX_ENTRIES) return `has more than ${MAX_ENTRIES} entries, so it cannot be shown to be free of a vault; point corpus.paths at a smaller folder`;
+        if (foldSegment(entry.name) === ".obsidian") return `${VAULT_REASON} (found ${join(dir, entry.name)})`;
+        const abs = join(dir, entry.name);
+        if (entry.isSymbolicLink()) { report.skipped.links++; continue; }
+        const rel = relative(root, abs).split(sep).join("/");
+        const protectedPath = () => isDenied(abs, deny) || (pii.length > 0 && (matchesPolicyLexically(this.ctx.cwd, abs, pii) || matchesPolicy(this.ctx.cwd, abs, pii)));
+        if (entry.isDirectory()) {
+          // A protected folder is never listed or read, so a vault anywhere below it cannot be ruled out: the root cannot be shown free of a vault, and it is refused
+          // (the owner decision: evidence that cannot be checked refuses). Point corpus.paths at a folder that holds no protected folder. A protected FILE is only skipped.
+          if (protectedPath()) return `holds the protected folder ${abs} (pii.paths, the deny rules or local.deny), which is never listed, so the root cannot be shown to be free of an Obsidian vault; point corpus.paths at a folder with no protected folder in it`;
+          subdirs.push(abs);
+        } else if (entry.isFile() && NOTE_EXT.test(entry.name)) {
+          if (protectedPath()) { report.skipped.protected++; continue; }
+          if (out.length >= MAX_NOTES) { report.skipped.overCap++; continue; }
+          let st: Stats;
+          try { st = lstatSync(abs); } catch { report.skipped.unreadable++; continue; }
+          if (!st.isFile() || st.nlink !== 1) { report.skipped.links++; continue; }
+          if (st.size > MAX_NOTE_BYTES) { report.skipped.large++; continue; }
+          out.push({ path: rel, st });
+        }
+      }
+      for (const sub of subdirs.reverse()) stack.push({ dir: sub, depth: depth + 1 });
+    }
+    return out;
+  }
+}
+
+/** The paths, spelled for a person: configured text as written. Exported for the status line and tests. */
+export const corpusRootLabel = (configured: string): string => isAbsolute(configured) ? configured : `./${configured}`;