neurain 0.1.0-alpha.0

package/src/core/recall_bench.mjs

@@ -0,0 +1,275 @@
+// Recall benchmark + scorecard runner (W-A). Repeatably scores a directory of
+// benchmark suites against the engine's routed lexical ranker, the apples-to-
+// apples comparison with the vault's neurain-search (same BM25 + boosts). The
+// suite files and the eval corpus stay external (passed as a runtime arg); this
+// module ships only synthetic fixtures in tests, never real vault data.
+//
+// Why the LEXICAL branch and not the full hybrid: the vault baseline (94.1%) was
+// produced by the vault lexical search. In the engine's routed hybrid the lexical
+// results come FIRST, so hybrid's top-K equals lexical's top-K whenever lexical
+// returns >= K results, and otherwise hybrid only ADDS exact/semantic catches.
+// Lexical source recall is therefore a faithful lower bound on hybrid; gating on
+// it is both fast (one corpus build, no per-case semantic walk) and conservative.
+import fs from 'node:fs';
+import path from 'node:path';
+import { performance } from 'node:perf_hooks';
+import { recallConfig } from './config.mjs';
+import { resolveAreaDir } from './recall_corpus.mjs';
+import { buildLexicalContext, lexicalSearchWithContext } from './recall_lexical.mjs';
+
+function resolveSuitesDir(root, suites) {
+  if (!suites) return '';
+  return path.isAbsolute(suites) ? suites : path.join(root, suites);
+}
+
+function loadSuites(dir) {
+  if (!dir || !fs.existsSync(dir)) return [];
+  const out = [];
+  for (const file of fs.readdirSync(dir).filter((f) => f.endsWith('.json')).sort()) {
+    let parsed;
+    try {
+      parsed = JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8'));
+    } catch {
+      continue;
+    }
+    const cases = Array.isArray(parsed) ? parsed : parsed.cases;
+    if (!Array.isArray(cases)) continue;
+    out.push({
+      file,
+      suite: parsed.suite || file.replace(/\.json$/, ''),
+      pass_threshold: parsed.pass_threshold || {},
+      cases,
+    });
+  }
+  return out;
+}
+
+const base = (p) => String(p || '').split('/').pop();
+
+// strict = exact path equality (the vault recall-check matcher, how 94.1% was
+// computed); loose = strict OR basename equality OR endsWith (the vault scorecard
+// matcher). Both are reported; the gate uses strict.
+function matchExpected(resultPaths, expected, mode) {
+  if (mode === 'loose') {
+    return expected.some((s) => resultPaths.some((p) => p === s || p.endsWith(`/${s}`) || base(p) === base(s)));
+  }
+  return expected.some((s) => resultPaths.includes(s));
+}
+
+function rankOfFirstHit(resultPaths, expected, mode) {
+  for (let i = 0; i < resultPaths.length; i += 1) {
+    if (matchExpected([resultPaths[i]], expected, mode)) return i + 1;
+  }
+  return 0;
+}
+
+function runCase(ctx, c, { top, maxPerLayer, prefix, mode }) {
+  const out = lexicalSearchWithContext(ctx, c.question, { top, maxPerLayer });
+  const paths = out.results.map((r) => (prefix && r.path.startsWith(prefix) ? r.path.slice(prefix.length) : r.path));
+  const entities = new Set(out.results.flatMap((r) => r.matched_entities || []));
+  const expSources = Array.isArray(c.expected_sources) ? c.expected_sources : [];
+  const expEntities = Array.isArray(c.expected_entities) ? c.expected_entities : [];
+  return {
+    id: c.id || '',
+    results: out.results,
+    paths,
+    source_ok_strict: !expSources.length || matchExpected(paths, expSources, 'strict'),
+    source_ok_loose: !expSources.length || matchExpected(paths, expSources, 'loose'),
+    entity_ok: !expEntities.length || expEntities.every((e) => entities.has(e)),
+    source_ok: !expSources.length || matchExpected(paths, expSources, mode),
+    expected_sources: expSources,
+    expected_entities: expEntities,
+  };
+}
+
+export function benchRecall(root, opts = {}) {
+  const recallCfg = recallConfig(root);
+  const areaDir = resolveAreaDir(root, opts.area || '', recallCfg);
+  const top = Math.max(1, Math.min(Number(opts.top || 5), 50));
+  const maxPerLayer = Number.isFinite(Number(opts.maxPerLayer)) ? Number(opts.maxPerLayer) : 3;
+  const mode = opts.matcher === 'loose' ? 'loose' : 'strict';
+  const suitesDir = resolveSuitesDir(root, opts.suites || opts.suiteDir || '');
+  const baseline = Number.isFinite(Number(opts.baseline)) ? Number(opts.baseline) : recallCfg.bench.baseline_source_recall;
+  const prefix = areaDir ? `${recallCfg.areas_dir}/${areaDir}/` : '';
+
+  if (!suitesDir) {
+    return { ok: false, command: 'recall bench', root, durable_write: false, reason: 'no_suites', note: 'Provide --suites <dir> pointing at a directory of benchmark suite JSON files.' };
+  }
+  const suites = loadSuites(suitesDir);
+  if (!suites.length) {
+    return { ok: false, command: 'recall bench', root, durable_write: false, reason: 'no_suites', suites_dir: suitesDir, note: 'No suite JSON files with a cases array were found.' };
+  }
+
+  const ctx = buildLexicalContext(root, { area: areaDir, recallCfg });
+  const corpusPaths = new Set(ctx.baseDocs.map((d) => (prefix && d.relPath.startsWith(prefix) ? d.relPath.slice(prefix.length) : d.relPath)));
+
+  // Single-case explain mode for parity debugging.
+  if (opts.caseId) {
+    for (const suite of suites) {
+      const c = suite.cases.find((x) => x.id === opts.caseId);
+      if (!c) continue;
+      const r = runCase(ctx, c, { top, maxPerLayer, prefix, mode });
+      const missing = r.expected_sources.filter((s) => !matchExpected(r.paths, [s], 'loose'));
+      return {
+        ok: r.source_ok && r.entity_ok,
+        command: 'recall bench',
+        root,
+        durable_write: false,
+        area: opts.area || '',
+        area_dir: areaDir,
+        suite: suite.suite,
+        case: {
+          id: c.id,
+          question: c.question,
+          expected_sources: r.expected_sources,
+          expected_entities: r.expected_entities,
+          source_ok_strict: r.source_ok_strict,
+          source_ok_loose: r.source_ok_loose,
+          entity_ok: r.entity_ok,
+          missing_expected: missing,
+          // distinguishes a corpus bug (expected source not indexed at all) from a
+          // ranking bug (indexed but below top-K).
+          expected_in_corpus: Object.fromEntries(r.expected_sources.map((s) => [s, corpusPaths.has(s)])),
+          top_results: opts.explain ? r.results.slice(0, top).map((x) => ({
+            path: prefix && x.path.startsWith(prefix) ? x.path.slice(prefix.length) : x.path,
+            matched_by: 'lexical',
+            layer: x.layer,
+            score: x.score,
+            signals: x.signals,
+            matched_entities: x.matched_entities,
+          })) : undefined,
+        },
+      };
+    }
+    return { ok: false, command: 'recall bench', root, durable_write: false, reason: 'case_not_found', case_id: opts.caseId };
+  }
+
+  const report = [];
+  let allSourcePass = 0;
+  let allSourcePassLoose = 0;
+  let allEntityPass = 0;
+  let allCases = 0;
+  const failures = [];
+  for (const suite of suites) {
+    let sPass = 0;
+    let sPassLoose = 0;
+    let ePass = 0;
+    for (const c of suite.cases) {
+      const r = runCase(ctx, c, { top, maxPerLayer, prefix, mode });
+      if (r.source_ok_strict) sPass += 1; else failures.push({ suite: suite.suite, id: r.id, expected_in_corpus: r.expected_sources.map((s) => corpusPaths.has(s)) });
+      if (r.source_ok_loose) sPassLoose += 1;
+      if (r.entity_ok) ePass += 1;
+    }
+    const n = suite.cases.length || 1;
+    const sThr = suite.pass_threshold?.source_recall ?? 0.8;
+    const eThr = suite.pass_threshold?.entity_recall ?? 0.8;
+    const sR = sPass / n;
+    const eR = ePass / n;
+    report.push({
+      suite: suite.suite,
+      cases: n,
+      source_recall: round3(sR),
+      source_recall_loose: round3(sPassLoose / n),
+      entity_recall: round3(eR),
+      passed: sR >= sThr && eR >= eThr,
+    });
+    allSourcePass += sPass;
+    allSourcePassLoose += sPassLoose;
+    allEntityPass += ePass;
+    allCases += n;
+  }
+
+  const overall = {
+    source_recall: allCases ? round3(allSourcePass / allCases) : 0,
+    source_recall_loose: allCases ? round3(allSourcePassLoose / allCases) : 0,
+    entity_recall: allCases ? round3(allEntityPass / allCases) : 0,
+    cases: allCases,
+  };
+  const suitesPassed = report.length > 0 && report.every((r) => r.passed);
+  const gate = baseline != null
+    ? { baseline, metric: 'source_recall_strict', value: overall.source_recall, passed: overall.source_recall >= baseline }
+    : null;
+  return {
+    ok: suitesPassed && (gate ? gate.passed : true),
+    command: 'recall bench',
+    root,
+    durable_write: false,
+    strategy: 'routed_lexical',
+    note: 'Measures the routed lexical branch (vault-equivalent). The runtime hybrid (lexical union exact union semantic) is a superset, so its source recall is >= these numbers.',
+    area: opts.area || '',
+    area_dir: areaDir,
+    top,
+    matcher: mode,
+    suites_dir: suitesDir,
+    suites: report,
+    overall,
+    gate,
+    suites_passed: suitesPassed,
+    failing_cases: failures,
+  };
+}
+
+export function scorecardRecall(root, opts = {}) {
+  const recallCfg = recallConfig(root);
+  const areaDir = resolveAreaDir(root, opts.area || '', recallCfg);
+  const top = Math.max(1, Math.min(Number(opts.top || 5), 50));
+  const maxPerLayer = Number.isFinite(Number(opts.maxPerLayer)) ? Number(opts.maxPerLayer) : 3;
+  const mode = opts.matcher === 'loose' ? 'loose' : 'strict';
+  const suitesDir = resolveSuitesDir(root, opts.suites || opts.suiteDir || '');
+  const prefix = areaDir ? `${recallCfg.areas_dir}/${areaDir}/` : '';
+  if (!suitesDir) return { ok: false, command: 'recall scorecard', root, durable_write: false, reason: 'no_suites' };
+  const suites = loadSuites(suitesDir);
+  if (!suites.length) return { ok: false, command: 'recall scorecard', root, durable_write: false, reason: 'no_suites', suites_dir: suitesDir };
+
+  const ctx = buildLexicalContext(root, { area: areaDir, recallCfg });
+  let hit = 0;
+  let rSum = 0;
+  let mrrSum = 0;
+  let entHit = 0;
+  let total = 0;
+  const latencies = [];
+  for (const suite of suites) {
+    for (const c of suite.cases) {
+      const started = performance.now();
+      const r = runCase(ctx, c, { top, maxPerLayer, prefix, mode });
+      latencies.push(performance.now() - started);
+      total += 1;
+      if (r.source_ok) hit += 1;
+      const exp = r.expected_sources;
+      if (exp.length) {
+        const found = exp.filter((s) => matchExpected(r.paths, [s], mode)).length;
+        rSum += found / exp.length;
+      } else {
+        rSum += 1;
+      }
+      const rank = rankOfFirstHit(r.paths, exp, mode);
+      mrrSum += rank ? 1 / rank : 0;
+      if (r.entity_ok) entHit += 1;
+    }
+  }
+  latencies.sort((a, b) => a - b);
+  const p95 = latencies.length ? latencies[Math.min(latencies.length - 1, Math.floor(latencies.length * 0.95))] : 0;
+  const avg = latencies.length ? latencies.reduce((s, x) => s + x, 0) / latencies.length : 0;
+  return {
+    ok: true,
+    command: 'recall scorecard',
+    root,
+    durable_write: false,
+    strategy: 'routed_lexical',
+    area: opts.area || '',
+    area_dir: areaDir,
+    top,
+    matcher: mode,
+    cases: total,
+    hit_at_k: round3(total ? hit / total : 0),
+    recall_at_k: round3(total ? rSum / total : 0),
+    mrr: round3(total ? mrrSum / total : 0),
+    entity_recall: round3(total ? entHit / total : 0),
+    latency_ms_avg: Math.round(avg * 100) / 100,
+    latency_ms_p95: Math.round(p95 * 100) / 100,
+  };
+}
+
+function round3(value) {
+  return Number(Number(value || 0).toFixed(3));
+}

package/src/core/recall_corpus.mjs

@@ -0,0 +1,152 @@
+// Shared recall corpus selection: the single definition of WHICH markdown files
+// enter the recall corpus (whitelist + config include/exclude + label-based
+// private exclusion + secret/injection content gate). Both the indexed/semantic
+// path (recall.collectMarkdownDocs) and the routed lexical ranker
+// (recall_lexical) consume listRecallMarkdownFiles so they share one corpus; they
+// differ only in how they use the text (normalized snippet vs raw scoring body).
+// Lives in its own module so recall.mjs and recall_lexical.mjs both import it
+// without a cycle.
+import fs from 'node:fs';
+import path from 'node:path';
+import { isTextFile, readText, relPath, walkFiles } from './fs.mjs';
+import { injectionLike, secretLike } from './safety.mjs';
+import { createSensitivityResolver } from './labels.mjs';
+import { recallConfig } from './config.mjs';
+
+export const reEsc = (value) => String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+// Compile config-supplied regex sources, skipping (and reporting) invalid ones
+// so a hand-edited config never throws mid-walk.
+export function compilePatterns(sources) {
+  const regexes = [];
+  const errors = [];
+  for (const source of sources || []) {
+    try {
+      regexes.push(new RegExp(String(source)));
+    } catch (error) {
+      errors.push({ pattern: String(source), error: error.message });
+    }
+  }
+  return { regexes, errors };
+}
+
+export function recallConfigErrors(recallCfg) {
+  return [...compilePatterns(recallCfg.include).errors, ...compilePatterns(recallCfg.exclude).errors];
+}
+
+// Build the markdown whitelist matcher once per corpus walk. Order: hard
+// excludes -> config excludes -> (built-in includes OR config includes).
+// Directory names are config-driven; defaults match the reference vault. The
+// built-in include set was expanded in W-A to the general area knowledge class
+// (areas/<area>/**.md), hubs, and the area registry.
+export function buildRecallPathMatcher(recallCfg = {}) {
+  const areasDir = recallCfg.areas_dir || '10_areas';
+  const wikiDir = recallCfg.wiki_dir || 'wiki';
+  const hubsDir = recallCfg.hubs_dir || '20_hubs';
+  const systemDir = recallCfg.system_dir || '00_system';
+  const archiveDir = recallCfg.archive_dir || '90_archive';
+  const include = compilePatterns(recallCfg.include).regexes;
+  const exclude = compilePatterns(recallCfg.exclude).regexes;
+  const hardExclude = [
+    new RegExp(`^${reEsc(recallCfg.raw_dir || 'raw')}\\/`),
+    new RegExp(`^${reEsc(recallCfg.output_dir || 'output')}\\/receipts\\/`),
+    /^node_modules\//,
+    /(^|\/)(_trash|_archive)(\/|$)/,
+    new RegExp(`^${reEsc(archiveDir)}\\/`),
+    /\/agent-mailbox\/runtime\//,
+  ];
+  const builtinInclude = [
+    /^README\.md$/,
+    /^index\.md$/,
+    /^log\.md$/,
+    new RegExp(`^${reEsc(wikiDir)}\\/`),
+    new RegExp(`^${reEsc(systemDir)}\\/neurain\\/lessons\\.md$`),
+    new RegExp(`^${reEsc(systemDir)}\\/sessions\\/handoffs\\/.+\\.md$`),
+    new RegExp(`^${reEsc(systemDir)}\\/area-registry\\.md$`),
+    new RegExp(`^${reEsc(areasDir)}\\/[^/]+\\/.+\\.md$`),
+    new RegExp(`^${reEsc(hubsDir)}\\/.+\\.md$`),
+  ];
+  return (rel) => {
+    if (hardExclude.some((re) => re.test(rel))) return false;
+    if (exclude.some((re) => re.test(rel))) return false;
+    if (builtinInclude.some((re) => re.test(rel))) return true;
+    return include.some((re) => re.test(rel));
+  };
+}
+
+export function safeToIndex(text) {
+  const value = String(text || '');
+  return !secretLike(value) && !injectionLike(value);
+}
+
+export function kindForPath(rel) {
+  if (rel.startsWith('wiki/')) return 'wiki';
+  if (rel.includes('/current/')) return 'handoff';
+  if (rel.includes('/product/')) return 'product';
+  if (rel === '00_system/neurain/lessons.md') return 'lessons';
+  if (rel.includes('/sessions/handoffs/')) return 'session';
+  if (rel.endsWith('log.md')) return 'log';
+  return 'markdown';
+}
+
+export function scopeForPath(rel) {
+  const area = rel.match(/^10_areas\/_?([^/]+)\//)?.[1];
+  if (area) return `area:${area}`;
+  const session = rel.match(/^00_system\/sessions\/handoffs\/(.+)\.md$/)?.[1];
+  if (session) return `session:${session}`;
+  return 'global';
+}
+
+export function titleForText(text, rel) {
+  return String(text || '').match(/^#\s+(.+)$/m)?.[1]?.trim() || path.basename(rel);
+}
+
+// Resolve a --area name to a real area directory: exact match, else the
+// underscore-prefixed convention (--area research -> _research). Empty -> no scope.
+// Unknown -> throw with the available area dirs.
+export function resolveAreaDir(root, name, recallCfg = recallConfig(root)) {
+  const value = String(name || '').trim();
+  if (!value) return '';
+  const areasDir = recallCfg.areas_dir || '10_areas';
+  const areasAbs = path.join(root, areasDir);
+  const exists = (dir) => {
+    try { return fs.statSync(path.join(areasAbs, dir)).isDirectory(); } catch { return false; }
+  };
+  if (exists(value)) return value;
+  if (!value.startsWith('_') && exists(`_${value}`)) return `_${value}`;
+  let available = [];
+  try {
+    available = fs.readdirSync(areasAbs, { withFileTypes: true }).filter((d) => d.isDirectory() && !d.name.startsWith('.')).map((d) => d.name);
+  } catch { /* areas dir absent */ }
+  throw new Error(`Unknown area "${name}". Available areas: ${available.join(', ') || '(none)'}`);
+}
+
+// area dir -> recall scope value, matching scopeForPath which strips one leading
+// underscore (10_areas/_research/... -> area:research).
+export function scopeForArea(areaDir) {
+  return areaDir ? `area:${areaDir.replace(/^_/, '')}` : '';
+}
+
+// The single corpus file selection. Returns the markdown files (with raw text
+// and resolved sensitivity) that belong in the recall corpus. When `area` (a
+// resolved area directory like "_research") is given, the walk is scoped to that
+// area for efficiency and area-local statistics; the whitelist still matches on
+// full root-relative paths.
+export function listRecallMarkdownFiles(root, recallCfg, { area = '' } = {}) {
+  const resolver = createSensitivityResolver(root, recallCfg);
+  const matches = buildRecallPathMatcher(recallCfg);
+  const areasDir = recallCfg.areas_dir || '10_areas';
+  const walkRoot = area ? path.join(root, areasDir, area) : root;
+  const out = [];
+  for (const file of walkFiles(walkRoot, { includeRaw: false, maxFiles: 50000 })) {
+    const rel = relPath(root, file);
+    if (!matches(rel)) continue;
+    if (!isTextFile(rel)) continue;
+    const text = readText(file, '');
+    const sensitivity = resolver.sensitivityFor(rel, text);
+    if (sensitivity === 'private') continue;
+    if (!safeToIndex(text)) continue;
+    out.push({ rel, text, sensitivity });
+  }
+  return out;
+}

package/src/core/recall_facts.mjs

@@ -0,0 +1,233 @@
+// Fact-ledger fusion for recall (W-A, minimal). A dependency-free, pure-JS port
+// of the vault's memory-intel adapter, scoped to what the lexical ranker needs:
+// it reads per-area fact ledgers registered in the search index registry,
+// header-driven markdown-table parsing (so heterogeneous ledger schemas are read
+// by column name), and exposes factsFor() so curated source docs and ledgers
+// behind a matching fact earn the memory_fusion boost and bypass the diversity
+// cap. Task lookup, conflict detection, and needs-verification (vault memory
+// command surface) are intentionally NOT ported here; they belong to a later
+// wave. Every file is optional: a missing ledger contributes nothing.
+import fs from 'node:fs';
+import path from 'node:path';
+import { includesTermBoundary, normalizeText } from './recall_intel.mjs';
+
+function readJsonSafe(absPath, fallback) {
+  try {
+    return JSON.parse(fs.readFileSync(absPath, 'utf8'));
+  } catch {
+    return fallback;
+  }
+}
+
+function readTextSafe(absPath) {
+  try {
+    return fs.readFileSync(absPath, 'utf8');
+  } catch {
+    return '';
+  }
+}
+
+// A registered fact_ledger may be a single .md file or a directory of .md tables.
+function memoryFiles(root, areaRoot, value) {
+  if (!value) return [];
+  const abs = path.join(root, `${areaRoot}/${value}`);
+  let stat;
+  try {
+    stat = fs.statSync(abs);
+  } catch {
+    return [];
+  }
+  if (stat.isFile()) return [abs];
+  if (stat.isDirectory()) {
+    return fs.readdirSync(abs).filter((f) => f.endsWith('.md')).map((f) => path.join(abs, f));
+  }
+  return [];
+}
+
+// Split a markdown table row into trimmed cells, honoring escaped pipes (\|).
+export function splitRow(line) {
+  const cells = line.split(/(?<!\\)\|/).map((c) => c.replace(/\\\|/g, '|').trim());
+  if (cells.length && cells[0] === '') cells.shift();
+  if (cells.length && cells[cells.length - 1] === '') cells.pop();
+  return cells;
+}
+
+function isSeparator(cells) {
+  return cells.length > 0 && cells.every((c) => /^:?-{2,}:?$/.test(c.replace(/\s/g, '')));
+}
+
+// Parse every markdown table into { headers, rows } keyed by normalized header.
+export function parseTables(text) {
+  const lines = String(text || '').split(/\r?\n/);
+  const tables = [];
+  for (let i = 0; i < lines.length; i += 1) {
+    if (!/^\s*\|/.test(lines[i])) continue;
+    if (i + 1 >= lines.length) break;
+    const header = splitRow(lines[i]);
+    const sep = splitRow(lines[i + 1]);
+    if (!isSeparator(sep) || sep.length !== header.length) continue;
+    const rows = [];
+    let j = i + 2;
+    for (; j < lines.length && /^\s*\|/.test(lines[j]); j += 1) {
+      const cells = splitRow(lines[j]);
+      if (isSeparator(cells)) continue;
+      const row = {};
+      header.forEach((h, k) => {
+        row[normalizeText(h)] = cells[k] ?? '';
+      });
+      rows.push(row);
+    }
+    tables.push({ headers: header.map(normalizeText), rows });
+    i = j - 1;
+  }
+  return tables;
+}
+
+function pick(row, names) {
+  for (const n of names) {
+    if (row[n] !== undefined && String(row[n]).trim() !== '') return String(row[n]).trim();
+  }
+  return '';
+}
+
+function tableHas(table, names) {
+  return table.headers.some((h) => names.includes(h));
+}
+
+function emptyCell(value) {
+  const t = normalizeText(value).trim();
+  return t === '' || t === '-' || t === 'n/a' || t === 'na' || t === 'none';
+}
+
+const FACT_ID = ['fact id', 'factid'];
+const SRC = ['source doc', 'source', 'ref', 'reference', 'sources'];
+
+// Optional: honor a registered foreign id_column so non-English fact tables are
+// recognized; the built-in FACT_ID headers already cover the reference vault.
+function factIdColumns(root, recallCfg, area) {
+  const ids = new Set(FACT_ID);
+  const registryRel = recallCfg?.intel?.memory_write_registry || '00_system/neurain/memory-write-registry.json';
+  const reg = readJsonSafe(path.join(root, registryRel), { areas: {} });
+  const areaDef = (reg.areas || {})[area];
+  for (const def of Object.values(areaDef?.tables || {})) {
+    const idc = String(def.id_column || '').toLowerCase().trim();
+    if (idc && !idc.includes('task')) ids.add(idc);
+  }
+  return [...ids];
+}
+
+// Resolve a Source Doc cell to real root-relative paths, applying the area's
+// path_map. A cell may hold multiple sources (backticked, comma/semicolon, or
+// markdown links); URLs and unresolvable tokens are skipped.
+function resolveSources(root, value, areaRoot, pathMap) {
+  const raw = String(value ?? '');
+  const tokens = [];
+  let m;
+  const backtick = /`([^`]+)`/g;
+  while ((m = backtick.exec(raw))) tokens.push(m[1]);
+  const link = /\]\(([^)\s]+)\)/g;
+  while ((m = link.exec(raw))) tokens.push(m[1]);
+  if (!tokens.length) for (const piece of raw.split(/[,;]/)) tokens.push(piece);
+  const out = [];
+  for (let token of tokens) {
+    token = token.replace(/`/g, '').replace(/\\/g, '/').replace(/^\.\//, '').trim();
+    if (!token || /^https?:\/\//i.test(token)) continue;
+    const candidates = [token];
+    for (const rule of pathMap || []) {
+      if (rule && rule.prefix && token.startsWith(rule.prefix)) candidates.push(rule.replace_with + token.slice(rule.prefix.length));
+    }
+    for (const candidate of candidates) {
+      const rel = `${areaRoot}/${candidate}`;
+      if (fs.existsSync(path.join(root, rel))) {
+        out.push(rel);
+        break;
+      }
+    }
+  }
+  return [...new Set(out)];
+}
+
+// Load curated facts from every registered area's fact ledger. No module cache;
+// the caller (lexical context builder) owns the snapshot.
+export function loadFactIntel(root, recallCfg) {
+  const registryRel = recallCfg?.intel?.registry || '00_system/neurain/search-index-registry.json';
+  const registry = readJsonSafe(path.join(root, registryRel), { areas: {} });
+  const facts = [];
+  const rel = (file) => path.relative(root, file).split(path.sep).join('/');
+  for (const [area, areaDef] of Object.entries(registry.areas || {})) {
+    if (!areaDef || !areaDef.area_root) continue;
+    const factIds = factIdColumns(root, recallCfg, area);
+    for (const file of memoryFiles(root, areaDef.area_root, areaDef.fact_ledger)) {
+      for (const table of parseTables(readTextSafe(file))) {
+        if (!tableHas(table, factIds)) continue;
+        for (const row of table.rows) {
+          const id = pick(row, factIds);
+          if (!id) continue;
+          facts.push({
+            area,
+            file: rel(file),
+            id,
+            entity: pick(row, ['entity', 'subject']),
+            domain: pick(row, ['domain']),
+            attribute: pick(row, ['attribute']),
+            value: pick(row, ['value']),
+            status: pick(row, ['status', 'state']),
+            recorded_at: pick(row, ['recorded at', 'logged']),
+            superseded_by: pick(row, ['superseded by']),
+            source: pick(row, SRC),
+            sources_resolved: resolveSources(root, pick(row, SRC), areaDef.area_root, areaDef.path_map),
+            fields: { ...row },
+          });
+        }
+      }
+    }
+  }
+  return { facts };
+}
+
+const OBSOLETE = ['obsolete', 'superseded', 'deprecated'];
+
+export function isOutdated(fact) {
+  const status = normalizeText(fact.status);
+  return OBSOLETE.some((o) => status.includes(o)) || !emptyCell(fact.superseded_by);
+}
+
+function tokens(query) {
+  return normalizeText(query).split(/\s+/).filter(Boolean);
+}
+
+function scoreFields(fieldsText, queryTokens) {
+  const lower = normalizeText(fieldsText);
+  let n = 0;
+  for (const t of queryTokens) if (includesTermBoundary(lower, t)) n += 1;
+  return n;
+}
+
+const EMPTY_FACTS = { facts: [] };
+
+// Facts whose fields match the query, current facts ranked above outdated ones,
+// then by score and recency. Mirrors the vault ordering exactly.
+export function factsFor(query, intel = EMPTY_FACTS, { area, top = 10 } = {}) {
+  const qt = tokens(query);
+  if (!qt.length) return [];
+  return intel.facts
+    .filter((f) => !area || f.area === area)
+    .map((f) => ({
+      f,
+      score: scoreFields(
+        f.fields && Object.keys(f.fields).length
+          ? Object.values(f.fields).join(' ')
+          : [f.entity, f.domain, f.attribute, f.value, f.id].join(' '),
+        qt
+      ),
+    }))
+    .filter((x) => x.score > 0)
+    .sort(
+      (a, b) =>
+        (isOutdated(a.f) ? 1 : 0) - (isOutdated(b.f) ? 1 : 0) ||
+        b.score - a.score ||
+        String(b.f.recorded_at).localeCompare(String(a.f.recorded_at))
+    )
+    .slice(0, top)
+    .map((x) => x.f);
+}