heyspex 0.1.0 → 0.2.0

package/spexharness/src/doctrine/vault-source.mjs

@@ -0,0 +1,574 @@
+// spexharness/src/doctrine/vault-source.mjs
+//
+// DR5 (epic [[HS-KNHZ-8CMW-EPIC-spexcode-doctrine-v0-the-foundational]], task
+// [[HS-7Z9E-8CMW-TASK-dr5-vault-knowledge-as-a-doctrine]]) — the Doctrine learns from
+// the project's own CURATED knowledge, not only from session transcripts: owner
+// rulings (memory/decisions), lessons, agent-facing protocol enforcement rules,
+// accepted ADRs, the north star / operating model / principles index, the naughty
+// list, this app repo's own always-loaded CLAUDE.md contract, and UI-pattern taste.
+//
+// Every extracted rule is emitted at a SEEDED confidence (never routed through the
+// `research_synthesis` seat — see synthesize.mjs's `source !== 'vault'` filter and
+// merge.mjs's {@link import('./merge.mjs').seedVaultRulesAsPrinciples}): the owner's
+// own written word does not need an LLM to re-derive it, and re-synthesizing it would
+// risk drifting the text away from what was actually decided.
+//
+// Supersession rules (survey table, [[HS-7Z9E-8CMW-TASK-dr5-vault-knowledge-as-a-doctrine]]):
+//   - A file whose frontmatter `status` is `superseded` or `closed` is skipped
+//     entirely — zero observations.
+//   - A body carrying a `> [!amendment]` or `> [!deprecated]` callout (regardless of
+//     `status`, e.g. a still-`open` decision the owner later reversed in-body) uses
+//     the amendment/deprecation text as the CURRENT rule and drops the original
+//     paragraph — the callout is what is actually still true.
+//
+// Full-recompute per run (the vault is git-tracked and small), with a per-file mtime
+// cache ({@link loadVaultCache}/{@link saveVaultCache}, `.heyspex/doctrine/vault-cache.json`)
+// so an unchanged file's content is never re-read — see {@link extractWithCache}.
+// Every observation's `text` passes through `redact.mjs` before it is ever returned.
+
+import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+
+import yaml from 'js-yaml';
+
+import { redactText } from '../redact.mjs';
+
+/**
+ * @typedef {{
+ *   id: string,
+ *   source: 'vault',
+ *   set: string,
+ *   kind: 'vault-rule',
+ *   stance: 'do'|'avoid',
+ *   seedConfidence: number,
+ *   text: string,
+ *   path: string,
+ *   updated: string|null,
+ * }} VaultObservation
+ */
+
+// ── paths ─────────────────────────────────────────────────────────────────────────
+
+/** The sibling vault repo checkout — see reference memory "the-spec-vault-lives-in-the-sibling". */
+export function defaultVaultRoot(repoRoot) {
+  return join(repoRoot, '..', 'heyspex-specs');
+}
+
+export function defaultVaultSpecsDir(vaultRoot) {
+  return join(vaultRoot, 'specs');
+}
+
+export function defaultVaultCachePath(repoRoot) {
+  return join(repoRoot, '.heyspex', 'doctrine', 'vault-cache.json');
+}
+
+// ── frontmatter + markdown helpers ──────────────────────────────────────────────────
+
+const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?/;
+
+/** Split a vault file's raw text into `{ data, body }`. Tolerant of missing/corrupt frontmatter. */
+export function parseFrontmatter(raw) {
+  const text = String(raw ?? '').replace(/\r\n/g, '\n');
+  const m = FRONTMATTER_RE.exec(text);
+  if (!m) return { data: {}, body: text };
+  let data = {};
+  try {
+    const parsed = yaml.load(m[1]);
+    if (parsed && typeof parsed === 'object') data = parsed;
+  } catch {
+    // corrupt frontmatter — treat as no frontmatter, body is the whole (unparsed) file
+  }
+  return { data, body: text.slice(m[0].length) };
+}
+
+/** A file whose frontmatter `status` marks it retired — skip entirely, per the module header. */
+export function isSuperseded(fm) {
+  const status = typeof fm?.status === 'string' ? fm.status.toLowerCase() : '';
+  return status === 'superseded' || status === 'closed';
+}
+
+/**
+ * A `> [!amendment] ...` / `> [!deprecated] ...` callout anywhere in the body — captures
+ * the marker's own same-line text plus every following `>`-prefixed line, joined into
+ * one string. `null` when no such callout is present.
+ */
+export function extractAmendmentCallout(body) {
+  const re = /^>\s*\[!(?:amendment|deprecated)\]\s?([^\n]*)\n((?:>.*(?:\n|$))*)/im;
+  const m = re.exec(body ?? '');
+  if (!m) return null;
+  const firstLine = m[1].trim();
+  const restLines = m[2]
+    .split('\n')
+    .map((l) => l.replace(/^>\s?/, '').trim())
+    .filter(Boolean);
+  const all = [firstLine, ...restLines].filter(Boolean);
+  const joined = all.join(' ').trim();
+  return joined || null;
+}
+
+/** First non-empty, non-heading-only paragraph of a body — skips a `# Title` line AND any leading `##`+ section heading with no prose of its own (e.g. a body opening straight into `## Lesson\n\n<content>` with no summary). */
+export function firstParagraph(body) {
+  const paras = String(body ?? '')
+    .split(/\n{2,}/)
+    .map((p) => p.trim())
+    .filter(Boolean);
+  for (const p of paras) {
+    if (/^#{1,6}\s+/.test(p)) continue;
+    return p;
+  }
+  return '';
+}
+
+/** Content of a `## <heading>` section (case-insensitive, exact heading text), up to the next `##`/`---`/EOF. */
+export function extractHeadingSection(body, headingName) {
+  const re = new RegExp(`^##\\s+${headingName}\\s*\\n([\\s\\S]*?)(?=\\n##\\s|\\n---\\s*$|$)`, 'im');
+  const m = re.exec(body ?? '');
+  return m ? m[1].trim() : null;
+}
+
+/**
+ * The **How to apply** block, either as a `## How to apply` / `## The general shape`
+ * heading section, or (the decisions-set shape) an inline `**How to apply:** ...`
+ * paragraph. `null` when neither is present.
+ */
+export function findHowToApplyBlock(body) {
+  const heading = extractHeadingSection(body, '(?:How to apply|The general shape)');
+  if (heading) return heading;
+  const paras = String(body ?? '').split(/\n{2,}/);
+  for (const p of paras) {
+    const m = /^\*\*How to apply:?\*\*\s*([\s\S]*)$/i.exec(p.trim());
+    if (m && m[1].trim()) return m[1].trim();
+  }
+  return null;
+}
+
+/** `fm.summary` when present, else the body's first paragraph. */
+function openingSpan(fm, body) {
+  const summary = typeof fm?.summary === 'string' ? fm.summary.trim() : '';
+  return summary || firstParagraph(body);
+}
+
+/** Shared decisions/lessons text builder: opening span (+ How to apply block when present), OR the amendment text when the body carries one. */
+function extractRulingSpan(fm, body) {
+  const amendment = extractAmendmentCallout(body);
+  if (amendment) return amendment;
+  const opening = openingSpan(fm, body);
+  const block = findHowToApplyBlock(body);
+  if (!opening && !block) return null;
+  return block ? `${opening}\n\n${block}`.trim() : opening;
+}
+
+/** `##`-level sections of a document, in order, `{ heading, content }`. Nested (`###`+) headings stay inside their parent's `content`. */
+export function splitTopHeadingSections(body) {
+  const text = String(body ?? '');
+  const re = /^##[ \t]+(.+?)\s*$/gm;
+  const matches = [...text.matchAll(re)];
+  const sections = [];
+  for (let i = 0; i < matches.length; i++) {
+    const heading = matches[i][1].trim();
+    const start = matches[i].index + matches[i][0].length;
+    const end = i + 1 < matches.length ? matches[i + 1].index : text.length;
+    sections.push({ heading, content: text.slice(start, end).trim() });
+  }
+  return sections;
+}
+
+/** Content between a named heading (any level 1-6, matched by `namePattern`) and the next heading of the same-or-shallower level. `null` when not found. */
+export function extractNamedSection(raw, namePattern) {
+  const lines = String(raw ?? '').split('\n');
+  const headingRe = /^(#{1,6})\s+(.*)$/;
+  let start = -1;
+  let level = 0;
+  for (let i = 0; i < lines.length; i++) {
+    const m = headingRe.exec(lines[i]);
+    if (m && namePattern.test(m[2])) {
+      start = i + 1;
+      level = m[1].length;
+      break;
+    }
+  }
+  if (start === -1) return null;
+  let end = lines.length;
+  for (let i = start; i < lines.length; i++) {
+    const m = headingRe.exec(lines[i]);
+    if (m && m[1].length <= level) {
+      end = i;
+      break;
+    }
+  }
+  return lines.slice(start, end).join('\n').trim();
+}
+
+/** Top-level (non-indented) `- ` bullets within a block of text, in order. */
+function topLevelBullets(block) {
+  if (!block) return [];
+  const out = [];
+  for (const line of block.split('\n')) {
+    const m = /^-\s+(.+)$/.exec(line);
+    if (m && m[1].trim()) out.push(m[1].trim());
+  }
+  return out;
+}
+
+function specIdFor(fm, filePath) {
+  if (typeof fm?.id === 'string' && fm.id.trim()) return fm.id.trim();
+  const base = filePath.split(/[\\/]/).pop() ?? filePath;
+  return base.replace(/\.md$/i, '');
+}
+
+function listMarkdownFiles(dir) {
+  let entries;
+  try {
+    entries = readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return [];
+  }
+  return entries
+    .filter((e) => e.isFile() && e.name.toLowerCase().endsWith('.md'))
+    .map((e) => join(dir, e.name))
+    .sort();
+}
+
+// ── per-set extractors — each returns zero or more `{ setId, stance, seedConfidence, text }` (unredacted) ──
+
+const SET = {
+  DECISIONS: 'decisions',
+  LESSONS: 'lessons',
+  PROTOCOLS_ENFORCEMENT: 'protocols-enforcement',
+  PROTOCOLS: 'protocols',
+  ADR: 'adr',
+  PRINCIPLES: 'principles',
+  NAUGHTY_LIST: 'naughty-list',
+  CLAUDE_MD: 'claude-md',
+  UI_PATTERNS: 'ui-patterns',
+};
+
+function extractDecisionFile(filePath, raw) {
+  const { data: fm, body } = parseFrontmatter(raw);
+  if (isSuperseded(fm)) return [];
+  const text = extractRulingSpan(fm, body);
+  if (!text) return [];
+  return [
+    {
+      id: `vault:${specIdFor(fm, filePath)}`,
+      set: SET.DECISIONS,
+      stance: 'do',
+      seedConfidence: 0.9,
+      text,
+      updated: fm?.updated ?? fm?.created ?? null,
+    },
+  ];
+}
+
+function extractLessonFile(filePath, raw) {
+  const { data: fm, body } = parseFrontmatter(raw);
+  if (isSuperseded(fm)) return [];
+  const text = extractRulingSpan(fm, body);
+  if (!text) return [];
+  return [
+    {
+      id: `vault:${specIdFor(fm, filePath)}`,
+      set: SET.LESSONS,
+      stance: 'do',
+      seedConfidence: 0.8,
+      text,
+      updated: fm?.updated ?? fm?.created ?? null,
+    },
+  ];
+}
+
+function extractProtocolFile(filePath, raw) {
+  const { data: fm, body } = parseFrontmatter(raw);
+  if (isSuperseded(fm)) return [];
+  const specId = specIdFor(fm, filePath);
+  const enforcement = Array.isArray(fm?.enforcement) ? fm.enforcement : [];
+  const injectRules = enforcement.filter((e) => e && typeof e.inject === 'string' && e.inject.trim());
+
+  if (injectRules.length) {
+    return injectRules.map((entry, i) => ({
+      id: `vault:${specId}#${entry.rule ?? i}`,
+      set: SET.PROTOCOLS_ENFORCEMENT,
+      stance: 'do',
+      seedConfidence: 0.9,
+      text: entry.inject.trim(),
+      updated: fm?.updated ?? fm?.created ?? null,
+    }));
+  }
+
+  const summary = typeof fm?.summary === 'string' ? fm.summary.trim() : '';
+  if (!summary) return [];
+  return [
+    {
+      id: `vault:${specId}`,
+      set: SET.PROTOCOLS,
+      stance: 'do',
+      seedConfidence: 0.6,
+      text: summary,
+      updated: fm?.updated ?? fm?.created ?? null,
+    },
+  ];
+}
+
+function extractAdrFile(filePath, raw) {
+  const { data: fm, body } = parseFrontmatter(raw);
+  const status = typeof fm?.status === 'string' ? fm.status.toLowerCase() : '';
+  if (status !== 'accepted') return [];
+  const decision = extractHeadingSection(body, 'Decision');
+  const text = decision || firstParagraph(body);
+  if (!text) return [];
+  return [
+    {
+      id: `vault:${specIdFor(fm, filePath)}`,
+      set: SET.ADR,
+      stance: 'do',
+      seedConfidence: 0.85,
+      text: text.slice(0, 2000),
+      updated: fm?.updated ?? fm?.created ?? null,
+    },
+  ];
+}
+
+const BOILERPLATE_HEADING_RE = /^(related specs?|change log|sources?)$/i;
+const LIST_ITEM_RE = /^(?:\d+\.|[-*])\s+\*\*/;
+
+function extractPrinciplesFile(filePath, raw) {
+  const { data: fm, body } = parseFrontmatter(raw);
+  if (isSuperseded(fm)) return [];
+  const specId = specIdFor(fm, filePath);
+  const sections = splitTopHeadingSections(body).filter((s) => !BOILERPLATE_HEADING_RE.test(s.heading));
+
+  const out = [];
+  let n = 0;
+  for (const section of sections) {
+    const paras = section.content
+      .split(/\n{2,}/)
+      .map((p) => p.trim())
+      .filter(Boolean);
+    const listItems = paras.filter((p) => LIST_ITEM_RE.test(p));
+    const texts = listItems.length ? listItems.map((p) => p.replace(/^(?:\d+\.|[-*])\s+/, '').trim()) : paras.length ? [paras[0]] : [];
+    for (const text of texts) {
+      n += 1;
+      out.push({
+        id: `vault:${specId}#${n}`,
+        set: SET.PRINCIPLES,
+        stance: 'do',
+        seedConfidence: 0.95,
+        text,
+        updated: fm?.updated ?? fm?.created ?? null,
+      });
+    }
+  }
+  return out;
+}
+
+const TABLE_ROW_RE = /^\|\s*(.+?)\s*\|\s*(.+?)\s*\|\s*(.+?)\s*\|\s*$/;
+
+function extractNaughtyListFile(filePath, raw) {
+  const { data: fm, body } = parseFrontmatter(raw);
+  const specId = specIdFor(fm, filePath);
+  const sections = splitTopHeadingSections(body).filter((s) => /^banned/i.test(s.heading));
+
+  const out = [];
+  let n = 0;
+  for (const section of sections) {
+    for (const line of section.content.split('\n')) {
+      const m = TABLE_ROW_RE.exec(line.trim());
+      if (!m) continue;
+      const term = m[1].replace(/\*\*/g, '').trim();
+      if (/^-+$/.test(term) || /^banned term$/i.test(term)) continue; // header/separator row
+      const why = m[2].replace(/\*\*/g, '').trim();
+      const useInstead = m[3].replace(/\*\*/g, '').trim();
+      n += 1;
+      out.push({
+        id: `vault:${specId}#${n}`,
+        set: SET.NAUGHTY_LIST,
+        stance: 'avoid',
+        seedConfidence: 0.95,
+        text: `Avoid "${term}" (${why}) — use ${useInstead} instead.`,
+        updated: fm?.updated ?? fm?.created ?? null,
+      });
+    }
+  }
+  return out;
+}
+
+function extractUiPatternFile(filePath, raw) {
+  const { data: fm, body } = parseFrontmatter(raw);
+  if (isSuperseded(fm)) return [];
+  const pattern = extractHeadingSection(body, 'Pattern');
+  const whenToApply = extractHeadingSection(body, 'When to apply');
+  const text = [pattern, whenToApply].filter(Boolean).join('\n\n').trim();
+  if (!text) return [];
+  return [
+    {
+      id: `vault:${specIdFor(fm, filePath)}`,
+      set: SET.UI_PATTERNS,
+      stance: 'do',
+      seedConfidence: 0.75,
+      text,
+      updated: fm?.updated ?? fm?.created ?? null,
+    },
+  ];
+}
+
+const CLAUDE_MD_SECTIONS = [/session start/i, /agent claim contract/i, /^conventions$/i];
+
+function extractClaudeMdFile(filePath, raw) {
+  const out = [];
+  let n = 0;
+  for (const pattern of CLAUDE_MD_SECTIONS) {
+    const section = extractNamedSection(raw, pattern);
+    for (const bullet of topLevelBullets(section)) {
+      n += 1;
+      out.push({
+        id: `vault:claude-md#${n}`,
+        set: SET.CLAUDE_MD,
+        stance: 'do',
+        seedConfidence: 0.95,
+        text: bullet,
+        updated: null,
+      });
+    }
+  }
+  return out;
+}
+
+// ── mtime cache ──────────────────────────────────────────────────────────────────
+
+export function loadVaultCache(path) {
+  if (!existsSync(path)) return {};
+  try {
+    const parsed = JSON.parse(readFileSync(path, 'utf8'));
+    return parsed && typeof parsed === 'object' && parsed.files && typeof parsed.files === 'object' ? parsed.files : {};
+  } catch {
+    return {};
+  }
+}
+
+export function saveVaultCache(path, files) {
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, `${JSON.stringify({ version: 1, files }, null, 2)}\n`);
+}
+
+function statMtime(filePath) {
+  try {
+    return statSync(filePath).mtimeMs;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Extract `filePath` via `extractor(raw, filePath)`, reusing `cache[filePath]` when its
+ * `mtimeMs` still matches the file on disk — the file is never re-read in that case.
+ * Always writes an entry into `newCache` for every file that still exists, so a file
+ * removed from the vault since the last run naturally drops out of the cache.
+ */
+function extractWithCache(filePath, cache, newCache, extractor, stats) {
+  const mtimeMs = statMtime(filePath);
+  if (mtimeMs === null) return [];
+  const prior = cache[filePath];
+  if (prior && prior.mtimeMs === mtimeMs) {
+    newCache[filePath] = prior;
+    stats.filesCached += 1;
+    return prior.observations;
+  }
+  const raw = readFileSync(filePath, 'utf8');
+  const observations = (extractor(filePath, raw) ?? []).map((o) => ({ ...o, path: filePath }));
+  newCache[filePath] = { mtimeMs, observations };
+  stats.filesRead += 1;
+  return observations;
+}
+
+// ── top-level entrypoint ────────────────────────────────────────────────────────
+
+/**
+ * Extract every vault-sourced Doctrine observation.
+ *
+ * @param {{
+ *   vaultRoot?: string,
+ *   repoRoot?: string,
+ *   cache?: object,
+ *   redact?: (text: string) => string,
+ * }} [opts]
+ * @returns {{
+ *   observations: VaultObservation[],
+ *   cache: object,
+ *   countsBySet: Record<string, number>,
+ *   stats: { filesRead: number, filesCached: number },
+ * }}
+ */
+export function extractVaultObservations({ vaultRoot, repoRoot = process.cwd(), cache = {}, redact = redactText } = {}) {
+  const resolvedVaultRoot = vaultRoot ?? defaultVaultRoot(repoRoot);
+  const specsDir = defaultVaultSpecsDir(resolvedVaultRoot);
+  const newCache = {};
+  const stats = { filesRead: 0, filesCached: 0 };
+  const rawObservations = [];
+
+  const run = (files, extractor) => {
+    for (const f of files) rawObservations.push(...extractWithCache(f, cache, newCache, extractor, stats));
+  };
+
+  run(listMarkdownFiles(join(specsDir, 'memory', 'decisions')), extractDecisionFile);
+  run(listMarkdownFiles(join(specsDir, 'memory', 'lessons')), extractLessonFile);
+  run(listMarkdownFiles(join(specsDir, 'protocols')), extractProtocolFile);
+  run(listMarkdownFiles(join(specsDir, 'adr')), extractAdrFile);
+  run(listMarkdownFiles(join(specsDir, 'memory', 'ui-patterns')), extractUiPatternFile);
+
+  for (const name of ['heyspex-north-star.md', 'heyspex-operating-model.md', 'principles-index.md']) {
+    run([join(specsDir, name)], extractPrinciplesFile);
+  }
+  run([join(specsDir, 'naughty-list.md')], extractNaughtyListFile);
+  run([join(repoRoot, 'CLAUDE.md')], extractClaudeMdFile);
+
+  const observations = rawObservations.map((o) => ({
+    id: o.id,
+    source: 'vault',
+    set: o.set,
+    kind: 'vault-rule',
+    stance: o.stance,
+    seedConfidence: o.seedConfidence,
+    text: redact(o.text),
+    path: o.path ?? '',
+    updated: o.updated ?? null,
+  }));
+
+  const countsBySet = {};
+  for (const o of observations) countsBySet[o.set] = (countsBySet[o.set] ?? 0) + 1;
+
+  return { observations, cache: newCache, countsBySet, stats };
+}
+
+// ── category classification (used by merge.mjs — never the seat, see module header) ──
+
+const CATEGORY_KEYWORDS = [
+  ['safety-permissions', ['permission', 'destructive', 'safety', 'credential', 'secret', 'redact', 'guardrail', 'settings write', 'write-permission']],
+  ['verification', ['verify', 'verification', 'acceptance', 'test', 'gate', 'proof', 'validate']],
+  ['delegation', ['delegat', 'subagent', 'seat', 'routing', 'route ', 'dispatch', 'fleet', 'model council', 'assign']],
+  ['research-grounding', ['research', 'ground', 'citation', 'evidence', 'cite', 'web search']],
+  ['naming', ['naming', 'brand', 'terminology', 'vocabulary', 'glossary', 'name the']],
+  ['communication', ['report', 'communicat', 'status update', 'concise', 'progress']],
+  ['session-hygiene', ['session', 'commit', 'handoff', 'close-out', 'claim', 'hygiene', 'wip']],
+];
+
+/**
+ * Category for a vault-rule observation — never the `research_synthesis` seat (vault
+ * observations are never sent to it, see synthesize.mjs). `naughty-list` and
+ * `ui-patterns` are fixed by set per the DR5 survey table; everything else is a
+ * keyword heuristic over the set's own protocol-domain vocabulary and the rule text,
+ * falling back to `session-hygiene`.
+ * @param {{ set: string, text: string }} observation
+ * @returns {string}
+ */
+export function classifyVaultCategory(observation) {
+  const set = observation?.set;
+  if (set === SET.NAUGHTY_LIST) return 'communication';
+  if (set === SET.UI_PATTERNS) return 'product-taste';
+  const haystack = String(observation?.text ?? '').toLowerCase();
+  for (const [category, keywords] of CATEGORY_KEYWORDS) {
+    if (keywords.some((kw) => haystack.includes(kw))) return category;
+  }
+  return 'session-hygiene';
+}
+
+export { SET as VAULT_SETS };