heyspex 0.1.0 → 0.2.0

package/spexharness/src/doctrine/corpus.mjs

@@ -0,0 +1,733 @@
+// spexharness/src/doctrine/corpus.mjs
+//
+// DR1 (epic [[HS-KNHZ-8CMW-EPIC-spexcode-doctrine-v0-the-foundational]], task
+// [[HS-TWTC-8CMW-TASK-dr1-doctrine-corpus-reader-stream-every]]) — the streaming,
+// memory-bounded extractor over every source sources.mjs enumerates. Never
+// `readFileSync`s a whole session file (some run past 100 MB): every JSONL/markdown
+// file is read line-by-line via `node:readline` over a `createReadStream`.
+//
+// One record per signal, with provenance: `{ id, source, project, session, ts, seq,
+// kind, text, context?, confidence? }`. `kind` is one of:
+//   - `instruction` — a user turn ≥ 12 words that directs work (imperative/preference
+//     language; approximated here as "12+ words, not already a ruling/correction").
+//   - `correction`  — a user turn that immediately follows an assistant turn and
+//     contains a reversal/reprimand marker ("no,", "not that", "wrong", "don't",
+//     "stop", "instead", "i said", "why did you"); carries a 240-char `context` of
+//     the assistant text it corrects.
+//   - `ruling`      — contains an explicit-decision marker ("always", "never", "from
+//     now on", "owner ruling", "yes -", "no -", "rule:", "decision:"), any length.
+//   - `approval`/`refusal` — a permission-mode record (Claude Code) or a tool-confirm
+//     answer.
+//   - `taste`       — one `- <text> Confidence: <n>` line from a Command Code
+//     taste.md, `confidence` carried.
+//
+// Every `text`/`context` is redacted via spexharness/src/redact.mjs BEFORE it is
+// written anywhere — this module never even holds an unredacted string past the
+// point of extraction. Assistant text is kept only transiently (as `context` for a
+// following correction), never emitted as its own observation. Pasted
+// screenshots/images and raw tool payloads are skipped (never even reach the
+// classifier — a `tool_result`/`tool_reference`-only content array is dropped whole).
+//
+// OWNER-AUTHORSHIP (2026-09-20): a `tool_result`-block check is not enough — machine text
+// also arrives as an ordinary `type:"text"` block in a `type:"user"` record. Two screens now
+// run before classification: `isOwnerAuthoredClaudeCodeRecord` (record shape) and
+// `machineTextMarker` (text shape). See the "owner-authored vs machine-authored" section.
+//
+// Incremental: a per-session state entry `{ mtime, lines, lastAssistantText,
+// lastRecordType }` lets a re-run (a) skip a session outright when its file's mtime
+// is unchanged, and (b) resume a grown session from the line it left off at, so only
+// the NEW lines are parsed and only the NEW observations are appended/returned.
+
+import { closeSync, createReadStream, existsSync, mkdirSync, openSync, readFileSync, writeSync } from 'node:fs';
+import { createInterface } from 'node:readline';
+import { dirname, join } from 'node:path';
+
+import { redactText } from '../redact.mjs';
+import { enumerateSources as defaultEnumerateSources, sourceKey } from './sources.mjs';
+
+export { sourceKey };
+
+/**
+ * @typedef {{
+ *   id: string,
+ *   source: string,
+ *   project: string,
+ *   session: string,
+ *   ts: string|null,
+ *   seq: number,
+ *   kind: 'instruction'|'correction'|'ruling'|'approval'|'refusal'|'taste',
+ *   text: string,
+ *   context?: string,
+ *   confidence?: number,
+ * }} DoctrineObservation
+ */
+
+// ── kind classification ────────────────────────────────────────────────────────────
+
+const RULING_MARKERS = [
+  /\balways\b/i,
+  /\bnever\b/i,
+  /\bfrom now on\b/i,
+  /\bowner ruling\b/i,
+  /\byes\s*-/i,
+  /\bno\s*-/i,
+  /\brule:/i,
+  /\bdecision:/i,
+];
+
+const CORRECTION_MARKERS = [
+  /\bno,/i,
+  /\bnot that\b/i,
+  /\bwrong\b/i,
+  /\bdon'?t\b/i,
+  /\bstop\b/i,
+  /\binstead\b/i,
+  /\bi said\b/i,
+  /\bwhy did you\b/i,
+];
+
+const MIN_INSTRUCTION_WORDS = 12;
+
+function wordCount(text) {
+  const trimmed = text.trim();
+  if (!trimmed) return 0;
+  return trimmed.split(/\s+/).filter(Boolean).length;
+}
+
+/**
+ * Classify one user-authored turn's text. Precedence: a CORRECTION only fires when
+ * the turn immediately follows an assistant turn (`precededByAssistant`) AND carries
+ * a reversal/reprimand marker; else a RULING marker (any length) wins; else 12+ words
+ * makes it an INSTRUCTION; otherwise the turn carries no signal and is dropped.
+ *
+ * @param {string} text
+ * @param {{ precededByAssistant?: boolean }} [opts]
+ * @returns {'correction'|'ruling'|'instruction'|null}
+ */
+export function classifyUserText(text, { precededByAssistant = false } = {}) {
+  if (typeof text !== 'string' || !text.trim()) return null;
+  if (precededByAssistant && CORRECTION_MARKERS.some((re) => re.test(text))) return 'correction';
+  if (RULING_MARKERS.some((re) => re.test(text))) return 'ruling';
+  if (wordCount(text) >= MIN_INSTRUCTION_WORDS) return 'instruction';
+  return null;
+}
+
+// ── content-shape helpers (shared by Claude Code + Command Code, same block shape) ──
+
+function extractTextFromContent(content) {
+  if (typeof content === 'string') return content;
+  if (!Array.isArray(content)) return '';
+  return content
+    .filter((b) => b && b.type === 'text' && typeof b.text === 'string')
+    .map((b) => b.text)
+    .join('\n');
+}
+
+/** True when every block in an array content is a tool result/reference — never the owner's words. */
+function isToolOnlyContent(content) {
+  if (!Array.isArray(content) || content.length === 0) return false;
+  return content.every((b) => b && (b.type === 'tool_result' || b.type === 'tool_reference'));
+}
+
+// ── owner-authored vs machine-authored text ────────────────────────────────────────
+//
+// `isToolOnlyContent` above only catches a literal `tool_result`/`tool_reference` BLOCK.
+// The far larger contamination channel is machine text that arrives as an ordinary
+// `type:"text"` block inside a `type:"user"` record, which sails straight through to the
+// classifier. Measured on this machine 2026-09-20: 862 of 5,328 stored observations (16.2%)
+// were not the owner's words at all — 483 copies of the sidecar title-generator's own system
+// prompt, 376 `<task-notification>` blobs, plus hook-injected context.
+//
+// Two independent screens, because each catches what the other cannot:
+//   1. RECORD SHAPE (`isOwnerAuthoredClaudeCodeRecord`) — Claude Code's own `promptSource`
+//      discriminator, the Claude Code counterpart of the `meta.source !== 'user'` check
+//      `handleCommandCodeRecord` has always had. Precise, but only present on newer records.
+//   2. TEXT SHAPE (`machineTextMarker`) — a small, documented denylist of the control
+//      envelopes and injected preambles the harness and host actually emit. Catches older
+//      records that predate `promptSource`, and already-stored observations (see
+//      `filterObservations`, the corpus-hygiene path).
+//
+// DELIBERATELY CONSERVATIVE. A genuine owner line dropped here costs one habit citation; a
+// machine line learned here is laundered into a confident-looking committed principle that
+// ships to every clone. When in doubt, drop it.
+
+/**
+ * `promptSource` values that mean "the owner put this text here". Verified on this machine
+ * 2026-09-20 over 5,803 Claude Code user records: `typed` (201), `queued` (11),
+ * `suggestion_accepted` (6) — against `system` (171: hook injections and task
+ * notifications) and, in the sidecar SDK sessions, `sdk`. `paste`/`slash_command` are
+ * listed defensively; they are owner-originated shapes the host may emit.
+ */
+export const OWNER_PROMPT_SOURCES = Object.freeze([
+  'typed',
+  'queued',
+  'paste',
+  'pasted',
+  'slash_command',
+  'suggestion_accepted',
+]);
+
+/**
+ * True when a Claude Code `type:"user"` record is a turn the OWNER authored, as opposed to
+ * one the host or a hook injected into the transcript in user position.
+ *
+ * A record with NO `promptSource` is allowed through (5,414 of 5,803 on this machine — the
+ * field is recent, and most of those are tool-result relays already dropped by
+ * `isToolOnlyContent`); those are caught by the text screen instead. A record WITH a
+ * `promptSource` must name an owner origin — an UNRECOGNISED value is rejected, which is the
+ * conservative direction: a future host adding a new owner-typed source costs us some genuine
+ * lines until it is listed here, whereas allowing unknowns re-opens exactly this bug.
+ *
+ * @param {object} rec
+ * @returns {boolean}
+ */
+export function isOwnerAuthoredClaudeCodeRecord(rec) {
+  if (!rec || typeof rec !== 'object') return false;
+  const promptSource = rec.promptSource;
+  if (typeof promptSource === 'string' && promptSource && !OWNER_PROMPT_SOURCES.includes(promptSource)) {
+    return false;
+  }
+  // Transcript scaffolding the host writes in user position, never a typed turn.
+  if (rec.isMeta === true) return false;
+  if (rec.isCompactSummary === true) return false;
+  // A sidechain is a subagent's own transcript: its "user" turns are the PARENT AGENT's
+  // prompts, not the owner's.
+  if (rec.isSidechain === true) return false;
+  if (typeof rec.userType === 'string' && rec.userType !== 'external') return false;
+  return true;
+}
+
+/**
+ * Control-envelope tag names. Machine content delivered in user position is wrapped in one of
+ * these XML-ish envelopes by the host or by this harness; the owner does not type them.
+ * Matched as a tag (`<name`, `<name>`, `</name>`) anywhere in the text — an envelope appended
+ * AFTER an owner sentence still means the blob as a whole is not a clean habit citation.
+ */
+export const CONTROL_ENVELOPE_TAGS = Object.freeze([
+  'task-notification',
+  'system-reminder',
+  'local-command-stdout',
+  'local-command-stderr',
+  'command-name',
+  'command-message',
+  'command-args',
+  'command-contents',
+  'user-prompt-submit-hook',
+  'session-start-hook',
+  'post-tool-use-hook',
+  'bash-stdout',
+  'bash-stderr',
+  'ide_selection',
+  'ide_opened_file',
+  'ide_diagnostics',
+  'channel',
+]);
+
+function envelopePattern(tag) {
+  return new RegExp(`<\\/?${tag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?=[\\s/>])`, 'i');
+}
+
+/**
+ * The denylist, one row per marker, each with the shape it anchors on and why it is here.
+ * Data-driven on purpose: a single unreadable mega-regex is how the next marker gets added
+ * wrong. Anchored on shapes the harness/host EMIT (an envelope tag, a rendered block header,
+ * a fixed generator prompt) rather than on incidental wording, so a genuine owner sentence
+ * that happens to mention "SpexCode Profile" in prose is not swept up.
+ *
+ * @type {ReadonlyArray<{ id: string, pattern: RegExp, why: string }>}
+ */
+export const MACHINE_TEXT_MARKERS = Object.freeze([
+  ...CONTROL_ENVELOPE_TAGS.map((tag) => ({
+    id: `envelope:${tag}`,
+    pattern: envelopePattern(tag),
+    why: `<${tag}> is a host/harness control envelope delivered in user position`,
+  })),
+  {
+    id: 'sticky-title-sidecar',
+    // The single largest contaminant measured: 483 copies of the session-title generator's own
+    // system prompt, harvested out of the sidecar SDK sessions under a Local\Temp cwd.
+    pattern: /You maintain a short, sticky title for an ongoing coding-assistant session/i,
+    why: "the session-title generator's system prompt, replayed into its own sidecar transcript",
+  },
+  {
+    id: 'hook-additional-context',
+    pattern: /UserPromptSubmit hook additional context:/i,
+    why: 'the literal preamble Claude Code prefixes to a UserPromptSubmit hook injection',
+  },
+  {
+    id: 'local-command-caveat',
+    pattern: /Caveat: The messages below were generated by the user while running local commands/i,
+    why: 'the host preamble wrapping local slash-command output, not a typed turn',
+  },
+  {
+    id: 'spexcode-profile-block',
+    // session-bootstrap.mjs's `renderProfileBlock` — the profile we WRITE, injected every user
+    // turn. Learning it back is a direct feedback loop. Line-anchored so prose that merely
+    // names the profile survives.
+    pattern: /^#[ \t]*SpexCode Profile\b/m,
+    why: "the injected SpexCode Profile block's own header — re-learning it feeds the profile back into itself",
+  },
+  {
+    id: 'spexcode-doctrine-block',
+    pattern: /^#[ \t]*SpexCode Doctrine\b/m,
+    why: 'the injected SpexCode Doctrine block header — same feedback loop as the profile block',
+  },
+  {
+    id: 'doctrine-injection',
+    // doctrine/inject.mjs's `renderDoctrineBlock` header: `Doctrine (learned <iso>, N principles):`
+    pattern: /^Doctrine \(learned .*\bprinciples?\):/m,
+    why: "the injected Doctrine block's rendered header line",
+  },
+  {
+    id: 'vault-context-injection',
+    pattern: /VAULT CONTEXT \(retrieved/i,
+    why: 'the inject-vault-context gate\'s retrieved-references block',
+  },
+  {
+    id: 'heyspex-gate-check',
+    pattern: /HEYSPEX GATE CHECK/i,
+    why: 'a HeySpex gate-check preamble injected ahead of the turn',
+  },
+  {
+    id: 'dev-factory-seat',
+    pattern: /^DEV-FACTORY SEAT:/m,
+    why: "dev-factory/lib/seat-context.mjs's seat mandate line, injected at session start",
+  },
+]);
+
+/**
+ * The id of the first {@link MACHINE_TEXT_MARKERS} row that matches `text`, or `null`.
+ * Returning the id (not just a boolean) is what lets the hygiene path report WHY each row
+ * was rejected.
+ * @param {unknown} text
+ * @returns {string|null}
+ */
+export function machineTextMarker(text) {
+  if (typeof text !== 'string' || !text) return null;
+  for (const marker of MACHINE_TEXT_MARKERS) {
+    if (marker.pattern.test(text)) return marker.id;
+  }
+  return null;
+}
+
+/** True when `text` carries any machine-authorship marker — see {@link MACHINE_TEXT_MARKERS}. */
+export function isMachineAuthoredText(text) {
+  return machineTextMarker(text) !== null;
+}
+
+function parseJsonLine(raw) {
+  try {
+    return JSON.parse(raw);
+  } catch {
+    return null;
+  }
+}
+
+// ── per-source-shape record handlers — each returns zero or more candidate
+// observations `{ kind, text, ts?, context?, confidence? }` and may mutate `ctx`
+// (the session's running "what came before" state, carried across incremental runs). ──
+
+/**
+ * Claude Code JSONL record shapes (verified on this machine 2026-09-11):
+ *   {"type":"user","message":{"role":"user","content": string | Array<block>}, "timestamp", ...}
+ *   {"type":"assistant","message":{"content":[{type:"text"|"tool_use"|"thinking"}]}}
+ *   {"type":"permission-mode","permissionMode": string, "sessionId"}
+ *   {"type":"mode", ...} — session UI state, not an observation; skipped.
+ * Any other `type` (ai-title, summary, environment, ...) is likewise skipped.
+ */
+export function handleClaudeCodeRecord(rec, ctx) {
+  const results = [];
+  if (!rec || typeof rec !== 'object') return results;
+
+  if (rec.type === 'assistant') {
+    const text = extractTextFromContent(rec.message?.content);
+    ctx.lastRecordType = 'assistant';
+    if (text) ctx.lastAssistantText = text;
+    return results;
+  }
+
+  if (rec.type === 'user') {
+    const content = rec.message?.content;
+    const precededByAssistant = ctx.lastRecordType === 'assistant';
+
+    // The two authorship screens run BEFORE `ctx.lastRecordType` is touched, so an injected
+    // machine record is TRANSPARENT to the correction chain. Otherwise a task notification
+    // landing between the assistant's reply and the owner's "No, stop — ..." made that
+    // correction look like it followed a user turn, and it was silently dropped: the host
+    // interrupting the owner cost us exactly the observations worth the most.
+    //
+    // Record-shape screen — the Claude Code counterpart of handleCommandCodeRecord's
+    // `meta.source !== 'user'` discriminator.
+    if (!isOwnerAuthoredClaudeCodeRecord(rec)) return results;
+    const text = isToolOnlyContent(content) ? '' : extractTextFromContent(content);
+    // Text-shape screen — catches the records that predate `promptSource`.
+    if (text && isMachineAuthoredText(text)) return results;
+
+    ctx.lastRecordType = 'user';
+    if (!text) return results;
+    const kind = classifyUserText(text, { precededByAssistant });
+    if (!kind) return results;
+    const obs = { kind, text, ts: rec.timestamp ?? null };
+    if (kind === 'correction' && ctx.lastAssistantText) obs.context = ctx.lastAssistantText.slice(0, 240);
+    results.push(obs);
+    return results;
+  }
+
+  if (rec.type === 'permission-mode') {
+    results.push({ kind: 'approval', text: `permission mode: ${rec.permissionMode ?? 'unknown'}`, ts: null });
+    return results;
+  }
+
+  return results;
+}
+
+/**
+ * Command Code JSONL record shapes (verified on this machine 2026-09-11):
+ *   {"type":"session", ...} — one per file, not an observation.
+ *   {"type":"message","message":{"role":"user"|"assistant","content":[block...],
+ *     "meta":{"source":"user"|"tool"|...}}}
+ * `meta.source === "user"` is the discriminator for a genuinely owner-typed turn — a
+ * `role:"user"` turn whose content is a tool_result relay carries `meta.source ===
+ * "tool"` and is never the owner's words.
+ */
+export function handleCommandCodeRecord(rec, ctx) {
+  const results = [];
+  if (!rec || rec.type !== 'message') return results;
+  const role = rec.message?.role;
+  const content = rec.message?.content;
+
+  if (role === 'assistant') {
+    const text = extractTextFromContent(content);
+    ctx.lastRecordType = 'assistant';
+    if (text) ctx.lastAssistantText = text;
+    return results;
+  }
+
+  if (role === 'user') {
+    const precededByAssistant = ctx.lastRecordType === 'assistant';
+    ctx.lastRecordType = 'user';
+    if (rec.message?.meta?.source !== 'user') return results; // tool-result relay, not the owner
+    if (isToolOnlyContent(content)) return results;
+    const text = extractTextFromContent(content);
+    if (!text) return results;
+    if (isMachineAuthoredText(text)) return results;
+    const kind = classifyUserText(text, { precededByAssistant });
+    if (!kind) return results;
+    const obs = { kind, text, ts: rec.timestamp ?? null };
+    if (kind === 'correction' && ctx.lastAssistantText) obs.context = ctx.lastAssistantText.slice(0, 240);
+    results.push(obs);
+    return results;
+  }
+
+  return results;
+}
+
+const TASTE_LINE_RE = /^-\s+(.*?)\s+Confidence:\s*([0-9]*\.?[0-9]+)\s*$/;
+
+/** One `- <text> Confidence: <n>` line from a Command Code taste.md. */
+export function handleTasteLine(rawLine) {
+  const m = TASTE_LINE_RE.exec(rawLine.trim());
+  if (!m) return null;
+  return { kind: 'taste', text: m[1].trim(), confidence: Number(m[2]), ts: null };
+}
+
+/**
+ * spexharness's own session-end export (session-record.mjs / session-end.mjs). Its
+ * current shape (`{ sessionId, exportedAt, events, toolCalls }`) carries no user-
+ * authored text at all — only gate decisions and tool-call name/input/output — so
+ * there is nothing to classify from it TODAY. This handler is forward-compatible: a
+ * future sink that adds a string `ask` field (mirroring session-record.mjs's
+ * `createSessionRecord({ ask })`) is picked up automatically, classified exactly like
+ * a Claude Code user turn, with no prior-assistant context available in this file.
+ */
+export function handleSpexcodeRecord(rec) {
+  if (!rec || typeof rec.ask !== 'string') return [];
+  if (isMachineAuthoredText(rec.ask)) return [];
+  const kind = classifyUserText(rec.ask, { precededByAssistant: false });
+  if (!kind) return [];
+  return [{ kind, text: rec.ask, ts: rec.exportedAt ?? null }];
+}
+
+// ── streaming line reader ────────────────────────────────────────────────────────
+
+/** Yield `{ n, line }` for every line of `filePath` past line number `skip` (1-indexed). */
+async function* streamNewLines(filePath, skip) {
+  const rl = createInterface({ input: createReadStream(filePath, { encoding: 'utf8' }), crlfDelay: Infinity });
+  let n = 0;
+  try {
+    for await (const line of rl) {
+      n++;
+      if (n <= skip) continue;
+      yield { n, line };
+    }
+  } finally {
+    rl.close();
+  }
+}
+
+function dispatchRecord(src, rec, rawLine, ctx) {
+  if (src.source === 'claude-code') return handleClaudeCodeRecord(rec, ctx);
+  if (src.source === 'command-code') return handleCommandCodeRecord(rec, ctx);
+  if (src.source === 'spexcode') return handleSpexcodeRecord(rec);
+  return [];
+}
+
+/**
+ * Extract new observations from ONE source (one session file), resuming from its
+ * prior state. Returns the per-session outcome; observations are pushed to `sink` as
+ * they are produced (never buffered whole here) so a caller can stream them straight
+ * to disk.
+ */
+async function extractFromSource(src, priorState, redact, sink) {
+  const key = sourceKey(src);
+  const prior = priorState[key] ?? { mtime: null, lines: 0, lastAssistantText: null, lastRecordType: null };
+
+  if (prior.mtime === src.mtime) {
+    return { key, newState: prior, added: 0, skipped: true };
+  }
+
+  const ctx = { lastAssistantText: prior.lastAssistantText ?? null, lastRecordType: prior.lastRecordType ?? null };
+  let added = 0;
+  let lastLine = prior.lines;
+
+  if (src.source === 'taste') {
+    for await (const { n, line } of streamNewLines(src.path, prior.lines)) {
+      lastLine = n;
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      const cand = handleTasteLine(line);
+      if (!cand) continue;
+      emit(src, n, cand, redact, sink);
+      added++;
+    }
+  } else {
+    for await (const { n, line } of streamNewLines(src.path, prior.lines)) {
+      lastLine = n;
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      const rec = parseJsonLine(trimmed);
+      if (!rec) continue;
+      const candidates = dispatchRecord(src, rec, trimmed, ctx);
+      for (const cand of candidates) {
+        emit(src, n, cand, redact, sink);
+        added++;
+      }
+    }
+  }
+
+  return {
+    key,
+    newState: {
+      mtime: src.mtime,
+      lines: Math.max(lastLine, prior.lines),
+      lastAssistantText: ctx.lastAssistantText,
+      lastRecordType: ctx.lastRecordType,
+    },
+    added,
+    skipped: false,
+  };
+}
+
+function emit(src, seq, cand, redact, sink) {
+  const obs = {
+    id: `${src.source}:${src.project}:${src.session}:${seq}`,
+    source: src.source,
+    project: src.project,
+    session: src.session,
+    ts: cand.ts ?? null,
+    seq,
+    kind: cand.kind,
+    text: redact(cand.text),
+  };
+  if (cand.context) obs.context = redact(cand.context);
+  if (typeof cand.confidence === 'number') obs.confidence = cand.confidence;
+  sink(obs);
+}
+
+// ── top-level entrypoint ────────────────────────────────────────────────────────
+
+/**
+ * Stream every source and extract observations, resuming from `state`.
+ *
+ * @param {{
+ *   sources?: Array<{source:string,project:string,session:string,path:string,mtime:number}>,
+ *   state?: object,
+ *   redact?: (text: string) => string,
+ *   since?: string|null,
+ *   project?: string|null,
+ *   onObservation?: ((obs: object) => void)|null,
+ *   onSourceDone?: ((result: { source: string, project: string, session: string, added: number, skipped: boolean }) => void)|null,
+ * }} [opts]
+ * @returns {Promise<{ observations: DoctrineObservation[], newState: object, countsBySource: object, countsByKind: object, sessionsProcessed: number, sessionsSkipped: number }>}
+ */
+export async function extractObservations({
+  sources = defaultEnumerateSources(),
+  state = {},
+  redact = redactText,
+  since = null,
+  project = null,
+  onObservation = null,
+  onSourceDone = null,
+} = {}) {
+  const sinceMs = since ? Date.parse(since) : NaN;
+  const filtered = sources.filter((s) => {
+    if (project && s.project !== project) return false;
+    if (Number.isFinite(sinceMs) && s.mtime < sinceMs) return false;
+    return true;
+  });
+
+  const observations = [];
+  const sink = (obs) => {
+    observations.push(obs);
+    if (onObservation) onObservation(obs);
+  };
+
+  const newState = { ...state };
+  const countsBySource = {};
+  const countsByKind = {};
+  let sessionsProcessed = 0;
+  let sessionsSkipped = 0;
+
+  for (const src of filtered) {
+    const result = await extractFromSource(src, state, redact, sink);
+    newState[result.key] = result.newState;
+    if (result.skipped) sessionsSkipped++;
+    else sessionsProcessed++;
+    countsBySource[src.source] = (countsBySource[src.source] ?? 0) + result.added;
+    if (onSourceDone) {
+      onSourceDone({ source: src.source, project: src.project, session: src.session, added: result.added, skipped: result.skipped });
+    }
+  }
+
+  for (const obs of observations) {
+    countsByKind[obs.kind] = (countsByKind[obs.kind] ?? 0) + 1;
+  }
+
+  return { observations, newState, countsBySource, countsByKind, sessionsProcessed, sessionsSkipped };
+}
+
+// ── incremental state + observation-file persistence ────────────────────────────
+
+export function defaultDoctrineDir(repoRoot) {
+  return join(repoRoot, '.heyspex', 'doctrine');
+}
+
+export function defaultStatePath(repoRoot) {
+  return join(defaultDoctrineDir(repoRoot), 'state.json');
+}
+
+export function defaultObservationsPath(repoRoot) {
+  return join(defaultDoctrineDir(repoRoot), 'observations.jsonl');
+}
+
+/** Read the incremental state file; missing/corrupt → `{}` (a fresh full read). */
+export function loadDoctrineState(path) {
+  if (!existsSync(path)) return {};
+  try {
+    const parsed = JSON.parse(readFileSync(path, 'utf8'));
+    return parsed && typeof parsed === 'object' ? parsed : {};
+  } catch {
+    return {};
+  }
+}
+
+/** Persist the incremental state file. */
+export function saveDoctrineState(path, state) {
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync_(path, `${JSON.stringify(state, null, 2)}\n`);
+}
+
+// writeFileSync isn't imported above (this module prefers streaming appends for the
+// observation log); a tiny local wrapper keeps the state-file write a plain one-shot
+// without adding a second import line for a single call site.
+function writeFileSync_(path, contents) {
+  const fd = openSync(path, 'w');
+  try {
+    writeSync(fd, contents);
+  } finally {
+    closeSync(fd);
+  }
+}
+
+/** Append observations to the observation JSONL log — line-atomic (open "a" + one writeSync + close). */
+export function appendObservations(path, observations) {
+  if (!observations || observations.length === 0) return;
+  mkdirSync(dirname(path), { recursive: true });
+  const fd = openSync(path, 'a');
+  try {
+    for (const obs of observations) writeSync(fd, `${JSON.stringify(obs)}\n`);
+  } finally {
+    closeSync(fd);
+  }
+}
+
+/**
+ * CORPUS HYGIENE — re-apply the machine-text screen to observations ALREADY stored in
+ * `observations.jsonl`, so a corpus harvested by an older build can be cleaned without
+ * re-reading every session file (and without discarding rows whose sessions have since been
+ * deleted). Pure: reads nothing, writes nothing — the caller decides what to do with `kept`.
+ *
+ * The record-shape screen cannot run here (a stored observation no longer carries its source
+ * record's `promptSource`), so this is the text screen plus, optionally, the project scope.
+ *
+ * `claudeCodeProjects` mirrors sources.mjs exactly: it scopes ONLY `source: 'claude-code'`
+ * rows, because that is the only source the cross-project harvest applies to. Command Code,
+ * taste and spexcode rows are left alone — their `project` is derived differently (taste and
+ * spexcode carry `basename(repoRoot)`, not the Claude Code directory slug), so scoping them
+ * by the same list would silently delete this repo's own taste profile.
+ *
+ * @param {Array<object>} observations
+ * @param {{ claudeCodeProjects?: string[]|null }} [opts]
+ * @returns {{
+ *   kept: Array<object>,
+ *   rejected: Array<{ observation: object, reason: string }>,
+ *   rejectedByReason: Record<string, number>,
+ * }}
+ */
+export function filterObservations(observations, { claudeCodeProjects = null } = {}) {
+  const allow = Array.isArray(claudeCodeProjects) ? new Set(claudeCodeProjects) : null;
+  const kept = [];
+  const rejected = [];
+  const rejectedByReason = {};
+  const reject = (observation, reason) => {
+    rejected.push({ observation, reason });
+    rejectedByReason[reason] = (rejectedByReason[reason] ?? 0) + 1;
+  };
+
+  for (const obs of observations ?? []) {
+    if (!obs || typeof obs !== 'object') continue;
+    if (allow && obs.source === 'claude-code' && !allow.has(obs.project)) {
+      reject(obs, 'out-of-scope-project');
+      continue;
+    }
+    const marker = machineTextMarker(obs.text) ?? machineTextMarker(obs.context);
+    if (marker) {
+      reject(obs, marker);
+      continue;
+    }
+    kept.push(obs);
+  }
+
+  return { kept, rejected, rejectedByReason };
+}
+
+/**
+ * Read all observations from the JSONL log, oldest first. Missing file → []. Corrupt lines are skipped.
+ * @param {string} path
+ * @returns {DoctrineObservation[]}
+ */
+export function readObservations(path) {
+  if (!existsSync(path)) return [];
+  const text = readFileSync(path, 'utf8');
+  const out = [];
+  for (const line of text.split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      out.push(JSON.parse(trimmed));
+    } catch {
+      // corrupt/partial line — skip, don't throw
+    }
+  }
+  return out;
+}