heyspex 0.1.0 → 0.2.0

package/spexharness/src/profile/learn.mjs

@@ -0,0 +1,551 @@
+// spexharness/src/profile/learn.mjs
+//
+// P1 (epic [[HS-HEBV-8CMW-EPIC-spexcode-profile-v0-learn-the-developer]], task
+// [[HS-3N98-8CMW-TASK-p1-spexcode-profile-learner-core]]) + P2 (task
+// [[HS-6CE7-8CMW-TASK-p2-profile-learner-incremental-learned]]) — the SpexCode
+// Profile learner core. Distils this machine's already-extracted Doctrine
+// observation stream into a human-readable, cited profile file
+// (`.heyspex/spexcode-profile.md`, machine-local, NEVER written to the vault
+// under `specs/`).
+//
+// v0 is heuristic only — no model call (epic's own scope boundary; the trained
+// classifier lives in [[HS-PYW7-PSHL-MIL-per-vault-ml-training-infrastructure]]
+// and is explicitly out of scope here).
+//
+// Reuse, never duplicate:
+//   - spexharness/src/doctrine/corpus.mjs's `readObservations` is the ONLY way
+//     this module reads the observation stream — it never re-parses a raw
+//     session file, never re-implements JSONL streaming, and never re-derives
+//     a citation. DR1 already redacted every `text`/`context` before it ever
+//     reached `.heyspex/doctrine/observations.jsonl`.
+//   - spexharness/src/context-budget.mjs's `recordContextInjection` is the
+//     one context-injection measurement path in the harness; this module only
+//     WIRES the profile block's size through it (see
+//     `registerProfileContextUsage`) — the actual injection into a seat's
+//     context is a later task (epic scope item 4).
+//
+// Kinds this module classifies into profile sections: `instruction`,
+// `correction`, `ruling`, `approval`, `refusal`, `taste` — the six kinds DR1's
+// corpus.mjs produces. `behaviour` (DR-behaviour) and `vault-rule` (DR5,
+// concurrent with this task) are deliberately SKIPPED: those are
+// session-shape/vault-sourced signals, not the per-developer habit
+// observations this v0 profile distils. A vault decision/lesson's automatic
+// inclusion in the profile is a separate task (epic scope item 8, P7) — this
+// module does not read the vault.
+//
+// Incremental (P2): `.heyspex/profile-state.json` (`loadProfileState` /
+// `saveProfileState`) carries a cursor (`lastObservationSeq` — a count of
+// observations already folded into the profile, not a per-session `seq`
+// field; the observation log is append-only so a simple position cursor is a
+// correct, and much simpler, "or equivalent" to a true monotonic sequence)
+// plus the already-classified section entries themselves, so a re-run only
+// classifies NEW observations and the previously-written section entries are
+// carried through unchanged — the rendered `.md` file's pre-existing
+// observation lines come out byte-identical, because they are re-rendered
+// from the exact same stored entries in the exact same order, never
+// re-derived from a fresh markdown parse.
+//
+// Cold-run cap (P2): `maxObservations` bounds how many NEW observations a
+// single `buildProfile` call folds in; anything beyond the cap is left for
+// the next call (the cursor only advances past what was actually processed),
+// and the return value reports how many were deferred.
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+
+import { readObservations } from '../doctrine/corpus.mjs';
+import { recordContextInjection } from '../context-budget.mjs';
+
+/** Observation kinds this v0 developer-profile deliberately does not learn from. */
+const SKIPPED_KINDS = new Set(['behaviour', 'vault-rule']);
+
+/** Cold-run cap: default max NEW observations a single `buildProfile` call folds in. */
+export const DEFAULT_MAX_OBSERVATIONS = 500;
+
+// ── size caps (HS — profile context blowout) ────────────────────────────────────────
+//
+// Three caps, because the profile blew the context window three different ways at once and
+// each one is independently sufficient to do it again:
+//
+//   1. MAX_ENTRY_CHARS — ONE observation's text. DR1's corpus happily extracts a whole
+//      pasted document as a single `instruction`/`taste` observation; the two largest
+//      entries on this machine were 244KB and 242KB EACH. An entry is a one-line habit
+//      citation, never a document, so it is clamped (and whitespace-collapsed — see
+//      `clampEntryText`, embedded newlines also broke the rendered markdown structure).
+//   2. MAX_ENTRIES_PER_SECTION — total retained entries. `maxObservations` only bounds how
+//      many NEW observations ONE run folds in; sections themselves were append-forever, so
+//      every run permanently added up to 500 more lines. Most recent entries win.
+//   3. PROFILE_MAX_TOKENS — the injection budget, enforced at render (session-bootstrap.mjs's
+//      `renderProfileBlock`), mirroring DOCTRINE_MAX_TOKENS. This is the load-bearing one:
+//      it bounds what reaches a seat's context REGARDLESS of what is on disk, so a profile
+//      file that grew huge under an older build can never blow a context window again.
+
+/** Max characters ONE observation's text keeps as a profile entry. */
+export const MAX_ENTRY_CHARS = 400;
+
+/** Max entries retained per section; the most recent are kept. */
+export const MAX_ENTRIES_PER_SECTION = 200;
+
+/**
+ * Injection budget for the rendered profile block, in tokens — the profile's counterpart to
+ * ../doctrine/inject.mjs's `DOCTRINE_MAX_TOKENS` (800). Enforced by session-bootstrap.mjs's
+ * `renderProfileBlock`, which imports it from here the same way that module imports
+ * `DOCTRINE_MAX_TOKENS` from doctrine/inject.mjs.
+ */
+export const PROFILE_MAX_TOKENS = 1200;
+
+/**
+ * Clamp ONE observation's text to a single, bounded profile line.
+ *
+ * Collapses whitespace FIRST: `citationLine` renders `- <text> _(citation)_` as one markdown
+ * list item, so a text carrying its own newlines (a pasted document, a stack trace) did not
+ * merely take up room — its embedded `## ...`/``` lines were re-parsed as real markdown
+ * structure, which is why the rendered profile grew headings like "Error Message" and
+ * "Code Frame" that no section in SECTION_ORDER ever defines.
+ *
+ * @param {string} text
+ * @param {{ maxChars?: number }} [opts]
+ * @returns {string}
+ */
+export function clampEntryText(text, { maxChars = MAX_ENTRY_CHARS } = {}) {
+  const collapsed = String(text ?? '').replace(/\s+/g, ' ').trim();
+  if (collapsed.length <= maxChars) return collapsed;
+  return `${collapsed.slice(0, Math.max(1, maxChars - 1)).trimEnd()}…`;
+}
+
+/** Keep a section within {@link MAX_ENTRIES_PER_SECTION}, preferring the most recent entries. */
+function trimSection(entries) {
+  return entries.length > MAX_ENTRIES_PER_SECTION ? entries.slice(-MAX_ENTRIES_PER_SECTION) : entries;
+}
+
+/**
+ * Fixed section order — every rendered profile uses this order; empty sections are omitted
+ * (both `renderProfileMarkdown` and `renderProfileDigest` skip a section with no entries, so
+ * `unclassified` is invisible on a profile that has none).
+ *
+ * `unclassified` exists because "Preferred languages and idioms" used to be the FALLBACK for
+ * everything that was not an approval/refusal/correction and carried no test/commit keyword.
+ * A plain bug report was therefore served to a seat under the heading "Preferred languages and
+ * idioms", i.e. as a language preference the owner never expressed. Nothing is dropped — an
+ * observation that fits no section is filed here, honestly labelled, instead of being
+ * mislabelled.
+ */
+export const SECTION_ORDER = Object.freeze([
+  { key: 'languages', heading: 'Preferred languages and idioms' },
+  { key: 'tests', heading: 'Test style' },
+  { key: 'commits', heading: 'Commit and message conventions' },
+  { key: 'tools', heading: 'Tools approved or refused' },
+  { key: 'corrections', heading: 'Recurring corrections' },
+  { key: 'unclassified', heading: 'Unclassified observations' },
+]);
+
+const SECTION_KEYS = SECTION_ORDER.map((s) => s.key);
+
+// ── keyword matching ──────────────────────────────────────────────────────────────
+//
+// WORD BOUNDARIES, not `String.includes`. A bare substring test made `test` match `latest`
+// and `fastest`, and `spec` match `specific` and `respect` — so "ship the latest build"
+// was filed under "Test style". Each list is compiled once into one alternation with
+// boundaries on both sides.
+
+const TEST_KEYWORDS = ['test', 'tests', 'testing', 'spec', 'specs', 'jest', 'vitest', 'mocha', 'assert', 'assertion', 'assertions', 'coverage', 'tdd'];
+const COMMIT_KEYWORDS = ['commit', 'commits', 'committed', 'pull request', 'pull requests', 'pr', 'prs', 'changelog', 'conventional commit'];
+/**
+ * Positive evidence that an observation really is about a language/idiom preference. Before
+ * this list `languages` was a fallback, so it collected everything; now it has to be EARNED,
+ * and what does not earn it lands in `unclassified`.
+ */
+const LANGUAGE_KEYWORDS = [
+  'typescript', 'javascript', 'esm', 'cjs', 'mjs', 'commonjs', 'python', 'rust', 'golang', 'bash', 'powershell', 'sql', 'css', 'html', 'jsx', 'tsx', 'react', 'node', 'npm',
+  'idiom', 'idioms', 'syntax', 'lint', 'linting', 'eslint', 'prettier', 'formatting', 'indentation', 'naming convention', 'naming conventions',
+  'arrow function', 'arrow functions', 'async', 'await', 'strict mode', 'typed', 'typing', 'interface', 'interfaces', 'generics', 'import', 'imports', 'module', 'modules',
+];
+
+function keywordMatcher(keywords) {
+  const alternation = keywords
+    .slice()
+    .sort((a, b) => b.length - a.length)
+    .map((k) => k.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
+    .join('|');
+  return new RegExp(`\\b(?:${alternation})\\b`, 'i');
+}
+
+const TEST_RE = keywordMatcher(TEST_KEYWORDS);
+const COMMIT_RE = keywordMatcher(COMMIT_KEYWORDS);
+const LANGUAGE_RE = keywordMatcher(LANGUAGE_KEYWORDS);
+
+// ── paths ─────────────────────────────────────────────────────────────────────────
+
+/** Mirrors DR1's `defaultObservationsPath` pattern — the profile file's canonical location. */
+export function defaultProfilePath(repoRoot) {
+  return join(repoRoot, '.heyspex', 'spexcode-profile.md');
+}
+
+/** The incremental-mark file's canonical location — separate from DR1's own `.heyspex/doctrine/state.json`. */
+export function defaultProfileStatePath(repoRoot) {
+  return join(repoRoot, '.heyspex', 'profile-state.json');
+}
+
+// ── section classification (heuristic, no model call) ───────────────────────────────
+
+/**
+ * Classify one observation into a profile section key. `approval`/`refusal` observations
+ * feed "tools approved or refused"; `correction` observations feed "recurring corrections"
+ * (each correction DR1 extracted already represents a correction event); everything else is
+ * classified by WORD-BOUNDARY keyword matching over `text`.
+ *
+ * There is no fallback section. An observation matching no keyword list is `unclassified` —
+ * see {@link SECTION_ORDER}. It is still kept and still cited; it is simply not asserted to
+ * be a language preference, a test-style rule or a commit convention when nothing in it says
+ * so.
+ *
+ * @param {{ kind?: string, text?: string }} obs
+ * @returns {'languages'|'tests'|'commits'|'tools'|'corrections'|'unclassified'}
+ */
+export function classifySection(obs) {
+  if (obs.kind === 'approval' || obs.kind === 'refusal') return 'tools';
+  if (obs.kind === 'correction') return 'corrections';
+  const text = String(obs.text ?? '');
+  if (TEST_RE.test(text)) return 'tests';
+  if (COMMIT_RE.test(text)) return 'commits';
+  if (LANGUAGE_RE.test(text)) return 'languages';
+  return 'unclassified';
+}
+
+// ── dedupe (the Profile path's own defect) ────────────────────────────────────────
+//
+// doctrine/merge.mjs dedupes principles; the Profile path never did, so identical text was
+// appended once per occurrence — `permission mode: bypassPermissions` alone accounted for
+// 3,996 of the 5,328 observations measured on this machine, and every one of them became its
+// own profile line, evicting genuine habits under MAX_ENTRIES_PER_SECTION.
+//
+// Applied in `cloneSections`, so it protects the STORED `profile-state.json` as well as the
+// rendered digest — a state file written by an older build is collapsed the first time it is
+// read, exactly like the size caps above.
+
+/**
+ * Normalized dedupe key: casing-, punctuation- and whitespace-insensitive. Shared with
+ * proposals.mjs's near-duplicate detection, which imports it from here — one fingerprint
+ * definition, so "already in the profile" and "already proposed" can never disagree.
+ * @param {unknown} text
+ * @returns {string}
+ */
+export function normalizeFingerprint(text) {
+  return String(text ?? '')
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}\s]/gu, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+/**
+ * Collapse entries sharing a {@link normalizeFingerprint}, keeping the MOST RECENT occurrence
+ * (and therefore its citation) at the most-recent position — so a survivor is never evicted
+ * by {@link trimSection} on account of where its first duplicate happened to sit.
+ * @param {Array<{ text?: string }>} entries
+ * @returns {Array<object>}
+ */
+export function dedupeSection(entries) {
+  const byFingerprint = new Map();
+  for (const entry of entries) {
+    const fingerprint = normalizeFingerprint(entry?.text);
+    if (byFingerprint.has(fingerprint)) byFingerprint.delete(fingerprint);
+    byFingerprint.set(fingerprint, entry);
+  }
+  return [...byFingerprint.values()];
+}
+
+// ── section-entry state (the structured, append-friendly store) ─────────────────────
+
+function emptySections() {
+  const out = {};
+  for (const key of SECTION_KEYS) out[key] = [];
+  return out;
+}
+
+function cloneSections(sections) {
+  const out = emptySections();
+  if (!sections || typeof sections !== 'object') return out;
+  for (const key of SECTION_KEYS) {
+    // Entries are clamped and trimmed HERE as well as at fold time, so a state file written
+    // by an older, uncapped build is normalized the first time it is read instead of
+    // re-rendering its oversized entries forever. This preserves the incremental
+    // "re-renders byte-identical" contract in the only way that stays true going forward:
+    // every stored entry is already within the caps, so re-rendering it is a no-op.
+    const entries = Array.isArray(sections[key]) ? sections[key] : [];
+    out[key] = trimSection(dedupeSection(entries.map((e) => ({ ...e, text: clampEntryText(e?.text) }))));
+  }
+  return out;
+}
+
+function emptyProfileState() {
+  return { lastObservationSeq: 0, lastLearnedAt: null, sections: emptySections() };
+}
+
+// ── incremental state persistence ────────────────────────────────────────────────
+
+/** Read the profile incremental-state file; missing/corrupt → a fresh empty state. */
+export function loadProfileState(path) {
+  if (!existsSync(path)) return emptyProfileState();
+  try {
+    const parsed = JSON.parse(readFileSync(path, 'utf8'));
+    if (!parsed || typeof parsed !== 'object') return emptyProfileState();
+    return {
+      lastObservationSeq: Number.isFinite(parsed.lastObservationSeq) ? parsed.lastObservationSeq : 0,
+      lastLearnedAt: typeof parsed.lastLearnedAt === 'string' ? parsed.lastLearnedAt : null,
+      sections: cloneSections(parsed.sections),
+    };
+  } catch {
+    return emptyProfileState();
+  }
+}
+
+/** Persist the profile incremental-state file. */
+export function saveProfileState(path, state) {
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, `${JSON.stringify(state, null, 2)}\n`);
+}
+
+// ── rendering ─────────────────────────────────────────────────────────────────────
+
+/**
+ * One profile line for a section entry — carries its session citation
+ * (`source`, `project`, `session`, `ts`) exactly as DR1's corpus.mjs recorded it on the
+ * observation. Never re-derived or guessed: every field is read straight off the
+ * `DoctrineObservation` this entry was built from.
+ *
+ * @param {{ text: string, source: string, project: string, session: string, ts: string|null }} entry
+ * @returns {string}
+ */
+export function citationLine(entry) {
+  return `- ${entry.text} _(source: ${entry.source}, project: ${entry.project}, session: ${entry.session}, ts: ${entry.ts ?? 'null'})_`;
+}
+
+/**
+ * Render the full profile markdown from a `sections` store (the same shape `buildProfile`
+ * carries on its returned `state.sections`). One heading per non-empty section, in
+ * {@link SECTION_ORDER}; a section with no entries is omitted entirely rather than printed
+ * with a "nothing yet" placeholder. Pure — no I/O.
+ *
+ * @param {Record<string, Array<{ text: string, source: string, project: string, session: string, ts: string|null }>>} sections
+ * @returns {string}
+ */
+export function renderProfileMarkdown(sections) {
+  const lines = [
+    '# SpexCode Profile',
+    '',
+    'Machine-local, heuristic v0 — never written to the vault. Every line cites the session it came from.',
+    '',
+  ];
+  for (const { key, heading } of SECTION_ORDER) {
+    const entries = sections?.[key] ?? [];
+    if (entries.length === 0) continue;
+    lines.push(`## ${heading}`, '');
+    for (const entry of entries) lines.push(citationLine(entry));
+    lines.push('');
+  }
+  return `${lines.join('\n').replace(/\n+$/, '')}\n`;
+}
+
+/**
+ * Render the INJECTED profile digest — what actually rides a seat's context — from the same
+ * structured `sections` store {@link renderProfileMarkdown} renders the full file from.
+ *
+ * Distinct from `renderProfileMarkdown` on purpose: that one renders the complete, browsable
+ * record to `.heyspex/spexcode-profile.md`, and is allowed to be long. This one is the
+ * bounded excerpt, and it allocates the budget FAIRLY ACROSS SECTIONS rather than filling it
+ * from the top. A plain top-down cut of the rendered file spends the entire budget inside
+ * "Preferred languages and idioms" (the first and largest section) and silently starves
+ * "Recurring corrections" and "Tools approved or refused" — the two sections whose whole
+ * point is to change how a seat behaves.
+ *
+ * Selection within a section is most-recent-first, since a developer's latest correction
+ * describes them better than their oldest one. Round-robin across sections then takes one
+ * entry from each in turn, so a section is never crowded out by a wordier neighbour.
+ *
+ * @param {Record<string, Array<{ text: string, source: string, project: string, session: string, ts: string|null }>>} sections
+ * @param {{ maxTokens?: number, estimate?: (text: string) => number }} [opts]
+ * @returns {{ text: string, includedCount: number, totalCount: number, truncated: boolean }}
+ */
+export function renderProfileDigest(sections, { maxTokens = PROFILE_MAX_TOKENS, estimate } = {}) {
+  const sizeOf = typeof estimate === 'function' ? estimate : (t) => Math.ceil(String(t).length / 4);
+  const budget = Number.isFinite(maxTokens) && maxTokens > 0 ? maxTokens : Infinity;
+
+  const header = '# SpexCode Profile';
+  const subtitle = 'Machine-local, heuristic v0 — never written to the vault. Every line cites the session it came from.';
+
+  // Most-recent-first queues, one per section, plus the running total for the footer.
+  const queues = new Map();
+  let totalCount = 0;
+  for (const { key } of SECTION_ORDER) {
+    const entries = Array.isArray(sections?.[key]) ? [...sections[key]].reverse() : [];
+    totalCount += entries.length;
+    if (entries.length > 0) queues.set(key, entries);
+  }
+
+  // Reserve the footer's own cost up front, so the digest always comes in AT OR UNDER
+  // `budget` INCLUDING its footer. Without the reservation the footer is the one line the
+  // outer clamp in session-bootstrap.mjs's `renderProfileBlock` trims off — replacing the
+  // informative "N of M entries shown" note with its own blunter one.
+  const footerReserve = totalCount > 0 ? sizeOf(`_(${totalCount} of ${totalCount} profile entries shown — bounded to a ${budget}-token injection budget.)_`) : 0;
+  let used = sizeOf(header) + sizeOf(subtitle) + footerReserve;
+  const picked = new Map(); // key -> lines, in the order they were selected
+  let includedCount = 0;
+  let truncated = false;
+
+  // Round-robin: one entry per section per pass, until the budget is spent or nothing is left.
+  while (queues.size > 0) {
+    let progressed = false;
+    for (const { key, heading } of SECTION_ORDER) {
+      const queue = queues.get(key);
+      if (!queue || queue.length === 0) continue;
+      const line = citationLine(queue[0]);
+      const needsHeading = !picked.has(key);
+      const cost = sizeOf(line) + (needsHeading ? sizeOf(`## ${heading}`) : 0);
+      if (used + cost > budget) {
+        truncated = true;
+        queues.delete(key);
+        continue;
+      }
+      queue.shift();
+      used += cost;
+      if (needsHeading) picked.set(key, []);
+      picked.get(key).push(line);
+      includedCount++;
+      progressed = true;
+      if (queue.length === 0) queues.delete(key);
+    }
+    if (!progressed) break;
+  }
+
+  const lines = [header, '', subtitle, ''];
+  for (const { key, heading } of SECTION_ORDER) {
+    const got = picked.get(key);
+    if (!got || got.length === 0) continue;
+    lines.push(`## ${heading}`, '', ...got, '');
+  }
+  if (truncated) {
+    lines.push(`_(${includedCount} of ${totalCount} profile entries shown — bounded to a ${budget}-token injection budget.)_`);
+  }
+
+  return { text: lines.join('\n').replace(/\n+$/, ''), includedCount, totalCount, truncated };
+}
+
+// ── context-budget registration (P2 scope item 3 — measurement only, no injection here) ──
+
+/**
+ * Register the profile block's size against the harness's single context-injection
+ * accounting path ({@link recordContextInjection}), the same pattern DR3 uses for the
+ * Doctrine block. This function only measures and records — actually injecting the
+ * profile into a seat's context is a later task (epic scope item 4, "Conditioning").
+ *
+ * @param {ReturnType<import('../session-record.mjs').createSessionRecord>} record
+ * @param {string} markdown
+ * @param {{ source?: string }} [opts]
+ * @returns {import('../context-budget.mjs').ContextInjectionMeasurement}
+ */
+export function registerProfileContextUsage(record, markdown, { source = 'spexcode-profile' } = {}) {
+  return recordContextInjection(record, { source, text: markdown });
+}
+
+// ── top-level entrypoint ────────────────────────────────────────────────────────
+
+/**
+ * Build (or incrementally extend) the developer's SpexCode Profile from this machine's
+ * Doctrine observation stream, and write it to `outPath`.
+ *
+ * State resolution order: an explicit `state` object wins; else, when `statePath` is
+ * given, it is loaded from disk; else a fresh empty state (a first/cold run). When
+ * `statePath` is given, the resulting state is also persisted back to it — a caller that
+ * wants a purely in-memory two-call test (no disk state file) passes `state` explicitly
+ * instead and threads the returned `state` into the next call itself.
+ *
+ * @param {{
+ *   observationsPath: string,
+ *   outPath: string,
+ *   statePath?: string|null,
+ *   state?: { lastObservationSeq: number, lastLearnedAt: string|null, sections: object }|null,
+ *   maxObservations?: number,
+ *   now?: () => string,
+ * }} opts
+ * @returns {{
+ *   profilePath: string,
+ *   markdown: string,
+ *   state: object,
+ *   observationsProcessed: number,
+ *   observationsDeferred: number,
+ * }}
+ */
+export function buildProfile({
+  observationsPath,
+  outPath,
+  statePath = null,
+  state = null,
+  maxObservations = DEFAULT_MAX_OBSERVATIONS,
+  now = () => new Date().toISOString(),
+} = {}) {
+  if (!observationsPath) throw new Error('buildProfile: observationsPath is required');
+  if (!outPath) throw new Error('buildProfile: outPath is required');
+
+  const priorState = state ?? (statePath ? loadProfileState(statePath) : null) ?? emptyProfileState();
+  const sections = cloneSections(priorState.sections);
+  const startIndex = Number.isFinite(priorState.lastObservationSeq) ? priorState.lastObservationSeq : 0;
+  const cap = Number.isFinite(maxObservations) && maxObservations > 0 ? maxObservations : Infinity;
+
+  const allObservations = readObservations(observationsPath);
+
+  let processed = 0;
+  let index = startIndex;
+  for (; index < allObservations.length; index++) {
+    if (processed >= cap) break;
+    const obs = allObservations[index];
+    if (!obs || SKIPPED_KINDS.has(obs.kind)) continue;
+    const key = classifySection(obs);
+    sections[key].push({
+      text: clampEntryText(obs.text),
+      source: obs.source,
+      project: obs.project,
+      session: obs.session,
+      ts: obs.ts ?? null,
+    });
+    sections[key] = trimSection(dedupeSection(sections[key]));
+    processed++;
+  }
+
+  const observationsDeferred = Math.max(0, allObservations.length - index);
+  const newState = { lastObservationSeq: index, lastLearnedAt: now(), sections };
+  const markdown = renderProfileMarkdown(sections);
+
+  mkdirSync(dirname(outPath), { recursive: true });
+  writeFileSync(outPath, markdown);
+  if (statePath) saveProfileState(statePath, newState);
+
+  return {
+    profilePath: outPath,
+    markdown,
+    state: newState,
+    observationsProcessed: processed,
+    observationsDeferred,
+  };
+}
+
+export default {
+  DEFAULT_MAX_OBSERVATIONS,
+  MAX_ENTRY_CHARS,
+  MAX_ENTRIES_PER_SECTION,
+  PROFILE_MAX_TOKENS,
+  clampEntryText,
+  SECTION_ORDER,
+  classifySection,
+  normalizeFingerprint,
+  dedupeSection,
+  loadProfileState,
+  saveProfileState,
+  citationLine,
+  renderProfileMarkdown,
+  renderProfileDigest,
+  registerProfileContextUsage,
+  buildProfile,
+  defaultProfilePath,
+  defaultProfileStatePath,
+};