@ijfw/memory-server 1.5.5 → 1.6.0

package/src/profile/eval/slug-quality.mjs

@@ -0,0 +1,464 @@
+/**
+ * profile/eval/slug-quality.mjs — SLICE S2: slug-quality PRECISION gate.
+ *
+ * THE DECISION THIS MODULE OWNS: may a learned preference slug EVER auto-inject?
+ *
+ * The personalization layer is an EVIDENCE-ADMISSION system — nothing becomes
+ * "you" without a citation. A derived preference slug is only allowed to
+ * auto-act (inject silently, high bar) when the SURFACED slug set is precise
+ * enough that we trust it not to put words in the user's mouth. The founder's
+ * call is PRECISION >= 0.8 to auto-act; below it we show-and-confirm.
+ *
+ * This module turns that policy into a measurable, pre-registered gate:
+ *
+ *   1. LABEL each surfaced slug, three-way, against a HELD-OUT TEST window:
+ *        - 'correct-actionable-preference' : the slug semantically matches a
+ *          preference the user expressed in the TEST window (an actionable "you").
+ *        - 'real-but-not-a-preference'     : the slug matches something the user
+ *          really said in the TEST window, but it is NOT a preference (grounded,
+ *          but not actionable as a profile directive).
+ *        - 'wrong'                         : the slug matches NOTHING in the TEST
+ *          window — fabricated / noise / hallucinated. Cannot inject, ever.
+ *
+ *   2. SCORE precision = #correct / #surfaced. Recall is reported HONESTLY
+ *      (high-precision / low-recall is a feature here, not a bug — a cite-or-drop
+ *      gate would rather drop a real preference than inject a wrong one).
+ *
+ *   3. GATE: eligible-to-inject ONLY when precision >= the pre-registered bar
+ *      (0.8). The bar is frozen + tamper-evident via the SAME prereg rig Gate B
+ *      uses (buildPreReg / hashPreReg), so the 0.8 cannot be quietly loosened to
+ *      pass — re-registering after an edit changes the runId hash.
+ *
+ * CIRCULARITY GUARD (the ea15479 void): the gold here is the HELD-OUT TEST
+ * window's preferences — NEVER the train target a brief injects. Scoring a slug
+ * against the very phrase the system was handed to inject is teaching-to-the-test
+ * and was voided once already. We assert the test ids are disjoint from any
+ * provided train ids before scoring; a leaky corpus throws.
+ *
+ * REUSE, do NOT re-derive:
+ *   - semanticPrecisionRecall  (gate-c-capture.mjs)  — lexical-semantic matcher
+ *   - precisionRecall          (harness.mjs)         — exact-slug floor + vectors
+ *   - bootstrapCI              (harness.mjs re-export)— CI on the per-unit vectors
+ *   - buildPreReg / hashPreReg (prereg.mjs)          — tamper-evident bar freeze
+ *   - wrongTargetControl shape (wrong-target-control.mjs) — negative-control idiom
+ *
+ * Zero deps. ESM. No serving/capture code is touched — this is eval-only infra
+ * the runtime CALLS (eligibleSlugsForInjection) but does not live inside.
+ *
+ * Cites (methodology): LaMP time-based split [2304.11406] (held-out TEST) ·
+ * pre-registration tamper-evidence (prereg.mjs, Gate B v2 T6).
+ */
+
+import crypto from 'node:crypto';
+import { semanticPrecisionRecall } from './gate-c-capture.mjs';
+import { bootstrapCI } from './harness.mjs';
+import { buildPreReg, hashPreReg } from './prereg.mjs';
+
+/** The founder's call: precision required to AUTO-act (inject silently). */
+export const SLUG_PRECISION_BAR = 0.8;
+/** Default lexical-semantic match threshold (reused matcher's documented default). */
+export const SLUG_SEMANTIC_THRESHOLD = 0.5;
+
+/** The three quality labels a surfaced slug can receive. */
+export const SLUG_LABELS = Object.freeze({
+  CORRECT: 'correct-actionable-preference',
+  REAL_NOT_PREF: 'real-but-not-a-preference',
+  WRONG: 'wrong',
+});
+
+/**
+ * Normalize a phrase to the SAME subject slug space the heuristic derive uses
+ * (phraseToSubject) so a TEST-window gold phrase compares apples-to-apples with a
+ * surfaced slug. Identical 5-line rule, re-stated here (derive does not export it)
+ * — matches gate-c-capture.toSubject so the two evals are on one scale.
+ */
+function toSubject(phrase) {
+  return String(phrase || '')
+    .toLowerCase()
+    .replace(/[^a-z0-9\s]+/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim()
+    .slice(0, 80);
+}
+
+// ---------------------------------------------------------------------------
+// POLARITY GUARD — the precision-gate-specific hardening over the reused Gate C
+// matcher. The lexical-semantic Jaccard match (semanticPrecisionRecall) is
+// POLARITY-BLIND: "use tabs not spaces" and "use spaces not tabs" reduce to the
+// SAME content tokens {tabs, spaces} and so match at Jaccard 1.0. For a CAPTURE-
+// RECALL metric (Gate C) that is tolerable. For a PRECISION gate that decides
+// silent AUTO-INJECTION it is not — labeling the user's OPPOSITE preference as
+// "correct" is exactly the false-positive this gate exists to prevent (it would
+// put the inverse of what the user wants into the profile). So before a semantic
+// match is ALLOWED to count as a correct preference, we reject it when the two
+// phrases share content tokens but disagree on NEGATION polarity.
+//
+// Signal (dependency-free, no antonym lexicon to maintain): for each content
+// token shared by both phrases, record whether it sits BEFORE or AFTER the first
+// negation marker ("not"/"never"/"no"/"avoid"/"instead of"/"rather than") in each
+// phrase. If a shared token flips sides (before in one, after in the other), the
+// two phrases assert OPPOSITE polarity over the same subject — a conflict. This
+// catches the "X not Y" vs "Y not X" inversion without hardcoding domain antonyms.
+// ---------------------------------------------------------------------------
+
+const NEGATION_MARKERS = new Set(['not', 'never', 'no', 'avoid', "don't", 'dont', 'without', 'instead', 'rather']);
+
+/** Side of the first negation marker each content token sits on: -1 before, 1 after. */
+function negationSides(phrase) {
+  const toks = toSubject(phrase).split(/\s+/).filter(Boolean);
+  let negAt = -1;
+  for (let i = 0; i < toks.length; i += 1) {
+    if (NEGATION_MARKERS.has(toks[i])) { negAt = i; break; }
+  }
+  const sides = new Map();
+  for (let i = 0; i < toks.length; i += 1) {
+    const t = toks[i];
+    if (t.length < 3 || STOPWORDS_LITE.has(t) || NEGATION_MARKERS.has(t)) continue;
+    // No negation in the phrase -> neutral side 0 (cannot conflict on its own).
+    sides.set(t, negAt === -1 ? 0 : (i < negAt ? -1 : 1));
+  }
+  return sides;
+}
+
+// Small content stoplist shared with the matcher's spirit (function words that
+// would otherwise pollute the polarity comparison). Kept local + minimal.
+const STOPWORDS_LITE = new Set([
+  'the', 'a', 'an', 'to', 'of', 'and', 'or', 'for', 'in', 'on', 'with', 'is',
+  'are', 'be', 'i', 'you', 'user', 'prefer', 'prefers', 'preference', 'like',
+  'likes', 'want', 'wants', 'use', 'uses', 'using', 'should', 'would', 'do',
+  'does', 'my', 'me', 'it', 'this', 'that', 'when', 'always', 'over', 'than',
+]);
+
+/**
+ * polarityConflict(a, b) -> boolean. True when a and b share at least one content
+ * token that sits on OPPOSITE sides of a negation marker (one before, one after)
+ * — i.e. they express opposite preferences over the same subject. Only fires when
+ * AT LEAST ONE phrase actually contains a negation (a pure positive-vs-positive
+ * pair never conflicts on this signal).
+ */
+export function polarityConflict(a, b) {
+  const sa = negationSides(a);
+  const sb = negationSides(b);
+  for (const [tok, side] of sa) {
+    if (!sb.has(tok)) continue;
+    const other = sb.get(tok);
+    // A flip from one side of a negation to the other = opposite polarity.
+    if (side !== 0 && other !== 0 && side !== other) return true;
+  }
+  return false;
+}
+
+/**
+ * A slug counts as a CORRECT match of a gold subject iff it lexical-semantically
+ * matches (reused Gate C matcher) AND does NOT polarity-conflict with it. The
+ * polarity guard is the precision-gate hardening; recall-side Gate C keeps its
+ * polarity-blind matcher unchanged.
+ */
+function correctMatch(slug, goldSubject, threshold) {
+  if (semanticPrecisionRecall([slug], [goldSubject], threshold).perPredCorrect[0] !== 1) return false;
+  return !polarityConflict(slug, goldSubject);
+}
+
+/**
+ * Build the PRE-REGISTERED gate config. The 0.8 bar (and the semantic threshold)
+ * flow into the prereg hash via the rig's input spread, so any post-hoc edit to
+ * the bar is tamper-evident: re-registering with a changed bar yields a different
+ * runId hash. We do NOT relax the bar to pass — that was the explicit lesson.
+ *
+ * @param {object} [opts]
+ *   @param {number} [opts.bar]               precision bar (default 0.8)
+ *   @param {number} [opts.semanticThreshold] Jaccard threshold (default 0.5)
+ *   @param {number} [opts.seed]              prereg seed (default 1)
+ * @returns {object} frozen prereg with .slugPrecisionBar, .slugSemanticThreshold,
+ *   .hash and .runId. hashPreReg over this object is stable + tamper-evident.
+ */
+export function buildSlugQualityPreReg(opts = {}) {
+  const bar = Number.isFinite(opts.bar) ? opts.bar : SLUG_PRECISION_BAR;
+  const semanticThreshold = Number.isFinite(opts.semanticThreshold)
+    ? opts.semanticThreshold : SLUG_SEMANTIC_THRESHOLD;
+  const seed = Number.isFinite(opts.seed) ? opts.seed : 1;
+
+  // Reuse the Gate B rig for the base prereg (seeds + Bonferroni + base hash).
+  const base = buildPreReg({
+    primaryEndpoint: 'slug-quality-precision-gate',
+    slugPrecisionBar: bar,
+    slugSemanticThreshold: semanticThreshold,
+    seed,
+    runId: opts.runId,
+  });
+
+  // buildPreReg only hashes its FIXED field list, so `slugPrecisionBar` /
+  // `slugSemanticThreshold` are present on the object but do NOT yet affect
+  // base.hash — a silent loosening of the bar would slip through. We make the bar
+  // GENUINELY tamper-evident with a COMPOSITE hash over (base prereg hash + the
+  // slug-specific knobs). Editing the bar now changes the composite hash and the
+  // runId, exactly as the founder's-call requires (the 0.8 cannot be quietly
+  // moved to pass). hashPreReg is still reused for the base leg.
+  const baseHash = hashPreReg(base);
+  const composite = crypto.createHash('sha256')
+    .update(`${baseHash}|slugPrecisionBar:${bar}|slugSemanticThreshold:${semanticThreshold}`)
+    .digest('hex');
+
+  return Object.freeze({
+    ...base,
+    slugPrecisionBar: bar,
+    slugSemanticThreshold: semanticThreshold,
+    baseHash,
+    hash: composite,
+    runId: opts.runId || `slugq-${composite.slice(0, 12)}`,
+  });
+}
+
+/**
+ * Assemble the HELD-OUT TEST gold from a corpus's probe/test window. Returns two
+ * disjoint slug sets in the same normalized space:
+ *   - preferenceSubjects     : actionable preferences the user expressed (gold +).
+ *   - nonPreferenceSubjects  : things the user really said that are NOT
+ *     preferences (real-but-not-a-preference targets).
+ *
+ * Accepts the makeHeldOutFixture corpus shape (probes[].goldSubjects /
+ * probes[].nonPreferenceSubjects) OR an explicit { testPreferences,
+ * testNonPreferences } pair. The TEST window — never the train target — is the
+ * scoring source (anti-circularity).
+ */
+export function testWindowGold(corpus = {}) {
+  const pref = new Set();
+  const nonPref = new Set();
+
+  if (Array.isArray(corpus.testPreferences)) {
+    for (const p of corpus.testPreferences) pref.add(toSubject(p));
+  }
+  if (Array.isArray(corpus.testNonPreferences)) {
+    for (const p of corpus.testNonPreferences) nonPref.add(toSubject(p));
+  }
+  const probes = Array.isArray(corpus.probes) ? corpus.probes
+    : (Array.isArray(corpus.test) ? corpus.test : []);
+  for (const probe of probes) {
+    for (const g of (probe.goldSubjects || [])) pref.add(toSubject(g));
+    for (const g of (probe.nonPreferenceSubjects || probe.nonPrefSubjects || [])) {
+      nonPref.add(toSubject(g));
+    }
+  }
+  // A subject can't be both a preference and a non-preference. Preference wins
+  // (it is the stronger, actionable claim); drop the overlap from nonPref so the
+  // labels are mutually exclusive.
+  for (const s of pref) nonPref.delete(s);
+  return { preferenceSubjects: [...pref], nonPreferenceSubjects: [...nonPref] };
+}
+
+/**
+ * labelSlugs(slugs, corpus, opts) -> per-slug quality labels + counts.
+ *
+ * Each surfaced slug is labeled three-way against the HELD-OUT TEST window:
+ *   - matches a TEST preference (semantic)            -> 'correct-actionable-preference'
+ *   - else matches a TEST non-preference utterance    -> 'real-but-not-a-preference'
+ *   - else                                            -> 'wrong'
+ *
+ * Matching reuses the SAME lexical-semantic matcher Gate C uses
+ * (semanticPrecisionRecall), so "captured-but-paraphrased" counts and a verbatim
+ * coincidence is not required. The preference set is checked FIRST so a slug that
+ * is genuinely a preference is never mislabeled as merely-real.
+ *
+ * @param {string[]} slugs   the SURFACED slugs (already at the brief floor)
+ * @param {object}   corpus  carries the TEST window (see testWindowGold)
+ * @param {object}   [opts]  @param {number} [opts.semanticThreshold]
+ *   @param {string[]} [opts.trainIds] / @param {string[]} [opts.testIds] for the
+ *     held-out disjointness assertion (throws on leakage).
+ * @returns {{ labels: Array<{slug,label,matchedPreference,matchedNonPreference}>,
+ *   counts: {correct,realNotPref,wrong,total} }}
+ */
+export function labelSlugs(slugs = [], corpus = {}, opts = {}) {
+  // HELD-OUT ENFORCEMENT (assertion, not comment): if the caller declares both
+  // train and test ids, no id may appear on both sides. A leaky split means the
+  // gold may include the very target a brief injected — the ea15479 void. Refuse.
+  const trainIds = Array.isArray(opts.trainIds) ? opts.trainIds : (corpus.trainIds || []);
+  const testIds = Array.isArray(opts.testIds) ? opts.testIds : (corpus.testIds || []);
+  if (trainIds.length && testIds.length) {
+    const train = new Set(trainIds.map(String));
+    for (const id of testIds) {
+      if (train.has(String(id))) {
+        throw new Error(
+          'slug-quality held-out violation: a TEST id also appears in TRAIN '
+          + '(the gold would include the injected train target — the ea15479 circularity). '
+          + 'Refusing to label slugs against a leaky split.',
+        );
+      }
+    }
+  }
+
+  const threshold = Number.isFinite(opts.semanticThreshold)
+    ? opts.semanticThreshold : SLUG_SEMANTIC_THRESHOLD;
+  const { preferenceSubjects, nonPreferenceSubjects } = testWindowGold(corpus);
+
+  const labels = [];
+  let correct = 0; let realNotPref = 0; let wrong = 0;
+  for (const raw of slugs) {
+    const slug = toSubject(raw);
+    // Preference FIRST (the stronger, actionable claim). A correct match must be
+    // semantic AND polarity-agreeing — the OPPOSITE preference is NOT correct.
+    const matchedPref = firstMatch(slug, preferenceSubjects, threshold);
+    if (matchedPref) {
+      correct += 1;
+      labels.push({
+        slug, raw, label: SLUG_LABELS.CORRECT,
+        matchedPreference: matchedPref,
+        matchedNonPreference: null,
+      });
+      continue;
+    }
+    const matchedNonPref = firstMatch(slug, nonPreferenceSubjects, threshold);
+    if (matchedNonPref) {
+      realNotPref += 1;
+      labels.push({
+        slug, raw, label: SLUG_LABELS.REAL_NOT_PREF,
+        matchedPreference: null,
+        matchedNonPreference: matchedNonPref,
+      });
+      continue;
+    }
+    wrong += 1;
+    labels.push({
+      slug, raw, label: SLUG_LABELS.WRONG, matchedPreference: null, matchedNonPreference: null,
+    });
+  }
+
+  return {
+    labels,
+    counts: { correct, realNotPref, wrong, total: slugs.length },
+  };
+}
+
+/**
+ * First TEST-window gold subject a slug CORRECTLY matches (semantic AND
+ * polarity-agreeing), for provenance. The OPPOSITE preference is NOT a match.
+ */
+function firstMatch(slug, goldSubjects, threshold) {
+  for (const g of goldSubjects) {
+    if (correctMatch(slug, g, threshold)) return g;
+  }
+  return null;
+}
+
+/**
+ * eligibleSlugsForInjection(slugs, corpus, opts) -> the RUNTIME gate.
+ *
+ * Given the current derived/surfaced slugs, decide which (if any) may auto-inject.
+ * The gate is precision-over-the-surfaced-set: a single 'wrong' slug among few can
+ * sink precision below 0.8 and block auto-injection — that is the intended
+ * safe-by-design behavior (cite-or-drop, high bar to auto-act). We return BOTH the
+ * set-level verdict (`cleared`) AND the per-slug labels, so a caller may choose to
+ * inject only the 'correct' subset, or fall back to show-and-confirm below the bar.
+ *
+ * PRECISION counts only 'correct-actionable-preference' as a true positive.
+ * 'real-but-not-a-preference' is NOT a preference and so is a precision MISS (it
+ * would put a non-preference into the profile) — counted against precision, never
+ * eligible to inject. 'wrong' is likewise a miss.
+ *
+ * RECALL is reported honestly against the TEST preference gold (how many of the
+ * user's actual TEST-window preferences the surfaced set recovered). Low recall is
+ * acceptable; the gate optimizes precision.
+ *
+ * @param {string[]} slugs   current derived/surfaced slugs
+ * @param {object}   corpus  HELD-OUT TEST window (see testWindowGold)
+ * @param {object}   [opts]
+ *   @param {number} [opts.bar]               precision bar (default 0.8, pre-registered)
+ *   @param {number} [opts.semanticThreshold] Jaccard threshold (default 0.5)
+ *   @param {number} [opts.bootstrapSeed]     CI seed (default 42)
+ *   @param {string[]} [opts.trainIds] / [opts.testIds]  held-out leakage check
+ * @returns {{
+ *   bar, preReg:{hash,runId,slugPrecisionBar},
+ *   cleared:boolean,
+ *   precision:{point,lo,hi}, recall:{point,lo,hi},
+ *   eligible:string[], ineligible:string[],
+ *   labels:Array, counts:{correct,realNotPref,wrong,total},
+ *   gold:{nPreferences,nNonPreferences}
+ * }}
+ */
+export function eligibleSlugsForInjection(slugs = [], corpus = {}, opts = {}) {
+  const preReg = buildSlugQualityPreReg({
+    bar: opts.bar,
+    semanticThreshold: opts.semanticThreshold,
+    seed: opts.seed,
+    runId: opts.runId,
+  });
+  const bar = preReg.slugPrecisionBar;
+  const threshold = preReg.slugSemanticThreshold;
+  const seed = Number.isFinite(opts.bootstrapSeed) ? opts.bootstrapSeed : 42;
+
+  const { labels, counts } = labelSlugs(slugs, corpus, {
+    semanticThreshold: threshold,
+    trainIds: opts.trainIds,
+    testIds: opts.testIds,
+  });
+
+  // PRECISION per-unit vector: 1 iff the slug is a correct actionable preference.
+  // (real-but-not-a-preference and wrong both count as 0 — a non-preference in the
+  // profile is exactly the harm the gate exists to prevent.)
+  const perPredCorrect = labels.map((l) => (l.label === SLUG_LABELS.CORRECT ? 1 : 0));
+  const precision = bootstrapCI(perPredCorrect, { seed });
+  // point precision is the honest fraction; bootstrapCI.point equals it.
+  const precisionPoint = counts.total > 0 ? counts.correct / counts.total : 0;
+
+  // RECALL against the TEST preference gold. A gold preference is "recovered"
+  // only if SOME surfaced slug CORRECTLY matches it (semantic AND polarity-
+  // agreeing) — the opposite preference does not count as recovery. perGoldHit is
+  // a 0/1 vector over the gold prefs (feeds the same CI helper). Reported honestly
+  // (the gate optimizes precision; low recall is acceptable).
+  const { preferenceSubjects } = testWindowGold(corpus);
+  const slugSubjects = slugs.map(toSubject);
+  const perGoldHit = preferenceSubjects.map(
+    (g) => (slugSubjects.some((s) => correctMatch(s, g, threshold)) ? 1 : 0),
+  );
+  const recall = bootstrapCI(perGoldHit, { seed: seed + 1 });
+
+  // SET-LEVEL verdict: clear the bar (>=, not >) to auto-inject. Use the honest
+  // point precision for the decision (the CI is reported alongside, not the gate).
+  const cleared = counts.total > 0 && precisionPoint >= bar;
+
+  // The per-slug eligible subset: the 'correct' slugs are the ones that COULD
+  // inject. We surface them regardless of the set verdict so a caller may inject
+  // the precise subset even when the whole set doesn't clear — but `cleared`
+  // remains the auto-act gate.
+  const eligible = labels.filter((l) => l.label === SLUG_LABELS.CORRECT).map((l) => l.raw);
+  const ineligible = labels.filter((l) => l.label !== SLUG_LABELS.CORRECT).map((l) => l.raw);
+
+  return {
+    bar,
+    preReg: { hash: preReg.hash, runId: preReg.runId, slugPrecisionBar: bar },
+    cleared,
+    precision: { point: precisionPoint, lo: precision.lo, hi: precision.hi },
+    recall: { point: recall.point, lo: recall.lo, hi: recall.hi },
+    eligible,
+    ineligible,
+    labels,
+    counts,
+    gold: {
+      nPreferences: preferenceSubjects.length,
+      nNonPreferences: testWindowGold(corpus).nonPreferenceSubjects.length,
+    },
+  };
+}
+
+/**
+ * negativeControlClears(slugs, invertedCorpus, opts) -> boolean.
+ *
+ * The precision-guard control, in the wrong-target-control idiom: slugs scored
+ * against an UNRELATED user's TEST window must NOT clear the bar. A gate that
+ * lights up on someone else's preferences is matching universals, not the user —
+ * a false PASS. Tests assert this returns false for a fabricated/foreign set.
+ */
+export function negativeControlClears(slugs = [], invertedCorpus = {}, opts = {}) {
+  return eligibleSlugsForInjection(slugs, invertedCorpus, opts).cleared;
+}
+
+export default {
+  SLUG_PRECISION_BAR,
+  SLUG_SEMANTIC_THRESHOLD,
+  SLUG_LABELS,
+  buildSlugQualityPreReg,
+  testWindowGold,
+  labelSlugs,
+  eligibleSlugsForInjection,
+  negativeControlClears,
+  hashPreReg,
+};

package/src/profile/eval/stylometry-features.js

@@ -0,0 +1,85 @@
+// stylometry-features.js — PURE, dependency-free feature extractors shared by the
+// authorship metric (stylometry.js) and the reference-stats generator
+// (tools/build-stylo-reference.mjs). Kept in its own module so BOTH consumers use the
+// IDENTICAL tokenization, and so the generator never has to import stylometry.js (which
+// imports the generated reference constants — that would be a load-order cycle).
+//
+// Nothing here references the frozen REF_MEAN/REF_SD. These functions only turn text
+// into raw relative-frequency vectors over a given key set. The z-standardization (the
+// step that MUST use frozen constants, never recomputed from the scored text) lives in
+// stylometry.js::authorVector and is the subject of the anti-recompute guard test.
+
+// Word tokens: lowercase alphanumeric+apostrophe runs.
+export function tokenizeWords(text) {
+  return (String(text).toLowerCase().match(/[a-z0-9']+/gi) || []);
+}
+
+// Whitespace-marked character trigrams: collapse runs of whitespace to a single space
+// so word boundaries become part of the trigram signal, then slide a 3-char window.
+export function charTrigrams(text) {
+  const s = String(text).toLowerCase().replace(/\s+/g, ' ').trim();
+  const out = [];
+  for (let i = 0; i + 3 <= s.length; i += 1) out.push(s.slice(i, i + 3));
+  return out;
+}
+
+// Relative frequency of each function word = count / total words. Returns an array
+// aligned to `funcWords`.
+export function relFreqFunc(text, funcWords) {
+  const w = tokenizeWords(text);
+  const n = w.length || 1;
+  const counts = Object.create(null);
+  for (const x of w) counts[x] = (counts[x] || 0) + 1;
+  return funcWords.map((fw) => (counts[fw] || 0) / n);
+}
+
+// Relative frequency of each trigram key = count / total trigrams. Returns an array
+// aligned to `keys`.
+export function relFreqTrigrams(text, keys) {
+  const tris = charTrigrams(text);
+  const n = tris.length || 1;
+  const counts = Object.create(null);
+  for (const t of tris) counts[t] = (counts[t] || 0) + 1;
+  return keys.map((k) => (counts[k] || 0) / n);
+}
+
+// Relative frequency of each punctuation n-gram = occurrences / total chars. Returns an
+// array aligned to `keys`. Keys may be multi-char (e.g. '...', '?!').
+export function relFreqPunct(text, keys) {
+  const s = String(text);
+  const denom = s.length || 1;
+  return keys.map((k) => {
+    let n = 0;
+    let idx = 0;
+    while ((idx = s.indexOf(k, idx)) !== -1) { n += 1; idx += k.length; }
+    return n / denom;
+  });
+}
+
+// Full trigram count map — used ONLY by the generator to pick the top-K trigram keys
+// from the reference corpus. Never used at scoring time.
+export function trigramCounts(text) {
+  const counts = Object.create(null);
+  for (const t of charTrigrams(text)) counts[t] = (counts[t] || 0) + 1;
+  return counts;
+}
+
+// Split a corpus string into `n` roughly equal word-count chunks (for computing
+// per-feature mean/SD ACROSS documents, Burrows's-Delta style).
+export function splitChunks(text, n) {
+  const w = tokenizeWordsPreservingRaw(text);
+  const per = Math.max(1, Math.floor(w.length / n));
+  const chunks = [];
+  for (let i = 0; i < n; i += 1) {
+    const start = i * per;
+    const slice = i === n - 1 ? w.slice(start) : w.slice(start, start + per);
+    if (slice.length) chunks.push(slice.join(' '));
+  }
+  return chunks;
+}
+
+// Raw whitespace split (keeps punctuation attached) so chunk text still carries
+// punctuation for the punct features.
+function tokenizeWordsPreservingRaw(text) {
+  return String(text).split(/\s+/).filter(Boolean);
+}

package/src/profile/eval/stylometry-reference.js

@@ -0,0 +1,16 @@
+// AUTO-GENERATED by tools/build-stylo-reference.mjs — DO NOT EDIT BY HAND.
+// Frozen reference statistics that z-standardize the authorship sub-vectors. Derived
+// from a NEUTRAL hand-authored public English baseline, DISJOINT from every Gate-B
+// persona author. Never recomputed from scored text. Regenerate for the confirmatory
+// run:  node tools/build-stylo-reference.mjs --corpus <broad-disjoint-corpus>
+/* eslint-disable */
+export const REFERENCE_SOURCE = "neutral-handauthored-baseline-v1 (disjoint from all personas)";
+export const FUNCTION_WORDS = ["the","a","an","this","that","these","those","my","your","his","her","its","our","their","some","any","no","every","each","either","neither","all","both","few","many","much","more","most","several","such","what","which","whose","i","you","he","she","it","we","they","me","him","them","us","who","whom","whoever","anyone","someone","everyone","nobody","anything","something","everything","nothing","myself","yourself","himself","herself","itself","ourselves","themselves","one","ones","mine","yours","hers","ours","theirs","of","in","to","for","with","on","at","by","from","up","about","into","over","after","beneath","under","above","below","between","among","through","during","before","behind","beyond","plus","except","around","near","since","until","against","without","within","along","across","toward","upon","off","out","down","and","but","or","nor","so","yet","because","although","though","while","whereas","unless","whether","if","then","than","as","is","am","are","was","were","be","been","being","have","has","had","do","does","did","will","would","shall","should","can","could","may","might","must","ought","not","very","too","also","just","only","even","still","again","ever","never","always","often","sometimes","here","there","where","when","how","why","now","once","quite","rather","almost","enough","indeed","however","therefore"];
+export const PUNCT_KEYS = [".",",",";",":","!","?","-","—","–","(",")","[","]","\"","'","...","?!","!?","!!","??","--",", ",". ","; "];
+export const TRIGRAM_KEYS = [" th","the","he ","nd ","and"," a "," an","ing"," wh"," of","er ","is "," is","e w","of ","d t"," it"," to"," be","e t","re "," st",", a","at ","en ","it ","one","to ","hat","ng ","st "," ca"," ma","t i","t t","tha"," fo"," ho"," on","are","e s","e, ","es ","le ","n t","s a","s t","t a"," ar"," in"," no"," re",", t","an ","ed ","ent","est","her","ly ","se ","whe"," mo"," pa"," we"," wi"," wo",". t","be ","d a","der","ds ","e a","e b","e i","e o","ere","et ","in ","ind","ion","ll ","ne ","ow ","r t","und","we ","wor","y t"," ha"," le","a s","all","can","e c","e p","f a","hen","how","lea","old","on ","oun","s i","s. ","sto","t, ","th ","thi","tho","tio"," co"," la"," li"," pe"," si"," tr"," wa","ati","att","ce ","con","e h","e m","e r","ear","for","hou","ill","kin","low","nds","ome","ose","ove","rou","s, ","t f","t. ","ter","ton","tte","y a"," do"," fi"," ki"," lo"," ol"," so"," ye",", o","a l","any","ate","ch ","d s","des","e e","e f","end","ers","eve","ge ","han","hin","ice","ide","ise","it,","ith","lde","les","lle","n a","n. ","nin","no ","not","nsi","nt ","ny ","ons","or ","ork","ows","res","s m","s o","s r","sta","t c","t w","ts ","ure","ut ","ven","ver","wha","who","wit"," ac"," ag"," at"," bu"," by"," de"," ea"," fe"," gr"," me"," mu"," pl"," qu"," ri"," ro"," se"," sl"," un",", w",". a",". i",". w","a c","a m","aga","age","ain","ang","ard","aro","ave","by ","car","cro","d b","d l","d o","d, ","de ","din","do ","e d","e. ","ead","ean","eas","ele","ey ","f s","f t","fel"];
+export const REF_MEAN_FUNC = [0.0795491,0.0425523,0,0.0037037,0.012963,0.0037037,0.00362319,0,0,0,0,0,0,0,0,0.00185185,0.00740741,0,0.00185185,0,0,0,0.00185185,0,0.00555556,0.00185185,0.00185185,0.00185185,0,0.00185185,0.00740741,0.00181159,0,0,0,0,0,0.0258857,0.012963,0.0055153,0,0,0.00181159,0.00185185,0.00547504,0,0,0,0,0,0,0,0.00185185,0,0,0,0,0,0,0,0,0,0.00740741,0,0,0,0,0,0,0.0296296,0.009219,0.0221014,0.00555556,0.00555556,0.00555556,0.00185185,0.00555556,0,0,0.00185185,0,0.0037037,0,0,0,0,0,0,0,0.00185185,0,0,0,0.00185185,0,0,0.0055153,0,0,0.00185185,0.00185185,0.00185185,0,0,0.0037037,0.00185185,0,0,0.00185185,0.00185185,0.0351047,0.00185185,0.00185185,0,0.00185185,0.00555556,0,0,0,0.00181159,0,0,0.00185185,0.00185185,0.00555556,0.00555556,0,0.0295894,0,0.00913849,0.00185185,0,0.012963,0,0,0.00185185,0,0,0.00555556,0,0,0.00185185,0,0,0,0.009219,0,0,0,0.0037037,0,0.00185185,0,0,0,0,0.0037037,0.00185185,0.00185185,0.00366345,0,0,0,0.00185185,0,0,0.00555556,0.00185185,0.00555556,0.00925926,0.00181159,0,0,0,0,0,0,0,0,0];
+export const REF_SD_FUNC = [0.0440053,0.0264162,0.0001,0.00828173,0.0142243,0.00828173,0.0120168,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0061419,0.013858,0.0001,0.0061419,0.0001,0.0001,0.0001,0.0061419,0.0001,0.0096225,0.0061419,0.0061419,0.0061419,0.0001,0.0061419,0.013858,0.00600838,0.0001,0.0001,0.0001,0.0001,0.0001,0.0253975,0.0191557,0.00955345,0.0001,0.0001,0.00600838,0.0061419,0.0129887,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0061419,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.013858,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0245676,0.0109088,0.0237814,0.0096225,0.0096225,0.0096225,0.0061419,0.0096225,0.0001,0.0001,0.0061419,0.0001,0.00828173,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0061419,0.0001,0.0001,0.0001,0.0061419,0.0001,0.0001,0.00955345,0.0001,0.0001,0.0061419,0.0061419,0.0061419,0.0001,0.0001,0.00828173,0.0061419,0.0001,0.0001,0.0061419,0.0061419,0.0141744,0.0061419,0.0061419,0.0001,0.0061419,0.0132249,0.0001,0.0001,0.0001,0.00600838,0.0001,0.0001,0.0061419,0.0061419,0.0096225,0.0132249,0.0001,0.020966,0.0001,0.0187945,0.0061419,0.0001,0.0168712,0.0001,0.0001,0.0061419,0.0001,0.0001,0.0096225,0.0001,0.0001,0.0061419,0.0001,0.0001,0.0001,0.0191289,0.0001,0.0001,0.0001,0.00828173,0.0001,0.0061419,0.0001,0.0001,0.0001,0.0001,0.0122838,0.0061419,0.0061419,0.00819231,0.0001,0.0001,0.0001,0.0061419,0.0001,0.0001,0.0096225,0.0061419,0.0132249,0.0142243,0.00600838,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001];
+export const REF_MEAN_TRI = [0.0262081,0.0197752,0.0146636,0.0115133,0.00835129,0.00803939,0.00732562,0.00664172,0.00606836,0.00561603,0.00558707,0.00554066,0.00552094,0.00547383,0.00561603,0.00539653,0.00481424,0.00492965,0.00456318,0.00459663,0.00416959,0.0041447,0.00417769,0.00420407,0.00405605,0.00412779,0.0041609,0.00423076,0.00385096,0.00378708,0.0037749,0.00360926,0.00247826,0.00343663,0.00340436,0.00360291,0.00242611,0.00303332,0.00329417,0.00305784,0.00315133,0.00309155,0.00311913,0.00310203,0.00306396,0.0031014,0.00314394,0.00313747,0.00270622,0.00280604,0.00239966,0.00275446,0.00278363,0.00295621,0.00278929,0.00275477,0.00269392,0.00272885,0.00284828,0.00246671,0.00271408,0.00249953,0.00238731,0.0024527,0.00243153,0.00236626,0.0024487,0.00246899,0.00238354,0.00237798,0.00242567,0.00242069,0.00244214,0.00244875,0.00245042,0.00238705,0.00243167,0.00243351,0.00252937,0.00238049,0.00244905,0.00246368,0.00241778,0.0024729,0.002351,0.00210108,0.00236626,0.00248027,0.00208468,0.00206919,0.00205711,0.00209973,0.00221216,0.00210887,0.00213963,0.00207907,0.00203758,0.00205546,0.00206781,0.00204544,0.00217957,0.0020741,0.00208354,0.00207131,0.00209342,0.00208017,0.00208575,0.00212053,0.00206517,0.00203327,0.00142341,0.0013864,0.00174495,0.00179818,0.00171403,0.00180406,0.00174467,0.00168028,0.00177547,0.00173062,0.00170734,0.00172911,0.00138968,0.00137104,0.00169966,0.0017718,0.00172374,0.00173951,0.0018642,0.00170752,0.00177068,0.00174655,0.00174916,0.0017948,0.00169859,0.0016997,0.00137502,0.00178033,0.0017992,0.00174032,0.00177547,0.00171722,0.00137989,0.00135388,0.00151109,0.0014362,0.00137068,0.00139493,0.00137898,0.0013948,0.00139847,0.00141357,0.00139372,0.00137584,0.00141283,0.00136114,0.00137149,0.00106195,0.00135572,0.00142169,0.00133885,0.00139629,0.00141613,0.00142169,0.0013834,0.00137395,0.00139933,0.00137544,0.0013513,0.00137068,0.00137,0.00141513,0.00139372,0.00135171,0.00138926,0.00137058,0.00136921,0.0014278,0.00102124,0.00141357,0.00137093,0.00144025,0.00135274,0.00132938,0.00137646,0.00144169,0.00141946,0.00106332,0.00135862,0.00147284,0.00136697,0.00136551,0.00139991,0.00137989,0.00138875,0.00138528,0.00133085,0.0013736,0.0013513,0.00105634,0.00102638,0.00103411,0.00104062,0.00102336,0.0010332,0.00102638,0.00110666,0.0010001,0.00101975,0.00102775,0.00104287,0.00103983,0.00102382,0.00072457,0.00105051,0.00104481,0.00100147,0.00101794,0.00109013,0.00104332,0.00102233,0.0010165,0.00105485,0.00102638,0.00104902,0.00102638,0.00104318,0.00101987,0.0010081,0.00104611,0.00102336,0.00103478,0.00103806,0.00101975,0.00104211,0.00103073,0.00101838,0.00105046,0.00104062,0.000680998,0.00105195,0.00103759,0.00102233,0.00103262,0.00102775,0.00104211,0.000684604,0.00105046,0.00106265,0.00110666];
+export const REF_SD_TRI = [0.00895104,0.00900031,0.00851377,0.00593693,0.00241578,0.00500421,0.00345127,0.00276521,0.00704054,0.00465794,0.00564577,0.00383774,0.0038419,0.00531826,0.00465794,0.00562228,0.00474384,0.00462869,0.00280884,0.00441612,0.00409947,0.00336316,0.0017046,0.00342941,0.00514773,0.00295658,0.00370557,0.00468011,0.00362062,0.00469265,0.00416407,0.00410278,0.00364846,0.00329229,0.00328679,0.00406721,0.00205821,0.003331,0.00534869,0.00374798,0.00396759,0.00414992,0.00347914,0.00348068,0.00291142,0.00298574,0.00246607,0.00298131,0.00447818,0.00256421,0.00264854,0.00255534,0.00316297,0.0054157,0.00262453,0.00384367,0.00469174,0.00475669,0.00278822,0.00403313,0.00378129,0.00323093,0.00311064,0.00358161,0.00206243,0.00308014,0.00207656,0.00326886,0.00307596,0.00309519,0.00262671,0.00352913,0.00316073,0.00318579,0.00269444,0.00385127,0.00314259,0.00318193,0.00427975,0.00308827,0.00269362,0.00359892,0.00263369,0.0031869,0.00348487,0.00315884,0.00308014,0.00267407,0.00265415,0.00207086,0.00205803,0.0032092,0.00399446,0.00268487,0.00334536,0.00316669,0.002601,0.0026423,0.00206954,0.00357692,0.00290796,0.00353941,0.002691,0.00269935,0.00266833,0.00319257,0.00209304,0.00212715,0.00314138,0.00310213,0.00201852,0.00196108,0.00267522,0.00213162,0.00203031,0.00213824,0.0026943,0.00258558,0.00365225,0.00315603,0.00202124,0.00261202,0.00261397,0.00193971,0.00262972,0.00269929,0.00265171,0.00268431,0.00398974,0.00258782,0.00210061,0.00363033,0.0026536,0.00212737,0.00260268,0.00347163,0.00194506,0.00211253,0.00366721,0.0026415,0.00365225,0.00203297,0.00195254,0.00252958,0.00395141,0.00267133,0.00351676,0.00262027,0.00258791,0.00311899,0.00197864,0.00200536,0.00261804,0.00194712,0.00199846,0.00192556,0.00194079,0.00184616,0.00191802,0.00201738,0.00189359,0.00260179,0.00262751,0.00201738,0.00309758,0.0025973,0.00258373,0.00258471,0.00191145,0.00351676,0.0019386,0.00267596,0.00197152,0.0019125,0.0019651,0.00257905,0.0025516,0.00265819,0.00176981,0.00200536,0.00193941,0.00267053,0.00191356,0.00339627,0.00194712,0.00204262,0.00201351,0.00184838,0.0019223,0.00283523,0.00193409,0.00252964,0.00198065,0.00195254,0.0025481,0.00195983,0.00247928,0.00255093,0.00191145,0.00251293,0.0017782,0.00179231,0.00249349,0.0017727,0.00246913,0.0017782,0.00191927,0.00173233,0.00176711,0.00178054,0.00180771,0.00244995,0.00177405,0.00162638,0.00181963,0.00181157,0.00173475,0.00176347,0.00189193,0.00180895,0.00177136,0.0017609,0.00251105,0.0017782,0.00250376,0.0017782,0.00247679,0.00176746,0.00174611,0.00181202,0.0017727,0.00179264,0.00179878,0.00176711,0.00180548,0.00178587,0.00176475,0.00250555,0.00249349,0.00152373,0.00182209,0.00179745,0.00177136,0.00178965,0.00178054,0.00180548,0.00153101,0.00181949,0.00184094,0.00191927];
+export const REF_MEAN_PUNCT = [0.00992986,0.0130429,0.000344353,0.000746792,0,0.000672043,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0.0130429,0.00959519,0.000344353];
+export const REF_SD_PUNCT = [0.00458435,0.00551939,0.00114209,0.00167133,0.0001,0.00222891,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.0001,0.00551939,0.0048496,0.00114209];