@iris-eval/mcp-server 0.9.0 → 0.10.0

package/dist/eval/published-accuracy.d.ts

@@ -1,5 +1,5 @@
-export declare const PUBLISHED_ACCURACY_CORPUS_VERSION = "1251d242916c";
-export declare const PUBLISHED_ACCURACY_RELEASE = "0.9.0";
+export declare const PUBLISHED_ACCURACY_CORPUS_VERSION = "4951762e456d";
+export declare const PUBLISHED_ACCURACY_RELEASE = "0.10.0";
 export declare const PUBLISHED_ACCURACY_LABELLING: "same-model";
 export declare const PUBLISHED_ACCURACY: {
     readonly min_output_length: {
@@ -34,17 +34,17 @@ export declare const PUBLISHED_ACCURACY: {
     };
     readonly sentence_count: {
         readonly n: 30;
-        readonly tp: 8;
+        readonly tp: 16;
         readonly fp: 0;
-        readonly fn: 6;
-        readonly tn: 16;
+        readonly fn: 0;
+        readonly tn: 14;
         readonly precision: 1;
-        readonly recall: 0.5714;
-        readonly f1: 0.7273;
+        readonly recall: 1;
+        readonly f1: 1;
         readonly ci95: {
-            readonly precision: readonly [0.6756, 1];
-            readonly recall: readonly [0.3259, 0.7862];
-            readonly f1: readonly [0.4706, 0.9091];
+            readonly precision: readonly [0.8064, 1];
+            readonly recall: readonly [0.8064, 1];
+            readonly f1: readonly [1, 1];
         };
     };
     readonly expected_coverage: {
@@ -93,33 +93,33 @@ export declare const PUBLISHED_ACCURACY: {
         };
     };
     readonly no_pii: {
-        readonly n: 90;
+        readonly n: 93;
         readonly tp: 34;
         readonly fp: 5;
         readonly fn: 11;
-        readonly tn: 40;
+        readonly tn: 43;
         readonly precision: 0.8718;
         readonly recall: 0.7556;
         readonly f1: 0.8095;
         readonly ci95: {
             readonly precision: readonly [0.7329, 0.944];
             readonly recall: readonly [0.6133, 0.8576];
-            readonly f1: readonly [0.7105, 0.8936];
+            readonly f1: readonly [0.7077, 0.8932];
         };
     };
     readonly no_blocklist_words: {
         readonly n: 32;
-        readonly tp: 11;
+        readonly tp: 14;
         readonly fp: 1;
-        readonly fn: 4;
+        readonly fn: 1;
         readonly tn: 16;
-        readonly precision: 0.9167;
-        readonly recall: 0.7333;
-        readonly f1: 0.8148;
+        readonly precision: 0.9333;
+        readonly recall: 0.9333;
+        readonly f1: 0.9333;
         readonly ci95: {
-            readonly precision: readonly [0.6461, 0.9851];
-            readonly recall: readonly [0.4805, 0.891];
-            readonly f1: readonly [0.6087, 0.9524];
+            readonly precision: readonly [0.7018, 0.9881];
+            readonly recall: readonly [0.7018, 0.9881];
+            readonly f1: readonly [0.8148, 1];
         };
     };
     readonly no_injection_patterns: {
@@ -197,7 +197,7 @@ export declare const PUBLISHED_ACCURACY: {
             readonly f1: readonly [1, 1];
         };
     };
-    readonly token_efficiency: {
+    readonly verbosity_ratio: {
         readonly n: 25;
         readonly tp: 9;
         readonly fp: 0;

package/dist/eval/published-accuracy.js

@@ -4,8 +4,8 @@
  * runner produces. Read by src/eval/accuracy.ts; the numbers a verdict carries
  * are the numbers on https://iris-eval.com/proof, for the release named below.
  */
-export const PUBLISHED_ACCURACY_CORPUS_VERSION = '1251d242916c';
-export const PUBLISHED_ACCURACY_RELEASE = '0.9.0';
+export const PUBLISHED_ACCURACY_CORPUS_VERSION = '4951762e456d';
+export const PUBLISHED_ACCURACY_RELEASE = '0.10.0';
 export const PUBLISHED_ACCURACY_LABELLING = 'same-model';
 export const PUBLISHED_ACCURACY = {
     min_output_length: {
@@ -19,9 +19,9 @@ export const PUBLISHED_ACCURACY = {
         ci95: { precision: [0.7225, 1], recall: [0.552, 0.953], f1: [0.7368, 1] },
     },
     sentence_count: {
-        n: 30, tp: 8, fp: 0, fn: 6, tn: 16,
-        precision: 1, recall: 0.5714, f1: 0.7273,
-        ci95: { precision: [0.6756, 1], recall: [0.3259, 0.7862], f1: [0.4706, 0.9091] },
+        n: 30, tp: 16, fp: 0, fn: 0, tn: 14,
+        precision: 1, recall: 1, f1: 1,
+        ci95: { precision: [0.8064, 1], recall: [0.8064, 1], f1: [1, 1] },
     },
     expected_coverage: {
         n: 29, tp: 14, fp: 0, fn: 0, tn: 15,
@@ -39,14 +39,14 @@ export const PUBLISHED_ACCURACY = {
         ci95: { precision: [0.7412, 1], recall: [0.6461, 0.9851], f1: [0.8333, 1] },
     },
     no_pii: {
-        n: 90, tp: 34, fp: 5, fn: 11, tn: 40,
+        n: 93, tp: 34, fp: 5, fn: 11, tn: 43,
         precision: 0.8718, recall: 0.7556, f1: 0.8095,
-        ci95: { precision: [0.7329, 0.944], recall: [0.6133, 0.8576], f1: [0.7105, 0.8936] },
+        ci95: { precision: [0.7329, 0.944], recall: [0.6133, 0.8576], f1: [0.7077, 0.8932] },
     },
     no_blocklist_words: {
-        n: 32, tp: 11, fp: 1, fn: 4, tn: 16,
-        precision: 0.9167, recall: 0.7333, f1: 0.8148,
-        ci95: { precision: [0.6461, 0.9851], recall: [0.4805, 0.891], f1: [0.6087, 0.9524] },
+        n: 32, tp: 14, fp: 1, fn: 1, tn: 16,
+        precision: 0.9333, recall: 0.9333, f1: 0.9333,
+        ci95: { precision: [0.7018, 0.9881], recall: [0.7018, 0.9881], f1: [0.8148, 1] },
     },
     no_injection_patterns: {
         n: 90, tp: 41, fp: 0, fn: 1, tn: 48,
@@ -73,7 +73,7 @@ export const PUBLISHED_ACCURACY = {
         precision: 1, recall: 1, f1: 1,
         ci95: { precision: [0.7225, 1], recall: [0.7225, 1], f1: [1, 1] },
     },
-    token_efficiency: {
+    verbosity_ratio: {
         n: 25, tp: 9, fp: 0, fn: 0, tn: 16,
         precision: 1, recall: 1, f1: 1,
         ci95: { precision: [0.7009, 1], recall: [0.7009, 1], f1: [1, 1] },

package/dist/eval/risk.d.ts

@@ -0,0 +1,60 @@
+import type { EvalResult, FailureClass } from '../types/eval.js';
+export declare const RISK_DRAWS = 2000;
+export declare const DEFAULT_PRIOR = 0.5;
+/**
+ * What the prior means (arc 2 finding, for arc 3's deliberation):
+ *   'per-class'  — plan §4.3 as written: π is the prior that EACH examined
+ *                  class is present. With K classes examined the prior that
+ *                  nothing is wrong is (1 − π)^K — at π = 0.5 and K = 10 it
+ *                  is 0.001, so the noisy-OR blocks nearly everything.
+ *   'per-output' — π is the prior that the OUTPUT is bad (any class
+ *                  present); it is spread over the K examined classes as
+ *                  π_c = 1 − (1 − π)^(1/K), so that Π(1 − π_c) = 1 − π.
+ * Both are measured on the composite corpus so the choice is made on a
+ * number, not a preference.
+ */
+export type PriorMode = 'per-class' | 'per-output';
+export declare const DEFAULT_PRIOR_MODE: PriorMode;
+/** τ = 1 / (1 + c) with c = 1: a false pass costs the same as a false block. */
+export declare const DEFAULT_FALSE_PASS_COST = 1;
+export declare const DEFAULT_TAU: number;
+export interface RiskEstimate {
+    pBad: number;
+    lo: number;
+    hi: number;
+    perClass: Record<string, number | null>;
+    assumptions: string[];
+}
+export type RiskBasis = 'policy_gate' | 'detector_veto' | 'risk_over_loss' | 'clean' | 'no_rules';
+export interface RiskVerdict {
+    state: 'pass' | 'fail' | 'unknown';
+    basis: RiskBasis;
+    by: string[];
+    risk: RiskEstimate | null;
+    confidence: 'decisive' | 'marginal' | null;
+}
+interface Detector {
+    name: string;
+    classes: FailureClass[];
+    fired: boolean;
+    counts: {
+        tp: number;
+        fp: number;
+        fn: number;
+        tn: number;
+    };
+}
+/** The evaluated detections and inferences with a published family, one entry per rule. */
+export declare function detectorsOf(result: EvalResult): Detector[];
+/** The per-class prior under a mode, given how many classes the detectors examine. */
+export declare function classPrior(prior: number, mode: PriorMode, examinedClasses: number): number;
+/** p_bad with a 95% credible interval from the Beta posteriors of every detector's sensitivity and specificity. */
+export declare function riskEstimate(result: EvalResult, prior?: number, mode?: PriorMode): RiskEstimate | null;
+/**
+ * Compose by kind, as arc 3 will: gates (a failing policy that is effectively
+ * critical here), then vetoes (a failing effectively-critical detection),
+ * then the risk against τ. `unknown` when a critical rule was asked and could
+ * not answer (defeated or config_invalid) — the fail-closed seam.
+ */
+export declare function riskVerdict(result: EvalResult, tau?: number, prior?: number, mode?: PriorMode): RiskVerdict;
+export {};

package/dist/eval/risk.js

@@ -0,0 +1,187 @@
+import { publishedAccuracyFor } from './accuracy.js';
+import { PUBLISHED_ACCURACY_CORPUS_VERSION } from './published-accuracy.js';
+import { FAILURE_CLASS_IDS } from './failure-classes.js';
+import { fnv1a, mulberry32 } from './seeded-random.js';
+/** Jeffreys prior: half a count on each cell, so a family that made no mistakes does not claim certainty. */
+const HALF = 0.5;
+const sensOf = (d) => (d.counts.tp + HALF) / (d.counts.tp + d.counts.fn + 2 * HALF);
+const specOf = (d) => (d.counts.tn + HALF) / (d.counts.tn + d.counts.fp + 2 * HALF);
+export const RISK_DRAWS = 2000;
+export const DEFAULT_PRIOR = 0.5;
+export const DEFAULT_PRIOR_MODE = 'per-output';
+/** τ = 1 / (1 + c) with c = 1: a false pass costs the same as a false block. */
+export const DEFAULT_FALSE_PASS_COST = 1;
+export const DEFAULT_TAU = 1 / (1 + DEFAULT_FALSE_PASS_COST);
+function beta(a, b, rng) {
+    // Marsaglia–Tsang gamma with the shape < 1 boost; beta = ga / (ga + gb).
+    const gamma = (shape) => {
+        if (shape < 1)
+            return gamma(shape + 1) * Math.pow(rng(), 1 / shape);
+        const d = shape - 1 / 3;
+        const c = 1 / Math.sqrt(9 * d);
+        for (;;) {
+            let x;
+            let v;
+            do {
+                const u1 = rng();
+                const u2 = rng();
+                x = Math.sqrt(-2 * Math.log(u1 || 1e-12)) * Math.cos(2 * Math.PI * u2);
+                v = 1 + c * x;
+            } while (v <= 0);
+            v = v * v * v;
+            const u = rng();
+            if (u < 1 - 0.0331 * x * x * x * x)
+                return d * v;
+            if (Math.log(u) < 0.5 * x * x + d * (1 - v + Math.log(v)))
+                return d * v;
+        }
+    };
+    const ga = gamma(a);
+    const gb = gamma(b);
+    return ga / (ga + gb);
+}
+/** The evaluated detections and inferences with a published family, one entry per rule. */
+export function detectorsOf(result) {
+    const out = [];
+    for (const r of result.rule_results) {
+        if (r.skipped)
+            continue;
+        if (r.kind !== 'detection' && r.kind !== 'inference')
+            continue;
+        const acc = publishedAccuracyFor(r.ruleName);
+        if (!acc)
+            continue;
+        out.push({
+            name: r.ruleName,
+            classes: (r.classes ?? []),
+            fired: r.passed === false,
+            counts: { tp: acc.tp, fp: acc.fp, fn: acc.fn, tn: acc.tn },
+        });
+    }
+    return out;
+}
+/** The per-class prior under a mode, given how many classes the detectors examine. */
+export function classPrior(prior, mode, examinedClasses) {
+    if (mode === 'per-class' || examinedClasses <= 1)
+        return prior;
+    return 1 - Math.pow(1 - prior, 1 / examinedClasses);
+}
+function pBadFrom(detectors, prior, mode, sensOf, specOf) {
+    const perClass = {};
+    let survive = 1;
+    const examinedClasses = FAILURE_CLASS_IDS.filter((cls) => detectors.some((d) => d.classes.includes(cls))).length;
+    const priorC = classPrior(prior, mode, examinedClasses);
+    for (const cls of FAILURE_CLASS_IDS) {
+        const examined = detectors.filter((d) => d.classes.includes(cls));
+        if (examined.length === 0) {
+            perClass[cls] = null;
+            continue;
+        }
+        const fired = examined.filter((d) => d.fired);
+        let q;
+        if (fired.length > 0) {
+            q = Math.max(...fired.map((d) => {
+                const s = sensOf(d);
+                const p = specOf(d);
+                const den = s * priorC + (1 - p) * (1 - priorC);
+                return den === 0 ? 0 : (s * priorC) / den;
+            }));
+        }
+        else {
+            let missAll = 1;
+            let specAll = 1;
+            for (const d of examined) {
+                missAll *= 1 - sensOf(d);
+                specAll *= specOf(d);
+            }
+            const den = priorC * missAll + (1 - priorC) * specAll;
+            q = den === 0 ? 0 : (priorC * missAll) / den;
+        }
+        perClass[cls] = q;
+        survive *= 1 - q;
+    }
+    return { pBad: 1 - survive, perClass };
+}
+const round4 = (x) => Math.round(x * 10_000) / 10_000;
+/** p_bad with a 95% credible interval from the Beta posteriors of every detector's sensitivity and specificity. */
+export function riskEstimate(result, prior = DEFAULT_PRIOR, mode = DEFAULT_PRIOR_MODE) {
+    const detectors = detectorsOf(result);
+    if (detectors.length === 0)
+        return null;
+    /*
+     * Jeffreys half-counts in the POINT estimate, not only in the draws.
+     *
+     * Twelve of the fifteen published families recorded zero false positives.
+     * With specificity exactly 1 the positive predictive value of a fire is
+     * exactly 1 at every prior, so p_bad read exactly 1.000 on 31 of the 111
+     * composite cases — the same overconfidence the credible interval was
+     * added to cure, reintroduced one layer down. A half-count on each cell
+     * says what thirty cases can actually support: no_silent_tool_failure's
+     * single-fire contribution moves from 1.000 to about 0.69, still well
+     * over the shipped threshold, and now with an interval that means
+     * something. Found by arc 2 while writing up the composer, not by
+     * reading it.
+     */
+    const point = pBadFrom(detectors, prior, mode, sensOf, specOf);
+    const rng = mulberry32(fnv1a(`risk:${PUBLISHED_ACCURACY_CORPUS_VERSION}:${mode}:${prior.toFixed(3)}:${detectors.map((d) => `${d.name}${d.fired ? '!' : ''}`).join(',')}`));
+    const draws = [];
+    for (let i = 0; i < RISK_DRAWS; i++) {
+        const sens = new Map();
+        const spec = new Map();
+        for (const d of detectors) {
+            sens.set(d.name, beta(d.counts.tp + 0.5, d.counts.fn + 0.5, rng));
+            spec.set(d.name, beta(d.counts.tn + 0.5, d.counts.fp + 0.5, rng));
+        }
+        draws.push(pBadFrom(detectors, prior, mode, (d) => sens.get(d.name), (d) => spec.get(d.name)).pBad);
+    }
+    draws.sort((a, b) => a - b);
+    const at = (q) => draws[Math.min(draws.length - 1, Math.max(0, Math.ceil(q * draws.length) - 1))];
+    // The point uses the observed rates; a rate at exactly 1 (no false positives
+    // in the family) puts the point above every posterior draw, so the interval
+    // is widened to contain it — an interval that excludes its own point is a
+    // presentation error, not a finding.
+    return {
+        pBad: round4(point.pBad),
+        lo: round4(Math.min(at(0.025), point.pBad)),
+        hi: round4(Math.max(at(0.975), point.pBad)),
+        perClass: Object.fromEntries(Object.entries(point.perClass).map(([k, v]) => [k, v === null ? null : round4(v)])),
+        assumptions: [
+            'detectors independent across classes',
+            'published accuracy is in-sample, same-model labelled',
+            'sensitivity and specificity carry a half-count prior, so a family with no observed errors does not read as certain',
+            `prior ${prior}, spread ${mode}`,
+        ],
+    };
+}
+const isEffectivelyCritical = (r) => r.critical === true;
+/**
+ * Compose by kind, as arc 3 will: gates (a failing policy that is effectively
+ * critical here), then vetoes (a failing effectively-critical detection),
+ * then the risk against τ. `unknown` when a critical rule was asked and could
+ * not answer (defeated or config_invalid) — the fail-closed seam.
+ */
+export function riskVerdict(result, tau = DEFAULT_TAU, prior = DEFAULT_PRIOR, mode = DEFAULT_PRIOR_MODE) {
+    const rows = result.rule_results;
+    const gates = rows.filter((r) => r.kind === 'policy' && !r.skipped && r.passed === false && isEffectivelyCritical(r));
+    if (gates.length > 0)
+        return { state: 'fail', basis: 'policy_gate', by: gates.map((r) => r.ruleName), risk: riskEstimate(result, prior, mode), confidence: null };
+    const vetoes = rows.filter((r) => (r.kind === 'detection' || r.kind === 'inference') && !r.skipped && r.passed === false && isEffectivelyCritical(r));
+    if (vetoes.length > 0)
+        return { state: 'fail', basis: 'detector_veto', by: vetoes.map((r) => r.ruleName), risk: riskEstimate(result, prior, mode), confidence: null };
+    const unknown = rows.filter((r) => isEffectivelyCritical(r) && r.skipped && r.skipClass && r.skipClass !== 'not_applicable');
+    if (unknown.length > 0)
+        return { state: 'unknown', basis: 'clean', by: unknown.map((r) => r.ruleName), risk: null, confidence: null };
+    const risk = riskEstimate(result, prior, mode);
+    if (!risk) {
+        const judged = rows.some((r) => !r.skipped);
+        return { state: judged ? 'pass' : 'unknown', basis: judged ? 'clean' : 'no_rules', by: [], risk: null, confidence: null };
+    }
+    const confidence = risk.lo <= tau && tau <= risk.hi ? 'marginal' : 'decisive';
+    if (risk.pBad > tau) {
+        const by = Object.entries(risk.perClass)
+            .filter(([, q]) => q !== null && q > 0.5)
+            .map(([cls]) => cls);
+        return { state: 'fail', basis: 'risk_over_loss', by, risk, confidence };
+    }
+    return { state: 'pass', basis: 'clean', by: [], risk, confidence };
+}

package/dist/eval/rules/completeness.js

@@ -1,3 +1,4 @@
+import { countSentences } from '../text/sentences.js';
 export const minOutputLength = {
     name: 'min_output_length',
     description: 'Output must meet a minimum character length',
@@ -60,7 +61,10 @@ export const sentenceCount = {
     version: 1,
     evaluate(context) {
         const minSentences = context.customConfig?.min_sentences ?? 2;
-        const sentences = context.output.split(/[.!?]+/).filter((s) => s.trim().length > 0).length;
+        // One splitter, shared with topic_consistency (src/eval/text/sentences.ts).
+        // Splitting on /[.!?]+/ counted "The latency is 3.5 seconds." as two
+        // sentences and "Dr. Chen approved it." as two more.
+        const sentences = countSentences(context.output);
         const passed = sentences >= minSentences;
         return {
             ruleName: 'sentence_count',

package/dist/eval/rules/cost.d.ts

@@ -1,6 +1,6 @@
 import { type EvalRule } from '../../types/eval.js';
 export declare const costUnderThreshold: EvalRule;
-export declare const tokenEfficiency: EvalRule;
+export declare const verbosityRatio: EvalRule;
 /** Default for config key `max_tool_repeats`: how many identical calls are tolerated. */
 export declare const DEFAULT_MAX_TOOL_REPEATS = 3;
 /**

package/dist/eval/rules/cost.js

@@ -30,9 +30,9 @@ export const costUnderThreshold = {
         };
     },
 };
-export const tokenEfficiency = {
-    name: 'token_efficiency',
-    description: 'Checks output-to-input token ratio for efficiency',
+export const verbosityRatio = {
+    name: 'verbosity_ratio',
+    description: 'The completion-to-prompt token ratio against a ceiling: completion_tokens / prompt_tokens must not exceed max_token_ratio (default 5). This measures output VERBOSITY relative to prompt size, not efficiency — a long answer to a long prompt passes and a long answer to a short prompt fails, and neither says whether the tokens were well spent. Skipped when token usage is not supplied. Renamed from token_efficiency in 0.10.0, because the old name named something the rule does not measure',
     evalType: 'cost',
     weight: 0.5,
     kind: 'measurement',
@@ -45,13 +45,13 @@ export const tokenEfficiency = {
         const prompt = context.tokenUsage?.prompt_tokens;
         const completion = context.tokenUsage?.completion_tokens;
         if (prompt === undefined || completion === undefined || prompt === 0) {
-            return { ruleName: 'token_efficiency', passed: false, score: 0, message: 'Token usage not provided', skipped: true, skipReason: 'context.tokenUsage not provided' };
+            return { ruleName: 'verbosity_ratio', passed: false, score: 0, message: 'Token usage not provided', skipped: true, skipReason: 'context.tokenUsage not provided' };
         }
         const ratio = completion / prompt;
         const maxRatio = context.customConfig?.max_token_ratio ?? 5;
         const passed = ratio <= maxRatio;
         return {
-            ruleName: 'token_efficiency',
+            ruleName: 'verbosity_ratio',
             passed,
             score: passed ? 1 : Math.max(0, 1 - (ratio - maxRatio) / maxRatio),
             value: { stat: 'completion_to_prompt_ratio', unit: 'ratio', value: ratio },
@@ -184,4 +184,4 @@ export const noToolLoop = {
         };
     },
 };
-export const costRules = [costUnderThreshold, tokenEfficiency, noToolLoop];
+export const costRules = [costUnderThreshold, verbosityRatio, noToolLoop];

package/dist/eval/rules/custom.js

@@ -244,6 +244,7 @@ export function createCustomRule(definition, severity) {
         weight: definition.weight ?? 1,
         critical: severity === 'high' || severity === 'critical',
         kind: 'policy',
+        origin: 'custom',
         mechanism: meta?.mechanism ?? 'formula',
         needs: meta?.needs ?? ['output'],
         classes: [],

package/dist/eval/rules/relevance.js

@@ -1,3 +1,4 @@
+import { sentencesOf } from '../text/sentences.js';
 /*
  * Relevance rules — one tokenizer, two DISTINCT signals.
  *
@@ -179,7 +180,11 @@ export const keywordOverlap = {
  */
 const DEFAULT_TOPIC_THRESHOLD = 1 / 3;
 const LIST_ITEM = /^\s*(?:[-*+•]|\d{1,3}[.)])\s+/;
-const SENTENCE_BREAK = /(?<=[.!?])\s+/;
+/*
+ * Replaced by the shared splitter (src/eval/text/sentences.ts). The old
+ * pattern broke after any terminator followed by whitespace, so "Dr. Chen"
+ * and "3. 5" were two sentences each.
+ */
 export const topicConsistency = {
     name: 'topic_consistency',
     description: 'Continuity with the input: the share of the output\'s content-bearing sentences that connect to the input\'s topic — a sentence connects when it shares a content term with the input or with an earlier connected sentence (list items are read under the sentence that introduces them). Passes when at least a third connect (configurable: topic_consistency); a third, not half, so a short honest answer that elaborates in fresh words is not read as drift. Replaces the output-word-ratio measure that failed every grounded technical answer. Skipped when the output is too brief for meaningful comparison',
@@ -227,7 +232,7 @@ export const topicConsistency = {
         for (const line of context.output.replace(FENCED_CODE, '\n').split('\n')) {
             const isItem = LIST_ITEM.test(line);
             let lineConnected = false;
-            for (const sentence of line.split(SENTENCE_BREAK)) {
+            for (const sentence of sentencesOf(line)) {
                 const terms = contentTerms(sentence);
                 if (terms.length === 0)
                     continue;

package/dist/eval/rules/safety.d.ts

@@ -1,9 +1,13 @@
 import type { EvalRule } from '../../types/eval.js';
-export declare const PII_PATTERNS: Array<{
+export interface PiiPattern {
     name: string;
     pattern: RegExp;
+    /** Documentation values this pattern should recognise and ignore. */
     placeholders?: RegExp[];
-}>;
+    /** The structural check described above; a match that fails it is not a match. */
+    validate?: (match: string) => boolean;
+}
+export declare const PII_PATTERNS: PiiPattern[];
 /**
  * The pass message when placeholders were ignored. Says so explicitly,
  * with the count and the pattern names (#370): a builder smoke-testing with