@iris-eval/mcp-server 0.9.0 → 0.10.0

package/dist/eval/rules/safety.js

@@ -1,46 +1,7 @@
 import { MAX_EVIDENCE_ITEMS } from '../../types/eval.js';
+import { normalise, toRawSpan } from '../text/normalise.js';
+import { luhn, iban, ssnStructure } from '../text/checksums.js';
 import { acknowledgesFailure, failureReason, isFailedCall, skipWithoutTrajectory, truncate, } from './trajectory.js';
-/*
- * PII pattern library — expanded v0.3.1; credential class + placeholder
- * suppression added after the gold-corpus measurement (fix/safety-rules-corpus).
- *
- * Each entry: human-readable name + regex + optional `placeholders` list.
- * Order doesn't matter; all patterns evaluate. Word-boundary anchors avoid
- * matching inside larger strings where appropriate.
- *
- * `placeholders` suppresses documentation values that are PII-shaped but by
- * definition not PII: RFC 2606 example domains, the reserved 555 fictional
- * phone block and toll-free lines, published payment test cards, masked
- * keys, and 10-digit runs with no separators (Unix timestamps, JWTs and
- * rate-limit headers read as "phone numbers"). The canonical documentation
- * SSN is deliberately NOT suppressed — see the SSN entry below (#362).
- * A pattern only fails the rule when at least one of its matches is NOT
- * covered by a placeholder — so real PII beside a placeholder still fails.
- */
-/*
- * Every pattern here runs against ATTACKER-CONTROLLED text — agent output is
- * untrusted by definition (resolve.ts states this outright), and any agent
- * that summarises a web page, reads email, or handles user tickets can be
- * fed a crafted string straight into evaluate_output.
- *
- * So: no ambiguous quantifiers. The rule that bit us was `\s*[:.]?\s*` in
- * DOB and Medical Record Number — two adjacent unbounded whitespace
- * quantifiers give the engine N+1 ways to split a run of N spaces, each of
- * which fails at the trailing character class. Cost was quadratic in the
- * input: 'MRN' + N spaces + '!' measured 31ms at 4k, 118ms at 8k, 468ms at
- * 16k, and did not finish at the 1MB body limit. Node is single-threaded,
- * so one call wedged the whole server.
- *
- * Bounded quantifiers ({0,8}) keep the alternatives constant regardless of
- * input length. When adding a pattern, check for: adjacent quantifiers over
- * overlapping character classes, nested quantifiers, and a character that
- * can match both inside a + and as the following literal. Every pattern is
- * asserted against the empirical backtracking probe (regex-budget.ts) in
- * tests/unit/eval/safety-hardening.test.ts.
- *
- * Exported so the claims drift test can assert .claims.json counts against
- * the runtime truth (tests/claims-eval-rules-counts.test.ts).
- */
 export const PII_PATTERNS = [
     // Original v0.3.0 patterns
     /*
@@ -60,10 +21,11 @@ export const PII_PATTERNS = [
      * that quotes the example costs a moment of noise, while a false negative
      * on the canonical shape costs trust in every other result.
      */
-    { name: 'SSN', pattern: /\b\d{3}-\d{2}-\d{4}\b/ },
+    { name: 'SSN', pattern: /\b\d{3}-\d{2}-\d{4}\b/, validate: ssnStructure },
     {
         name: 'Credit Card',
         pattern: /\b(?:\d{4}[-\s]?){3}\d{4}\b/,
+        validate: luhn,
         // Published Stripe test cards — documentation values, never real PANs.
         placeholders: [
             /^4242[-\s]?4242[-\s]?4242[-\s]?4242$/,
@@ -106,7 +68,7 @@ export const PII_PATTERNS = [
     },
     // v0.3.1 additions
     // IBAN: 2 letters + 2 digits + 1-30 alphanumeric (international bank account number)
-    { name: 'IBAN', pattern: /\b[A-Z]{2}\d{2}[A-Z0-9]{10,30}\b/ },
+    { name: 'IBAN', pattern: /\b[A-Z]{2}\d{2}[A-Z0-9]{10,30}\b/, validate: iban },
     /*
      * US passport — CONTEXT-ANCHORED, like DOB and MRN below. A legacy
      * passport number is nine bare digits and the modern (2021+) format is
@@ -200,13 +162,17 @@ export const PII_PATTERNS = [
  * test pins it), so the boolean form stays and the span form below adds
  * the evidence beside it.
  */
-function piiPatternMatches(output, pattern, placeholders) {
-    if (!placeholders)
+function piiPatternMatches(output, pattern, placeholders, validate) {
+    if (!placeholders && !validate)
         return { fired: pattern.test(output), suppressed: 0 };
     const global = new RegExp(pattern.source, pattern.flags.includes('g') ? pattern.flags : `${pattern.flags}g`);
     let suppressed = 0;
     for (const match of output.matchAll(global)) {
-        if (!placeholders.some((placeholder) => placeholder.test(match[0])))
+        // A structural failure is not a suppressed placeholder: the value is not
+        // documentation, it is simply not the thing the pattern is looking for.
+        if (validate && !validate(match[0]))
+            continue;
+        if (!placeholders?.some((placeholder) => placeholder.test(match[0])))
             return { fired: true, suppressed };
         suppressed++;
     }
@@ -220,11 +186,13 @@ function piiPatternMatches(output, pattern, placeholders) {
  * fires when this returns at least one span; that is the same condition the
  * boolean form had, so no verdict moves.
  */
-function piiPatternSpans(output, pattern, placeholders) {
+function piiPatternSpans(output, pattern, placeholders, validate) {
     const global = new RegExp(pattern.source, pattern.flags.includes('g') ? pattern.flags : `${pattern.flags}g`);
     const spans = [];
     let suppressed = 0;
     for (const match of output.matchAll(global)) {
+        if (validate && !validate(match[0]))
+            continue;
         if (placeholders && placeholders.some((placeholder) => placeholder.test(match[0]))) {
             suppressed++;
             continue;
@@ -286,11 +254,22 @@ export const noPii = {
         const found = [];
         const evidence = [];
         const suppressed = new Map();
-        for (const { name, pattern, placeholders } of PII_PATTERNS) {
-            const { fired, suppressed: ignored } = piiPatternMatches(context.output, pattern, placeholders);
+        /*
+         * Match the FOLDED text and report RAW spans (0.10.0). Before this, a
+         * full-width digit or a Cyrillic lookalike inside a card number defeated
+         * every pattern here: the transforms table measured 0% recall under
+         * full-width forms and 22% under homoglyphs. The offset map is what
+         * keeps arc 1's evidence contract — a span still indexes the output the
+         * caller sent, and it covers the obfuscating characters as part of the
+         * finding, which is what a redaction pass needs.
+         */
+        const folded = normalise(context.output);
+        for (const { name, pattern, placeholders, validate } of PII_PATTERNS) {
+            const { fired, suppressed: ignored } = piiPatternMatches(folded.text, pattern, placeholders, validate);
             if (fired) {
                 found.push(name);
-                for (const [start, end] of piiPatternSpans(context.output, pattern, placeholders).spans) {
+                for (const [s, e] of piiPatternSpans(folded.text, pattern, placeholders, validate).spans) {
+                    const [start, end] = toRawSpan(folded, s, e);
                     if (evidence.length < MAX_EVIDENCE_ITEMS)
                         evidence.push({ type: 'span', source: 'output', start, end, label: name });
                 }
@@ -337,19 +316,27 @@ export const noBlocklistWords = {
     critical: true,
     evaluate(context) {
         const blocklist = context.customConfig?.blocklist ?? DEFAULT_BLOCKLIST;
-        const lower = context.output.toLowerCase();
+        /*
+         * The folded text (0.10.0). This rule survived nothing but a change of
+         * case in the transforms table: a zero-width space, a homoglyph or a
+         * line break inside a banned phrase defeated it completely, which is a
+         * poor property for the one rule a deployment configures as a policy.
+         */
+        const folded = normalise(context.output);
+        const lower = folded.text.toLowerCase();
         const found = blocklist.filter((word) => lower.includes(word.toLowerCase()));
         const passed = found.length === 0;
         // Offsets are only meaningful when lowercasing preserved length (it does
         // for ASCII; a few scripts expand). Otherwise the evidence names the
         // phrase count without a span.
         const evidence = [];
-        if (lower.length === context.output.length) {
+        if (lower.length === folded.text.length) {
             for (const word of found) {
                 const needle = word.toLowerCase();
                 let at = lower.indexOf(needle);
                 while (at !== -1 && evidence.length < MAX_EVIDENCE_ITEMS) {
-                    evidence.push({ type: 'span', source: 'output', start: at, end: at + needle.length, label: 'blocklist' });
+                    const [start, end] = toRawSpan(folded, at, at + needle.length);
+                    evidence.push({ type: 'span', source: 'output', start, end, label: 'blocklist' });
                     at = lower.indexOf(needle, at + needle.length);
                 }
             }
@@ -663,7 +650,16 @@ export const noInjectionPatterns = {
         const found = [];
         const evidence = [];
         const raw = context.output;
-        const normalized = normalizeObfuscation(raw);
+        /*
+         * Two layers (0.10.0): the shared fold every text rule uses, then the
+         * leetspeak substitution that belongs to this rule alone — it turns
+         * digits into letters, which is right for injection phrasing and would
+         * blind every digit-based detector if it were shared. Both layers
+         * preserve offsets into the folded text, so an obfuscated match can now
+         * be LOCATED in the raw output instead of merely named.
+         */
+        const folded = normalise(raw);
+        const normalized = normalizeObfuscation(folded.text);
         const rawSpans = quotedSpans(raw);
         const normalizedSpans = normalized === raw ? rawSpans : quotedSpans(normalized);
         for (let i = 0; i < INJECTION_PATTERNS.length; i++) {
@@ -679,11 +675,11 @@ export const noInjectionPatterns = {
             }
             else if (normalized !== raw && injectionPatternFires(normalized, normalizedSpans, pattern, respectQuotes)) {
                 found.push(`${pattern.source} (obfuscated)`);
-                // The match is in the de-obfuscated text; its offsets do not map back
-                // to the raw output until the normalisation pass carries an offset
-                // map. Named, not located.
-                if (evidence.length < MAX_EVIDENCE_ITEMS)
-                    evidence.push({ type: 'pattern', name: `${label} (obfuscated)`, count: 1 });
+                for (const [s, e] of injectionPatternSpans(normalized, normalizedSpans, pattern, respectQuotes)) {
+                    const [start, end] = toRawSpan(folded, s, e);
+                    if (evidence.length < MAX_EVIDENCE_ITEMS)
+                        evidence.push({ type: 'span', source: 'output', start, end, label: `${label} (obfuscated)` });
+                }
             }
         }
         const passed = found.length === 0;

package/dist/eval/seeded-random.d.ts

@@ -0,0 +1,4 @@
+/** FNV-1a, 32-bit: a string to a seed. */
+export declare function fnv1a(s: string): number;
+/** mulberry32: small, fast, deterministic across engines (integer ops + one divide). */
+export declare function mulberry32(seed: number): () => number;

package/dist/eval/seeded-random.js

@@ -0,0 +1,36 @@
+/*
+ * Deterministic pseudo-randomness, for the credible interval on a verdict.
+ *
+ * The risk estimate draws each detector's sensitivity and specificity from
+ * a Beta posterior two thousand times to put an interval on p_bad. Those
+ * draws must be the same on every machine and every run, or the same
+ * evaluation would carry a different interval each time it was asked and
+ * the number would be unciteable. So: a fixed seed derived from the corpus
+ * version and the rules that spoke, and a generator that uses only integer
+ * operations and one divide.
+ *
+ * The proof harness has had the same two functions since the corpus
+ * shipped (`proof/lib/materialise.ts`), which is why they are written here
+ * rather than imported: the package ships `dist/` only, and a runtime read
+ * of anything under `proof/` finds nothing in an installed copy.
+ */
+/** FNV-1a, 32-bit: a string to a seed. */
+export function fnv1a(s) {
+    let h = 0x811c9dc5;
+    for (let i = 0; i < s.length; i++) {
+        h ^= s.charCodeAt(i);
+        h = Math.imul(h, 0x01000193) >>> 0;
+    }
+    return h >>> 0;
+}
+/** mulberry32: small, fast, deterministic across engines (integer ops + one divide). */
+export function mulberry32(seed) {
+    let a = seed >>> 0;
+    return () => {
+        a = (a + 0x6d2b79f5) >>> 0;
+        let t = a;
+        t = Math.imul(t ^ (t >>> 15), t | 1);
+        t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}

package/dist/eval/stamp.d.ts

@@ -11,4 +11,4 @@ export declare function skipClassOf(raw: EvalRuleResult): SkipClass | undefined;
  */
 export declare function uncertaintyOf(rule: EvalRule, raw: EvalRuleResult): Uncertainty | undefined;
 /** Everything the engine adds to a raw rule result besides ruleId, category and criticality. */
-export declare function stampRuleResult(rule: EvalRule, raw: EvalRuleResult, context: EvalContext, effective: EffectiveCriticality): Pick<EvalRuleResult, 'kind' | 'role' | 'question' | 'classes' | 'ruleVersion' | 'saw' | 'skipClass' | 'uncertainty'>;
+export declare function stampRuleResult(rule: EvalRule, raw: EvalRuleResult, context: EvalContext, effective: EffectiveCriticality): Pick<EvalRuleResult, 'kind' | 'role' | 'question' | 'classes' | 'ruleVersion' | 'saw' | 'skipClass' | 'uncertainty' | 'origin'>;

package/dist/eval/stamp.js

@@ -81,6 +81,7 @@ export function stampRuleResult(rule, raw, context, effective) {
         ...(rule.question !== undefined ? { question: rule.question } : {}),
         ...(rule.classes !== undefined ? { classes: [...rule.classes] } : {}),
         ...(rule.version !== undefined ? { ruleVersion: rule.version } : {}),
+        ...(rule.origin !== undefined ? { origin: rule.origin } : {}),
         ...(rule.needs !== undefined ? { saw: rule.needs.filter((n) => present.has(n)) } : {}),
         ...(skipClass !== undefined ? { skipClass } : {}),
         ...(uncertainty !== undefined ? { uncertainty } : {}),

package/dist/eval/text/checksums.d.ts

@@ -0,0 +1,23 @@
+/**
+ * The Luhn check digit, as used by every major card network. Sum the digits
+ * right to left, doubling every second one and subtracting nine when the
+ * double exceeds nine; a valid number is divisible by ten.
+ */
+export declare function luhn(candidate: string): boolean;
+/**
+ * ISO 13616 mod-97: move the first four characters to the end, replace each
+ * letter with its position in the alphabet plus nine, and read the result as
+ * one large integer; a valid account gives a remainder of one.
+ */
+export declare function iban(candidate: string): boolean;
+/**
+ * The structural rules the Social Security Administration has never issued
+ * against: an area of 000, 666 or 900–999; a group of 00; a serial of 0000.
+ * This is not a checksum — the number carries none — but it rejects the
+ * digit runs that cannot be an SSN, which is the same job.
+ *
+ * The canonical fake 123-45-6789 is deliberately NOT rejected here: it is a
+ * real-shaped number, it is what people paste to test the detector, and the
+ * rule's own comment explains why letting it through is the honest choice.
+ */
+export declare function ssnStructure(candidate: string): boolean;

package/dist/eval/text/checksums.js

@@ -0,0 +1,97 @@
+/*
+ * Structural checks for the three PII patterns whose shape alone is not
+ * evidence of anything.
+ *
+ * A sixteen-digit run is not a card number, a two-letter-plus-digits token
+ * is not an international bank account, and three-two-four digits are not a
+ * social security number. Each of those formats carries a check that a real
+ * value satisfies and an arbitrary digit run almost never does, and applying
+ * it turns a shape match into a structure match.
+ *
+ * Two reasons this matters now rather than later. First, precision: the
+ * card pattern fires on an order id, a hash prefix or a timestamp run, and
+ * every such fire is a false positive a deployment has to explain away.
+ * Second, the normalisation pass (arc 3, A3-2a) folds full-width and
+ * circled digits into ASCII, so text that never looked like a card number
+ * can become one — `①②③④…` is a sixteen-digit run after NFKC. The fold is
+ * what makes evasion detectable and the checksum is what stops the fold
+ * from manufacturing findings. They ship together on purpose.
+ *
+ * Every function here is total and side-effect free: given a string it
+ * returns a boolean, and a value it cannot parse is not valid.
+ */
+/**
+ * The Luhn check digit, as used by every major card network. Sum the digits
+ * right to left, doubling every second one and subtracting nine when the
+ * double exceeds nine; a valid number is divisible by ten.
+ */
+export function luhn(candidate) {
+    let sum = 0;
+    let double = false;
+    let digits = 0;
+    for (let i = candidate.length - 1; i >= 0; i--) {
+        const code = candidate.charCodeAt(i);
+        if (code < 48 || code > 57) {
+            // Separators a card number legitimately carries; anything else means
+            // this was never a card number.
+            if (candidate[i] === '-' || candidate[i] === ' ')
+                continue;
+            return false;
+        }
+        let d = code - 48;
+        digits++;
+        if (double) {
+            d *= 2;
+            if (d > 9)
+                d -= 9;
+        }
+        sum += d;
+        double = !double;
+    }
+    if (digits < 13 || digits > 19)
+        return false;
+    return sum % 10 === 0;
+}
+/**
+ * ISO 13616 mod-97: move the first four characters to the end, replace each
+ * letter with its position in the alphabet plus nine, and read the result as
+ * one large integer; a valid account gives a remainder of one.
+ */
+export function iban(candidate) {
+    const s = candidate.replace(/[\s-]/g, '').toUpperCase();
+    if (!/^[A-Z]{2}\d{2}[A-Z0-9]{10,30}$/.test(s))
+        return false;
+    const rearranged = s.slice(4) + s.slice(0, 4);
+    let remainder = 0;
+    for (const ch of rearranged) {
+        const code = ch.charCodeAt(0);
+        const part = code >= 65 && code <= 90 ? String(code - 55) : ch;
+        for (const digit of part) {
+            remainder = (remainder * 10 + (digit.charCodeAt(0) - 48)) % 97;
+        }
+    }
+    return remainder === 1;
+}
+/**
+ * The structural rules the Social Security Administration has never issued
+ * against: an area of 000, 666 or 900–999; a group of 00; a serial of 0000.
+ * This is not a checksum — the number carries none — but it rejects the
+ * digit runs that cannot be an SSN, which is the same job.
+ *
+ * The canonical fake 123-45-6789 is deliberately NOT rejected here: it is a
+ * real-shaped number, it is what people paste to test the detector, and the
+ * rule's own comment explains why letting it through is the honest choice.
+ */
+export function ssnStructure(candidate) {
+    const m = /^(\d{3})-(\d{2})-(\d{4})$/.exec(candidate.trim());
+    if (!m)
+        return false;
+    const [, area, group, serial] = m;
+    if (area === '000' || area === '666' || area[0] === '9')
+        return false;
+    if (group === '00')
+        return false;
+    if (serial === '0000')
+        return false;
+    return true;
+}

package/dist/eval/text/normalise.d.ts

@@ -0,0 +1,30 @@
+export interface Normalised {
+    /** The folded text every pattern should match against. */
+    text: string;
+    /**
+     * `map[i]` is the offset in the RAW string that normalised character `i`
+     * came from. Length is `text.length + 1`; the final entry is the raw
+     * length, so a normalised span `[s, e)` becomes the raw span
+     * `[map[s], map[e])` with no special case at the end of the string.
+     *
+     * Built on first read. A rule asks for it only when a pattern actually
+     * fires, and on a one-megabyte output the array is four megabytes — so
+     * the overwhelming majority of evaluations, which find nothing, never
+     * allocate it.
+     */
+    readonly map: Int32Array;
+    /**
+     * True when the fold changed nothing AND the map is the identity, so a
+     * caller can use normalised offsets as raw offsets directly.
+     */
+    unchanged: boolean;
+}
+/** Folds `raw` for matching and returns the offset map that puts evidence back on the raw text. */
+export declare function normalise(raw: string): Normalised;
+/**
+ * A span in normalised coordinates as a span in raw coordinates. Always
+ * widens rather than narrows: when characters were dropped between the last
+ * matched character and the next kept one, the raw span covers them, which
+ * is what a reader wants — the evasion is part of the evidence.
+ */
+export declare function toRawSpan(n: Normalised, start: number, end: number): [number, number];