@iris-eval/mcp-server 0.8.2 → 0.10.0

package/dist/eval/seeded-random.js

@@ -0,0 +1,36 @@
+/*
+ * Deterministic pseudo-randomness, for the credible interval on a verdict.
+ *
+ * The risk estimate draws each detector's sensitivity and specificity from
+ * a Beta posterior two thousand times to put an interval on p_bad. Those
+ * draws must be the same on every machine and every run, or the same
+ * evaluation would carry a different interval each time it was asked and
+ * the number would be unciteable. So: a fixed seed derived from the corpus
+ * version and the rules that spoke, and a generator that uses only integer
+ * operations and one divide.
+ *
+ * The proof harness has had the same two functions since the corpus
+ * shipped (`proof/lib/materialise.ts`), which is why they are written here
+ * rather than imported: the package ships `dist/` only, and a runtime read
+ * of anything under `proof/` finds nothing in an installed copy.
+ */
+/** FNV-1a, 32-bit: a string to a seed. */
+export function fnv1a(s) {
+    let h = 0x811c9dc5;
+    for (let i = 0; i < s.length; i++) {
+        h ^= s.charCodeAt(i);
+        h = Math.imul(h, 0x01000193) >>> 0;
+    }
+    return h >>> 0;
+}
+/** mulberry32: small, fast, deterministic across engines (integer ops + one divide). */
+export function mulberry32(seed) {
+    let a = seed >>> 0;
+    return () => {
+        a = (a + 0x6d2b79f5) >>> 0;
+        let t = a;
+        t = Math.imul(t ^ (t >>> 15), t | 1);
+        t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}

package/dist/eval/stamp.d.ts

@@ -0,0 +1,14 @@
+import type { EvalContext, EvalRule, EvalRuleResult, Need, SkipClass, Uncertainty } from '../types/eval.js';
+import type { EffectiveCriticality } from './criticality.js';
+/** Which needs the call actually carried. `tools_catalogue` and `citations` arrive with later releases. */
+export declare function inputsPresent(context: EvalContext): Set<Need>;
+export declare function skipClassOf(raw: EvalRuleResult): SkipClass | undefined;
+/**
+ * The uncertainty a result carries, by the kind of claim it makes. A skipped
+ * rule made no claim and gets none. The prior is the corpus default until a
+ * deployment states its own prevalence (the compose-by-kind release) or the
+ * own-traffic labels estimate one.
+ */
+export declare function uncertaintyOf(rule: EvalRule, raw: EvalRuleResult): Uncertainty | undefined;
+/** Everything the engine adds to a raw rule result besides ruleId, category and criticality. */
+export declare function stampRuleResult(rule: EvalRule, raw: EvalRuleResult, context: EvalContext, effective: EffectiveCriticality): Pick<EvalRuleResult, 'kind' | 'role' | 'question' | 'classes' | 'ruleVersion' | 'saw' | 'skipClass' | 'uncertainty' | 'origin'>;

package/dist/eval/stamp.js

@@ -0,0 +1,89 @@
+import { DEFAULT_PREVALENCE, missRateInterval, ppvInterval, publishedAccuracyFor, publishedProvenance } from './accuracy.js';
+/** Which needs the call actually carried. `tools_catalogue` and `citations` arrive with later releases. */
+export function inputsPresent(context) {
+    const present = new Set(['output']);
+    if (typeof context.input === 'string' && context.input.length > 0)
+        present.add('input');
+    if (typeof context.expected === 'string' && context.expected.length > 0)
+        present.add('expected');
+    if (Array.isArray(context.toolCalls) && context.toolCalls.length > 0) {
+        present.add('tool_calls');
+        if (context.toolCalls.some((c) => c && c.output !== undefined))
+            present.add('tool_outputs');
+    }
+    if (typeof context.costUsd === 'number')
+        present.add('cost');
+    if (context.tokenUsage && (context.tokenUsage.prompt_tokens !== undefined || context.tokenUsage.completion_tokens !== undefined || context.tokenUsage.total_tokens !== undefined)) {
+        present.add('tokens');
+    }
+    return present;
+}
+export function skipClassOf(raw) {
+    if (!raw.skipped)
+        return undefined;
+    if (raw.budgetExceeded)
+        return 'defeated';
+    if (raw.configInvalid)
+        return 'config_invalid';
+    return 'not_applicable';
+}
+/**
+ * The uncertainty a result carries, by the kind of claim it makes. A skipped
+ * rule made no claim and gets none. The prior is the corpus default until a
+ * deployment states its own prevalence (the compose-by-kind release) or the
+ * own-traffic labels estimate one.
+ */
+export function uncertaintyOf(rule, raw) {
+    if (raw.skipped || rule.kind === undefined)
+        return undefined;
+    switch (rule.kind) {
+        case 'policy':
+            return { basis: 'policy' };
+        case 'measurement': {
+            const published = publishedAccuracyFor(rule.name);
+            if (!published)
+                return { basis: 'unmeasured', why: 'no proof family for this rule' };
+            // A measurement's family checks that the formula is implemented right:
+            // its "accuracy" is conformance, not the badness of an output.
+            return { basis: 'definition', conformance: { n: published.n, matched: published.tp + published.tn } };
+        }
+        case 'detection':
+        case 'inference': {
+            const published = publishedAccuracyFor(rule.name);
+            if (!published)
+                return { basis: 'unmeasured', why: 'no proof family for this rule' };
+            const prov = publishedProvenance();
+            const corpus = { n: published.n, tp: published.tp, fp: published.fp, fn: published.fn, tn: published.tn, version: prov.corpusVersion, release: prov.release, labelling: prov.labelling };
+            const prior = { pi: DEFAULT_PREVALENCE, source: 'default' };
+            const fired = raw.passed === false;
+            const interval = fired ? ppvInterval(rule.name, DEFAULT_PREVALENCE) : missRateInterval(rule.name, DEFAULT_PREVALENCE);
+            if (!interval)
+                return { basis: 'unmeasured', why: 'the proof family has no positives or no negatives' };
+            return fired ? { basis: 'published_accuracy', fired: true, ppv: interval, prior, corpus } : { basis: 'published_accuracy', fired: false, missRate: interval, prior, corpus };
+        }
+        case 'judgment':
+            return { basis: 'unmeasured', why: 'judge accuracy is measurable on a key you supply (npm run proof:judge) and not yet published' };
+        case 'verification':
+            return { basis: 'unmeasured', why: 'verification accuracy is measurable on a key you supply and not yet published' };
+        default:
+            return undefined;
+    }
+}
+/** Everything the engine adds to a raw rule result besides ruleId, category and criticality. */
+export function stampRuleResult(rule, raw, context, effective) {
+    const present = inputsPresent(context);
+    const role = effective.critical ? 'veto' : 'term';
+    const skipClass = skipClassOf(raw);
+    const uncertainty = uncertaintyOf(rule, raw);
+    return {
+        ...(rule.kind !== undefined ? { kind: rule.kind } : {}),
+        role,
+        ...(rule.question !== undefined ? { question: rule.question } : {}),
+        ...(rule.classes !== undefined ? { classes: [...rule.classes] } : {}),
+        ...(rule.version !== undefined ? { ruleVersion: rule.version } : {}),
+        ...(rule.origin !== undefined ? { origin: rule.origin } : {}),
+        ...(rule.needs !== undefined ? { saw: rule.needs.filter((n) => present.has(n)) } : {}),
+        ...(skipClass !== undefined ? { skipClass } : {}),
+        ...(uncertainty !== undefined ? { uncertainty } : {}),
+    };
+}

package/dist/eval/stats.d.ts

@@ -0,0 +1,33 @@
+export declare function fnv1a(s: string): number;
+export declare function mulberry32(seed: number): () => number;
+/** A standard normal draw (Box–Muller) from a uniform generator; the uniform is kept away from 0. */
+export declare function normal(rng: () => number): number;
+/**
+ * Gamma(shape, 1) by Marsaglia–Tsang. For shape < 1 the standard boost:
+ * draw Gamma(shape + 1) and scale by U^(1/shape).
+ */
+export declare function gamma(shape: number, rng: () => number): number;
+/** Beta(a, b) as X / (X + Y) with X ~ Gamma(a), Y ~ Gamma(b). */
+export declare function beta(a: number, b: number, rng: () => number): number;
+/** The 2.5th and 97.5th percentiles of a sample (nearest-rank, sorted in place). */
+export declare function percentile95(values: number[]): [number, number];
+export interface Confusion {
+    tp: number;
+    fp: number;
+    fn: number;
+    tn: number;
+}
+/** Sensitivity (recall on the positive class) and specificity from a confusion matrix; null where the denominator is zero. */
+export declare function sensitivity(c: Confusion): number | null;
+export declare function specificity(c: Confusion): number | null;
+/**
+ * Positive predictive value at prevalence π: of the outputs the rule fires
+ * on, the share that are real violations, when a share π of all outputs are
+ * violations. The published precision is the PPV at the corpus prevalence
+ * (about one half); at one percent prevalence the same rule's fire is worth
+ * far less, and this is the arithmetic that says how much.
+ */
+export declare function ppv(sens: number, spec: number, prevalence: number): number;
+/** P(violation | the rule did not fire) at prevalence π — the residual miss rate. */
+export declare function missRate(sens: number, spec: number, prevalence: number): number;
+export declare const round4: (x: number) => number;

package/dist/eval/stats.js

@@ -0,0 +1,109 @@
+/*
+ * Small, seeded statistics for the product side of the proof.
+ *
+ * The proof harness (proof/lib/) computes the published intervals; this
+ * module lets the SHIPPED server reason with them: a seeded generator so a
+ * Monte Carlo interval is a pure function of its inputs (the same on every
+ * machine, every request), Gamma and Beta draws for posterior sampling, and
+ * the diagnostic-test arithmetic (sensitivity, specificity, positive
+ * predictive value at a prevalence) that turns a published confusion matrix
+ * into "how often a fire is right for you".
+ *
+ * `fnv1a` and `mulberry32` are byte-identical twins of proof/lib/materialise.ts
+ * (src/ cannot import proof/); tests/unit/eval/stats.test.ts pins the two
+ * pairs to each other on a fixed seed.
+ */
+export function fnv1a(s) {
+    let h = 0x811c9dc5;
+    for (let i = 0; i < s.length; i++) {
+        h ^= s.charCodeAt(i);
+        h = Math.imul(h, 0x01000193) >>> 0;
+    }
+    return h >>> 0;
+}
+export function mulberry32(seed) {
+    let a = seed >>> 0;
+    return () => {
+        a = (a + 0x6d2b79f5) >>> 0;
+        let t = a;
+        t = Math.imul(t ^ (t >>> 15), t | 1);
+        t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
+/** A standard normal draw (Box–Muller) from a uniform generator; the uniform is kept away from 0. */
+export function normal(rng) {
+    let u = 0;
+    while (u === 0)
+        u = rng();
+    const v = rng();
+    return Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v);
+}
+/**
+ * Gamma(shape, 1) by Marsaglia–Tsang. For shape < 1 the standard boost:
+ * draw Gamma(shape + 1) and scale by U^(1/shape).
+ */
+export function gamma(shape, rng) {
+    if (!(shape > 0))
+        throw new Error(`gamma: shape must be positive, got ${shape}`);
+    if (shape < 1) {
+        let u = 0;
+        while (u === 0)
+            u = rng();
+        return gamma(shape + 1, rng) * Math.pow(u, 1 / shape);
+    }
+    const d = shape - 1 / 3;
+    const c = 1 / Math.sqrt(9 * d);
+    for (;;) {
+        let x;
+        let v;
+        do {
+            x = normal(rng);
+            v = 1 + c * x;
+        } while (v <= 0);
+        v = v * v * v;
+        const u = rng();
+        if (u < 1 - 0.0331 * x * x * x * x)
+            return d * v;
+        if (Math.log(u) < 0.5 * x * x + d * (1 - v + Math.log(v)))
+            return d * v;
+    }
+}
+/** Beta(a, b) as X / (X + Y) with X ~ Gamma(a), Y ~ Gamma(b). */
+export function beta(a, b, rng) {
+    const x = gamma(a, rng);
+    const y = gamma(b, rng);
+    return x / (x + y);
+}
+/** The 2.5th and 97.5th percentiles of a sample (nearest-rank, sorted in place). */
+export function percentile95(values) {
+    const sorted = [...values].sort((p, q) => p - q);
+    const at = (q) => sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(q * sorted.length) - 1))];
+    return [at(0.025), at(0.975)];
+}
+/** Sensitivity (recall on the positive class) and specificity from a confusion matrix; null where the denominator is zero. */
+export function sensitivity(c) {
+    return c.tp + c.fn === 0 ? null : c.tp / (c.tp + c.fn);
+}
+export function specificity(c) {
+    return c.tn + c.fp === 0 ? null : c.tn / (c.tn + c.fp);
+}
+/**
+ * Positive predictive value at prevalence π: of the outputs the rule fires
+ * on, the share that are real violations, when a share π of all outputs are
+ * violations. The published precision is the PPV at the corpus prevalence
+ * (about one half); at one percent prevalence the same rule's fire is worth
+ * far less, and this is the arithmetic that says how much.
+ */
+export function ppv(sens, spec, prevalence) {
+    const truePos = sens * prevalence;
+    const falsePos = (1 - spec) * (1 - prevalence);
+    return truePos + falsePos === 0 ? 0 : truePos / (truePos + falsePos);
+}
+/** P(violation | the rule did not fire) at prevalence π — the residual miss rate. */
+export function missRate(sens, spec, prevalence) {
+    const missed = (1 - sens) * prevalence;
+    const trueNeg = spec * (1 - prevalence);
+    return missed + trueNeg === 0 ? 0 : missed / (missed + trueNeg);
+}
+export const round4 = (x) => Math.round(x * 10_000) / 10_000;

package/dist/eval/text/checksums.d.ts

@@ -0,0 +1,23 @@
+/**
+ * The Luhn check digit, as used by every major card network. Sum the digits
+ * right to left, doubling every second one and subtracting nine when the
+ * double exceeds nine; a valid number is divisible by ten.
+ */
+export declare function luhn(candidate: string): boolean;
+/**
+ * ISO 13616 mod-97: move the first four characters to the end, replace each
+ * letter with its position in the alphabet plus nine, and read the result as
+ * one large integer; a valid account gives a remainder of one.
+ */
+export declare function iban(candidate: string): boolean;
+/**
+ * The structural rules the Social Security Administration has never issued
+ * against: an area of 000, 666 or 900–999; a group of 00; a serial of 0000.
+ * This is not a checksum — the number carries none — but it rejects the
+ * digit runs that cannot be an SSN, which is the same job.
+ *
+ * The canonical fake 123-45-6789 is deliberately NOT rejected here: it is a
+ * real-shaped number, it is what people paste to test the detector, and the
+ * rule's own comment explains why letting it through is the honest choice.
+ */
+export declare function ssnStructure(candidate: string): boolean;

package/dist/eval/text/checksums.js

@@ -0,0 +1,97 @@
+/*
+ * Structural checks for the three PII patterns whose shape alone is not
+ * evidence of anything.
+ *
+ * A sixteen-digit run is not a card number, a two-letter-plus-digits token
+ * is not an international bank account, and three-two-four digits are not a
+ * social security number. Each of those formats carries a check that a real
+ * value satisfies and an arbitrary digit run almost never does, and applying
+ * it turns a shape match into a structure match.
+ *
+ * Two reasons this matters now rather than later. First, precision: the
+ * card pattern fires on an order id, a hash prefix or a timestamp run, and
+ * every such fire is a false positive a deployment has to explain away.
+ * Second, the normalisation pass (arc 3, A3-2a) folds full-width and
+ * circled digits into ASCII, so text that never looked like a card number
+ * can become one — `①②③④…` is a sixteen-digit run after NFKC. The fold is
+ * what makes evasion detectable and the checksum is what stops the fold
+ * from manufacturing findings. They ship together on purpose.
+ *
+ * Every function here is total and side-effect free: given a string it
+ * returns a boolean, and a value it cannot parse is not valid.
+ */
+/**
+ * The Luhn check digit, as used by every major card network. Sum the digits
+ * right to left, doubling every second one and subtracting nine when the
+ * double exceeds nine; a valid number is divisible by ten.
+ */
+export function luhn(candidate) {
+    let sum = 0;
+    let double = false;
+    let digits = 0;
+    for (let i = candidate.length - 1; i >= 0; i--) {
+        const code = candidate.charCodeAt(i);
+        if (code < 48 || code > 57) {
+            // Separators a card number legitimately carries; anything else means
+            // this was never a card number.
+            if (candidate[i] === '-' || candidate[i] === ' ')
+                continue;
+            return false;
+        }
+        let d = code - 48;
+        digits++;
+        if (double) {
+            d *= 2;
+            if (d > 9)
+                d -= 9;
+        }
+        sum += d;
+        double = !double;
+    }
+    if (digits < 13 || digits > 19)
+        return false;
+    return sum % 10 === 0;
+}
+/**
+ * ISO 13616 mod-97: move the first four characters to the end, replace each
+ * letter with its position in the alphabet plus nine, and read the result as
+ * one large integer; a valid account gives a remainder of one.
+ */
+export function iban(candidate) {
+    const s = candidate.replace(/[\s-]/g, '').toUpperCase();
+    if (!/^[A-Z]{2}\d{2}[A-Z0-9]{10,30}$/.test(s))
+        return false;
+    const rearranged = s.slice(4) + s.slice(0, 4);
+    let remainder = 0;
+    for (const ch of rearranged) {
+        const code = ch.charCodeAt(0);
+        const part = code >= 65 && code <= 90 ? String(code - 55) : ch;
+        for (const digit of part) {
+            remainder = (remainder * 10 + (digit.charCodeAt(0) - 48)) % 97;
+        }
+    }
+    return remainder === 1;
+}
+/**
+ * The structural rules the Social Security Administration has never issued
+ * against: an area of 000, 666 or 900–999; a group of 00; a serial of 0000.
+ * This is not a checksum — the number carries none — but it rejects the
+ * digit runs that cannot be an SSN, which is the same job.
+ *
+ * The canonical fake 123-45-6789 is deliberately NOT rejected here: it is a
+ * real-shaped number, it is what people paste to test the detector, and the
+ * rule's own comment explains why letting it through is the honest choice.
+ */
+export function ssnStructure(candidate) {
+    const m = /^(\d{3})-(\d{2})-(\d{4})$/.exec(candidate.trim());
+    if (!m)
+        return false;
+    const [, area, group, serial] = m;
+    if (area === '000' || area === '666' || area[0] === '9')
+        return false;
+    if (group === '00')
+        return false;
+    if (serial === '0000')
+        return false;
+    return true;
+}

package/dist/eval/text/normalise.d.ts

@@ -0,0 +1,30 @@
+export interface Normalised {
+    /** The folded text every pattern should match against. */
+    text: string;
+    /**
+     * `map[i]` is the offset in the RAW string that normalised character `i`
+     * came from. Length is `text.length + 1`; the final entry is the raw
+     * length, so a normalised span `[s, e)` becomes the raw span
+     * `[map[s], map[e])` with no special case at the end of the string.
+     *
+     * Built on first read. A rule asks for it only when a pattern actually
+     * fires, and on a one-megabyte output the array is four megabytes — so
+     * the overwhelming majority of evaluations, which find nothing, never
+     * allocate it.
+     */
+    readonly map: Int32Array;
+    /**
+     * True when the fold changed nothing AND the map is the identity, so a
+     * caller can use normalised offsets as raw offsets directly.
+     */
+    unchanged: boolean;
+}
+/** Folds `raw` for matching and returns the offset map that puts evidence back on the raw text. */
+export declare function normalise(raw: string): Normalised;
+/**
+ * A span in normalised coordinates as a span in raw coordinates. Always
+ * widens rather than narrows: when characters were dropped between the last
+ * matched character and the next kept one, the raw span covers them, which
+ * is what a reader wants — the evasion is part of the evidence.
+ */
+export declare function toRawSpan(n: Normalised, start: number, end: number): [number, number];