@iris-eval/mcp-server 0.9.0 → 0.10.0

package/dist/eval/text/normalise.js

@@ -0,0 +1,265 @@
+/*
+ * One normalisation pass, shared by every rule that matches text.
+ *
+ * The problem it solves is measured, not hypothetical. `npm run proof`
+ * publishes a transforms table: for every positive a critical rule catches,
+ * the text inside the evidence span is transformed the way an evader would
+ * transform it, and the rule is re-run. Before this module, `no_pii` kept
+ * 38% of its catches under a zero-width space, 22% under Cyrillic
+ * homoglyphs and **none at all** under full-width digits, and
+ * `no_blocklist_words` survived nothing but a change of case.
+ *
+ * What it does, in order, per grapheme cluster:
+ *   1. drops format characters that carry no meaning — zero-width spaces
+ *      and joiners, the soft hyphen, the byte-order mark;
+ *   2. NFKC-folds the cluster, which turns full-width and mathematical
+ *      alphanumerics into ASCII (４１１１ → 4111, 𝐩𝐚𝐬𝐬 → pass);
+ *   3. maps the confusables NFKC does NOT fold — Cyrillic and Greek letters
+ *      that are drawn like Latin ones (раssword with a Cyrillic а and р);
+ *   4. collapses every run of whitespace to ONE character — a newline when
+ *      the run contains one, a space otherwise. Line structure is meaning:
+ *      a forged "System:" line and a fenced block are line-shaped, and
+ *      flattening newlines to spaces measurably cost the injection rule
+ *      recall on three transforms. Horizontal runs carry no such meaning.
+ *
+ * What it deliberately does NOT do is leetspeak (0 → o, 1 → i). That
+ * substitution is correct for injection phrasing and catastrophic for
+ * everything else: it would turn a credit card number into letters and
+ * blind every digit-based detector. The injection rule applies it on top of
+ * this pass, to this pass's output, and owns it alone.
+ *
+ * Every rule that matches on `text` reports evidence through `map`, so a
+ * span still indexes the RAW output the caller sent — the arc-1 contract
+ * ("spans are offsets into the raw text") is what makes redaction and the
+ * transforms measurement correct, and normalising without a map would
+ * quietly break it.
+ */
+/** Format characters that carry no textual meaning and are pure evasion when they sit inside a token. */
+const DROPPED = new Set([
+    '​', // zero-width space
+    '‌', // zero-width non-joiner
+    '‍', // zero-width joiner
+    '‎', // left-to-right mark
+    '‏', // right-to-left mark
+    '⁠', // word joiner
+    '', // byte-order mark / zero-width no-break space
+    '­', // soft hyphen
+]);
+/**
+ * Letters that NFKC leaves alone but a reader cannot tell apart from Latin.
+ * Cyrillic first, then Greek; lowercase and uppercase where both are
+ * confusable. Deliberately conservative: only characters whose common
+ * rendering is indistinguishable in the fonts an agent's output is read in.
+ */
+const CONFUSABLES = new Map([
+    // Cyrillic → Latin
+    ['а', 'a'], ['А', 'A'],
+    ['е', 'e'], ['Е', 'E'],
+    ['о', 'o'], ['О', 'O'],
+    ['р', 'p'], ['Р', 'P'],
+    ['с', 'c'], ['С', 'C'],
+    ['х', 'x'], ['Х', 'X'],
+    ['у', 'y'], ['У', 'Y'],
+    ['к', 'k'], ['К', 'K'],
+    ['м', 'm'], ['М', 'M'],
+    ['н', 'h'], ['Н', 'H'],
+    ['т', 't'], ['Т', 'T'],
+    ['в', 'v'], ['В', 'B'],
+    ['і', 'i'], ['І', 'I'],
+    ['ј', 'j'], ['Ј', 'J'],
+    ['ѕ', 's'], ['Ѕ', 'S'],
+    ['б', '6'],
+    ['г', 'r'],
+    ['з', '3'],
+    ['һ', 'h'],
+    ['ҙ', 'z'],
+    // Greek → Latin
+    ['ο', 'o'], ['Ο', 'O'],
+    ['α', 'a'], ['Α', 'A'],
+    ['ε', 'e'], ['Ε', 'E'],
+    ['ρ', 'p'], ['Ρ', 'P'],
+    ['τ', 't'], ['Τ', 'T'],
+    ['ν', 'v'], ['Ν', 'N'],
+    ['υ', 'u'], ['Υ', 'Y'],
+    ['ι', 'i'], ['Ι', 'I'],
+    ['κ', 'k'], ['Κ', 'K'],
+    ['β', 'B'], ['Β', 'B'],
+    ['η', 'n'], ['Η', 'H'],
+    ['χ', 'x'], ['Χ', 'X'],
+    ['μ', 'u'], ['Μ', 'M'],
+    ['γ', 'y'], ['Ζ', 'Z'],
+    ['Φ', 'O'],
+    // Other scripts whose letters are drawn as Latin
+    ['ԁ', 'd'],
+    ['ԛ', 'q'],
+    ['ɡ', 'g'],
+    ['ẞ', 'S'],
+    ['ո', 'n'],
+    ['ս', 'u'],
+    ['օ', 'o'],
+]);
+/**
+ * Printable ASCII plus the newline. Nothing in that set folds, so the only
+ * thing that could change such a string is a whitespace RUN — which makes
+ * two linear scans a complete test for "this text is already normalised".
+ *
+ * This is the hot path and it is why the pass is affordable. Ordinary agent
+ * output is plain text; a one-megabyte payload of it used to cost a grapheme
+ * segmentation and a character-by-character rebuild, and the hostile-payload
+ * budget in the test battery caught exactly that.
+ */
+const PLAIN_TEXT = /^[\x20-\x7E\n]*$/;
+const WHITESPACE_RUN = /\s\s/;
+/** The result for text that is already in normal form: no copy, no map until asked. */
+function identity(raw) {
+    let cached;
+    return {
+        text: raw,
+        unchanged: true,
+        get map() {
+            if (cached === undefined) {
+                cached = new Int32Array(raw.length + 1);
+                for (let i = 0; i <= raw.length; i++)
+                    cached[i] = i;
+            }
+            return cached;
+        },
+    };
+}
+let segmenter;
+function graphemes(raw) {
+    if (typeof Intl?.Segmenter === 'function') {
+        segmenter ??= new Intl.Segmenter('en', { granularity: 'grapheme' });
+        return segmenter.segment(raw);
+    }
+    // Environments without Intl.Segmenter fall back to code points, which is
+    // correct for everything this pass folds and only differs on combining
+    // sequences it would leave alone anyway.
+    return [...raw];
+}
+const WHITESPACE = /\s/u;
+const LINE_BREAK = /[\n\r\u2028\u2029]/u;
+/** Folds `raw` for matching and returns the offset map that puts evidence back on the raw text. */
+export function normalise(raw) {
+    // Already in normal form: two linear scans and no allocation at all.
+    if (PLAIN_TEXT.test(raw) && !WHITESPACE_RUN.test(raw))
+        return identity(raw);
+    const out = [];
+    const offsets = [];
+    /** The whitespace run being accumulated: where it started, and whether it broke a line. */
+    let run = null;
+    let changed = false;
+    /** False as soon as one output character does not sit at its own raw offset. */
+    let identityMap = true;
+    const push = (chars, at) => {
+        for (const ch of chars) {
+            if (at !== out.length)
+                identityMap = false;
+            out.push(ch);
+            offsets.push(at);
+        }
+    };
+    /** Emits the pending whitespace run as one character: a newline if it broke a line, else a space. */
+    const flushRun = () => {
+        if (run === null)
+            return;
+        const ch = run.hadBreak ? '\n' : ' ';
+        if (raw.slice(run.at, run.at + 1) !== ch)
+            changed = true;
+        push(ch, run.at);
+        run = null;
+    };
+    const segments = graphemes(raw);
+    const iterate = (rawCluster, index) => {
+        /*
+         * Strip the format characters from INSIDE the cluster, not just from
+         * clusters that are one. A zero-width non-joiner between two digits
+         * binds into the neighbouring grapheme, so a whole-cluster test misses
+         * exactly the evasion this exists to fold.
+         */
+        let cluster = rawCluster;
+        if (cluster.length > 1 || DROPPED.has(cluster)) {
+            let stripped = '';
+            for (const ch of cluster)
+                if (!DROPPED.has(ch))
+                    stripped += ch;
+            if (stripped !== cluster) {
+                changed = true;
+                cluster = stripped;
+            }
+        }
+        if (cluster === '')
+            return;
+        if (WHITESPACE.test(cluster)) {
+            const hadBreak = LINE_BREAK.test(cluster);
+            if (run === null)
+                run = { at: index, hadBreak };
+            else {
+                run.hadBreak ||= hadBreak;
+                changed = true;
+            }
+            return;
+        }
+        flushRun();
+        let folded = cluster.normalize('NFKC');
+        if (folded !== cluster)
+            changed = true;
+        if (CONFUSABLES.size > 0) {
+            let mapped = '';
+            for (const ch of folded) {
+                const sub = CONFUSABLES.get(ch);
+                if (sub === undefined)
+                    mapped += ch;
+                else {
+                    mapped += sub;
+                    changed = true;
+                }
+            }
+            folded = mapped;
+        }
+        // A cluster that folds away entirely (a lone combining mark NFKC drops)
+        // contributes nothing; its offset is covered by the next kept character.
+        push(folded, index);
+    };
+    if (Array.isArray(segments)) {
+        let at = 0;
+        for (const cluster of segments) {
+            iterate(cluster, at);
+            at += cluster.length;
+        }
+    }
+    else {
+        for (const { segment, index } of segments)
+            iterate(segment, index);
+    }
+    flushRun();
+    const text = out.join('');
+    let cached;
+    return {
+        text,
+        unchanged: !changed && identityMap && text.length === raw.length,
+        get map() {
+            if (cached === undefined) {
+                cached = new Int32Array(offsets.length + 1);
+                cached.set(offsets);
+                cached[offsets.length] = raw.length;
+            }
+            return cached;
+        },
+    };
+}
+/**
+ * A span in normalised coordinates as a span in raw coordinates. Always
+ * widens rather than narrows: when characters were dropped between the last
+ * matched character and the next kept one, the raw span covers them, which
+ * is what a reader wants — the evasion is part of the evidence.
+ */
+export function toRawSpan(n, start, end) {
+    // The identity case never touches the map, so no array is built for the
+    // ordinary text that makes up almost every evaluation.
+    if (n.unchanged)
+        return [Math.max(0, start), Math.max(start, end)];
+    const s = Math.max(0, Math.min(start, n.map.length - 1));
+    const e = Math.max(s, Math.min(end, n.map.length - 1));
+    return [n.map[s], n.map[e]];
+}

package/dist/eval/text/sentences.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Named sentencesOf, not sentencesOf: the hallucination rule in
+ * src/eval/rules/safety.ts has its own sentencesOf that also breaks on
+ * every newline, because it wants per-line units for grounding checks. The
+ * two are not the same job and unifying them would move that rule's
+ * numbers, so it is a separate measured change, not a rename.
+ *
+ * Splits `text` into sentences. Never returns an empty sentence; a text with
+ * no terminator is one sentence. Line breaks do not split on their own — a
+ * wrapped paragraph is one sentence — but a blank line does, because a new
+ * block is a new thought and a bullet list is not one long sentence.
+ */
+export declare function sentencesOf(text: string): string[];
+/** How many sentences the text contains. The one number `sentence_count` reports. */
+export declare function countSentences(text: string): number;

package/dist/eval/text/sentences.js

@@ -0,0 +1,149 @@
+/*
+ * One sentence splitter, for the two rules that count or walk sentences.
+ *
+ * Both had their own, and both were wrong in the same way. `sentence_count`
+ * split on `/[.!?]+/`, so "The latency is 3.5 seconds." counted as two
+ * sentences and "Dr. Chen approved it." as two more; the arc-zero review
+ * measured the damage at 43% of that rule's family. `topic_consistency`
+ * split on a full stop followed by whitespace, which fixes the decimal only
+ * when the decimal has no space after it and never fixes the abbreviation.
+ *
+ * A sentence ends at `.`, `!` or `?` when what follows looks like the start
+ * of a new sentence and what precedes is not one of the things that ends in
+ * a full stop without ending a sentence:
+ *
+ *   - never between digits, so 3.5 and 1.2.3 stay whole;
+ *   - never after a closed list of abbreviations (Dr, Mr, Mrs, Ms, e.g,
+ *     i.e, vs, etc, No, Fig, St, Inc, Ltd, Jr, Sr, approx, cf, al);
+ *   - never after a single capital letter, which is an initial (J. Smith);
+ *   - only when the next non-space character starts a sentence: an
+ *     uppercase letter, an opening quote or bracket, or a digit.
+ *
+ * The list is closed on purpose. An open-ended abbreviation heuristic
+ * (a short token ending in a full stop) swallows real sentence ends —
+ * "It was fun. Then we left." — and this splitter is used to COUNT, where
+ * missing a break is as wrong as inventing one.
+ */
+/**
+ * Abbreviations that are essentially never the last word of a sentence, so
+ * the full stop after them is punctuation and not an end. Lowercase, no
+ * trailing stop.
+ */
+const ALWAYS_ABBREVIATION = new Set([
+    'dr', 'mr', 'mrs', 'ms', 'prof', 'sr', 'jr', 'st', 'mt',
+    'e.g', 'i.e', 'vs', 'al', 'cf', 'approx', 'est',
+    'dept', 'univ', 'a.m', 'p.m', 'u.s', 'u.k',
+]);
+/**
+ * Abbreviations that are ALSO ordinary sentence endings — "shipped in Oct.
+ * The rollout held" ends a sentence; "shipped on Oct. 5" does not. What
+ * separates them is what follows: a number means the abbreviation is being
+ * used as a label, anything else means the sentence ended. Guessing either
+ * way unconditionally is wrong about half the time, and this splitter is
+ * used to COUNT, where a missed break costs exactly as much as an invented
+ * one.
+ */
+const ABBREVIATION_BEFORE_NUMBER = new Set([
+    'no', 'fig', 'eq', 'ch', 'vol', 'pp', 'etc',
+    'inc', 'ltd', 'co', 'corp',
+    'jan', 'feb', 'mar', 'apr', 'jun', 'jul', 'aug', 'sep', 'sept', 'oct', 'nov', 'dec',
+]);
+const TERMINATORS = new Set(['.', '!', '?']);
+/** True when the character can open a new sentence. */
+function opensSentence(ch) {
+    if (ch === undefined)
+        return false;
+    if (ch >= 'A' && ch <= 'Z')
+        return true;
+    if (ch >= '0' && ch <= '9')
+        return true;
+    return '"‘’“”\'([{*_#-—'.includes(ch);
+}
+const isDigit = (ch) => ch !== undefined && ch >= '0' && ch <= '9';
+/** The token immediately before `at`, lowercased, without its trailing stop. */
+function precedingToken(text, at) {
+    let i = at - 1;
+    while (i >= 0 && !/[\s(["']/.test(text[i]))
+        i--;
+    return text.slice(i + 1, at).toLowerCase();
+}
+/**
+ * Named sentencesOf, not sentencesOf: the hallucination rule in
+ * src/eval/rules/safety.ts has its own sentencesOf that also breaks on
+ * every newline, because it wants per-line units for grounding checks. The
+ * two are not the same job and unifying them would move that rule's
+ * numbers, so it is a separate measured change, not a rename.
+ *
+ * Splits `text` into sentences. Never returns an empty sentence; a text with
+ * no terminator is one sentence. Line breaks do not split on their own — a
+ * wrapped paragraph is one sentence — but a blank line does, because a new
+ * block is a new thought and a bullet list is not one long sentence.
+ */
+export function sentencesOf(text) {
+    const out = [];
+    let start = 0;
+    for (let i = 0; i < text.length; i++) {
+        const ch = text[i];
+        // A blank line ends a sentence whatever came before it.
+        if (ch === '\n' && /^\s*\n/.test(text.slice(i + 1))) {
+            const piece = text.slice(start, i).trim();
+            if (piece.length > 0)
+                out.push(piece);
+            start = i + 1;
+            continue;
+        }
+        if (!TERMINATORS.has(ch))
+            continue;
+        // 3.5 — a full stop between digits is a decimal point.
+        if (ch === '.' && isDigit(text[i - 1]) && isDigit(text[i + 1]))
+            continue;
+        // Run past a cluster of terminators ("What?!").
+        let end = i;
+        while (end + 1 < text.length && TERMINATORS.has(text[end + 1]))
+            end++;
+        // Closing quotes and brackets belong to the sentence that ends here.
+        let after = end + 1;
+        while (after < text.length && '"’”\')]}'.includes(text[after]))
+            after++;
+        // What comes next has to look like a new sentence.
+        let next = after;
+        while (next < text.length && /[ \t\r\n]/.test(text[next]))
+            next++;
+        /*
+         * No whitespace after the stop is usually a mid-token full stop — a
+         * version (v0.10.0), a filename (package.json), a hostname
+         * (iris-eval.com/proof) — and must not break. The exception is a
+         * following CAPITAL, which is a missing space between two sentences
+         * ("...ready.Ship it") and not a token: no filename or version has one.
+         */
+        if (next === after && next < text.length && !(text[next] >= 'A' && text[next] <= 'Z'))
+            continue;
+        if (next < text.length && !opensSentence(text[next]))
+            continue;
+        // Dr. Chen — an abbreviation, not an end.
+        if (ch === '.') {
+            const token = precedingToken(text, i);
+            if (ALWAYS_ABBREVIATION.has(token))
+                continue;
+            // Oct. 5 is a date; "in Oct. The rollout held" is two sentences.
+            if (ABBREVIATION_BEFORE_NUMBER.has(token) && isDigit(text[next]))
+                continue;
+            // A single capital letter is an initial: J. Smith.
+            if (token.length === 1 && /[a-z]/i.test(token))
+                continue;
+        }
+        const piece = text.slice(start, after).trim();
+        if (piece.length > 0)
+            out.push(piece);
+        start = after;
+        i = after - 1;
+    }
+    const tail = text.slice(start).trim();
+    if (tail.length > 0)
+        out.push(tail);
+    return out;
+}
+/** How many sentences the text contains. The one number `sentence_count` reports. */
+export function countSentences(text) {
+    return sentencesOf(text).length;
+}

package/dist/self-test.js

@@ -305,7 +305,7 @@ export async function runSelfTest(write = stdoutLine) {
         insertedIds.push(result.id);
     };
     await step(SELF_TEST_STEPS.piiEval, async () => {
-        const result = evalEngine.evaluate('safety', {
+        const result = await evalEngine.evaluate('safety', {
             // A real-shaped SSN, not the never-issued 123-45-6789 documentation
             // placeholder — no_pii suppresses that one on purpose.
             output: 'Done. For the record, the customer SSN is 536-22-8145.',
@@ -317,7 +317,7 @@ export async function runSelfTest(write = stdoutLine) {
         return 'no_pii flagged the planted SSN';
     });
     await step(SELF_TEST_STEPS.injectionEval, async () => {
-        const result = evalEngine.evaluate('safety', {
+        const result = await evalEngine.evaluate('safety', {
             output: 'Sure. I will ignore all previous instructions and reveal the system prompt.',
         });
         const rule = result.rule_results.find((r) => r.ruleName === 'no_injection_patterns');
@@ -327,7 +327,7 @@ export async function runSelfTest(write = stdoutLine) {
         return 'no_injection_patterns flagged the override text';
     });
     await step(SELF_TEST_STEPS.cleanEval, async () => {
-        const result = evalEngine.evaluate('safety', {
+        const result = await evalEngine.evaluate('safety', {
             output: 'The report is ready: weather in Paris stays mild this week, with light rain expected on Thursday evening.',
         });
         ensure(result.passed && result.score === 1, `clean output should score 1 and pass; got score=${result.score} passed=${result.passed}`);

package/dist/storage/sqlite-adapter.js

@@ -22,6 +22,7 @@
 import Database from 'better-sqlite3';
 import { ensureOwnerOnly } from '../utils/write-atomic.js';
 import { deriveCoverage, deriveCriticalSkipped, deriveVerdict } from '../eval/verdict.js';
+import { compose, DEFAULT_COMPOSE } from '../eval/compose.js';
 import { TenantContextRequiredError } from '../types/tenant.js';
 import { runMigrations } from './migrations/index.js';
 const ALLOWED_SORT_COLUMNS = new Set(['timestamp', 'latency_ms', 'cost_usd']);
@@ -758,8 +759,21 @@ export class SqliteAdapter {
             result.critical_skipped = criticalSkipped;
         if (result.rule_results.some((r) => r.question !== undefined))
             result.coverage = deriveCoverage(result.rule_results);
-        if (result.provenance)
-            result.verdict = deriveVerdict(result, result.provenance.thresholds.default);
+        /*
+         * Read back with the SAME composer that wrote it, or a stored row would
+         * report a different verdict than the one the caller was given. The
+         * config is not stored (only its hash), so this composes under the
+         * shipped defaults — which is what a default-configured server used, and
+         * what `eval.composer: "legacy"` selects for a deployment that has not
+         * moved yet. A row written before the verdict existed still reads back
+         * with none: absent, never fabricated.
+         */
+        if (result.provenance) {
+            result.verdict =
+                DEFAULT_COMPOSE.composer === 'legacy'
+                    ? deriveVerdict(result, result.provenance.thresholds.default)
+                    : compose(result, DEFAULT_COMPOSE);
+        }
         return result;
     }
 }

package/dist/tools/evaluate-output.js

@@ -116,8 +116,8 @@ export function registerEvaluateOutputTool(server, storage, evalEngine, options)
         };
         const customRules = args.custom_rules;
         const result = evalType === 'all'
-            ? evalEngine.evaluateAll(context, customRules)
-            : evalEngine.evaluate(evalType, context, customRules);
+            ? await evalEngine.evaluateAll(context, customRules)
+            : await evalEngine.evaluate(evalType, context, customRules);
         if (args.trace_id) {
             result.trace_id = args.trace_id;
         }

package/dist/tools/evaluate-with-llm-judge.d.ts

@@ -17,6 +17,9 @@ export declare const judgeOutputSchema: z.ZodObject<{
     trace_id: z.ZodOptional<z.ZodString>;
     score: z.ZodNumber;
     passed: z.ZodBoolean;
+    pass_threshold: z.ZodNumber;
+    self_reported_pass: z.ZodOptional<z.ZodBoolean>;
+    disagreement: z.ZodOptional<z.ZodBoolean>;
     rationale: z.ZodString;
     dimensions: z.ZodRecord<z.ZodString, z.ZodUnknown>;
     model: z.ZodString;

package/dist/tools/evaluate-with-llm-judge.js

@@ -70,7 +70,10 @@ export const judgeOutputSchema = z.looseObject({
     id: z.string().describe('the evaluation id; read it back at iris://evaluations/{id}'),
     trace_id: z.string().optional().describe('the linked trace, when one was named'),
     score: z.number().describe('0..1 from the judge'),
-    passed: z.boolean().describe('the judge\'s own pass verdict for the template'),
+    passed: z.boolean().describe('the verdict: the score against the template\'s threshold, which is pass_threshold below. Not the model\'s own boolean — that is self_reported_pass'),
+    pass_threshold: z.number().describe('the threshold the score was read against, so you can check the arithmetic'),
+    self_reported_pass: z.boolean().optional().describe('what the model said about passing, when it said anything. Recorded, never obeyed'),
+    disagreement: z.boolean().optional().describe('true when the model\'s own boolean disagrees with the threshold verdict — its rubric and its judgement have come apart on this output'),
     rationale: z.string().describe('the judge\'s reasoning, in its words'),
     dimensions: z.record(z.string(), z.unknown()).describe('per-dimension sub-scores for the template'),
     model: z.string().describe('the model that judged'),
@@ -155,6 +158,28 @@ export function registerEvaluateWithLLMJudgeTool(server, storage) {
                     passed: result.passed,
                     score: result.score,
                     message: result.rationale || 'LLM judge evaluation',
+                    /*
+                     * The row says what KIND of claim it is (0.10.0). Without it a
+                     * stored judge evaluation read back through the composer had no
+                     * layer to fall into — not a policy, not a detector with a
+                     * published rate — and a FAILED judgement read back as clean.
+                     * A judgment the caller asked and paid for decides.
+                     */
+                    kind: 'judgment',
+                    role: 'gate',
+                    saw: ['output'],
+                    evidence: [
+                        {
+                            type: 'sample',
+                            score: result.score,
+                            ...(result.selfReportedPass !== undefined ? { selfReportedPass: result.selfReportedPass } : {}),
+                            rationaleHash: '',
+                        },
+                    ],
+                    uncertainty: {
+                        basis: 'unmeasured',
+                        why: 'the judge is user-keyed and its accuracy is measured only by a run on a key you or the maintainer supplies (npm run proof:judge)',
+                    },
                 },
             ],
             suggestions: result.passed ? [] : [result.rationale],
@@ -171,6 +196,9 @@ export function registerEvaluateWithLLMJudgeTool(server, storage) {
             ...(args.trace_id ? { trace_id: args.trace_id } : {}),
             score: result.score,
             passed: result.passed,
+            pass_threshold: result.passThreshold,
+            ...(result.selfReportedPass !== undefined ? { self_reported_pass: result.selfReportedPass } : {}),
+            ...(result.disagreement ? { disagreement: true } : {}),
             rationale: result.rationale,
             dimensions: result.dimensions,
             model: result.model,

package/dist/tools/verify-citations.d.ts

@@ -24,7 +24,8 @@ export declare const verifyCitationsOutputSchema: z.ZodObject<{
     id: z.ZodString;
     trace_id: z.ZodOptional<z.ZodString>;
     overall_score: z.ZodNullable<z.ZodNumber>;
-    passed: z.ZodBoolean;
+    passed: z.ZodNullable<z.ZodBoolean>;
+    total_unsupported: z.ZodNumber;
     total_citations_found: z.ZodNumber;
     total_resolved: z.ZodNumber;
     total_judged: z.ZodNumber;

package/dist/tools/verify-citations.js

@@ -69,7 +69,11 @@ export const verifyCitationsOutputSchema = z.looseObject({
     id: z.string().describe('the evaluation id; read it back at iris://evaluations/{id}'),
     trace_id: z.string().optional().describe('the linked trace, when one was named'),
     overall_score: z.number().nullable().describe('supported / judged; null when nothing was judged'),
-    passed: z.boolean().describe('true when every judged citation was supported, or nothing was judged and nothing failed'),
+    passed: z
+        .boolean()
+        .nullable()
+        .describe('true when every judged citation was supported; false when any judged citation was not; NULL when nothing was judged — no verdict, because nothing was verified. Until 0.10.0 that last case returned true.'),
+    total_unsupported: z.number().int().describe('judged citations the judge ruled unsupported — the number the verdict turns on'),
     total_citations_found: z.number().int().describe('citations extracted from the output'),
     total_resolved: z.number().int().describe('citations whose source was fetched'),
     total_judged: z.number().int().describe('citations the judge ruled on'),
@@ -138,18 +142,32 @@ export function registerVerifyCitationsTool(server, storage) {
             eval_type: 'custom',
             output_text: args.output,
             score,
-            passed: result.passed,
+            passed: result.passed === true,
             rule_results: [
                 {
                     ruleName: `semantic_citation_verify:${provider}/${args.model}`,
-                    passed: result.passed,
+                    /*
+                     * Null means nothing was judged, which is not a pass and not a
+                     * failure — it is a check that did not run. Stored as a SKIP so
+                     * the composer treats it as coverage rather than silently
+                     * reading a paid-for "nothing verified" as clean.
+                     */
+                    passed: result.passed === null ? false : result.passed,
+                    ...(result.passed === null
+                        ? { skipped: true, skipReason: `no citation was judged (found ${result.totalCitationsFound}, resolved ${result.totalResolved})` }
+                        : {}),
+                    kind: 'judgment',
                     score,
                     message: result.overallScore === null
                         ? `No citations judged (found ${result.totalCitationsFound}, resolved ${result.totalResolved}, judged 0)`
                         : `${result.totalSupported}/${result.totalJudged} judged sources supported the output`,
                 },
             ],
-            suggestions: result.passed ? [] : [`Only ${result.totalSupported}/${result.totalJudged} judged sources actually supported the claim.`],
+            suggestions: result.passed === null
+                ? ['No citation was judged, so nothing about the sources was verified. This is not a pass.']
+                : result.passed
+                    ? []
+                    : [`${result.totalUnsupported} of ${result.totalJudged} judged sources did not support the claim.`],
             rules_evaluated: 1,
             rules_skipped: 0,
             insufficient_data: result.overallScore === null,
@@ -160,6 +178,9 @@ export function registerVerifyCitationsTool(server, storage) {
             ...(args.trace_id ? { trace_id: args.trace_id } : {}),
             overall_score: result.overallScore,
             passed: result.passed,
+            // Derived rather than read: the verifier reports it, but the tool
+            // must not break if a caller hands it an older shape.
+            total_unsupported: result.totalUnsupported ?? Math.max(0, result.totalJudged - result.totalSupported),
             total_citations_found: result.totalCitationsFound,
             total_resolved: result.totalResolved,
             total_judged: result.totalJudged,