@iris-eval/mcp-server 0.8.2 → 0.10.0

package/dist/eval/criticality.d.ts

@@ -1,4 +1,4 @@
-import type { EvalRule, EvalType } from '../types/eval.js';
+import type { EvalRule, EvalType, ClaimKind, Mechanism, Need, QuestionId, FailureClass } from '../types/eval.js';
 /** Where a rule's EFFECTIVE criticality came from. */
 export type CriticalitySource = 'default' | 'config';
 export interface CriticalityOverrides {
@@ -6,6 +6,13 @@ export interface CriticalityOverrides {
     criticalRules?: string[];
     /** Built-in rule names demoted from critical. */
     nonCriticalRules?: string[];
+    composer?: 'risk' | 'legacy';
+    falsePassCost?: number;
+    onCriticalSkipped?: 'unknown' | 'fail' | 'pass';
+    requiredEvidence?: string[];
+    defaultsGate?: boolean;
+    prior?: number;
+    priorMode?: 'per-output' | 'per-class';
 }
 export interface EffectiveCriticality {
     critical: boolean;
@@ -50,6 +57,13 @@ export interface BuiltInRuleMeta {
     critical: boolean;
     /** Who decided it: the rule's own declaration, or one of the config lists. */
     criticalSource: CriticalitySource;
+    /** The rule's declared metadata (see EvalRule): the kind of claim, its mechanism, what it reads, the question it answers, the failure classes, the definition version. */
+    kind?: ClaimKind;
+    mechanism?: Mechanism;
+    needs?: readonly Need[];
+    question?: QuestionId;
+    classes?: readonly FailureClass[];
+    version?: number;
 }
 /**
  * The whole built-in roster, one entry per rule.

package/dist/eval/criticality.js

@@ -147,6 +147,12 @@ export function builtInRuleRoster(resolve) {
                 weight: rule.weight,
                 critical: effective.critical,
                 criticalSource: effective.source,
+                kind: rule.kind,
+                mechanism: rule.mechanism,
+                needs: rule.needs,
+                question: rule.question,
+                classes: rule.classes,
+                version: rule.version,
             });
         }
     }

package/dist/eval/decision-moment.js

@@ -104,14 +104,34 @@ function computeRuleSnapshot(evals) {
     }
     return { failed, skipped, passedCount, totalCount };
 }
+/*
+ * The moment SHOWS the verdict each evaluation reached; it does not compute
+ * a second one.
+ *
+ * It used to count failed rules: no failures meant pass, no passes meant
+ * fail, anything else meant partial. From 0.10.0 those two answers diverge,
+ * and the divergence is the whole point of the composer. An evaluation can
+ * pass with a rule visibly failed — a shipped default that only advises, or
+ * evidence too weak to carry the risk past the deployment's loss threshold
+ * — and the old arithmetic would have called that "partial", contradicting
+ * the verdict the tool returned for the same evaluation.
+ *
+ * "partial" now means what it says: several evaluations of one trace and
+ * they did not agree. An `unknown` verdict reads as unevaluated, because
+ * that is what it is — asked, and unable to answer.
+ */
 function computeVerdict(evals, snapshot) {
     if (evals.length === 0)
         return 'unevaluated';
     if (snapshot.totalCount - snapshot.skipped.length === 0)
         return 'unevaluated';
-    if (snapshot.failed.length === 0)
+    const decided = evals.filter((e) => e.verdict === undefined || e.verdict.state !== 'unknown');
+    if (decided.length === 0)
+        return 'unevaluated';
+    const passed = decided.filter((e) => e.passed).length;
+    if (passed === decided.length)
         return 'pass';
-    if (snapshot.passedCount === 0)
+    if (passed === 0)
         return 'fail';
     return 'partial';
 }
@@ -122,8 +142,17 @@ function computeOverallScore(evals) {
     return sum / evals.length;
 }
 function classifySignificance({ trace, evals, ruleSnapshot, verdict, }) {
-    // 1. Safety violation — any safety rule failed → top priority.
-    const safetyFailed = ruleSnapshot.failed.filter((name) => SAFETY_RULE_NAMES.has(name));
+    /*
+     * 1. Safety violation — a rule that VETOES failed, or a safety-bundle rule
+     * did. Bundle membership alone was the old test, and it is the weaker
+     * one: from 0.10.0 which rules veto is the deployment's call
+     * (eval.criticalRules), so a rule promoted to critical outside the safety
+     * bundle is exactly as serious and used to rank as a plain failure. The
+     * bundle list stays as well, because a safety rule that a deployment
+     * DEMOTED still describes what it found.
+     */
+    const vetoed = new Set(evals.flatMap((e) => e.rule_results.filter((r) => !r.skipped && r.passed === false && r.role === 'veto').map((r) => r.ruleName)));
+    const safetyFailed = ruleSnapshot.failed.filter((name) => SAFETY_RULE_NAMES.has(name) || vetoed.has(name));
     if (safetyFailed.length > 0) {
         return {
             kind: 'safety-violation',

package/dist/eval/dormant.d.ts

@@ -0,0 +1,4 @@
+import type { Coverage } from '../types/eval.js';
+export type DormantRule = NonNullable<Coverage['dormant']>[number];
+/** The quarantined entries that would have gated, as `coverage.dormant` rows. */
+export declare function dormantRulesFrom(quarantined: readonly unknown[]): DormantRule[];

package/dist/eval/dormant.js

@@ -0,0 +1,22 @@
+const GATING = new Set(['high', 'critical']);
+function field(entry, key) {
+    if (!entry || typeof entry !== 'object')
+        return undefined;
+    const value = entry[key];
+    return typeof value === 'string' ? value : undefined;
+}
+/** The quarantined entries that would have gated, as `coverage.dormant` rows. */
+export function dormantRulesFrom(quarantined) {
+    const out = [];
+    for (const entry of quarantined) {
+        const severity = field(entry, 'severity');
+        if (!severity || !GATING.has(severity))
+            continue;
+        out.push({
+            ruleId: field(entry, 'id') ?? 'unknown',
+            name: field(entry, 'name') ?? 'unnamed',
+            reason: `quarantined: the stored definition failed validation in this version, so this ${severity} rule is not running`,
+        });
+    }
+    return out;
+}

package/dist/eval/engine.d.ts

@@ -39,12 +39,15 @@ export declare class EvalEngine {
     private idByRule;
     private threshold;
     private ruleThresholds?;
+    private criticalityOverrides?;
     /**
      * Effective criticality per rule, bound to this engine's config overrides.
      * Every veto decision reads THIS, never `rule.critical` directly, so a
      * promotion or demotion cannot apply on one code path and not another.
      */
     private criticality;
+    /** The verdict's six defaults, resolved once from the config this engine was built with. */
+    private compose;
     /**
      * `criticalityOverrides` are `config.eval` — the criticalRules /
      * nonCriticalRules lists. Validated here as well as in loadConfig, so an
@@ -52,6 +55,7 @@ export declare class EvalEngine {
      * misspelled rule name.
      */
     constructor(threshold?: number, ruleThresholds?: Record<string, unknown>, criticalityOverrides?: CriticalityOverrides);
+    private decide;
     /** The effective criticality of one rule under this engine's config. Read by the rule roster surfaces. */
     effectiveCriticality(rule: EvalRule): EffectiveCriticality;
     /**
@@ -71,7 +75,7 @@ export declare class EvalEngine {
     unregisterRule(ruleId: string): boolean;
     /** Whether a deployed rule id is currently registered (and therefore firing). */
     hasRule(ruleId: string): boolean;
-    evaluate(evalType: EvalType, context: EvalContext, customRules?: CustomRuleDefinition[]): EvalResult;
+    evaluate(evalType: EvalType, context: EvalContext, customRules?: CustomRuleDefinition[]): Promise<EvalResult>;
     /**
      * eval_type="all" (#370): every built-in bundle, each with the deployed
      * rules registered under it, plus the rules deployed under "custom" and
@@ -81,7 +85,7 @@ export declare class EvalEngine {
      * that ran (weighted score against the threshold, critical veto across
      * all bundles); `categories` carries the same arithmetic per bundle.
      */
-    evaluateAll(context: EvalContext, customRules?: CustomRuleDefinition[]): EvalResult;
+    evaluateAll(context: EvalContext, customRules?: CustomRuleDefinition[]): Promise<EvalResult>;
     private run;
     /**
      * Weighted average over the rules that ran, plus the critical veto.

package/dist/eval/engine.js

@@ -1,5 +1,9 @@
 import { getRulesForType, createCustomRule } from './rules/index.js';
 import { criticalityResolver } from './criticality.js';
+import { compose, interpretations, DEFAULT_COMPOSE } from './compose.js';
+import { inputsPresent, stampRuleResult } from './stamp.js';
+import { buildProvenance, configHash, deriveCoverage, deriveVerdict, rulesetHash } from './verdict.js';
+import { PKG_VERSION } from '../config/defaults.js';
 import { generateEvalId } from '../utils/ids.js';
 /**
  * Every bundle eval_type="all" walks, in the order their categories are
@@ -40,12 +44,15 @@ export class EvalEngine {
     idByRule = new Map();
     threshold;
     ruleThresholds;
+    criticalityOverrides;
     /**
      * Effective criticality per rule, bound to this engine's config overrides.
      * Every veto decision reads THIS, never `rule.critical` directly, so a
      * promotion or demotion cannot apply on one code path and not another.
      */
     criticality;
+    /** The verdict's six defaults, resolved once from the config this engine was built with. */
+    compose;
     /**
      * `criticalityOverrides` are `config.eval` — the criticalRules /
      * nonCriticalRules lists. Validated here as well as in loadConfig, so an
@@ -55,7 +62,35 @@ export class EvalEngine {
     constructor(threshold = 0.7, ruleThresholds, criticalityOverrides) {
         this.threshold = threshold;
         this.ruleThresholds = ruleThresholds;
+        this.criticalityOverrides = criticalityOverrides;
         this.criticality = criticalityResolver(criticalityOverrides);
+        this.compose = {
+            composer: criticalityOverrides?.composer ?? DEFAULT_COMPOSE.composer,
+            falsePassCost: criticalityOverrides?.falsePassCost ?? DEFAULT_COMPOSE.falsePassCost,
+            onCriticalSkipped: criticalityOverrides?.onCriticalSkipped ?? DEFAULT_COMPOSE.onCriticalSkipped,
+            requiredEvidence: criticalityOverrides?.requiredEvidence ?? DEFAULT_COMPOSE.requiredEvidence,
+            defaultsGate: criticalityOverrides?.defaultsGate ?? DEFAULT_COMPOSE.defaultsGate,
+            prior: criticalityOverrides?.prior ?? DEFAULT_COMPOSE.prior,
+            priorMode: criticalityOverrides?.priorMode ?? DEFAULT_COMPOSE.priorMode,
+        };
+    }
+    /*
+     * The verdict, and `passed` with it.
+     *
+     * From 0.10.0 `passed` IS `verdict.state === 'pass'` — one definition, on
+     * every surface. The weighted `score` survives untouched as a quality
+     * gradient over the rules that ran, and is never re-meant: a reader who
+     * was using it as a gradient keeps it, and a reader who was using it as a
+     * safety signal was reading a number that arc zero measured as inert.
+     */
+    decide(result) {
+        const verdict = this.compose.composer === 'legacy' ? deriveVerdict(result, this.threshold) : compose(result, this.compose);
+        result.verdict = verdict;
+        result.passed = verdict.passed;
+        const notes = interpretations(result, verdict, this.compose);
+        if (notes.length > 0)
+            result.interpretations = notes;
+        return result;
     }
     /** The effective criticality of one rule under this engine's config. Read by the rule roster surfaces. */
     effectiveCriticality(rule) {
@@ -104,7 +139,7 @@ export class EvalEngine {
     hasRule(ruleId) {
         return this.rulesById.has(ruleId);
     }
-    evaluate(evalType, context, customRules) {
+    async evaluate(evalType, context, customRules) {
         /*
          * Inline custom_rules are ADDITIVE, which is what evaluate_output's
          * description promises in two places: "fires REGARDLESS of eval_type"
@@ -141,7 +176,7 @@ export class EvalEngine {
      * that ran (weighted score against the threshold, critical veto across
      * all bundles); `categories` carries the same arithmetic per bundle.
      */
-    evaluateAll(context, customRules) {
+    async evaluateAll(context, customRules) {
         const rules = [];
         const categories = [];
         for (const type of ALL_EVAL_TYPES) {
@@ -156,7 +191,16 @@ export class EvalEngine {
         }
         return this.run('all', rules, categories, context);
     }
-    run(evalType, rules, categories, context) {
+    /*
+     * Async from 0.10.0. Nothing it awaits yet: every rule the package ships
+     * is synchronous, and `EvalRule.evaluate` stays synchronous so the type
+     * system keeps proving that a deterministic rule cannot reach the network
+     * — which is what makes "evaluate_output never spends" a compile-time
+     * fact rather than a test. The signature moves first, in one mechanical
+     * change, so the judgment rule that DOES call a provider can be added
+     * without re-touching every caller a second time.
+     */
+    async run(evalType, rules, categories, context) {
         // Merge system-level thresholds into customConfig (user-provided values take precedence)
         if (this.ruleThresholds) {
             context = {
@@ -177,6 +221,7 @@ export class EvalEngine {
                 rules_evaluated: 0,
                 rules_skipped: 0,
                 insufficient_data: true,
+                verdict: { state: 'unknown', passed: false, basis: 'no_rules', by: [], risk: null },
             };
         }
         /*
@@ -188,10 +233,41 @@ export class EvalEngine {
          * rule it carries.
          */
         const evalContext = { ...context, regexBudget: { breaches: 0 } };
-        const ruleResults = rules.map((rule, i) => {
-            const raw = rule.evaluate(evalContext);
+        /*
+         * Sequential, and it must stay sequential when a rule becomes awaitable:
+         * every rule in one evaluation shares the regex circuit breaker above,
+         * and running them concurrently would race the breach count that bounds
+         * a hostile output.
+         */
+        const ruleResults = [];
+        for (const [i, rule] of rules.entries()) {
+            /*
+             * A judgment rule calls a provider and costs money. It runs only when
+             * the caller has said this evaluation may spend — which the free
+             * evaluation path never does. Enforced here, on the one path every
+             * evaluation takes, so no tool can forget it and no future rule can
+             * quietly opt itself in.
+             */
+            const raw = rule.kind === 'judgment' && evalContext.allowPaid !== true
+                ? {
+                    ruleName: rule.name,
+                    passed: false,
+                    score: 0,
+                    message: 'Judgment rules are not run on this path: it may not call a paid provider.',
+                    skipped: true,
+                    skipReason: 'this evaluation may not spend (context.allowPaid is not set)',
+                }
+                : rule.evaluate(evalContext);
             const ruleId = this.idByRule.get(rule);
-            const category = categories?.[i];
+            /*
+             * The bundle this rule ran under. `categories` is only supplied for
+             * eval_type="all"; for a single bundle the rule's own evalType is the
+             * answer and is just as true. It used to be left off, which meant a
+             * single-bundle call could not tell a custom rule from a built-in one
+             * — and the composer needs that, because a custom rule's severity is
+             * the deployment's own statement of how much it matters.
+             */
+            const category = categories?.[i] ?? (evalType === 'all' ? rule.evalType : evalType);
             /*
              * Every result says whether THIS rule vetoes and who decided that.
              * Without it, a reader holding a failed evaluation cannot tell a
@@ -201,21 +277,53 @@ export class EvalEngine {
              * the one path every evaluation takes, so a surface cannot render the
              * declared criticality where the engine applied a configured one.
              */
-            const { critical, source } = this.criticality(rule);
+            const effective = this.criticality(rule);
+            const { critical, source } = effective;
             // ruleId / category sit right after the name so a reader scanning
             // rule_results sees WHICH deployed rule (and which bundle) spoke.
             const { ruleName, ...rest } = raw;
-            return {
+            /*
+             * The stamp (0.9.0): what kind of claim this is, what the composer
+             * did with it, which question it answers, what it saw, why it skipped,
+             * and how wrong it tends to be — from the rule's declaration, the
+             * inputs this call carried, and the published accuracy that ships in
+             * the package. Computed here, on the one path every evaluation takes,
+             * so no surface can show a result without its receipt. It changes no
+             * verdict: summarize() below still decides passed exactly as before.
+             */
+            ruleResults.push({
                 ruleName,
                 ...(ruleId !== undefined ? { ruleId } : {}),
                 ...(category !== undefined ? { category } : {}),
                 critical,
                 criticalSource: source,
                 ...rest,
-            };
-        });
+                ...stampRuleResult(rule, raw, context, effective),
+            });
+        }
         const overall = this.summarize(rules, ruleResults);
         const perCategory = categories ? this.categorize(rules, ruleResults, categories) : undefined;
+        /*
+         * The receipt for the whole evaluation (0.9.0): what produced it, which
+         * questions it judged, and the basis of its verdict. Computed here from
+         * what the engine already holds; persisted as provenance and derived
+         * again on every read, so a stored row answers "why did this pass on
+         * that day" without a backfill. Changes no verdict.
+         */
+        const provenance = buildProvenance({
+            irisVersion: PKG_VERSION,
+            rulesetHash: rulesetHash(rules, (r) => this.criticality(r)),
+            configHash: configHash({
+                threshold: this.threshold,
+                ruleThresholds: this.ruleThresholds,
+                criticalRules: this.criticalityOverrides?.criticalRules,
+                nonCriticalRules: this.criticalityOverrides?.nonCriticalRules,
+            }),
+            threshold: this.threshold,
+            ruleThresholds: this.ruleThresholds,
+            judgedAt: new Date().toISOString(),
+        });
+        const coverage = deriveCoverage(ruleResults, inputsPresent(context));
         // Handle "all rules skipped" — insufficient data
         if (overall.rulesEvaluated === 0) {
             const skipMessages = ruleResults
@@ -225,7 +333,7 @@ export class EvalEngine {
             // that EVERY critical rule that skipped is named here, and a caller
             // whose only rules were critical ones should not have to infer that
             // from insufficient_data alone.
-            return {
+            const unknown = {
                 id: generateEvalId(),
                 eval_type: evalType,
                 output_text: context.output,
@@ -242,7 +350,10 @@ export class EvalEngine {
                 insufficient_data: true,
                 ...(overall.criticalSkipped.length > 0 ? { critical_skipped: overall.criticalSkipped } : {}),
                 ...(perCategory ? { categories: perCategory } : {}),
+                coverage,
+                provenance,
             };
+            return this.decide(unknown);
         }
         const suggestions = [];
         for (const result of ruleResults) {
@@ -273,7 +384,7 @@ export class EvalEngine {
                 'checks, not "clean"; a gate that must fail closed should treat critical_skipped ' +
                 'as a failure.');
         }
-        return {
+        const result = {
             id: generateEvalId(),
             eval_type: evalType,
             output_text: context.output,
@@ -288,7 +399,10 @@ export class EvalEngine {
             ...(overall.criticalFailures.length > 0 ? { critical_failures: overall.criticalFailures } : {}),
             ...(overall.criticalSkipped.length > 0 ? { critical_skipped: overall.criticalSkipped } : {}),
             ...(perCategory ? { categories: perCategory } : {}),
+            coverage,
+            provenance,
         };
+        return this.decide(result);
     }
     /**
      * Weighted average over the rules that ran, plus the critical veto.

package/dist/eval/failure-classes.d.ts

@@ -0,0 +1,8 @@
+import type { FailureClass, Need } from '../types/eval.js';
+export interface FailureClassEntry {
+    id: FailureClass;
+    text: string;
+}
+export declare const FAILURE_CLASSES: readonly FailureClassEntry[];
+export declare const FAILURE_CLASS_IDS: readonly FailureClass[];
+export declare const NEEDS: readonly Need[];

package/dist/eval/failure-classes.js

@@ -0,0 +1,18 @@
+export const FAILURE_CLASSES = [
+    { id: 'pii_leak', text: 'Personal data in the output (SSN, card, phone, email, date of birth, address …)' },
+    { id: 'credential_leak', text: 'A secret in the output (API key, token, private key, seed phrase)' },
+    { id: 'injection', text: 'Injection-shaped content in the output: attack phrasing or a directive aimed at an evaluator or a downstream system' },
+    { id: 'injection_compliance', text: 'The agent read an injected instruction in a tool result and obeyed it' },
+    { id: 'silent_tool_failure', text: 'A tool call failed and the output never acknowledges it' },
+    { id: 'tool_loop', text: 'The same call repeated past the configured limit with the same result' },
+    { id: 'stub', text: 'A placeholder, a deferral or a promise instead of the work' },
+    { id: 'fabrication', text: 'A claim that contradicts the material the agent was given' },
+    { id: 'ungrounded', text: 'An identifier, number or citation in the output that nothing the agent read supports' },
+    { id: 'incomplete_ask', text: 'Part of a multi-part ask was not answered' },
+    { id: 'off_task', text: 'The output does not address what was asked' },
+    { id: 'over_budget', text: 'The run cost more than the deployment allows, in money or tokens' },
+    { id: 'format', text: 'The output fails a shape or size requirement (empty, too short, too few sentences, invalid JSON)' },
+    { id: 'invalid_tool_call', text: 'A tool was called that is not in the catalogue, or with arguments its schema rejects' },
+];
+export const FAILURE_CLASS_IDS = FAILURE_CLASSES.map((c) => c.id);
+export const NEEDS = ['output', 'input', 'expected', 'tool_calls', 'tool_outputs', 'tools_catalogue', 'cost', 'tokens', 'citations'];

package/dist/eval/llm-judge/evaluator.d.ts

@@ -1,5 +1,15 @@
 import { type LLMProvider } from './client.js';
 import { type TemplateName } from './templates/index.js';
+/**
+ * The pre-check refused the call: the worst-case spend (two attempts) would
+ * exceed the cap. Typed so the tool can answer IRIS_BUDGET_EXCEEDED with both
+ * numbers; nothing was spent.
+ */
+export declare class CostCapError extends Error {
+    readonly estimatedUsd: number;
+    readonly capUsd: number;
+    constructor(estimatedUsd: number, capUsd: number);
+}
 export interface LLMJudgeEvaluateParams {
     output: string;
     template: TemplateName;
@@ -16,8 +26,28 @@ export interface LLMJudgeEvaluateParams {
     maxInputTokensEstimate?: number;
 }
 export interface LLMJudgeEvaluationResult {
+    /**
+     * The verdict, and it is the THRESHOLD's, not the model's.
+     *
+     * Until 0.10.0 the model's own `passed` boolean won whenever it supplied
+     * one, and the template's threshold was a fallback the product rarely
+     * reached. That let a judge return `score: 0.2` with `passed: true` and
+     * be believed — a scoring rubric whose score did not decide anything.
+     * Now the score is the measurement and the threshold is the rule.
+     */
     passed: boolean;
     score: number;
+    /** The threshold the score was read against, so a reader can check the arithmetic. */
+    passThreshold: number;
+    /** What the model said about passing, when it said anything. Recorded, never obeyed. */
+    selfReportedPass?: boolean;
+    /**
+     * True when the model's own boolean disagrees with the threshold verdict.
+     * Worth surfacing: a judge that scores 0.95 and says "fail", or scores
+     * 0.2 and says "pass", is telling you its rubric and its judgement have
+     * come apart on this output.
+     */
+    disagreement?: boolean;
     rationale: string;
     dimensions: Record<string, number>;
     model: string;

package/dist/eval/llm-judge/evaluator.js

@@ -1,6 +1,21 @@
 import { callLLMJudge, estimateInputTokens, LLMJudgeError } from './client.js';
 import { estimateCostUsd, findPricing } from './pricing.js';
 import { getTemplate } from './templates/index.js';
+/**
+ * The pre-check refused the call: the worst-case spend (two attempts) would
+ * exceed the cap. Typed so the tool can answer IRIS_BUDGET_EXCEEDED with both
+ * numbers; nothing was spent.
+ */
+export class CostCapError extends Error {
+    estimatedUsd;
+    capUsd;
+    constructor(estimatedUsd, capUsd) {
+        super(`Estimated max cost ${estimatedUsd.toFixed(4)} USD (including one retry on a malformed judge reply) exceeds cap ${capUsd.toFixed(4)} USD — refusing to call. Raise IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL or max_cost_usd, or trim prompts/maxOutputTokens.`);
+        this.estimatedUsd = estimatedUsd;
+        this.capUsd = capUsd;
+        this.name = 'CostCapError';
+    }
+}
 // Malformed judge response — retried once by `evaluate`, surfaced as
 // LLMJudgeError('malformed_response') if the retry also fails.
 function parseJudgeResponse(raw) {
@@ -88,7 +103,7 @@ export async function evaluateWithLLMJudge(params) {
     const retryCost = estimateCostUsd(params.model, estimateInputTokens(strictSystem, userPrompt), retryMaxOutputTokens);
     const estimatedCost = firstAttemptCost === null || retryCost === null ? null : firstAttemptCost + retryCost;
     if (estimatedCost !== null && estimatedCost > maxCost) {
-        throw new Error(`Estimated max cost ${estimatedCost.toFixed(4)} USD (including one retry on a malformed judge reply) exceeds cap ${maxCost.toFixed(4)} USD — refusing to call. Raise IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL or trim prompts/maxOutputTokens.`);
+        throw new CostCapError(estimatedCost, maxCost);
     }
     // First attempt
     let raw = await callLLMJudge({
@@ -136,10 +151,19 @@ export async function evaluateWithLLMJudge(params) {
         latencyMs += raw.latencyMs;
         parsed = parseJudgeResponse(raw.content);
     }
-    const passed = parsed.passed ?? parsed.score >= template.passThreshold;
+    /*
+     * The threshold decides. The model's own boolean is evidence about the
+     * model, not about the output, and it is recorded beside the verdict
+     * rather than substituted for it.
+     */
+    const passed = parsed.score >= template.passThreshold;
+    const disagreement = parsed.passed !== undefined && parsed.passed !== passed;
     const costUsd = estimateCostUsd(params.model, inputTokens, outputTokens);
     return {
         passed,
+        passThreshold: template.passThreshold,
+        ...(parsed.passed !== undefined ? { selfReportedPass: parsed.passed } : {}),
+        ...(disagreement ? { disagreement: true } : {}),
         score: parsed.score,
         rationale: parsed.rationale,
         dimensions: parsed.dimensions,