@iris-eval/mcp-server 0.8.2 → 0.9.0

package/dist/tools/verify-citations.js

@@ -1,41 +1,31 @@
 import { z } from 'zod';
 import { LOCAL_TENANT } from '../types/tenant.js';
 import { verifyCitations } from '../eval/citation-verify/verifier.js';
-import { findPricing } from '../eval/llm-judge/pricing.js';
 import { generateEvalId } from '../utils/ids.js';
+import { JUDGE_KEY_VARS } from '../judge-enablement.js';
 import { strictInput } from './strict-input.js';
 import { assertTraceExists, insertLinkedEvalResult } from './trace-link.js';
+import { inferProvider, resolveApiKey } from './evaluate-with-llm-judge.js';
+import { describeTool, ERROR_ENVELOPE_SENTENCE } from './describe.js';
+import { irisError } from './errors.js';
+import { evaluationLinks, guarded, respond } from './respond.js';
 const inputSchema = {
     output: z.string().min(1).describe('The agent output containing citations to verify'),
     model: z
         .string()
-        .describe('Judge model for per-citation verification. Supported: anthropic = claude-opus-4-7 | claude-sonnet-4-6 | claude-haiku-4-5-20251001; openai = gpt-4o | gpt-4o-mini | o1-mini.'),
+        .describe('Judge model for per-citation verification. Supported: anthropic = claude-opus-4-7 | claude-sonnet-4-6 | claude-haiku-4-5 | claude-haiku-4-5-20251001; openai = gpt-4o | gpt-4o-mini | o1-mini.'),
     provider: z.enum(['anthropic', 'openai']).optional().describe('Auto-detected from model when omitted'),
     allow_fetch: z.boolean().optional().describe('Permit outbound HTTP to resolve URLs/DOIs. Defaults to IRIS_CITATION_ALLOW_FETCH=1; false otherwise. SSRF-guarded regardless.'),
     domain_allowlist: z
         .array(z.string())
         .optional()
         .describe('Restrict fetches to hostnames in this list (suffix match allowed). Merged with IRIS_CITATION_DOMAINS env.'),
-    max_cost_usd_total: z.number().positive().optional().describe('Cap TOTAL judge cost across all citations in this call; default $1.00'),
-    max_citations: z.number().int().positive().max(50).optional().describe('Max citations to verify (extras skipped); default 20'),
+    max_cost_usd_total: z.number().positive().optional().describe('Cap TOTAL judge cost across all citations in this call; default 1.00 USD — the pipeline stops when the next call would exceed it'),
+    max_citations: z.number().int().positive().max(50).optional().describe('Max citations to verify (extras skipped, not errored); default 20, at most 50'),
     per_source_timeout_ms: z.number().int().positive().optional().describe('Per-URL fetch timeout; default 10_000'),
     per_source_max_bytes: z.number().int().positive().optional().describe('Per-URL body cap; default 5MB'),
     trace_id: z.string().optional().describe('Link verification result to a stored trace (id from log_trace / get_traces); an unknown id is rejected before any fetch or judge call'),
 };
-function inferProvider(model) {
-    const pricing = findPricing(model);
-    if (!pricing) {
-        throw new Error(`Unknown model "${model}". Provider cannot be inferred. Supported models: src/eval/llm-judge/pricing.ts.`);
-    }
-    return pricing.provider;
-}
-function resolveApiKey(provider) {
-    const key = provider === 'anthropic' ? process.env.IRIS_ANTHROPIC_API_KEY : process.env.IRIS_OPENAI_API_KEY;
-    if (!key) {
-        throw new Error(`${provider === 'anthropic' ? 'Anthropic' : 'OpenAI'} judge requires IRIS_${provider === 'anthropic' ? 'ANTHROPIC' : 'OPENAI'}_API_KEY for verify_citations.`);
-    }
-    return key;
-}
 function resolveAllowFetch(paramValue) {
     if (paramValue !== undefined)
         return paramValue;
@@ -54,8 +44,8 @@ function resolveDomainAllowlist(paramValue) {
  * was nothing to judge (no citations, none resolved). It is NOT the honest
  * answer when citations resolved and the judge then failed on every one —
  * a wrong API key, a model the provider refused, a parse failure — because
- * the caller reads "passed" and ships. That case is an error naming the
- * cause; nothing is stored. (v0.6.0 acceptance pass, observation 3.)
+ * the caller reads "passed" and ships. That case is IRIS_JUDGE_FAILED
+ * naming the cause; nothing is stored. (v0.6.0 acceptance pass, observation 3.)
  */
 export function assertJudgeRan(result) {
     if (result.totalResolved === 0 || result.totalJudged > 0)
@@ -65,39 +55,59 @@ export function assertJudgeRan(result) {
         return;
     const kinds = [...new Set(judgeFailures.map((c) => c.resolveError.kind))].join(', ');
     const first = judgeFailures[0].resolveError.message;
-    throw new Error(`verify_citations could not judge any of the ${result.totalResolved} resolved citation(s): the judge failed on every one (${kinds}). ` +
-        `Nothing was verified and nothing was stored, so there is no verdict. First error: ${first}`);
+    throw irisError('IRIS_JUDGE_FAILED', `verify_citations could not judge any of the ${result.totalResolved} resolved citation(s): the judge failed on every one (${kinds}). ` +
+        `Nothing was verified and nothing was stored, so there is no verdict. First error: ${first}`, {
+        retryable: /timeout|rate_limit|server_error/.test(kinds),
+        recovery: [
+            'Check the key and the model: a refused key or an unknown model fails every citation the same way.',
+            'Retry when the kind is a timeout, a rate limit or a provider server error.',
+            'Raise max_cost_usd_total when the kind is cost_cap_reached.',
+        ],
+    });
 }
+export const verifyCitationsOutputSchema = z.looseObject({
+    id: z.string().describe('the evaluation id; read it back at iris://evaluations/{id}'),
+    trace_id: z.string().optional().describe('the linked trace, when one was named'),
+    overall_score: z.number().nullable().describe('supported / judged; null when nothing was judged'),
+    passed: z.boolean().describe('true when every judged citation was supported, or nothing was judged and nothing failed'),
+    total_citations_found: z.number().int().describe('citations extracted from the output'),
+    total_resolved: z.number().int().describe('citations whose source was fetched'),
+    total_judged: z.number().int().describe('citations the judge ruled on'),
+    total_supported: z.number().int().describe('citations the judge found supported'),
+    total_cost_usd: z.number().describe('the spend across every judge call'),
+    citations: z.array(z.looseObject({ resolve_status: z.string() })).describe('per citation: the citation (raw, kind, identifier, offsets), resolve_status ok | skipped | error, resolve_error, source (url, status, content_type, bytes_fetched, truncated), judge (supported, confidence, rationale, cost_usd, latency_ms, tokens)'),
+});
 export function registerVerifyCitationsTool(server, storage) {
     server.registerTool('verify_citations', {
         title: 'Verify Citations',
-        description: [
-            'Extract citations from agent output, fetch the cited sources, and use an LLM judge to check whether each source supports the claim in context. Returns per-citation verdicts + an overall support ratio.',
-            '',
-            'Sibling tools — evaluate_with_llm_judge runs general semantic scoring (accuracy, helpfulness, correctness, faithfulness); this tool is specifically for citation grounding (does the cited source actually support the claim). evaluate_output\'s no_hallucination_markers heuristic detects FABRICATED-looking citations cheaply (free, no fetch); this tool resolves and verifies them (paid, opt-in fetch, SSRF-guarded). log_trace / get_traces handle trace I/O. verify_citations is the GROUNDING-CHECK path — narrowest in scope, deepest in rigor.',
-            '',
-            'Behavior. Three-phase pipeline: (1) regex extraction of [N] numbered refs, (Author, Year) parentheticals, bare URLs, and DOIs (in-process, no network); (2) SSRF-guarded fetch of URL + DOI citations, with scheme allowlist, private/link-local/cloud-metadata IP blocking, optional domain allowlist (IRIS_CITATION_DOMAINS), 10s timeout, 5MB body cap, manual redirect chase (max 3, re-checked), in-process LRU cache; (3) per-citation LLM judge call asking "does this source support this claim?" with a 256-token verdict. Opt-in via allow_fetch=true or IRIS_CITATION_ALLOW_FETCH=1 — Iris refuses outbound HTTP by default. Cost-capped across the entire call by max_cost_usd_total (default $1.00) — the pipeline stops when the cap would be exceeded. Rate-limited to 20 req/min on HTTP MCP. Writes one eval_result row tagged with per-citation provenance.',
-            '',
-            'Output shape. Returns JSON: `{ "id": "<uuid>", "overall_score": 0..1|null, "passed": boolean, "total_citations_found": number, "total_resolved": number, "total_judged": number, "total_supported": number, "total_cost_usd": number, "citations": [{ "citation": { "raw", "kind", "identifier", "offset_start", "offset_end" }, "resolve_status": "ok"|"skipped"|"error", "resolve_error"?, "source"?: { "url", "status", "content_type", "bytes_fetched", "truncated" }, "judge"?: { "supported", "confidence", "rationale", "cost_usd", "latency_ms", "input_tokens", "output_tokens" } }] }`. `overall_score = supported / judged`; `null` when nothing was judged (no resolvable citations, or every judge call failed). Infrastructure failures (cost cap, judge timeout/error, malformed verdict) leave a citation resolved-but-unjudged — reported per-citation via resolve_error, never scored as unsupported.',
-            '',
-            'Use when the output makes factual claims backed by [1]-style references, DOIs, or URLs and you want to separate "cited correctly" from "cited and wrong" from "cited but unresolvable". Particularly useful for research/legal/medical agents where fabricated citations are the dominant failure mode.',
-            "",
-            "Don't use when the agent output has no citations at all (overall_score will be null; the tool degrades gracefully but a heuristic rule is cheaper). Don't use without allow_fetch=true or IRIS_CITATION_ALLOW_FETCH=1 — the tool refuses outbound HTTP unless explicitly enabled. Don't use with an open allowlist + untrusted output on the public internet; you are effectively running a user-directed fetcher. For stricter safety set IRIS_CITATION_DOMAINS to a curated list.",
-            '',
-            'Parameters. model is required; provider auto-detected from model name (override only for ambiguous IDs). allow_fetch=false by default — outbound HTTP is REFUSED unless explicitly true OR IRIS_CITATION_ALLOW_FETCH=1 env. domain_allowlist suffix-matches hostnames (e.g., "wikipedia.org" allows en.wikipedia.org); merged with IRIS_CITATION_DOMAINS env (UNION — either source permits). max_citations defaults 20, hard cap 50 (extras are skipped silently, NOT errored — check total_citations_found in the response if precise). max_cost_usd_total defaults $1.00 — the pipeline stops mid-citation when the next judge call would exceed the cap (returns partial verdicts). per_source_timeout_ms defaults 10000 (10s); per_source_max_bytes defaults 5MB (truncates at boundary, judges still run on truncated content); independently of that, the judge reads at most the first 12,000 characters of each fetched source, and the per-citation cost estimate is taken on that truncated prompt, not on the full body. trace_id optional but recommended. Defaults: max_citations=20, max_cost_usd_total=$1.00, per_source_timeout_ms=10000, per_source_max_bytes=5242880, allow_fetch=false.',
-            '',
-            'Error modes. Throws when the API key env var is missing. Throws "Unknown model" on unsupported model IDs. Throws when trace_id does not match a stored trace (checked before any fetch or judge call; nothing is written). Per-citation errors are collected (resolve_error.kind = bad_scheme / ssrf / not_allowed_domain / timeout / too_large / bad_status / redirect_loop / not_text / fetch_disabled / malformed_judge_response / cost_cap_reached / unresolvable_kind) and returned in the response rather than thrown. An empty output or output with zero extractable citations returns overall_score=null + passed=true (nothing to fail).',
-        ].join('\n'),
+        description: describeTool({
+            summary: 'Extract the citations in an output, fetch the sources (opt-in, SSRF-guarded) and ask an LLM judge on your key whether each source supports its claim.',
+            does: 'Three phases. Extraction, no network: [N] references, (Author, Year), bare URLs and DOIs. Fetch of URL and DOI citations only when allow_fetch is true or IRIS_CITATION_ALLOW_FETCH=1, through a scheme allowlist, private and cloud-metadata address blocking, an optional hostname allowlist (domain_allowlist, merged with IRIS_CITATION_DOMAINS), a per-source timeout and byte cap, and at most three re-checked redirects. ' +
+                'Then one judge call per resolved citation on your own key, reading the first part of each source, capped in total by max_cost_usd_total. Up to max_citations are verified; extras are skipped, not errored. ' +
+                'overall_score is supported / judged and null when nothing was judged. Per-citation failures (bad scheme, blocked address, timeout, too large, cost cap, fetch disabled) are reported on the citation, never scored as unsupported. One evaluation row is stored.',
+            whenNot: "When the output has no citations: the score is null, and evaluate_output's hallucination signals are the cheap check. " +
+                `Without a key (${JUDGE_KEY_VARS.anthropic} or ${JUDGE_KEY_VARS.openai}): the call returns IRIS_JUDGE_NOT_ENABLED with the enable steps. ` +
+                'With fetch enabled and an open allowlist on untrusted output: you are running a user-directed fetcher — set IRIS_CITATION_DOMAINS.',
+            returns: verifyCitationsOutputSchema,
+            errors: 'IRIS_JUDGE_NOT_ENABLED, IRIS_JUDGE_UNKNOWN_MODEL and IRIS_UNKNOWN_TRACE before any fetch or spend. IRIS_JUDGE_FAILED when citations resolved but the judge failed on every one — an error, not a passing verdict; nothing is stored. ' +
+                ERROR_ENVELOPE_SENTENCE,
+            siblings: {
+                evaluate_with_llm_judge: 'general semantic scoring',
+                evaluate_output: 'the free deterministic path, including the hallucination signals',
+                log_trace: 'record the execution first',
+            },
+        }),
         inputSchema: strictInput(inputSchema),
+        outputSchema: verifyCitationsOutputSchema,
         annotations: {
             readOnlyHint: false, // Writes eval_result + spends money
             destructiveHint: false, // Creates data; doesn't overwrite/delete
             idempotentHint: false, // External fetches + provider non-determinism
             openWorldHint: true, // Outbound HTTP to citation URLs + LLM provider API
         },
-    }, async (args) => {
+    }, guarded(async (args) => {
         const provider = args.provider ?? inferProvider(args.model);
-        const apiKey = resolveApiKey(provider);
+        const apiKey = resolveApiKey(provider, 'verify_citations');
         const allowFetch = resolveAllowFetch(args.allow_fetch);
         const domainAllowlist = resolveDomainAllowlist(args.domain_allowlist);
         // Refused before any fetch or judge call spends anything (#376).
@@ -143,58 +153,53 @@ export function registerVerifyCitationsTool(server, storage) {
             rules_evaluated: 1,
             rules_skipped: 0,
             insufficient_data: result.overallScore === null,
+            eval_cost_usd: result.totalCostUsd,
         });
-        return {
-            content: [
-                {
-                    type: 'text',
-                    text: JSON.stringify({
-                        id: evalId,
-                        overall_score: result.overallScore,
-                        passed: result.passed,
-                        total_citations_found: result.totalCitationsFound,
-                        total_resolved: result.totalResolved,
-                        total_judged: result.totalJudged,
-                        total_supported: result.totalSupported,
-                        total_cost_usd: result.totalCostUsd,
-                        citations: result.citations.map((c) => ({
-                            citation: {
-                                raw: c.citation.raw,
-                                kind: c.citation.kind,
-                                identifier: c.citation.identifier,
-                                offset_start: c.citation.offsetStart,
-                                offset_end: c.citation.offsetEnd,
-                            },
-                            resolve_status: c.resolveStatus,
-                            resolve_error: c.resolveError,
-                            // Mapped to the documented snake_case keys. The verifier's
-                            // internal shape is camelCase (contentType, bytesFetched) and
-                            // used to be passed through verbatim, so a client parsing
-                            // `source.content_type` per the description read undefined.
-                            source: c.source
-                                ? {
-                                    url: c.source.url,
-                                    status: c.source.status,
-                                    content_type: c.source.contentType,
-                                    bytes_fetched: c.source.bytesFetched,
-                                    truncated: c.source.truncated,
-                                }
-                                : undefined,
-                            judge: c.judge
-                                ? {
-                                    supported: c.judge.supported,
-                                    confidence: c.judge.confidence,
-                                    rationale: c.judge.rationale,
-                                    cost_usd: c.judge.costUsd,
-                                    latency_ms: c.judge.latencyMs,
-                                    input_tokens: c.judge.inputTokens,
-                                    output_tokens: c.judge.outputTokens,
-                                }
-                                : undefined,
-                        })),
-                    }),
+        return respond(verifyCitationsOutputSchema, {
+            id: evalId,
+            ...(args.trace_id ? { trace_id: args.trace_id } : {}),
+            overall_score: result.overallScore,
+            passed: result.passed,
+            total_citations_found: result.totalCitationsFound,
+            total_resolved: result.totalResolved,
+            total_judged: result.totalJudged,
+            total_supported: result.totalSupported,
+            total_cost_usd: result.totalCostUsd,
+            citations: result.citations.map((c) => ({
+                citation: {
+                    raw: c.citation.raw,
+                    kind: c.citation.kind,
+                    identifier: c.citation.identifier,
+                    offset_start: c.citation.offsetStart,
+                    offset_end: c.citation.offsetEnd,
                 },
-            ],
-        };
-    });
+                resolve_status: c.resolveStatus,
+                resolve_error: c.resolveError,
+                // Mapped to the documented snake_case keys. The verifier's
+                // internal shape is camelCase (contentType, bytesFetched) and
+                // used to be passed through verbatim, so a client parsing
+                // `source.content_type` per the description read undefined.
+                source: c.source
+                    ? {
+                        url: c.source.url,
+                        status: c.source.status,
+                        content_type: c.source.contentType,
+                        bytes_fetched: c.source.bytesFetched,
+                        truncated: c.source.truncated,
+                    }
+                    : undefined,
+                judge: c.judge
+                    ? {
+                        supported: c.judge.supported,
+                        confidence: c.judge.confidence,
+                        rationale: c.judge.rationale,
+                        cost_usd: c.judge.costUsd,
+                        latency_ms: c.judge.latencyMs,
+                        input_tokens: c.judge.inputTokens,
+                        output_tokens: c.judge.outputTokens,
+                    }
+                    : undefined,
+            })),
+        }, evaluationLinks(evalId, args.trace_id));
+    }));
 }

package/dist/types/config.d.ts

@@ -2,6 +2,13 @@ export interface IrisConfig {
     storage: {
         type: 'sqlite';
         path: string;
+        /**
+         * `critical_spans` stores each evaluation's output text with the spans a
+         * critical detector flagged replaced by `[REDACTED:<pattern>]`, so a tool
+         * that detects leaks need not keep the leak it found. The evidence
+         * offsets still index the original text the caller saw. Default `none`.
+         */
+        redact?: 'none' | 'critical_spans';
     };
     server: {
         name: string;
@@ -53,6 +60,8 @@ export interface IrisConfig {
     };
     retention: {
         days: number;
+        /** How often the sweep re-runs after boot, in hours; 0 disables the timer (the boot sweep still runs). Default 24. */
+        sweepIntervalHours: number;
     };
     security: {
         apiKey?: string;

package/dist/types/eval.d.ts

@@ -8,6 +8,23 @@ export type EvalType = 'completeness' | 'relevance' | 'safety' | 'cost' | 'custo
  * under a real bundle, never under 'all'.
  */
 export type EvalResultType = EvalType | 'all';
+/**
+ * What KIND of claim a rule makes — the mandate's distinction between a
+ * measurement (a statistic against a threshold), a detection (a pattern is
+ * present, with a measured error rate), an inference (a signal standing in
+ * for an unobservable property), a judgment (a model's reasoning), a policy
+ * (the deployment's own constraint) and an external verification. Kind is
+ * the claim; `mechanism` is how the claim is measured. The composer decides
+ * by kind and never averages kinds together.
+ */
+export type ClaimKind = 'measurement' | 'detection' | 'inference' | 'judgment' | 'policy' | 'verification';
+export type Mechanism = 'formula' | 'pattern' | 'heuristic' | 'model' | 'external';
+/** An input a rule reads. A rule skips — never passes — when a declared need is absent. */
+export type Need = 'output' | 'input' | 'expected' | 'tool_calls' | 'tool_outputs' | 'tools_catalogue' | 'cost' | 'tokens' | 'citations';
+/** The evaluation question a rule answers; the registry is src/eval/questions.ts. */
+export type QuestionId = 'safe_output' | 'grounded' | 'complete' | 'relevant' | 'task_completed' | 'tool_use_correct' | 'within_budget';
+/** What went wrong, in the reader's words, independent of which rule caught it; the registry is src/eval/failure-classes.ts. */
+export type FailureClass = 'pii_leak' | 'credential_leak' | 'injection' | 'injection_compliance' | 'silent_tool_failure' | 'tool_loop' | 'stub' | 'fabrication' | 'ungrounded' | 'incomplete_ask' | 'off_task' | 'over_budget' | 'format' | 'invalid_tool_call';
 export interface EvalRule {
     name: string;
     description: string;
@@ -25,6 +42,23 @@ export interface EvalRule {
      * is the verdict, and a critical violation must never be averaged away.
      */
     critical?: boolean;
+    /**
+     * The rule's metadata — what kind of claim it makes, how it measures it,
+     * what it reads, which question it answers, which failure classes a
+     * failing result belongs to, and the version of its definition. Every
+     * built-in declares all six (tests/unit/eval/rule-metadata.test.ts);
+     * custom types declare kind, mechanism, needs and version and leave the
+     * question to their author. Optional on the interface so a rule built
+     * elsewhere still compiles; a result from a rule without them carries no
+     * `kind`, which reads as unknown — never as a measurement.
+     */
+    kind?: ClaimKind;
+    mechanism?: Mechanism;
+    needs?: readonly Need[];
+    question?: QuestionId;
+    classes?: readonly FailureClass[];
+    /** Bumped when the rule's meaning changes, so a stored result names the definition that produced it. */
+    version?: number;
     evaluate(context: EvalContext): EvalRuleResult;
 }
 export interface EvalContext {
@@ -64,8 +98,221 @@ export interface EvalContext {
         breaches: number;
     };
 }
+/**
+ * What the composer DID with a result under this deployment's configuration
+ * — distinct from `kind`, which is what the rule claims. Today's composer
+ * (a weighted mean plus the critical veto) knows two roles: `veto` for an
+ * effectively critical rule and `term` for one that feeds the score. The
+ * compose-by-kind release adds `gate` (a configured policy that decides),
+ * `risk` (a detection or inference feeding the risk estimate) and
+ * `advisory` (reported, deciding nothing).
+ */
+export type Role = 'gate' | 'veto' | 'risk' | 'advisory' | 'term';
+/**
+ * Why a rule skipped. `not_applicable`: the evidence it needs was not
+ * supplied (never asked — coverage). `defeated`: asked and could not answer,
+ * because this output stalled its pattern past the sandbox budget.
+ * `config_invalid`: asked and could not answer, because its definition is
+ * broken. A gate that fails closed treats the last two as unknown; the first
+ * is a coverage fact, not a verdict.
+ */
+export type SkipClass = 'not_applicable' | 'defeated' | 'config_invalid';
+export interface Interval {
+    point: number;
+    lo: number;
+    hi: number;
+}
+/**
+ * What a rule saw — typed, locatable, never an excerpt. A detection reports
+ * the OFFSETS of what it matched (into the raw text, so a leak detector can
+ * redact the span it found without ever repeating it); a trajectory rule the
+ * index of the call it judged; a measurement its statistic with a unit and
+ * the threshold it was held to; a signal that yields no offset yet reports
+ * its name and count. The reader can locate every claim; the stored row
+ * can be redacted; nothing here restates the offending text.
+ */
+export type Evidence = {
+    type: 'span';
+    source: 'output' | 'input' | `tool_outputs[${number}]`;
+    start: number;
+    end: number;
+    label: string;
+} | {
+    type: 'pattern';
+    name: string;
+    count: number;
+} | {
+    type: 'toolCall';
+    index: number;
+    toolName: string;
+    label: string;
+} | {
+    type: 'citation';
+    url: string;
+    status: 'resolved' | 'dead' | 'unverifiable' | 'supported' | 'unsupported';
+} | {
+    type: 'count';
+    stat: string;
+    unit: string;
+    value: number;
+    threshold?: number;
+    thresholdSource?: 'default' | 'config' | 'call' | 'rule';
+};
+/** A measurement's statistic — the number the rule computed, with its unit, before any score transform. */
+export interface MeasuredValue {
+    stat: string;
+    unit: string;
+    value: number;
+}
+/** Evidence lists are capped so a pathological output cannot balloon a stored row. */
+export declare const MAX_EVIDENCE_ITEMS = 25;
+/**
+ * Which evaluation questions this evaluation judged, which it did not and
+ * why — coverage by question, not by rule count. `inputs` says what the
+ * call carried; a question is `judged` when at least one rule that answers
+ * it ran, `unjudged` when every such rule skipped (the reason names the
+ * missing input, or that the rule was defeated or broken), and
+ * `not_applicable` when no rule for it was in the selected bundles.
+ */
+export interface Coverage {
+    inputs: Record<Need, boolean>;
+    questions: Array<{
+        id: QuestionId;
+        status: 'judged' | 'unjudged' | 'not_applicable';
+        why?: string;
+    }>;
+    /** Quarantined critical rules that did not run (surfaced by the rule-store release). */
+    dormant?: Array<{
+        ruleId: string;
+        name: string;
+        reason: string;
+    }>;
+}
+/**
+ * The verdict with its basis. `passed` is `state === 'pass'` and equals the
+ * top-level `passed`; `basis` says which layer decided — a configured policy,
+ * a detector's veto, nothing judged, or the score against the threshold.
+ * `risk` is null until the compose-by-kind release computes it.
+ */
+export interface Verdict {
+    state: 'pass' | 'fail' | 'unknown';
+    passed: boolean;
+    basis: 'policy_gate' | 'detector_veto' | 'critical_unknown' | 'required_evidence_missing' | 'risk_over_loss' | 'score_below_threshold' | 'clean' | 'no_rules';
+    by: string[];
+    risk: {
+        pBad: number;
+        lo: number;
+        hi: number;
+        perClass: Partial<Record<FailureClass, number | null>>;
+        assumptions: string[];
+    } | null;
+    confidence?: 'decisive' | 'marginal';
+}
+/** What produced this verdict, so it can be replayed or compared: the release, the ruleset, the configuration, the thresholds, the proof corpus, the time. */
+export interface Provenance {
+    irisVersion: string;
+    rulesetHash: string;
+    configHash: string;
+    thresholds: {
+        default: number;
+        perRule?: Record<string, unknown>;
+    };
+    corpusVersion: string;
+    judgedAt: string;
+}
+/**
+ * How wrong this result tends to be, and on what basis. `published_accuracy`
+ * carries the rule's measured numbers from the shipped proof (src/eval/
+ * published-accuracy.ts): for a fired detection or inference the positive
+ * predictive value at the stated prior, for one that did not fire the
+ * residual miss rate, each with a 95% credible interval. `definition` is a
+ * measurement's conformance to its formula (n cases, matched). `policy` is
+ * the deployment's own constraint — no error rate applies. `self_consistency`
+ * and `local_labels` arrive with the judge-through-the-engine and the
+ * own-traffic labels releases. `unmeasured` says why nothing can be stated.
+ */
+export type Uncertainty = {
+    basis: 'published_accuracy';
+    fired: true;
+    ppv: Interval;
+    prior: {
+        pi: number;
+        source: 'default' | 'config' | 'estimated';
+    };
+    corpus: {
+        n: number;
+        tp: number;
+        fp: number;
+        fn: number;
+        tn: number;
+        version: string;
+        release: string;
+        labelling: 'same-model' | 'human-verified';
+    };
+} | {
+    basis: 'published_accuracy';
+    fired: false;
+    missRate: Interval;
+    prior: {
+        pi: number;
+        source: 'default' | 'config' | 'estimated';
+    };
+    corpus: {
+        n: number;
+        tp: number;
+        fp: number;
+        fn: number;
+        tn: number;
+        version: string;
+        release: string;
+        labelling: 'same-model' | 'human-verified';
+    };
+} | {
+    basis: 'definition';
+    conformance: {
+        n: number;
+        matched: number;
+    };
+} | {
+    basis: 'self_consistency';
+    samples: number;
+    voteFraction: number;
+    scoreSd: number;
+} | {
+    basis: 'local_labels';
+    precision: Interval;
+    n: number;
+} | {
+    basis: 'policy';
+} | {
+    basis: 'unmeasured';
+    why: string;
+};
 export interface EvalRuleResult {
     ruleName: string;
+    /**
+     * What kind of claim this result makes, what the composer did with it,
+     * which question it answers and which failure classes a failure belongs
+     * to — stamped by the engine from the rule's declaration (0.9.0). Absent
+     * on results written before that release and on rules that declare no
+     * metadata; never fabricated on read.
+     */
+    kind?: ClaimKind;
+    role?: Role;
+    question?: QuestionId;
+    classes?: FailureClass[];
+    /** The version of the rule definition that produced this result. */
+    ruleVersion?: number;
+    /** Which of the rule's declared needs the call actually carried — what the rule SAW. */
+    saw?: Need[];
+    /** Present only when `skipped`; says whether the rule was never asked or was asked and could not answer. */
+    skipClass?: SkipClass;
+    /** How wrong this result tends to be, and on what basis. Present on every result that made a claim (not on skips). */
+    uncertainty?: Uncertainty;
+    /** What the rule saw: spans (offsets, never text), tool-call indices, pattern names, counts. Present on every fired detection or inference, and on measurements. */
+    evidence?: Evidence[];
+    /** A measurement's statistic and unit — the number before the score transform. */
+    value?: MeasuredValue;
     /**
      * Deployed rule id (rule-<hex>) when the rule came from the custom-rule
      * store. Absent for built-in rules and for inline custom_rules. Names are
@@ -169,6 +416,17 @@ export interface EvalResult {
      * rule_results carry a `category` per rule so a reader can regroup.
      */
     categories?: Partial<Record<EvalType, EvalCategoryResult>>;
+    /** The verdict with its basis (0.9.0) — computed by the engine, derived on read for stored rows that carry provenance. */
+    verdict?: Verdict;
+    /** Coverage by evaluation question (0.9.0) — computed by the engine, derived on read from the stamped rule results. */
+    coverage?: Coverage;
+    /** What produced this verdict (0.9.0) — persisted; absent on rows written before it, never fabricated. */
+    provenance?: Provenance;
+    /** What the evaluation itself cost (the judge's spend); undefined for the free rules. */
+    eval_cost_usd?: number;
+    eval_tokens?: number;
+    /** Set when the linked trace was deleted (delete_trace or the retention sweep) and this row's text was erased. */
+    erased_at?: string;
 }
 export type CustomRuleType = 'regex_match' | 'regex_no_match' | 'min_length' | 'max_length' | 'contains_keywords' | 'excludes_keywords' | 'json_schema' | 'cost_threshold';
 export interface CustomRuleDefinition {

package/dist/types/eval.js

@@ -1 +1,2 @@
-export {};
+/** Evidence lists are capped so a pathological output cannot balloon a stored row. */
+export const MAX_EVIDENCE_ITEMS = 25;

package/dist/types/query.d.ts

@@ -82,6 +82,8 @@ export interface IStorageAdapter {
     getSpansByTraceId(tenantId: TenantId, traceId: string): Promise<Span[]>;
     insertEvalResult(tenantId: TenantId, result: EvalResult): Promise<void>;
     getEvalsByTraceId(tenantId: TenantId, traceId: string): Promise<EvalResult[]>;
+    /** One stored evaluation by id, in the same derived-on-read shape as every other reader; null when absent. */
+    getEvalById(tenantId: TenantId, id: string): Promise<EvalResult | null>;
     queryEvalResults(tenantId: TenantId, options: {
         eval_type?: string;
         passed?: boolean;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@iris-eval/mcp-server",
-  "version": "0.8.2",
+  "version": "0.9.0",
   "description": "Stop shipping agents on vibes. Score every agent output for quality, safety, and cost.",
   "mcpName": "io.github.iris-eval/mcp-server",
   "type": "module",

package/server.json

@@ -6,12 +6,12 @@
     "url": "https://github.com/iris-eval/mcp-server",
     "source": "github"
   },
-  "version": "0.8.2",
+  "version": "0.9.0",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "@iris-eval/mcp-server",
-      "version": "0.8.2",
+      "version": "0.9.0",
       "transport": {
         "type": "stdio"
       },

package/dist/resources/dashboard-summary.d.ts

@@ -1,3 +0,0 @@
-import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
-import type { IStorageAdapter } from '../types/query.js';
-export declare function registerDashboardSummaryResource(server: McpServer, storage: IStorageAdapter): void;

package/dist/resources/dashboard-summary.js

@@ -1,16 +0,0 @@
-import { LOCAL_TENANT } from '../types/tenant.js';
-export function registerDashboardSummaryResource(server, storage) {
-    server.resource('dashboard-summary', 'iris://dashboard/summary', { description: 'Dashboard summary with key metrics and trends' }, async () => {
-        // OSS single-tenant: summary scopes to the local user.
-        const summary = await storage.getDashboardSummary(LOCAL_TENANT);
-        return {
-            contents: [
-                {
-                    uri: 'iris://dashboard/summary',
-                    mimeType: 'application/json',
-                    text: JSON.stringify(summary, null, 2),
-                },
-            ],
-        };
-    });
-}

package/dist/resources/trace-detail.d.ts

@@ -1,3 +0,0 @@
-import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
-import type { IStorageAdapter } from '../types/query.js';
-export declare function registerTraceDetailResource(server: McpServer, storage: IStorageAdapter): void;