@iris-eval/mcp-server 0.8.2 → 0.10.0

package/dist/tools/log-trace.js

@@ -3,6 +3,9 @@ import { generateTraceId, generateSpanId } from '../utils/ids.js';
 import { LOCAL_TENANT } from '../types/tenant.js';
 import { bestEffortExport } from '../otel/lazy.js';
 import { strictInput, strictNested } from './strict-input.js';
+import { describeTool, ERROR_ENVELOPE_SENTENCE } from './describe.js';
+import { guarded, respond } from './respond.js';
+import { traceUri } from '../resources/uris.js';
 /*
  * The tool-call record — one entry of `tool_calls[]`.
  *
@@ -58,47 +61,51 @@ export const logTraceInputShape = {
     framework: z.string().optional().describe('Agent framework identifier (e.g., langchain, autogen, custom)'),
     input: z.string().optional().describe('Agent input text — the user prompt or upstream input that produced this output'),
     output: z.string().optional().describe('Agent output text — what the agent produced (pass to evaluate_output for scoring)'),
-    tool_calls: z.array(toolCallSchema).optional().describe('Tool calls made during execution (per-call latency, errors, input/output)'),
+    tool_calls: z.array(toolCallSchema).optional().describe('Tool calls made during execution, in order, each { tool_name, input?, output?, latency_ms?, error? } — what the trajectory rules judge; evaluate_output reuses them when given this trace_id'),
     latency_ms: z.number().optional().describe('Total execution time in milliseconds (end-to-end agent latency)'),
     token_usage: TokenUsageSchema.optional().describe('Token usage breakdown (prompt/completion/total — used for cost analysis)'),
     cost_usd: z.number().optional().describe('Total cost in USD — overrides per-span aggregation when provided (treated as authoritative)'),
     metadata: z.record(z.string(), z.unknown()).optional().describe('Opaque key-value tags (e.g. {requestId, userId, env}) — queryable in dashboard, not via get_traces filters'),
-    spans: z.array(SpanSchema).optional().describe('Detailed execution spans (hierarchical span tree with timings, attributes, events)'),
+    spans: z.array(SpanSchema).optional().describe('Detailed execution spans (hierarchical span tree with timings, attributes, events); a span without start_time takes the trace timestamp'),
     timestamp: z.string().optional().describe('Trace timestamp (ISO 8601); defaults to now() when omitted'),
 };
+export const logTraceOutputSchema = z.looseObject({
+    trace_id: z.string().describe('the stored trace id, 32 hex — pass it to evaluate_output, get_traces or delete_trace'),
+    status: z.literal('stored').describe('always "stored" on success'),
+});
 export function registerLogTraceTool(server, storage) {
     server.registerTool('log_trace', {
         title: 'Log Trace',
-        description: [
-            'Persist a single agent execution trace (input, output, spans, tool calls, cost, latency, token usage).',
-            '',
-            'Sibling tools — evaluate_output runs heuristic scoring on the trace; evaluate_with_llm_judge runs semantic LLM-based scoring; verify_citations checks citation grounding; get_traces queries stored traces; delete_trace removes a single trace; list_rules / deploy_rule / delete_rule manage custom evaluation rules. log_trace is the WRITE path that records executions; everything else reads, scores, or manages around it.',
-            '',
-            'Behavior. Writes one row to Iris storage (SQLite). When IRIS_OTEL_ENDPOINT is set, ALSO fires a best-effort async export to the configured OTLP/HTTP collector (Jaeger, Tempo, Datadog OTLP, OTEL Collector). The OTel export is fire-and-forget — its success does not affect the tool response; failures are logged but the trace is still stored locally. No authentication in stdio mode. HTTP mode requires a Bearer token ONLY when --api-key / IRIS_API_KEY is set (recommended); with no key configured the auth middleware is a pass-through and writes are unauthenticated — a default HTTP server is protected by its loopback bind (127.0.0.1) and Origin validation, not by a credential. Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio. Not idempotent: each call mints a fresh trace_id, so resubmitting the same payload creates a duplicate trace.',
-            '',
-            'Output shape. Returns a JSON string: `{ "trace_id": "<32-hex>", "status": "stored" }`. The trace_id is the key you pass to evaluate_output or get_traces afterwards.',
-            '',
-            'Use when you want to record an agent execution for later evaluation, analysis, or audit. Call it AFTER the agent has produced output; call evaluate_output afterwards to score it; call get_traces to query historical traces. Store rich context: spans (span tree), tool_calls (which tools were invoked with latency/errors), token_usage, cost_usd, metadata (arbitrary key-value). All optional except agent_name.',
-            '',
-            'Don\'t use when you only need a transient log (use console logging). Don\'t use to update an existing trace — there is no update path (traces are immutable once stored).',
-            '',
-            'Parameters. agent_name is required; everything else is optional. token_usage and cost_usd are summary fields — if you ALSO pass spans with per-tool-call costs, the summary fields are treated as authoritative (no auto-aggregation). spans without an explicit start_time fall back to the trace timestamp; spans with an end_time get a duration_ms derived. metadata is opaque key-value (queryable in the dashboard, not via get_traces filters). tool_calls record per-tool latency + errors; missing latency_ms means "not reported," not "zero." Defaults: span.kind="INTERNAL", span.status_code="UNSET", timestamp=now() if omitted.',
-            '',
-            'Error modes. Throws on missing agent_name. Throws on malformed span or tool_call objects (Zod rejects). Returns 500 on storage failure (disk full, DB locked). Never blocks on the agent — the local write is synchronous and the OTel export is asynchronous.',
-        ].join('\n'),
+        description: describeTool({
+            summary: 'Store one agent execution — input, output, tool calls, spans, cost, latency, token usage — and get the trace_id every later call keys on.',
+            does: 'Writes one trace row to local SQLite and mints a fresh trace_id; nothing is deduplicated, so resubmitting the same payload stores a second trace. ' +
+                'Only agent_name is required. Store what you have: tool_calls so the trajectory rules can later judge what the agent did, cost_usd and token_usage so the cost rules can, input and output so everything else can. ' +
+                'When IRIS_OTEL_ENDPOINT is set the trace is also exported to that collector, best-effort and asynchronous; the local write never waits on it. ' +
+                'Traces are immutable: there is no update path. In stdio mode nothing authenticates the caller; over HTTP a Bearer token is required only when an API key is configured.',
+            whenNot: 'For a transient log line (use your logger). To score an output: log first, then call evaluate_output with the trace_id, which also lets it reuse the stored tool_calls. To change a stored trace: delete_trace and log again.',
+            returns: logTraceOutputSchema,
+            errors: 'IRIS_STORAGE_ERROR when the database cannot be written. An unknown argument or a malformed span or tool_calls entry is refused before the handler runs, naming the valid keys. ' +
+                ERROR_ENVELOPE_SENTENCE,
+            siblings: {
+                evaluate_output: 'score the stored output',
+                get_traces: 'query what was logged',
+                delete_trace: 'remove one trace',
+            },
+        }),
         // Strict at the MCP boundary (unknown args rejected, not stripped).
         // The dashboard's HTTP ingest builds its own — equally strict —
         // schema FROM this shape (dashboard/validation.ts): a client-supplied
         // trace_id is rejected there with a 400 whose message says the server
         // mints it, exactly as this tool mints its own in the handler below.
         inputSchema: strictInput(logTraceInputShape),
+        outputSchema: logTraceOutputSchema,
         annotations: {
             readOnlyHint: false, // Writes a row to storage
             destructiveHint: false, // Creates new data; doesn't overwrite or delete
             idempotentHint: false, // Each call mints a fresh trace_id; duplicate payloads produce distinct traces
             openWorldHint: false, // Local storage first. When IRIS_OTEL_ENDPOINT is set a best-effort async OTel export runs but is non-blocking (tool succeeds even if export fails).
         },
-    }, async (args) => {
+    }, guarded(async (args) => {
         const traceId = generateTraceId();
         const timestamp = args.timestamp ?? new Date().toISOString();
         const trace = {
@@ -128,13 +135,8 @@ export function registerLogTraceTool(server, storage) {
             // eslint-disable-next-line no-console
             console.warn(`[iris.otel] ${err.message}`);
         });
-        return {
-            content: [
-                {
-                    type: 'text',
-                    text: JSON.stringify({ trace_id: traceId, status: 'stored' }),
-                },
-            ],
-        };
-    });
+        return respond(logTraceOutputSchema, { trace_id: traceId, status: 'stored' }, [
+            { uri: traceUri(traceId), name: `trace ${traceId}`, description: 'The stored trace with its spans and, later, its evaluations' },
+        ]);
+    }));
 }

package/dist/tools/respond.d.ts

@@ -0,0 +1,42 @@
+import { z } from 'zod';
+import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
+import { IrisError } from './errors.js';
+export declare const CAPABILITIES_URI = "iris://capabilities";
+export interface ResourceLink {
+    uri: string;
+    name: string;
+    description?: string;
+}
+export declare const CAPABILITIES_LINK: ResourceLink;
+export declare function respond<S extends z.ZodType>(schema: S, payload: object, links?: ResourceLink[]): CallToolResult;
+export declare const errorEnvelopeSchema: z.ZodObject<{
+    error: z.ZodObject<{
+        code: z.ZodEnum<{
+            IRIS_INVALID_ARGUMENT: "IRIS_INVALID_ARGUMENT";
+            IRIS_UNKNOWN_TRACE: "IRIS_UNKNOWN_TRACE";
+            IRIS_DUPLICATE_RULE: "IRIS_DUPLICATE_RULE";
+            IRIS_INVALID_RULE_CONFIG: "IRIS_INVALID_RULE_CONFIG";
+            IRIS_JUDGE_NOT_ENABLED: "IRIS_JUDGE_NOT_ENABLED";
+            IRIS_JUDGE_UNKNOWN_MODEL: "IRIS_JUDGE_UNKNOWN_MODEL";
+            IRIS_BUDGET_EXCEEDED: "IRIS_BUDGET_EXCEEDED";
+            IRIS_PROVIDER_ERROR: "IRIS_PROVIDER_ERROR";
+            IRIS_JUDGE_FAILED: "IRIS_JUDGE_FAILED";
+            IRIS_STORAGE_ERROR: "IRIS_STORAGE_ERROR";
+            IRIS_INTERNAL_ERROR: "IRIS_INTERNAL_ERROR";
+        }>;
+        message: z.ZodString;
+        recovery: z.ZodArray<z.ZodString>;
+        retryable: z.ZodBoolean;
+        field: z.ZodOptional<z.ZodString>;
+        valid: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        see: z.ZodOptional<z.ZodString>;
+        kind: z.ZodOptional<z.ZodString>;
+        retryAfterMs: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$loose>;
+}, z.core.$loose>;
+export type ErrorEnvelope = z.infer<typeof errorEnvelopeSchema>;
+export declare function errorResult(err: IrisError): CallToolResult;
+/** Wrap a handler so every failure returns an envelope instead of a flattened line. */
+export declare function guarded<A extends unknown[]>(fn: (...args: A) => Promise<CallToolResult> | CallToolResult): (...args: A) => Promise<CallToolResult>;
+/** Links for what an evaluation created: the evaluation, and the trace when linked. */
+export declare function evaluationLinks(evalId: string, traceId?: string): ResourceLink[];

package/dist/tools/respond.js

@@ -0,0 +1,90 @@
+/*
+ * One way to answer a tool call.
+ *
+ * `respond` parses the payload through the tool's own output schema
+ * BEFORE serialising it — a field the schema does not describe fails a
+ * test, not a user — and emits the same object twice: as the text a
+ * client without structured-content support reads, and as
+ * `structuredContent` for one that has it. Beside the payload go
+ * `resource_link` items for what the call created and what explains its
+ * limits, so a client can follow them with resources/read instead of
+ * guessing a URI.
+ *
+ * `errorResult` is the failure shape: the IrisError envelope as the text
+ * and as structuredContent, `isError: true`, and a link to
+ * iris://capabilities. `guarded` wraps a handler so nothing thrown inside
+ * it reaches the SDK's flattener.
+ */
+import { z } from 'zod';
+import { ERROR_CODE_CATALOGUE, toIrisError } from './errors.js';
+import { CAPABILITIES_RESOURCE_URI, evaluationUri, traceUri } from '../resources/uris.js';
+export const CAPABILITIES_URI = CAPABILITIES_RESOURCE_URI;
+export const CAPABILITIES_LINK = {
+    uri: CAPABILITIES_URI,
+    name: 'capabilities',
+    description: 'What this server can judge, what each rule needs, judge state, limits, tools and resources',
+};
+const linkItem = (l) => ({
+    type: 'resource_link',
+    uri: l.uri,
+    name: l.name,
+    ...(l.description ? { description: l.description } : {}),
+    mimeType: 'application/json',
+});
+/** JSON round-trip: what the text carries is exactly what structuredContent carries. */
+function normalise(payload) {
+    return JSON.parse(JSON.stringify(payload));
+}
+export function respond(schema, payload, links = []) {
+    const body = normalise(payload);
+    const parsed = schema.safeParse(body);
+    if (!parsed.success) {
+        // A programming error: the tool built a response its own schema does
+        // not describe. Loud on purpose — the drift-lock tests catch it.
+        const issues = parsed.error.issues.slice(0, 3).map((i) => `${i.path.map(String).join('.') || '(root)'}: ${i.message}`);
+        throw new Error(`response does not match the tool's output schema: ${issues.join('; ')}`);
+    }
+    return {
+        content: [{ type: 'text', text: JSON.stringify(body) }, ...links.map(linkItem)],
+        structuredContent: body,
+    };
+}
+export const errorEnvelopeSchema = z.looseObject({
+    error: z.looseObject({
+        code: z.enum(ERROR_CODE_CATALOGUE),
+        message: z.string(),
+        recovery: z.array(z.string()),
+        retryable: z.boolean(),
+        field: z.string().optional(),
+        valid: z.array(z.string()).optional(),
+        see: z.string().optional(),
+        kind: z.string().optional(),
+        retryAfterMs: z.number().optional(),
+    }),
+});
+export function errorResult(err) {
+    const body = normalise({ error: err.envelope });
+    return {
+        content: [{ type: 'text', text: JSON.stringify(body) }, linkItem(CAPABILITIES_LINK)],
+        structuredContent: body,
+        isError: true,
+    };
+}
+/** Wrap a handler so every failure returns an envelope instead of a flattened line. */
+export function guarded(fn) {
+    return async (...args) => {
+        try {
+            return await fn(...args);
+        }
+        catch (err) {
+            return errorResult(toIrisError(err));
+        }
+    };
+}
+/** Links for what an evaluation created: the evaluation, and the trace when linked. */
+export function evaluationLinks(evalId, traceId) {
+    const links = [{ uri: evaluationUri(evalId), name: `evaluation ${evalId}`, description: 'The stored evaluation, as every reader sees it' }];
+    if (traceId)
+        links.push({ uri: traceUri(traceId), name: `trace ${traceId}`, description: 'The trace this evaluation is linked to, with its spans and every evaluation' });
+    return links;
+}

package/dist/tools/strict-input.js

@@ -29,7 +29,7 @@ export function strictInput(shape) {
                 `Valid arguments: ${validKeys}. ` +
                 'Unknown arguments are rejected rather than silently ignored, so a misspelled ' +
                 'argument name cannot change what gets evaluated — check the spelling against ' +
-                "the tool's input schema and retry."
+                "the tool's input schema and retry. Code IRIS_INVALID_ARGUMENT."
             : undefined,
     });
 }

package/dist/tools/trace-link.d.ts

@@ -11,6 +11,8 @@ export declare function unknownTraceMessage(traceId: string): string;
  * existence check had to load the row anyway. Fetching it twice would be
  * two reads for one fact — and two chances for them to disagree.
  */
+/** The IRIS_UNKNOWN_TRACE error, built once for both the pre-check and the insert race. */
+export declare function unknownTraceError(traceId: string): import("./errors.js").IrisError;
 export declare function getTraceOrThrow(storage: IStorageAdapter, tenantId: TenantId, traceId: string): Promise<Trace>;
 export declare function assertTraceExists(storage: IStorageAdapter, tenantId: TenantId, traceId: string): Promise<void>;
 /** insertEvalResult with the foreign-key race translated into the same clear message. */

package/dist/tools/trace-link.js

@@ -1,3 +1,4 @@
+import { irisError } from './errors.js';
 /*
  * Linking an evaluation to a trace that does not exist.
  *
@@ -26,10 +27,20 @@ export function unknownTraceMessage(traceId) {
  * existence check had to load the row anyway. Fetching it twice would be
  * two reads for one fact — and two chances for them to disagree.
  */
+/** The IRIS_UNKNOWN_TRACE error, built once for both the pre-check and the insert race. */
+export function unknownTraceError(traceId) {
+    return irisError('IRIS_UNKNOWN_TRACE', unknownTraceMessage(traceId), {
+        field: 'trace_id',
+        recovery: [
+            'Pass the trace_id that log_trace returned, or one listed by get_traces.',
+            'Or omit trace_id to store an unlinked evaluation.',
+        ],
+    });
+}
 export async function getTraceOrThrow(storage, tenantId, traceId) {
     const trace = await storage.getTrace(tenantId, traceId);
     if (!trace)
-        throw new Error(unknownTraceMessage(traceId));
+        throw unknownTraceError(traceId);
     return trace;
 }
 export async function assertTraceExists(storage, tenantId, traceId) {
@@ -44,7 +55,7 @@ export async function insertLinkedEvalResult(storage, tenantId, result) {
         const code = err.code;
         const message = err instanceof Error ? err.message : String(err);
         if (result.trace_id && (code === 'SQLITE_CONSTRAINT_FOREIGNKEY' || /FOREIGN KEY constraint failed/i.test(message))) {
-            throw new Error(unknownTraceMessage(result.trace_id));
+            throw unknownTraceError(result.trace_id);
         }
         throw err;
     }

package/dist/tools/verify-citations.d.ts

@@ -1,3 +1,4 @@
+import { z } from 'zod';
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { IStorageAdapter } from '../types/query.js';
 /**
@@ -5,8 +6,8 @@ import type { IStorageAdapter } from '../types/query.js';
  * was nothing to judge (no citations, none resolved). It is NOT the honest
  * answer when citations resolved and the judge then failed on every one —
  * a wrong API key, a model the provider refused, a parse failure — because
- * the caller reads "passed" and ships. That case is an error naming the
- * cause; nothing is stored. (v0.6.0 acceptance pass, observation 3.)
+ * the caller reads "passed" and ships. That case is IRIS_JUDGE_FAILED
+ * naming the cause; nothing is stored. (v0.6.0 acceptance pass, observation 3.)
  */
 export declare function assertJudgeRan(result: {
     totalResolved: number;
@@ -19,4 +20,19 @@ export declare function assertJudgeRan(result: {
         };
     }>;
 }): void;
+export declare const verifyCitationsOutputSchema: z.ZodObject<{
+    id: z.ZodString;
+    trace_id: z.ZodOptional<z.ZodString>;
+    overall_score: z.ZodNullable<z.ZodNumber>;
+    passed: z.ZodNullable<z.ZodBoolean>;
+    total_unsupported: z.ZodNumber;
+    total_citations_found: z.ZodNumber;
+    total_resolved: z.ZodNumber;
+    total_judged: z.ZodNumber;
+    total_supported: z.ZodNumber;
+    total_cost_usd: z.ZodNumber;
+    citations: z.ZodArray<z.ZodObject<{
+        resolve_status: z.ZodString;
+    }, z.core.$loose>>;
+}, z.core.$loose>;
 export declare function registerVerifyCitationsTool(server: McpServer, storage: IStorageAdapter): void;

package/dist/tools/verify-citations.js

@@ -1,41 +1,31 @@
 import { z } from 'zod';
 import { LOCAL_TENANT } from '../types/tenant.js';
 import { verifyCitations } from '../eval/citation-verify/verifier.js';
-import { findPricing } from '../eval/llm-judge/pricing.js';
 import { generateEvalId } from '../utils/ids.js';
+import { JUDGE_KEY_VARS } from '../judge-enablement.js';
 import { strictInput } from './strict-input.js';
 import { assertTraceExists, insertLinkedEvalResult } from './trace-link.js';
+import { inferProvider, resolveApiKey } from './evaluate-with-llm-judge.js';
+import { describeTool, ERROR_ENVELOPE_SENTENCE } from './describe.js';
+import { irisError } from './errors.js';
+import { evaluationLinks, guarded, respond } from './respond.js';
 const inputSchema = {
     output: z.string().min(1).describe('The agent output containing citations to verify'),
     model: z
         .string()
-        .describe('Judge model for per-citation verification. Supported: anthropic = claude-opus-4-7 | claude-sonnet-4-6 | claude-haiku-4-5-20251001; openai = gpt-4o | gpt-4o-mini | o1-mini.'),
+        .describe('Judge model for per-citation verification. Supported: anthropic = claude-opus-4-7 | claude-sonnet-4-6 | claude-haiku-4-5 | claude-haiku-4-5-20251001; openai = gpt-4o | gpt-4o-mini | o1-mini.'),
     provider: z.enum(['anthropic', 'openai']).optional().describe('Auto-detected from model when omitted'),
     allow_fetch: z.boolean().optional().describe('Permit outbound HTTP to resolve URLs/DOIs. Defaults to IRIS_CITATION_ALLOW_FETCH=1; false otherwise. SSRF-guarded regardless.'),
     domain_allowlist: z
         .array(z.string())
         .optional()
         .describe('Restrict fetches to hostnames in this list (suffix match allowed). Merged with IRIS_CITATION_DOMAINS env.'),
-    max_cost_usd_total: z.number().positive().optional().describe('Cap TOTAL judge cost across all citations in this call; default $1.00'),
-    max_citations: z.number().int().positive().max(50).optional().describe('Max citations to verify (extras skipped); default 20'),
+    max_cost_usd_total: z.number().positive().optional().describe('Cap TOTAL judge cost across all citations in this call; default 1.00 USD — the pipeline stops when the next call would exceed it'),
+    max_citations: z.number().int().positive().max(50).optional().describe('Max citations to verify (extras skipped, not errored); default 20, at most 50'),
     per_source_timeout_ms: z.number().int().positive().optional().describe('Per-URL fetch timeout; default 10_000'),
     per_source_max_bytes: z.number().int().positive().optional().describe('Per-URL body cap; default 5MB'),
     trace_id: z.string().optional().describe('Link verification result to a stored trace (id from log_trace / get_traces); an unknown id is rejected before any fetch or judge call'),
 };
-function inferProvider(model) {
-    const pricing = findPricing(model);
-    if (!pricing) {
-        throw new Error(`Unknown model "${model}". Provider cannot be inferred. Supported models: src/eval/llm-judge/pricing.ts.`);
-    }
-    return pricing.provider;
-}
-function resolveApiKey(provider) {
-    const key = provider === 'anthropic' ? process.env.IRIS_ANTHROPIC_API_KEY : process.env.IRIS_OPENAI_API_KEY;
-    if (!key) {
-        throw new Error(`${provider === 'anthropic' ? 'Anthropic' : 'OpenAI'} judge requires IRIS_${provider === 'anthropic' ? 'ANTHROPIC' : 'OPENAI'}_API_KEY for verify_citations.`);
-    }
-    return key;
-}
 function resolveAllowFetch(paramValue) {
     if (paramValue !== undefined)
         return paramValue;
@@ -54,8 +44,8 @@ function resolveDomainAllowlist(paramValue) {
  * was nothing to judge (no citations, none resolved). It is NOT the honest
  * answer when citations resolved and the judge then failed on every one —
  * a wrong API key, a model the provider refused, a parse failure — because
- * the caller reads "passed" and ships. That case is an error naming the
- * cause; nothing is stored. (v0.6.0 acceptance pass, observation 3.)
+ * the caller reads "passed" and ships. That case is IRIS_JUDGE_FAILED
+ * naming the cause; nothing is stored. (v0.6.0 acceptance pass, observation 3.)
  */
 export function assertJudgeRan(result) {
     if (result.totalResolved === 0 || result.totalJudged > 0)
@@ -65,39 +55,63 @@ export function assertJudgeRan(result) {
         return;
     const kinds = [...new Set(judgeFailures.map((c) => c.resolveError.kind))].join(', ');
     const first = judgeFailures[0].resolveError.message;
-    throw new Error(`verify_citations could not judge any of the ${result.totalResolved} resolved citation(s): the judge failed on every one (${kinds}). ` +
-        `Nothing was verified and nothing was stored, so there is no verdict. First error: ${first}`);
+    throw irisError('IRIS_JUDGE_FAILED', `verify_citations could not judge any of the ${result.totalResolved} resolved citation(s): the judge failed on every one (${kinds}). ` +
+        `Nothing was verified and nothing was stored, so there is no verdict. First error: ${first}`, {
+        retryable: /timeout|rate_limit|server_error/.test(kinds),
+        recovery: [
+            'Check the key and the model: a refused key or an unknown model fails every citation the same way.',
+            'Retry when the kind is a timeout, a rate limit or a provider server error.',
+            'Raise max_cost_usd_total when the kind is cost_cap_reached.',
+        ],
+    });
 }
+export const verifyCitationsOutputSchema = z.looseObject({
+    id: z.string().describe('the evaluation id; read it back at iris://evaluations/{id}'),
+    trace_id: z.string().optional().describe('the linked trace, when one was named'),
+    overall_score: z.number().nullable().describe('supported / judged; null when nothing was judged'),
+    passed: z
+        .boolean()
+        .nullable()
+        .describe('true when every judged citation was supported; false when any judged citation was not; NULL when nothing was judged — no verdict, because nothing was verified. Until 0.10.0 that last case returned true.'),
+    total_unsupported: z.number().int().describe('judged citations the judge ruled unsupported — the number the verdict turns on'),
+    total_citations_found: z.number().int().describe('citations extracted from the output'),
+    total_resolved: z.number().int().describe('citations whose source was fetched'),
+    total_judged: z.number().int().describe('citations the judge ruled on'),
+    total_supported: z.number().int().describe('citations the judge found supported'),
+    total_cost_usd: z.number().describe('the spend across every judge call'),
+    citations: z.array(z.looseObject({ resolve_status: z.string() })).describe('per citation: the citation (raw, kind, identifier, offsets), resolve_status ok | skipped | error, resolve_error, source (url, status, content_type, bytes_fetched, truncated), judge (supported, confidence, rationale, cost_usd, latency_ms, tokens)'),
+});
 export function registerVerifyCitationsTool(server, storage) {
     server.registerTool('verify_citations', {
         title: 'Verify Citations',
-        description: [
-            'Extract citations from agent output, fetch the cited sources, and use an LLM judge to check whether each source supports the claim in context. Returns per-citation verdicts + an overall support ratio.',
-            '',
-            'Sibling tools — evaluate_with_llm_judge runs general semantic scoring (accuracy, helpfulness, correctness, faithfulness); this tool is specifically for citation grounding (does the cited source actually support the claim). evaluate_output\'s no_hallucination_markers heuristic detects FABRICATED-looking citations cheaply (free, no fetch); this tool resolves and verifies them (paid, opt-in fetch, SSRF-guarded). log_trace / get_traces handle trace I/O. verify_citations is the GROUNDING-CHECK path — narrowest in scope, deepest in rigor.',
-            '',
-            'Behavior. Three-phase pipeline: (1) regex extraction of [N] numbered refs, (Author, Year) parentheticals, bare URLs, and DOIs (in-process, no network); (2) SSRF-guarded fetch of URL + DOI citations, with scheme allowlist, private/link-local/cloud-metadata IP blocking, optional domain allowlist (IRIS_CITATION_DOMAINS), 10s timeout, 5MB body cap, manual redirect chase (max 3, re-checked), in-process LRU cache; (3) per-citation LLM judge call asking "does this source support this claim?" with a 256-token verdict. Opt-in via allow_fetch=true or IRIS_CITATION_ALLOW_FETCH=1 — Iris refuses outbound HTTP by default. Cost-capped across the entire call by max_cost_usd_total (default $1.00) — the pipeline stops when the cap would be exceeded. Rate-limited to 20 req/min on HTTP MCP. Writes one eval_result row tagged with per-citation provenance.',
-            '',
-            'Output shape. Returns JSON: `{ "id": "<uuid>", "overall_score": 0..1|null, "passed": boolean, "total_citations_found": number, "total_resolved": number, "total_judged": number, "total_supported": number, "total_cost_usd": number, "citations": [{ "citation": { "raw", "kind", "identifier", "offset_start", "offset_end" }, "resolve_status": "ok"|"skipped"|"error", "resolve_error"?, "source"?: { "url", "status", "content_type", "bytes_fetched", "truncated" }, "judge"?: { "supported", "confidence", "rationale", "cost_usd", "latency_ms", "input_tokens", "output_tokens" } }] }`. `overall_score = supported / judged`; `null` when nothing was judged (no resolvable citations, or every judge call failed). Infrastructure failures (cost cap, judge timeout/error, malformed verdict) leave a citation resolved-but-unjudged — reported per-citation via resolve_error, never scored as unsupported.',
-            '',
-            'Use when the output makes factual claims backed by [1]-style references, DOIs, or URLs and you want to separate "cited correctly" from "cited and wrong" from "cited but unresolvable". Particularly useful for research/legal/medical agents where fabricated citations are the dominant failure mode.',
-            "",
-            "Don't use when the agent output has no citations at all (overall_score will be null; the tool degrades gracefully but a heuristic rule is cheaper). Don't use without allow_fetch=true or IRIS_CITATION_ALLOW_FETCH=1 — the tool refuses outbound HTTP unless explicitly enabled. Don't use with an open allowlist + untrusted output on the public internet; you are effectively running a user-directed fetcher. For stricter safety set IRIS_CITATION_DOMAINS to a curated list.",
-            '',
-            'Parameters. model is required; provider auto-detected from model name (override only for ambiguous IDs). allow_fetch=false by default — outbound HTTP is REFUSED unless explicitly true OR IRIS_CITATION_ALLOW_FETCH=1 env. domain_allowlist suffix-matches hostnames (e.g., "wikipedia.org" allows en.wikipedia.org); merged with IRIS_CITATION_DOMAINS env (UNION — either source permits). max_citations defaults 20, hard cap 50 (extras are skipped silently, NOT errored — check total_citations_found in the response if precise). max_cost_usd_total defaults $1.00 — the pipeline stops mid-citation when the next judge call would exceed the cap (returns partial verdicts). per_source_timeout_ms defaults 10000 (10s); per_source_max_bytes defaults 5MB (truncates at boundary, judges still run on truncated content); independently of that, the judge reads at most the first 12,000 characters of each fetched source, and the per-citation cost estimate is taken on that truncated prompt, not on the full body. trace_id optional but recommended. Defaults: max_citations=20, max_cost_usd_total=$1.00, per_source_timeout_ms=10000, per_source_max_bytes=5242880, allow_fetch=false.',
-            '',
-            'Error modes. Throws when the API key env var is missing. Throws "Unknown model" on unsupported model IDs. Throws when trace_id does not match a stored trace (checked before any fetch or judge call; nothing is written). Per-citation errors are collected (resolve_error.kind = bad_scheme / ssrf / not_allowed_domain / timeout / too_large / bad_status / redirect_loop / not_text / fetch_disabled / malformed_judge_response / cost_cap_reached / unresolvable_kind) and returned in the response rather than thrown. An empty output or output with zero extractable citations returns overall_score=null + passed=true (nothing to fail).',
-        ].join('\n'),
+        description: describeTool({
+            summary: 'Extract the citations in an output, fetch the sources (opt-in, SSRF-guarded) and ask an LLM judge on your key whether each source supports its claim.',
+            does: 'Three phases. Extraction, no network: [N] references, (Author, Year), bare URLs and DOIs. Fetch of URL and DOI citations only when allow_fetch is true or IRIS_CITATION_ALLOW_FETCH=1, through a scheme allowlist, private and cloud-metadata address blocking, an optional hostname allowlist (domain_allowlist, merged with IRIS_CITATION_DOMAINS), a per-source timeout and byte cap, and at most three re-checked redirects. ' +
+                'Then one judge call per resolved citation on your own key, reading the first part of each source, capped in total by max_cost_usd_total. Up to max_citations are verified; extras are skipped, not errored. ' +
+                'overall_score is supported / judged and null when nothing was judged. Per-citation failures (bad scheme, blocked address, timeout, too large, cost cap, fetch disabled) are reported on the citation, never scored as unsupported. One evaluation row is stored.',
+            whenNot: "When the output has no citations: the score is null, and evaluate_output's hallucination signals are the cheap check. " +
+                `Without a key (${JUDGE_KEY_VARS.anthropic} or ${JUDGE_KEY_VARS.openai}): the call returns IRIS_JUDGE_NOT_ENABLED with the enable steps. ` +
+                'With fetch enabled and an open allowlist on untrusted output: you are running a user-directed fetcher — set IRIS_CITATION_DOMAINS.',
+            returns: verifyCitationsOutputSchema,
+            errors: 'IRIS_JUDGE_NOT_ENABLED, IRIS_JUDGE_UNKNOWN_MODEL and IRIS_UNKNOWN_TRACE before any fetch or spend. IRIS_JUDGE_FAILED when citations resolved but the judge failed on every one — an error, not a passing verdict; nothing is stored. ' +
+                ERROR_ENVELOPE_SENTENCE,
+            siblings: {
+                evaluate_with_llm_judge: 'general semantic scoring',
+                evaluate_output: 'the free deterministic path, including the hallucination signals',
+                log_trace: 'record the execution first',
+            },
+        }),
         inputSchema: strictInput(inputSchema),
+        outputSchema: verifyCitationsOutputSchema,
         annotations: {
             readOnlyHint: false, // Writes eval_result + spends money
             destructiveHint: false, // Creates data; doesn't overwrite/delete
             idempotentHint: false, // External fetches + provider non-determinism
             openWorldHint: true, // Outbound HTTP to citation URLs + LLM provider API
         },
-    }, async (args) => {
+    }, guarded(async (args) => {
         const provider = args.provider ?? inferProvider(args.model);
-        const apiKey = resolveApiKey(provider);
+        const apiKey = resolveApiKey(provider, 'verify_citations');
         const allowFetch = resolveAllowFetch(args.allow_fetch);
         const domainAllowlist = resolveDomainAllowlist(args.domain_allowlist);
         // Refused before any fetch or judge call spends anything (#376).
@@ -128,73 +142,85 @@ export function registerVerifyCitationsTool(server, storage) {
             eval_type: 'custom',
             output_text: args.output,
             score,
-            passed: result.passed,
+            passed: result.passed === true,
             rule_results: [
                 {
                     ruleName: `semantic_citation_verify:${provider}/${args.model}`,
-                    passed: result.passed,
+                    /*
+                     * Null means nothing was judged, which is not a pass and not a
+                     * failure — it is a check that did not run. Stored as a SKIP so
+                     * the composer treats it as coverage rather than silently
+                     * reading a paid-for "nothing verified" as clean.
+                     */
+                    passed: result.passed === null ? false : result.passed,
+                    ...(result.passed === null
+                        ? { skipped: true, skipReason: `no citation was judged (found ${result.totalCitationsFound}, resolved ${result.totalResolved})` }
+                        : {}),
+                    kind: 'judgment',
                     score,
                     message: result.overallScore === null
                         ? `No citations judged (found ${result.totalCitationsFound}, resolved ${result.totalResolved}, judged 0)`
                         : `${result.totalSupported}/${result.totalJudged} judged sources supported the output`,
                 },
             ],
-            suggestions: result.passed ? [] : [`Only ${result.totalSupported}/${result.totalJudged} judged sources actually supported the claim.`],
+            suggestions: result.passed === null
+                ? ['No citation was judged, so nothing about the sources was verified. This is not a pass.']
+                : result.passed
+                    ? []
+                    : [`${result.totalUnsupported} of ${result.totalJudged} judged sources did not support the claim.`],
             rules_evaluated: 1,
             rules_skipped: 0,
             insufficient_data: result.overallScore === null,
+            eval_cost_usd: result.totalCostUsd,
         });
-        return {
-            content: [
-                {
-                    type: 'text',
-                    text: JSON.stringify({
-                        id: evalId,
-                        overall_score: result.overallScore,
-                        passed: result.passed,
-                        total_citations_found: result.totalCitationsFound,
-                        total_resolved: result.totalResolved,
-                        total_judged: result.totalJudged,
-                        total_supported: result.totalSupported,
-                        total_cost_usd: result.totalCostUsd,
-                        citations: result.citations.map((c) => ({
-                            citation: {
-                                raw: c.citation.raw,
-                                kind: c.citation.kind,
-                                identifier: c.citation.identifier,
-                                offset_start: c.citation.offsetStart,
-                                offset_end: c.citation.offsetEnd,
-                            },
-                            resolve_status: c.resolveStatus,
-                            resolve_error: c.resolveError,
-                            // Mapped to the documented snake_case keys. The verifier's
-                            // internal shape is camelCase (contentType, bytesFetched) and
-                            // used to be passed through verbatim, so a client parsing
-                            // `source.content_type` per the description read undefined.
-                            source: c.source
-                                ? {
-                                    url: c.source.url,
-                                    status: c.source.status,
-                                    content_type: c.source.contentType,
-                                    bytes_fetched: c.source.bytesFetched,
-                                    truncated: c.source.truncated,
-                                }
-                                : undefined,
-                            judge: c.judge
-                                ? {
-                                    supported: c.judge.supported,
-                                    confidence: c.judge.confidence,
-                                    rationale: c.judge.rationale,
-                                    cost_usd: c.judge.costUsd,
-                                    latency_ms: c.judge.latencyMs,
-                                    input_tokens: c.judge.inputTokens,
-                                    output_tokens: c.judge.outputTokens,
-                                }
-                                : undefined,
-                        })),
-                    }),
+        return respond(verifyCitationsOutputSchema, {
+            id: evalId,
+            ...(args.trace_id ? { trace_id: args.trace_id } : {}),
+            overall_score: result.overallScore,
+            passed: result.passed,
+            // Derived rather than read: the verifier reports it, but the tool
+            // must not break if a caller hands it an older shape.
+            total_unsupported: result.totalUnsupported ?? Math.max(0, result.totalJudged - result.totalSupported),
+            total_citations_found: result.totalCitationsFound,
+            total_resolved: result.totalResolved,
+            total_judged: result.totalJudged,
+            total_supported: result.totalSupported,
+            total_cost_usd: result.totalCostUsd,
+            citations: result.citations.map((c) => ({
+                citation: {
+                    raw: c.citation.raw,
+                    kind: c.citation.kind,
+                    identifier: c.citation.identifier,
+                    offset_start: c.citation.offsetStart,
+                    offset_end: c.citation.offsetEnd,
                 },
-            ],
-        };
-    });
+                resolve_status: c.resolveStatus,
+                resolve_error: c.resolveError,
+                // Mapped to the documented snake_case keys. The verifier's
+                // internal shape is camelCase (contentType, bytesFetched) and
+                // used to be passed through verbatim, so a client parsing
+                // `source.content_type` per the description read undefined.
+                source: c.source
+                    ? {
+                        url: c.source.url,
+                        status: c.source.status,
+                        content_type: c.source.contentType,
+                        bytes_fetched: c.source.bytesFetched,
+                        truncated: c.source.truncated,
+                    }
+                    : undefined,
+                judge: c.judge
+                    ? {
+                        supported: c.judge.supported,
+                        confidence: c.judge.confidence,
+                        rationale: c.judge.rationale,
+                        cost_usd: c.judge.costUsd,
+                        latency_ms: c.judge.latencyMs,
+                        input_tokens: c.judge.inputTokens,
+                        output_tokens: c.judge.outputTokens,
+                    }
+                    : undefined,
+            })),
+        }, evaluationLinks(evalId, args.trace_id));
+    }));
 }