@iris-eval/mcp-server 0.8.2 → 0.10.0

package/dist/tools/evaluate-with-llm-judge.js

@@ -1,10 +1,15 @@
 import { z } from 'zod';
 import { LOCAL_TENANT } from '../types/tenant.js';
 import { evaluateWithLLMJudge } from '../eval/llm-judge/evaluator.js';
-import { findPricing } from '../eval/llm-judge/pricing.js';
+import { findPricing, MODEL_PRICING } from '../eval/llm-judge/pricing.js';
 import { generateEvalId } from '../utils/ids.js';
+import { JUDGE_COST_CAP_VAR, JUDGE_DEFAULT_COST_CAP_USD, JUDGE_KEY_VARS, judgeCostCapUsd, judgeRecovery } from '../judge-enablement.js';
 import { strictInput } from './strict-input.js';
 import { assertTraceExists, insertLinkedEvalResult } from './trace-link.js';
+import { describeTool, ERROR_ENVELOPE_SENTENCE } from './describe.js';
+import { irisError } from './errors.js';
+import { evaluationLinks, guarded, respond } from './respond.js';
+import { CAPABILITIES_RESOURCE_URI } from '../resources/uris.js';
 const inputSchema = {
     output: z.string().min(1).describe('The agent output text to evaluate'),
     template: z
@@ -12,77 +17,104 @@ const inputSchema = {
         .describe('Judge dimension: accuracy (factual correctness), helpfulness (does it address the ask), safety (harm potential), correctness (vs reference answer — requires `expected`), faithfulness (RAG grounding — requires `source_material`).'),
     model: z
         .string()
-        .describe('Model ID. Supported: anthropic = claude-opus-4-7 | claude-sonnet-4-6 | claude-haiku-4-5 | claude-haiku-4-5-20251001; openai = gpt-4o | gpt-4o-mini | o1-mini.'),
+        .describe('Model ID. Supported: anthropic = claude-opus-4-7 | claude-sonnet-4-6 | claude-haiku-4-5 | claude-haiku-4-5-20251001; openai = gpt-4o | gpt-4o-mini | o1-mini. Required — cost varies a hundredfold across models'),
     provider: z.enum(['anthropic', 'openai']).optional().describe('Auto-detected from model when omitted'),
     input: z.string().optional().describe('User question / prompt that produced the output (improves accuracy for helpfulness/safety)'),
     expected: z.string().optional().describe('Reference answer (required for correctness template)'),
     source_material: z.string().optional().describe('Provided RAG sources (required for faithfulness template)'),
     trace_id: z.string().optional().describe('Link this evaluation to a stored trace (id from log_trace / get_traces); an unknown id is rejected BEFORE the judge is called'),
-    max_cost_usd: z.number().positive().optional().describe('Cost cap in USD; defaults to IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL or 0.25'),
+    max_cost_usd: z.number().positive().optional().describe(`Cost cap in USD for this call; defaults to ${JUDGE_COST_CAP_VAR} or ${JUDGE_DEFAULT_COST_CAP_USD}. The worst case (two attempts, full max_output_tokens) is computed before the call and refused if it exceeds the cap`),
     max_output_tokens: z.number().int().positive().max(4096).optional().describe('Judge output token cap; default 512'),
     temperature: z.number().min(0).max(2).optional().describe('Sampling temperature; default 0 (deterministic)'),
     timeout_ms: z.number().int().positive().optional().describe('Per-request timeout; default 60_000'),
 };
-function inferProvider(model) {
+/** The models the pricing table knows, so an unknown one can be refused with the valid list. */
+export function supportedModels() {
+    return MODEL_PRICING.map((m) => m.model);
+}
+export function inferProvider(model) {
     const pricing = findPricing(model);
     if (!pricing) {
-        throw new Error(`Unknown model "${model}". Provider cannot be inferred. Supported models are listed in src/eval/llm-judge/pricing.ts.`);
+        throw irisError('IRIS_JUDGE_UNKNOWN_MODEL', `Unknown model "${model}": its provider and price are not known, so the cost cap cannot be enforced.`, {
+            field: 'model',
+            valid: supportedModels(),
+            recovery: ['Pass one of the supported models (see valid).', 'Pass provider explicitly only for a model in the list whose id is ambiguous.'],
+        });
     }
     return pricing.provider;
 }
-function resolveApiKey(provider) {
-    if (provider === 'anthropic') {
-        const key = process.env.IRIS_ANTHROPIC_API_KEY;
-        if (!key) {
-            throw new Error('Anthropic judge requires IRIS_ANTHROPIC_API_KEY. Set it in the environment or use a different provider.');
-        }
-        return key;
-    }
-    const key = process.env.IRIS_OPENAI_API_KEY;
+/**
+ * The key for the provider, from this process's environment. Missing is
+ * IRIS_JUDGE_NOT_ENABLED with the enable steps as recovery — the fact
+ * users get wrong is that a shell export does not reach the process an
+ * MCP client spawns, and the steps say so.
+ */
+export function resolveApiKey(provider, toolName = 'evaluate_with_llm_judge') {
+    const variable = JUDGE_KEY_VARS[provider];
+    const key = process.env[variable];
     if (!key) {
-        throw new Error('OpenAI judge requires IRIS_OPENAI_API_KEY. Set it in the environment or use a different provider.');
+        throw irisError('IRIS_JUDGE_NOT_ENABLED', `${toolName} needs ${variable} in the environment of the process that runs Iris; no key for ${provider} reached this process. Nothing was spent.`, {
+            field: variable,
+            recovery: judgeRecovery(provider),
+            see: CAPABILITIES_RESOURCE_URI,
+        });
     }
     return key;
 }
 function resolveMaxCost(paramValue) {
     if (paramValue !== undefined)
         return paramValue;
-    const envRaw = process.env.IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL;
-    if (envRaw) {
-        const parsed = Number(envRaw);
-        if (Number.isFinite(parsed) && parsed > 0)
-            return parsed;
-    }
-    return 0.25;
+    return judgeCostCapUsd();
 }
+export const judgeOutputSchema = z.looseObject({
+    id: z.string().describe('the evaluation id; read it back at iris://evaluations/{id}'),
+    trace_id: z.string().optional().describe('the linked trace, when one was named'),
+    score: z.number().describe('0..1 from the judge'),
+    passed: z.boolean().describe('the verdict: the score against the template\'s threshold, which is pass_threshold below. Not the model\'s own boolean — that is self_reported_pass'),
+    pass_threshold: z.number().describe('the threshold the score was read against, so you can check the arithmetic'),
+    self_reported_pass: z.boolean().optional().describe('what the model said about passing, when it said anything. Recorded, never obeyed'),
+    disagreement: z.boolean().optional().describe('true when the model\'s own boolean disagrees with the threshold verdict — its rubric and its judgement have come apart on this output'),
+    rationale: z.string().describe('the judge\'s reasoning, in its words'),
+    dimensions: z.record(z.string(), z.unknown()).describe('per-dimension sub-scores for the template'),
+    model: z.string().describe('the model that judged'),
+    provider: z.enum(['anthropic', 'openai']).describe('the provider called'),
+    template: z.string().describe('the template used'),
+    input_tokens: z.number().describe('tokens sent, across both attempts when a retry ran'),
+    output_tokens: z.number().describe('tokens received, across both attempts when a retry ran'),
+    cost_usd: z.number().nullable().describe('the exact spend from the pricing table'),
+    latency_ms: z.number().describe('wall time of the provider call(s)'),
+    raw_response_id: z.string().optional().describe('the provider\'s response id, for your own audit'),
+});
 export function registerEvaluateWithLLMJudgeTool(server, storage) {
     server.registerTool('evaluate_with_llm_judge', {
         title: 'Evaluate With LLM Judge',
-        description: [
-            'Score agent output using an LLM as the judge (Anthropic or OpenAI). Returns a 0..1 score with rationale, per-dimension breakdown, and exact cost. The judge\'s own accuracy is measurable on a key you supply and is not yet published — see https://iris-eval.com/proof.',
-            '',
-            'Sibling tools — evaluate_output runs heuristic rules (free, deterministic, no API key needed); this tool runs LLM-based semantic scoring (paid, requires an API key). verify_citations is a SPECIALIZED form of LLM judging that focuses on citation grounding only. log_trace / get_traces handle trace I/O; list_rules / deploy_rule / delete_rule manage heuristic-rule lifecycle. evaluate_with_llm_judge is the GENERAL semantic-scoring path.',
-            '',
-            'Behavior. Calls an external LLM API (Anthropic or OpenAI) — costs money per call, takes as long as the provider takes, respects an IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL cap. Non-deterministic at temperature > 0; default temperature=0 gives near-deterministic scores. Writes one eval_result row to Iris storage (linked to trace_id if provided) plus captures provider response id + latency + token counts + cost in the rule_results payload. Rate-limited to 20 req/min on HTTP MCP; your LLM provider also enforces its own rate limits (we transparently retry once on 429).',
-            '',
-            'Output shape. Returns JSON: `{ "id": "<uuid>", "score": 0..1, "passed": boolean, "rationale": string, "dimensions": {...}, "model": string, "provider": "anthropic"|"openai", "template": string, "input_tokens": number, "output_tokens": number, "cost_usd": number, "latency_ms": number }`. `dimensions` has per-dimension sub-scores (e.g., accuracy template returns `{factual_claims, citations, internal_consistency}`).',
-            '',
-            'Use when heuristic rules (via evaluate_output) are too coarse for the quality signal you need — semantic correctness, factual accuracy vs a reference, RAG faithfulness to sources, nuanced safety/helpfulness. Pick the template that matches: `accuracy` (hallucination detection), `helpfulness` (does it address the ask), `safety` (harm potential beyond regex PII), `correctness` (vs reference answer — pass `expected`), `faithfulness` (RAG grounding — pass `source_material`).',
-            '',
-            "Don't use for simple regex/length/keyword checks (use evaluate_output with heuristic rules — they're free, deterministic and in-process). Don't use without an API key set (IRIS_ANTHROPIC_API_KEY or IRIS_OPENAI_API_KEY). Don't use on very large outputs (>8K tokens) without raising max_cost_usd — the pre-check will refuse the call.",
-            '',
-            'Parameters. model is required (no default — pick consciously since cost varies 100x across models). provider is auto-detected from the model name; override only for ambiguous IDs. expected is REQUIRED when template="correctness" (the reference answer to compare against); ignored for other templates. source_material is REQUIRED when template="faithfulness" (the RAG sources to ground against); ignored otherwise. input is optional but improves scoring on helpfulness/safety templates (gives the judge the user prompt that produced the output). max_cost_usd defaults to env var IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL or $0.25 — the worst-case cost is computed BEFORE the call (input_tokens × prompt_price + max_output_tokens × completion_price, PLUS the same for the one retry that fires if the judge\'s first reply is not valid JSON); call refused upfront if that two-attempt worst case would exceed. When a retry does run, the reported input_tokens / output_tokens / cost_usd / latency_ms are totals across both attempts. max_output_tokens caps the judge response (default 512, max 4096); higher = more rationale detail + more cost. temperature default 0 (deterministic). timeout_ms default 60000. trace_id optional but recommended (links eval to trace in dashboard). Defaults: temperature=0, max_output_tokens=512, max_cost_usd=$0.25, timeout_ms=60000.',
-            '',
-            'Error modes. Throws when the required API key env var is missing. Throws when trace_id does not match a stored trace — checked before the provider call, so no money is spent and nothing is written. Throws when the estimated worst-case cost exceeds max_cost_usd (raise the cap or trim prompts). Throws LLMJudgeError on provider errors — kind=`auth` on 401/403, `rate_limit` on 429 (auto-retried once), `server_error` on 5xx, `timeout` on abort, `malformed_response` when the judge fails to emit valid JSON on both attempts. Throws "Unknown model" for unsupported model IDs — update src/eval/llm-judge/pricing.ts first.',
-        ].join('\n'),
+        description: describeTool({
+            summary: 'Score an output with an LLM judge on your own provider key: a 0..1 score, a rationale, per-dimension sub-scores and the exact spend.',
+            does: `Calls Anthropic or OpenAI directly with the key in this process's environment (${JUDGE_KEY_VARS.anthropic} or ${JUDGE_KEY_VARS.openai}); Iris never proxies. ` +
+                'template picks the question: accuracy, helpfulness, safety, correctness (needs expected) or faithfulness (needs source_material); input improves helpfulness and safety. model is required; provider is inferred from it. ' +
+                `The worst-case spend — both attempts, full max_output_tokens — is computed BEFORE the call and refused if it exceeds max_cost_usd (default ${JUDGE_COST_CAP_VAR} or ${JUDGE_DEFAULT_COST_CAP_USD}). ` +
+                'temperature defaults to 0; a rate-limited call is retried once. One evaluation row is stored with the provider response id, tokens, cost and latency, linked to trace_id when given. ' +
+                "The judge's own accuracy is measurable on a key you supply and is not yet published (see iris://proof).",
+            whenNot: 'For length, keyword, PII, injection or cost checks: evaluate_output is free and deterministic. Without a key: the call returns IRIS_JUDGE_NOT_ENABLED with the enable steps — do not search for them. On very large outputs without raising max_cost_usd: the pre-check refuses.',
+            returns: judgeOutputSchema,
+            errors: 'IRIS_JUDGE_NOT_ENABLED (no key for the provider reached this process; recovery carries the steps). IRIS_JUDGE_UNKNOWN_MODEL (valid lists the models). IRIS_UNKNOWN_TRACE, checked before any spend. ' +
+                'IRIS_BUDGET_EXCEEDED (nothing spent; the message carries both numbers). IRIS_PROVIDER_ERROR with kind auth, rate_limit, bad_request, server_error, timeout or malformed_response, and retryable set. ' +
+                ERROR_ENVELOPE_SENTENCE,
+            siblings: {
+                evaluate_output: 'the free deterministic path',
+                verify_citations: 'citation grounding, the narrower judge',
+                log_trace: 'record the execution first',
+            },
+        }),
         inputSchema: strictInput(inputSchema),
+        outputSchema: judgeOutputSchema,
         annotations: {
             readOnlyHint: false, // Writes eval_result; also spends money (external API cost)
             destructiveHint: false, // Creates data; doesn't overwrite or delete
             idempotentHint: false, // Temperature > 0 may vary; even at T=0 provider non-determinism is possible; cost also varies per call
             openWorldHint: true, // Calls external APIs (Anthropic / OpenAI) — touches the world beyond local process
         },
-    }, async (args) => {
+    }, guarded(async (args) => {
         const provider = args.provider ?? inferProvider(args.model);
         const apiKey = resolveApiKey(provider);
         const maxCostUsd = resolveMaxCost(args.max_cost_usd);
@@ -95,8 +127,8 @@ export function registerEvaluateWithLLMJudgeTool(server, storage) {
         const result = await evaluateWithLLMJudge({
             output: args.output,
             template: args.template,
-            provider,
             model: args.model,
+            provider,
             apiKey,
             input: args.input,
             expected: args.expected,
@@ -106,12 +138,12 @@ export function registerEvaluateWithLLMJudgeTool(server, storage) {
             temperature: args.temperature,
             timeoutMs: args.timeout_ms,
         });
+        // Persist to eval_results so the dashboard can surface it.
+        // eval_type='custom' — LLM judge scores span all 4 heuristic
+        // categories (accuracy, helpfulness, safety, faithfulness); 'custom'
+        // is the honest bucket. rule_results[0] captures per-dimension
+        // breakdown + provider metadata for audit.
         const evalId = generateEvalId();
-        // Persist as a normal eval_result so the dashboard picks it up
-        // alongside heuristic scores. eval_type is 'custom' because LLM
-        // judge doesn't fit completeness/relevance/safety/cost taxonomy
-        // cleanly — it spans all four. The rule_results payload carries
-        // the full judge provenance.
         await insertLinkedEvalResult(storage, LOCAL_TENANT, {
             id: evalId,
             trace_id: args.trace_id,
@@ -126,34 +158,57 @@ export function registerEvaluateWithLLMJudgeTool(server, storage) {
                     passed: result.passed,
                     score: result.score,
                     message: result.rationale || 'LLM judge evaluation',
+                    /*
+                     * The row says what KIND of claim it is (0.10.0). Without it a
+                     * stored judge evaluation read back through the composer had no
+                     * layer to fall into — not a policy, not a detector with a
+                     * published rate — and a FAILED judgement read back as clean.
+                     * A judgment the caller asked and paid for decides.
+                     */
+                    kind: 'judgment',
+                    role: 'gate',
+                    saw: ['output'],
+                    evidence: [
+                        {
+                            type: 'sample',
+                            score: result.score,
+                            ...(result.selfReportedPass !== undefined ? { selfReportedPass: result.selfReportedPass } : {}),
+                            rationaleHash: '',
+                        },
+                    ],
+                    uncertainty: {
+                        basis: 'unmeasured',
+                        why: 'the judge is user-keyed and its accuracy is measured only by a run on a key you or the maintainer supplies (npm run proof:judge)',
+                    },
                 },
             ],
             suggestions: result.passed ? [] : [result.rationale],
             rules_evaluated: 1,
             rules_skipped: 0,
             insufficient_data: false,
+            // What the evaluation itself cost — the description promised it was
+            // kept and the write path stored none of it (arc zero, G15).
+            eval_cost_usd: result.costUsd ?? undefined,
+            eval_tokens: result.inputTokens + result.outputTokens,
         });
-        return {
-            content: [
-                {
-                    type: 'text',
-                    text: JSON.stringify({
-                        id: evalId,
-                        score: result.score,
-                        passed: result.passed,
-                        rationale: result.rationale,
-                        dimensions: result.dimensions,
-                        model: result.model,
-                        provider: result.provider,
-                        template: result.template,
-                        input_tokens: result.inputTokens,
-                        output_tokens: result.outputTokens,
-                        cost_usd: result.costUsd,
-                        latency_ms: result.latencyMs,
-                        raw_response_id: result.rawResponseId,
-                    }),
-                },
-            ],
-        };
-    });
+        return respond(judgeOutputSchema, {
+            id: evalId,
+            ...(args.trace_id ? { trace_id: args.trace_id } : {}),
+            score: result.score,
+            passed: result.passed,
+            pass_threshold: result.passThreshold,
+            ...(result.selfReportedPass !== undefined ? { self_reported_pass: result.selfReportedPass } : {}),
+            ...(result.disagreement ? { disagreement: true } : {}),
+            rationale: result.rationale,
+            dimensions: result.dimensions,
+            model: result.model,
+            provider: result.provider,
+            template: result.template,
+            input_tokens: result.inputTokens,
+            output_tokens: result.outputTokens,
+            cost_usd: result.costUsd,
+            latency_ms: result.latencyMs,
+            raw_response_id: result.rawResponseId,
+        }, evaluationLinks(evalId, args.trace_id));
+    }));
 }

package/dist/tools/get-traces.d.ts

@@ -27,4 +27,13 @@ export interface TraceRangeArgs {
  * bound the tool rejects is never one the HTTP query quietly accepts.
  */
 export declare function addTraceRangeIssues(args: TraceRangeArgs, ctx: z.RefinementCtx): void;
+export declare const getTracesOutputSchema: z.ZodObject<{
+    traces: z.ZodArray<z.ZodObject<{
+        trace_id: z.ZodString;
+    }, z.core.$loose>>;
+    total: z.ZodNumber;
+    limit: z.ZodNumber;
+    offset: z.ZodNumber;
+    summary: z.ZodOptional<z.ZodObject<{}, z.core.$loose>>;
+}, z.core.$loose>;
 export declare function registerGetTracesTool(server: McpServer, storage: IStorageAdapter): void;

package/dist/tools/get-traces.js

@@ -1,6 +1,8 @@
 import { z } from 'zod';
 import { LOCAL_TENANT } from '../types/tenant.js';
 import { strictInput } from './strict-input.js';
+import { describeTool, ERROR_ENVELOPE_SENTENCE } from './describe.js';
+import { guarded, respond } from './respond.js';
 /*
  * An ISO-8601 instant (2026-08-01T00:00:00Z, offsets allowed) or calendar
  * date (2026-08-01). Stored timestamps are ISO strings and the adapter
@@ -61,7 +63,7 @@ const inputSchema = {
     // Mirrors traceQuerySchema in dashboard/validation.ts — both capture paths
     // (MCP tool, HTTP query) enforce the same 1..1000 bound. Unclamped, limit:-1
     // meant "LIMIT -1" in SQLite, i.e. every row (#332).
-    limit: z.number().int().min(1).max(1000).default(50).describe('Results per page (default 50, max 1000 — values >1000 return 400)'),
+    limit: z.number().int().min(1).max(1000).default(50).describe('Results per page (default 50, max 1000 — values above are rejected)'),
     offset: z.number().int().min(0).default(0).describe('Zero-based pagination offset — skip first N results (non-negative integer)'),
     sort_by: z.enum(['timestamp', 'latency_ms', 'cost_usd']).default('timestamp').describe('Sort by timestamp | latency_ms | cost_usd (default timestamp)'),
     sort_order: z.enum(['asc', 'desc']).default('desc').describe('Sort order: asc | desc (default desc — most recent / highest first)'),
@@ -69,34 +71,40 @@ const inputSchema = {
 };
 // Cross-field range checks — see addTraceRangeIssues above.
 const inputSchemaWithRanges = strictInput(inputSchema).superRefine(addTraceRangeIssues);
+export const getTracesOutputSchema = z.looseObject({
+    traces: z.array(z.looseObject({ trace_id: z.string() })).describe('the page of traces: trace_id, agent_name, framework, input, output, tool_calls, latency_ms, token_usage, cost_usd, metadata, timestamp'),
+    total: z.number().int().describe('how many traces match the filters, across every page'),
+    limit: z.number().int().describe('the page size applied'),
+    offset: z.number().int().describe('the offset applied'),
+    summary: z.looseObject({}).optional().describe('the dashboard aggregates for the last hour, when include_summary was true'),
+});
 export function registerGetTracesTool(server, storage) {
     server.registerTool('get_traces', {
         title: 'Get Traces',
-        description: [
-            'Query stored agent-execution traces with filters, pagination, and optional dashboard summary.',
-            '',
-            'Sibling tools — log_trace creates traces, delete_trace removes a single trace, evaluate_output / evaluate_with_llm_judge / verify_citations score them, list_rules / deploy_rule / delete_rule manage the custom-rule lifecycle. get_traces is the READ path for historical agent executions — never mutates anything.',
-            '',
-            'Behavior. Read-only: never mutates storage, never calls external services. Idempotent: repeated calls with the same args return consistent results (new traces logged after the call obviously show up on subsequent calls). Tenant-scoped: queries only the caller\'s tenant rows (LOCAL_TENANT in OSS). Paginates results (default limit 50, max 1000). Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio.',
-            '',
-            'Output shape. Returns JSON: `{ "traces": [{...traceRow}], "total": number, "limit": number, "offset": number, "summary"?: { total_traces, avg_latency_ms, total_cost_usd, error_rate, eval_pass_rate, traces_per_hour, top_agents } }`. Each trace row includes trace_id, agent_name, framework, input, output, tool_calls, latency_ms, token_usage, cost_usd, metadata, timestamp. `summary` only included when `include_summary: true`.',
-            '',
-            'Use when you need historical data: investigating a past failure, computing quality trends, comparing agents, or feeding an analytics job. Set `agent_name` / `framework` / `since` / `until` to narrow the query. Set `min_score` / `max_score` to surface outliers. Set `sort_by: "cost_usd"` + `sort_order: "desc"` to find the most expensive traces. Set `include_summary: true` when you want dashboard-style aggregates in one round-trip.',
-            '',
-            'Don\'t use to score a trace (use evaluate_output). Don\'t use to create a trace (use log_trace). Don\'t use as a live event stream — it\'s a query, not a subscription, and Iris has no event-stream endpoint; poll with exponential backoff.',
-            '',
-            'Parameters. limit defaults to 50, max 1000 (anything higher returns 400). offset is zero-based pagination (non-negative integer). since / until must be ISO 8601 timestamps or dates — `since` is inclusive (timestamp >= since), `until` is inclusive (timestamp <= until), and `since` may not be later than `until`. min_score / max_score are 0..1 and filter on the LATEST eval per trace, not all evals (so a trace with one failing + one passing eval may or may not match depending on which landed last); min_score may not exceed max_score. Combining since + sort_by="latency_ms" + sort_order="desc" is the canonical "find slow recent traces" query. include_summary returns dashboard-style aggregates in the SAME response (saves a round-trip; use true for dashboard ingest, false for analytics queries that don\'t need them). agent_name and framework are exact-match (no wildcards). Defaults: limit=50, offset=0, sort_by="timestamp", sort_order="desc", include_summary=false.',
-            '',
-            'Error modes. Returns 400 on invalid sort_by / sort_order (Zod enum). Returns 400 if limit > 1000 or offset < 0. Returns 400 — naming both values — on an empty range: min_score > max_score, since later than until, a score outside 0..1, or a since/until that is not an ISO 8601 timestamp or date (an unparseable bound is refused, never silently ignored). Returns 429 when HTTP rate limit exceeded. Storage failures propagate as 500. Empty result with `total: 0` on no matches (not an error).',
-        ].join('\n'),
+        description: describeTool({
+            summary: 'Query stored traces with filters, pagination and sorting; optionally include the dashboard summary in the same response.',
+            does: 'Read-only, local storage only. Filters are exact-match (agent_name, framework), inclusive time bounds (since, until — an ISO 8601 timestamp or date) and a score range applied to the LATEST evaluation of each trace (min_score, max_score, 0..1). ' +
+                'limit is 1..1000 (default 50), offset counts from 0, sort_by is timestamp, latency_ms or cost_usd, sort_order asc or desc (default: newest first). include_summary adds the one-hour dashboard aggregates. ' +
+                'A crossed range (min above max, since after until) is refused naming both values rather than returning an empty page that reads as "no such traces".',
+            whenNot: 'To score a trace (evaluate_output). To create one (log_trace). As a live stream: this is a query, and Iris has no event stream — poll with backoff.',
+            returns: getTracesOutputSchema,
+            errors: 'IRIS_STORAGE_ERROR when the database cannot be read. An out-of-range or crossed bound is refused before the handler runs, naming the values. An empty result is total 0, not an error. ' +
+                ERROR_ENVELOPE_SENTENCE,
+            siblings: {
+                log_trace: 'record an execution',
+                evaluate_output: 'score one output',
+                delete_trace: 'remove one trace',
+            },
+        }),
         inputSchema: inputSchemaWithRanges,
+        outputSchema: getTracesOutputSchema,
         annotations: {
             readOnlyHint: true, // Pure query: never writes, never deletes
             destructiveHint: false, // Inverse of readOnly — trivially false
             idempotentHint: true, // Same args → same result (modulo new traces that may have landed since)
             openWorldHint: false, // Queries local storage only; no external network
         },
-    }, async (args) => {
+    }, guarded(async (args) => {
         // OSS single-tenant: MCP caller is the local user.
         const result = await storage.queryTraces(LOCAL_TENANT, {
             filter: {
@@ -121,13 +129,6 @@ export function registerGetTracesTool(server, storage) {
         if (args.include_summary) {
             response.summary = await storage.getDashboardSummary(LOCAL_TENANT);
         }
-        return {
-            content: [
-                {
-                    type: 'text',
-                    text: JSON.stringify(response),
-                },
-            ],
-        };
-    });
+        return respond(getTracesOutputSchema, response);
+    }));
 }

package/dist/tools/index.d.ts

@@ -2,4 +2,12 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { IStorageAdapter } from '../types/query.js';
 import type { EvalEngine } from '../eval/engine.js';
 import type { CustomRuleStore } from '../custom-rule-store.js';
+/**
+ * Every tool this server registers, by name. The capabilities object
+ * lists it, the docs contract checks prose against it, and a test asserts
+ * it equals what tools/list returns — so a tool added below without a
+ * name here (or the reverse) fails before it ships.
+ */
+export declare const TOOL_NAMES: readonly ["log_trace", "evaluate_output", "get_traces", "list_rules", "deploy_rule", "delete_rule", "delete_trace", "evaluate_with_llm_judge", "verify_citations"];
+export type ToolName = (typeof TOOL_NAMES)[number];
 export declare function registerAllTools(server: McpServer, storage: IStorageAdapter, evalEngine: EvalEngine, customRuleStore: CustomRuleStore): void;

package/dist/tools/index.js

@@ -7,9 +7,30 @@ import { registerDeleteRuleTool } from './delete-rule.js';
 import { registerDeleteTraceTool } from './delete-trace.js';
 import { registerEvaluateWithLLMJudgeTool } from './evaluate-with-llm-judge.js';
 import { registerVerifyCitationsTool } from './verify-citations.js';
+import { dormantRulesFrom } from '../eval/dormant.js';
+import { LOCAL_TENANT } from '../types/tenant.js';
+/**
+ * Every tool this server registers, by name. The capabilities object
+ * lists it, the docs contract checks prose against it, and a test asserts
+ * it equals what tools/list returns — so a tool added below without a
+ * name here (or the reverse) fails before it ships.
+ */
+export const TOOL_NAMES = [
+    'log_trace',
+    'evaluate_output',
+    'get_traces',
+    'list_rules',
+    'deploy_rule',
+    'delete_rule',
+    'delete_trace',
+    'evaluate_with_llm_judge',
+    'verify_citations',
+];
 export function registerAllTools(server, storage, evalEngine, customRuleStore) {
     registerLogTraceTool(server, storage);
-    registerEvaluateOutputTool(server, storage, evalEngine);
+    registerEvaluateOutputTool(server, storage, evalEngine, {
+        dormant: () => dormantRulesFrom(customRuleStore.quarantined(LOCAL_TENANT)),
+    });
     registerGetTracesTool(server, storage);
     registerListRulesTool(server, customRuleStore, evalEngine);
     registerDeployRuleTool(server, customRuleStore, evalEngine);

package/dist/tools/list-rules.d.ts

@@ -1,4 +1,17 @@
+import { z } from 'zod';
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { CustomRuleStore } from '../custom-rule-store.js';
 import type { EvalEngine } from '../eval/engine.js';
+export declare const listRulesOutputSchema: z.ZodObject<{
+    rules: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        name: z.ZodString;
+    }, z.core.$loose>>;
+    total: z.ZodNumber;
+    enabled_count: z.ZodNumber;
+    built_in: z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+    }, z.core.$loose>>;
+    quarantined: z.ZodArray<z.ZodUnknown>;
+}, z.core.$loose>;
 export declare function registerListRulesTool(server: McpServer, customRuleStore: CustomRuleStore, evalEngine: EvalEngine): void;

package/dist/tools/list-rules.js

@@ -1,59 +1,66 @@
 /*
- * list_rules MCP tool — enumerate deployed custom rules.
+ * list_rules MCP tool — the rule inventory.
  *
- * Read-only view into the custom-rule store (~/.iris/custom-rules.json).
- * Lets agents discover what rules are deployed, what each one evaluates,
- * and which are enabled — so an agent can decide whether to call
- * evaluate_output at all, and which eval_type to route through.
- *
- * Companion to deploy_rule / delete_rule. Together these replace the
- * dashboard-only Make-This-A-Rule composer when an agent (not a human)
- * needs to manage the rule set programmatically.
+ * Two halves. `built_in` is the shipped roster with everything a caller
+ * needs to trust a verdict: what each rule is (kind, mechanism), what it
+ * reads (needs — absent means it skips), the question it answers, the
+ * criticality THIS server applies and who decided it, and its published
+ * accuracy. `rules` is the custom-rule store (~/.iris/custom-rules.json),
+ * the read path deploy_rule and delete_rule write to.
  */
 import { z } from 'zod';
 import { builtInRuleRoster } from '../eval/criticality.js';
+import { ruleProof } from '../capabilities.js';
 import { LOCAL_TENANT } from '../types/tenant.js';
 import { strictInput } from './strict-input.js';
+import { describeTool, ERROR_ENVELOPE_SENTENCE } from './describe.js';
+import { guarded, respond } from './respond.js';
+import { PROOF_RESOURCE_URI } from '../resources/uris.js';
 const inputSchema = {
     eval_type: z
         .enum(['completeness', 'relevance', 'safety', 'cost', 'custom'])
         .optional()
-        .describe('Filter to rules of a specific eval category'),
+        .describe('Filter the custom rules to one eval category (exact match); built_in is never filtered'),
     enabled_only: z
         .boolean()
         .default(false)
-        .describe('Return only enabled rules (excludes disabled ones)'),
+        .describe('Return only enabled custom rules (a rule disabled with delete_rule stays in the store and does not fire)'),
 };
+export const listRulesOutputSchema = z.looseObject({
+    rules: z.array(z.looseObject({ id: z.string(), name: z.string() })).describe('the deployed custom rules after the filters: id, name, description, evalType, severity, definition, enabled, createdAt, updatedAt, version, sourceMomentId'),
+    total: z.number().int().describe('custom rules after the filters'),
+    enabled_count: z.number().int().describe('of those, how many are enabled'),
+    built_in: z.array(z.looseObject({ name: z.string() })).describe('the shipped roster, never filtered: name, category, description, weight, kind, mechanism, needs, question, classes, version, the EFFECTIVE critical flag with criticalSource, and proof (published precision, recall, intervals and ppvAt from https://iris-eval.com/proof; null where the proof is a conformance check)'),
+    quarantined: z.array(z.unknown()).describe('entries in the store this version could not validate; they do not fire and are never deleted by a deploy'),
+});
 export function registerListRulesTool(server, customRuleStore, evalEngine) {
     server.registerTool('list_rules', {
-        title: 'List Custom Rules',
-        description: [
-            'Enumerate deployed custom evaluation rules from the local rule store.',
-            '',
-            'Sibling tools — deploy_rule adds custom rules, delete_rule removes them, evaluate_output runs them against agent output. log_trace / get_traces / delete_trace handle the trace lifecycle separately. list_rules is the READ path for the custom-rule store; nothing else exposes the inventory.',
-            '',
-            'Behavior. Pure read of ~/.iris/custom-rules.json (in-memory cached; no disk read per call after server boot). No mutation, no external network. Returns the rules of the local tenant. Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio.',
-            '',
-            'Output shape. Returns JSON: `{ "rules": [{ "id": "rule-XXXX", "name", "description", "evalType", "severity", "definition": { name, type, config, weight? }, "enabled": boolean, "createdAt": ISO timestamp, "updatedAt": ISO timestamp, "version": number, "sourceMomentId?": string }], "total": number, "enabled_count": number, "built_in": [{ "name", "category", "weight", "critical": boolean, "criticalSource": "default" | "config" }] }`. Empty `rules` array + total=0 when no custom rules are deployed. A deployed rule fires only on evaluate_output calls whose eval_type equals its evalType (or eval_type="all", which runs every bundle). `built_in` is the shipped rule set, always present and NOT narrowed by the filters; `total` and `enabled_count` count custom rules only.',
-            '',
-            'Why `built_in` carries criticality. A critical rule vetoes `passed` regardless of the weighted score, and which built-in rules are critical is configurable (`eval.criticalRules` / `eval.nonCriticalRules`). `critical` is the EFFECTIVE value this server applies and `criticalSource` says who decided it: `default` is the declaration on the rule itself, `config` means one of those lists named it. Read it before trusting a `passed: true` — it is how you tell "nothing was violated" from "the rule that would have vetoed is demoted on this server".',
-            '',
-            'Use when you need to know what custom rules are currently live (before calling evaluate_output, before deploying a similar rule to avoid duplicates, or when building a dashboard view). Filter with `eval_type` to scope to a specific category, or `enabled_only: true` to exclude disabled rules. Use get_traces to see trace data; use evaluate_output to run scoring; use list_rules only when you need the RULE INVENTORY.',
-            '',
-            "Don't use to count traces or evals (that's get_traces). Don't use to deploy a rule (use deploy_rule); don't use to remove one (use delete_rule). Built-in rules are not in the store and cannot be deployed, deleted or disabled — they appear under `built_in` for reference, carrying the criticality this server applies.",
-            '',
-            'Parameters. eval_type filter is exact-match against each rule\'s evalType field (no wildcards). enabled_only excludes rules that are deployed-but-disabled — a rule is disabled without deleting it via delete_rule with `enabled: false` (and re-enabled with `enabled: true`), or from the dashboard; disabled rules stay in the store with their history but do not fire. Both filters are AND-combined when both are set. Both are optional; with no filter, all rules return. Defaults: eval_type=undefined (no filter), enabled_only=false (returns all rules including disabled).',
-            '',
-            "Error modes. Returns empty list if the rule store file doesn't exist (first run). Returns 429 if HTTP rate limit exceeded. Never throws on valid input.",
-        ].join('\n'),
+        title: 'List Rules',
+        description: describeTool({
+            summary: 'The rule inventory: the built-in roster with what each rule needs, the criticality this server applies and its published accuracy, plus every deployed custom rule.',
+            does: 'Read-only, no network. built_in is the shipped roster and is never narrowed by the filters. For each rule: kind (measurement, detection, inference, judgment, policy, verification), mechanism, needs (the inputs it reads — absent means the rule skips, never passes), question, classes, version, weight, ' +
+                'the EFFECTIVE critical flag with criticalSource (default, or config when eval.criticalRules / eval.nonCriticalRules changed it on this server — read it before trusting a passed: true), ' +
+                'and proof: precision and recall with 95% intervals and the positive predictive value at four prevalences, the numbers published at https://iris-eval.com/proof. ' +
+                'rules is the custom-rule store, filterable by eval_type and enabled_only; total and enabled_count count custom rules. quarantined lists store entries this version could not validate; they do not fire.',
+            whenNot: 'To count traces (get_traces). To add, remove or pause a rule (deploy_rule, delete_rule). Built-in rules are not in the store and cannot be deployed, deleted or disabled.',
+            returns: listRulesOutputSchema,
+            errors: 'IRIS_INTERNAL_ERROR if the store file cannot be read. A missing store file is an empty list, not an error. ' +
+                ERROR_ENVELOPE_SENTENCE,
+            siblings: {
+                deploy_rule: 'add a custom rule',
+                delete_rule: 'remove, disable or re-enable one',
+                evaluate_output: 'run the rules',
+            },
+        }),
         inputSchema: strictInput(inputSchema),
+        outputSchema: listRulesOutputSchema,
         annotations: {
             readOnlyHint: true,
             destructiveHint: false,
             idempotentHint: true,
             openWorldHint: false,
         },
-    }, async (args) => {
+    }, guarded(async (args) => {
         // OSS: MCP tools operate under LOCAL_TENANT. Cloud multi-tenant
         // exposure is a v0.5 architectural item (MCP SDK doesn't pass
         // session/tenant context to tool handlers).
@@ -75,19 +82,9 @@ export function registerListRulesTool(server, customRuleStore, evalEngine) {
          * purpose — the filters describe the custom-rule store.
          */
         const built_in = builtInRuleRoster((rule) => evalEngine.effectiveCriticality(rule)).map((r) => ({
-            name: r.name,
-            category: r.category,
-            weight: r.weight,
-            critical: r.critical,
-            criticalSource: r.criticalSource,
+            ...r,
+            proof: ruleProof(r.name),
         }));
-        return {
-            content: [
-                {
-                    type: 'text',
-                    text: JSON.stringify({ rules, total, enabled_count, built_in }),
-                },
-            ],
-        };
-    });
+        return respond(listRulesOutputSchema, { rules, total, enabled_count, built_in, quarantined: customRuleStore.quarantined(LOCAL_TENANT) }, [{ uri: PROOF_RESOURCE_URI, name: 'proof', description: 'The published accuracy of every measured rule, with the corpus it was measured on' }]);
+    }));
 }

package/dist/tools/log-trace.d.ts

@@ -58,4 +58,8 @@ export declare const logTraceInputShape: {
     }, z.core.$strip>>>;
     timestamp: z.ZodOptional<z.ZodString>;
 };
+export declare const logTraceOutputSchema: z.ZodObject<{
+    trace_id: z.ZodString;
+    status: z.ZodLiteral<"stored">;
+}, z.core.$loose>;
 export declare function registerLogTraceTool(server: McpServer, storage: IStorageAdapter): void;