@iris-eval/mcp-server 0.8.2 → 0.9.0

package/dist/tools/evaluate-output.js

@@ -1,10 +1,16 @@
 import { z } from 'zod';
+import { toEvaluationResponse } from '../eval/response.js';
+import { evaluateOutputResponseSchema } from '../eval/response-schema.js';
 import { DEFAULT_EVAL_TYPE, DEFAULT_EVAL_TYPE_NOTE } from '../eval/engine.js';
 import { INJECTION_SCOPE_SENTENCE } from '../eval/rules/safety.js';
 import { LOCAL_TENANT } from '../types/tenant.js';
 import { strictInput, strictNested } from './strict-input.js';
 import { toolCallSchema } from './log-trace.js';
 import { getTraceOrThrow, insertLinkedEvalResult } from './trace-link.js';
+import { describeTool, ERROR_ENVELOPE_SENTENCE } from './describe.js';
+import { evaluationLinks, guarded, respond } from './respond.js';
+/** The most inline custom rules one call may carry (see the argument description). */
+export const MAX_INLINE_CUSTOM_RULES = 10;
 /*
  * Strict one level down (#376): `{ name, type, config, wieght: 5 }` used to
  * parse with `wieght` silently discarded, so the rule ran at weight 1 and
@@ -31,13 +37,13 @@ const inputSchema = {
     eval_type: z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom', 'all']).optional().describe('Rule bundle to apply: completeness | relevance | safety | cost | custom | all — picks which built-in rules fire. "all" runs every bundle in one call and adds a per-category breakdown. Defaults to "all" when omitted — every bundle runs, safety included, and the response carries a note saying the default ran'),
     expected: z.string().optional().describe('Expected output for comparison — consulted only by the completeness bundle\'s expected_coverage rule; NOT used by relevance (the relevance rules compare the output against `input`)'),
     input: z.string().optional().describe('Original input for context (the ask + any source material the agent was given) — REQUIRED when eval_type="relevance" (keyword_overlap and topic_consistency compare the output against it and skip without it); also grounds the safety bundle\'s hallucination signals'),
-    trace_id: z.string().optional().describe('Link evaluation to a trace — surfaces this eval in the dashboard\'s trace drill-through. Must be the id of a stored trace (from log_trace / get_traces); an unknown id is rejected before anything is evaluated'),
+    trace_id: z.string().optional().describe('Link evaluation to a trace — surfaces this eval in the dashboard\'s trace drill-through and lets the tool reuse the trace\'s stored tool_calls. Must be the id of a stored trace (from log_trace / get_traces); an unknown id is rejected before anything is evaluated'),
     // .max(10): inline rules skip the deploy-time probe, and the engine runs
     // rules synchronously — without a cap, one request carrying N sandbox-
     // defeating regex rules stalls the server linearly in N (measured 9.3s at
     // N=50). Ten is ample for per-call rules; persistent sets belong in
     // deploy_rule, where deploy-time validation probes each pattern.
-    custom_rules: z.array(CustomRuleSchema).max(10).optional().describe('Custom evaluation rules, max 10 per call (deploy persistent rule sets via deploy_rule instead) — fires REGARDLESS of eval_type; pass eval_type="custom" if you want ONLY these. Each entry accepts exactly name, type, config, weight — an unknown key (e.g. a misspelled weight) is rejected'),
+    custom_rules: z.array(CustomRuleSchema).max(MAX_INLINE_CUSTOM_RULES).optional().describe('Custom evaluation rules, max 10 per call (deploy persistent rule sets via deploy_rule instead) — fires REGARDLESS of eval_type; pass eval_type="custom" if you want ONLY these. Each entry accepts exactly name, type, config, weight — an unknown key (e.g. a misspelled weight) is rejected'),
     cost_usd: z.number().optional().describe('Cost in USD — consulted by the cost bundle (eval_type="cost" or "all") AND by any cost_threshold custom rule regardless of eval_type; omit it and such a rule skips rather than passes (a critical one is listed in critical_skipped)'),
     token_usage: z.object({
         prompt_tokens: z.number().optional(),
@@ -49,38 +55,38 @@ const inputSchema = {
     // is how that field goes missing on one path and not the other.
     tool_calls: z.array(toolCallSchema).optional().describe('What the agent DID — the tool calls it made, in order, each { tool_name, input?, output?, latency_ms?, error? } exactly as log_trace records them. Read by the trajectory rules — the rules that judge what the agent DID rather than what it wrote. Omit it and those rules SKIP rather than pass — an evaluation with no trajectory data reports "not judged", never "clean". When trace_id names a stored trace and this argument is omitted, the tool_calls stored on that trace are loaded and used, so a caller who already logged them need not resend them'),
 };
-export function registerEvaluateOutputTool(server, storage, evalEngine) {
+export function registerEvaluateOutputTool(server, storage, evalEngine, options) {
     server.registerTool('evaluate_output', {
         title: 'Evaluate Output',
-        description: [
-            'Score agent output against configurable eval rules and return a 0..1 score + per-rule breakdown.',
-            '',
-            'Sibling tools — evaluate_with_llm_judge runs semantic LLM-based scoring (slower, costs money; this tool is heuristic, free, deterministic), verify_citations checks citation grounding specifically, log_trace records executions, get_traces queries them, list_rules / deploy_rule / delete_rule manage the custom-rule lifecycle. evaluate_output is the FAST PATH for length / keyword / PII / injection / cost-threshold checks where rules are sufficient.',
-            '',
-            'Behavior. Deterministic, in-process scoring — same inputs always produce the same result. Writes one eval_result row to Iris storage (linked to trace_id if provided; unlinked otherwise). No external network calls — semantic scoring is a separate tool (evaluate_with_llm_judge) that needs an API key you supply. Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio. Runs in-process; no provider is called.',
-            '',
-            'Output shape. Returns JSON: `{ "id": "<uuid>", "eval_type": "<bundle that ran>", "score": 0..1, "passed": boolean, "critical_failures?": string[], "critical_skipped?": string[], "rule_results": [{ "ruleName", "ruleId?", "category?", "critical", "criticalSource", "passed", "score", "message", "skipped?", "skipReason?", "budgetExceeded?", "configInvalid?" }], "suggestions": string[], "rules_evaluated": number, "rules_skipped": number, "insufficient_data": boolean, "categories?": { "<bundle>": { "score": number|null, "passed": boolean|null, "rules_evaluated", "rules_skipped", "insufficient_data", "critical_failures?", "critical_skipped?" } }, "note?": string }`. `ruleId` is present on results produced by a deployed rule (rule-XXXX) so two rules sharing a name stay distinguishable. `categories` appears only for eval_type="all" and carries one entry per bundle that had rules, each with the same threshold + critical-veto semantics as a single-bundle run; `category` on each rule result says which bundle it came from. `insufficient_data=true` means no applicable rules fired (e.g., safety eval with only cost data). Inside `categories`, a bundle that evaluated no rule (every rule skipped for missing context — cost without `cost_usd`, relevance without `input`) reports `passed: null` and `score: null` with `insufficient_data: true`: it was not judged, so it is neither passing nor failing, and it does not count toward the overall verdict. The top-level `passed` stays a boolean and is false when NOTHING was evaluated — a gate keyed on it fails closed; read `insufficient_data` to tell "failed" from "not judged". `note` appears only when eval_type was omitted, saying that the default ran every bundle.',
-            '',
-            'What `passed` means. `score` and `passed` answer different questions. `score` is the weighted average across the rules that ran — a 0..1 quality gradient. `passed` is the ship/no-ship verdict: true only when the score clears the pass threshold (default 0.7, configurable via config `eval.defaultThreshold`) AND no critical rule failed. Critical rules HARD-FAIL: if one fails, `passed` is false regardless of the weighted score, and the culprits are listed in `critical_failures`. The critical rules are the genuine safety violations — `no_pii`, `no_injection_patterns`, `no_blocklist_words` — plus any deployed custom rule with severity high/critical. Which built-in rules are critical is CONFIGURABLE per deployment (`eval.criticalRules` / `eval.nonCriticalRules`), so do not infer it from this list: every rule result carries `critical` (the effective value) and `criticalSource` (`default` when the declaration on the rule itself decided it, `config` when this server promoted or demoted it), and `list_rules` reports the same for the whole built-in roster. A leaked SSN can never be averaged away by other rules passing. For eval_type="all" the veto spans every bundle: one critical failure anywhere forces the overall `passed` to false. One caveat, stated because it is reachable on purpose: a critical rule that SKIPPED did not judge the output and therefore cannot veto — a regex rule whose match blew the 100ms sandbox budget on crafted output skips, so `passed` can be true with no `critical_failures`. Every such rule is named in `critical_skipped`. If your gate must fail closed, treat a non-empty `critical_skipped` as UNKNOWN, not clean.',
-            '',
-            'Use when you want a quality score on a specific output — typically after log_trace records the execution. Pass `eval_type` to route to the right rule bundle: `completeness` (length, non-empty output, sentence count, coverage of `expected`), `relevance` (keyword overlap and topic consistency against `input`), `safety` (PII leak, prompt injection, hallucination markers, stub-output detection — pass `input` so the hallucination signals can cross-check the output against the material the agent was given), `cost` (budget threshold), `custom` (bring your own rules via `custom_rules`), or `all` (every bundle above in one call — completeness, relevance, safety, cost, plus rules deployed under "custom" and any inline custom_rules — with per-category scores in `categories` and one overall verdict; rules whose context is missing, such as relevance without `input` or cost without `cost_usd`, skip and are excluded from the score exactly as in a single-bundle run).',
-            '',
-            'Injection scope. ' + INJECTION_SCOPE_SENTENCE + ' Screening what reaches the agent is a different control, outside this tool.',
-            '',
-            'Don\'t use when the output is empty or has no applicable rules — the eval_type decides which rules apply, and invalid combinations return score=0 + insufficient_data=true (not an error, but not actionable). Don\'t use to VALIDATE JSON schemas directly (use your language\'s JSON Schema validator — Iris\'s `json_schema` custom rule type is for output-shape assertions, not arbitrary validation).',
-            '',
-            'Parameters. input is REQUIRED when eval_type="relevance" (keyword_overlap and topic_consistency compare the output against it; without it both rules skip and the response reports insufficient_data=true) AND grounds the safety bundle\'s hallucination signals (without it those signals stay silent rather than guess); ignored otherwise. expected is consulted only by the completeness bundle\'s expected_coverage rule; ignored for other eval_types — it is NOT the relevance target. cost_usd is consulted by the cost bundle AND by any cost_threshold custom rule regardless of eval_type — omit it and such a rule skips rather than passes (a critical one is listed in critical_skipped); token_usage is ONLY consulted by the cost bundle. tool_calls is what the agent DID (the trajectory) and is read only by the trajectory rules; omit it and those rules SKIP rather than pass, so an evaluation with no trajectory data reports "not judged" instead of "clean", and when trace_id names a stored trace the tool_calls stored on it are used unless this argument overrides them. custom_rules ALWAYS fires regardless of eval_type — pass eval_type="custom" if you want ONLY your rules to run (otherwise both your rules AND the eval_type bundle run together); each entry takes exactly name, type, config and weight. trace_id is optional but recommended (linking the eval to its trace surfaces it in the dashboard\'s drill-through) and must name a stored trace. Defaults: eval_type="all" — every bundle runs, safety included, and when you rely on that default the response carries a `note` saying so; pass a single bundle name to narrow the run.',
-            '',
-            'Error modes. Throws on unknown argument names (strict schema — a misspelled argument is rejected with the valid argument list, never silently dropped), and likewise on an unknown key inside a custom_rules entry (e.g. `wieght`) or inside a tool_calls entry (e.g. `err` for `error`) — the valid keys are listed; a rule\'s `config` keys are free-form and are not checked here. Throws on malformed custom_rules (Zod rejects the shape: missing name/type, unknown type, non-object config, non-positive weight) and on more than 10 custom_rules in one call (use deploy_rule for persistent rule sets). Throws when trace_id does not match a stored trace — checked BEFORE evaluating, so nothing is scored or written; the message names the trace_id. An inline rule whose CONFIG is unusable — a regex that fails the safe-regex2 ReDoS check or exceeds the 1000-char limit, a missing or non-string config.pattern, non-string keywords — does NOT error: that rule reports skipped with configInvalid=true and a skipReason naming the field, and the other rules still run (deploy_rule rejects the same configs with a 400 at deploy time). Returns 429 when HTTP rate limit exceeded. Storage failures propagate as 500. The eval itself never throws — failing rules report `passed: false` with a message, they don\'t bubble exceptions. A regex that exceeds the 100ms sandbox matching budget on a given output reports skipped with budgetExceeded=true instead of hanging the server (fail-open per rule — gate on that flag if you must fail closed).',
-        ].join('\n'),
+        description: describeTool({
+            summary: 'Score an agent output against the deterministic rule bundles: the ship verdict with its basis, every rule result with evidence and uncertainty, and what was not judged.',
+            does: 'In-process, no network, no key. eval_type picks one bundle (completeness, relevance, safety, cost, custom) or all (the default): every bundle plus deployed and inline custom rules, with a per-bundle breakdown in categories. ' +
+                'Inputs decide what can be judged: input is REQUIRED when eval_type="relevance" (keyword_overlap and topic_consistency compare the output against it and skip without it) and grounds the hallucination signals; ' +
+                'tool_calls, or a trace_id whose stored tool_calls are reused, feed the trajectory rules; cost_usd and token_usage feed the cost rules; expected feeds only expected_coverage. ' +
+                'A rule without its input SKIPS, is named, and never counts as a pass. custom_rules always fire. One row is stored, linked to trace_id when given.',
+            whenNot: 'To validate arbitrary JSON Schema (the json_schema custom type asserts an output\'s shape only). ' +
+                `To screen inputs before they reach an agent: ${INJECTION_SCOPE_SENTENCE} ` +
+                'For semantic judgment, evaluate_with_llm_judge and verify_citations need a key you supply.',
+            returns: evaluateOutputResponseSchema,
+            errors: 'IRIS_UNKNOWN_TRACE when trace_id names no stored trace — checked first, nothing scored or written. IRIS_STORAGE_ERROR when the row cannot be written. ' +
+                'Unknown arguments or keys are refused before the handler runs, naming the valid ones; a regex rule over its budget or with a broken config reports skipped, not an error. ' +
+                ERROR_ENVELOPE_SENTENCE,
+            siblings: {
+                log_trace: 'record the execution first',
+                evaluate_with_llm_judge: 'semantic scoring on your key',
+                verify_citations: 'citation grounding on your key',
+                list_rules: 'the roster, needs and published accuracy',
+            },
+        }),
         inputSchema: strictInput(inputSchema),
+        outputSchema: evaluateOutputResponseSchema,
         annotations: {
             readOnlyHint: false, // Writes an eval_result row
             destructiveHint: false, // Creates new data; doesn't overwrite or delete
             idempotentHint: true, // Deterministic: same inputs → same score (each call writes a distinct result row, but the SCORE is stable)
             openWorldHint: false, // No external network in heuristic mode; LLM-as-judge has its own tool with openWorldHint:true
         },
-    }, async (args) => {
+    }, guarded(async (args) => {
         // Refuse an unknown trace_id up front (#376): the old path ran the
         // evaluation and then surfaced SQLite's "FOREIGN KEY constraint
         // failed", which names neither the field nor the fix.
@@ -118,37 +124,10 @@ export function registerEvaluateOutputTool(server, storage, evalEngine) {
         // OSS single-tenant: MCP tool callers are the local user. Cloud
         // will derive tenant from the authenticated MCP session.
         await insertLinkedEvalResult(storage, LOCAL_TENANT, result);
-        return {
-            content: [
-                {
-                    type: 'text',
-                    text: JSON.stringify({
-                        id: result.id,
-                        // Echo which bundle actually ran. Without this, a caller who
-                        // omitted eval_type could not tell a "safety pass" from a
-                        // completeness eval that never ran a single safety rule.
-                        eval_type: result.eval_type,
-                        score: result.score,
-                        passed: result.passed,
-                        ...(result.critical_failures ? { critical_failures: result.critical_failures } : {}),
-                        // The other half of the veto contract. The engine names every
-                        // critical rule that SKIPPED (budget-killed regex, missing cost
-                        // data) so a fail-closed gate can treat the eval as unknown;
-                        // this response used to drop the field, so the gate the
-                        // description tells users to write keyed on something that
-                        // never arrived and read passed:true as clean.
-                        ...(result.critical_skipped ? { critical_skipped: result.critical_skipped } : {}),
-                        rule_results: result.rule_results,
-                        suggestions: result.suggestions,
-                        rules_evaluated: result.rules_evaluated,
-                        rules_skipped: result.rules_skipped,
-                        insufficient_data: result.insufficient_data,
-                        // Per-bundle breakdown — eval_type="all" only.
-                        ...(result.categories ? { categories: result.categories } : {}),
-                        ...(evalTypeOmitted ? { note: DEFAULT_EVAL_TYPE_NOTE } : {}),
-                    }),
-                },
-            ],
-        };
-    });
+        // One serializer for every evaluation surface (src/eval/response.ts):
+        // the tool, the HTTP ingest route, the resources and the drift-lock
+        // all read the same object, so a field added there reaches every
+        // reader at once.
+        return respond(evaluateOutputResponseSchema, toEvaluationResponse(result, { traceId: args.trace_id, dormant: options?.dormant?.(), ...(evalTypeOmitted ? { note: DEFAULT_EVAL_TYPE_NOTE } : {}) }), evaluationLinks(result.id, args.trace_id));
+    }));
 }

package/dist/tools/evaluate-with-llm-judge.d.ts

@@ -1,3 +1,34 @@
+import { z } from 'zod';
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { IStorageAdapter } from '../types/query.js';
+import type { LLMProvider } from '../eval/llm-judge/client.js';
+/** The models the pricing table knows, so an unknown one can be refused with the valid list. */
+export declare function supportedModels(): string[];
+export declare function inferProvider(model: string): LLMProvider;
+/**
+ * The key for the provider, from this process's environment. Missing is
+ * IRIS_JUDGE_NOT_ENABLED with the enable steps as recovery — the fact
+ * users get wrong is that a shell export does not reach the process an
+ * MCP client spawns, and the steps say so.
+ */
+export declare function resolveApiKey(provider: LLMProvider, toolName?: string): string;
+export declare const judgeOutputSchema: z.ZodObject<{
+    id: z.ZodString;
+    trace_id: z.ZodOptional<z.ZodString>;
+    score: z.ZodNumber;
+    passed: z.ZodBoolean;
+    rationale: z.ZodString;
+    dimensions: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    model: z.ZodString;
+    provider: z.ZodEnum<{
+        anthropic: "anthropic";
+        openai: "openai";
+    }>;
+    template: z.ZodString;
+    input_tokens: z.ZodNumber;
+    output_tokens: z.ZodNumber;
+    cost_usd: z.ZodNullable<z.ZodNumber>;
+    latency_ms: z.ZodNumber;
+    raw_response_id: z.ZodOptional<z.ZodString>;
+}, z.core.$loose>;
 export declare function registerEvaluateWithLLMJudgeTool(server: McpServer, storage: IStorageAdapter): void;

package/dist/tools/evaluate-with-llm-judge.js

@@ -1,10 +1,15 @@
 import { z } from 'zod';
 import { LOCAL_TENANT } from '../types/tenant.js';
 import { evaluateWithLLMJudge } from '../eval/llm-judge/evaluator.js';
-import { findPricing } from '../eval/llm-judge/pricing.js';
+import { findPricing, MODEL_PRICING } from '../eval/llm-judge/pricing.js';
 import { generateEvalId } from '../utils/ids.js';
+import { JUDGE_COST_CAP_VAR, JUDGE_DEFAULT_COST_CAP_USD, JUDGE_KEY_VARS, judgeCostCapUsd, judgeRecovery } from '../judge-enablement.js';
 import { strictInput } from './strict-input.js';
 import { assertTraceExists, insertLinkedEvalResult } from './trace-link.js';
+import { describeTool, ERROR_ENVELOPE_SENTENCE } from './describe.js';
+import { irisError } from './errors.js';
+import { evaluationLinks, guarded, respond } from './respond.js';
+import { CAPABILITIES_RESOURCE_URI } from '../resources/uris.js';
 const inputSchema = {
     output: z.string().min(1).describe('The agent output text to evaluate'),
     template: z
@@ -12,77 +17,101 @@ const inputSchema = {
         .describe('Judge dimension: accuracy (factual correctness), helpfulness (does it address the ask), safety (harm potential), correctness (vs reference answer — requires `expected`), faithfulness (RAG grounding — requires `source_material`).'),
     model: z
         .string()
-        .describe('Model ID. Supported: anthropic = claude-opus-4-7 | claude-sonnet-4-6 | claude-haiku-4-5 | claude-haiku-4-5-20251001; openai = gpt-4o | gpt-4o-mini | o1-mini.'),
+        .describe('Model ID. Supported: anthropic = claude-opus-4-7 | claude-sonnet-4-6 | claude-haiku-4-5 | claude-haiku-4-5-20251001; openai = gpt-4o | gpt-4o-mini | o1-mini. Required — cost varies a hundredfold across models'),
     provider: z.enum(['anthropic', 'openai']).optional().describe('Auto-detected from model when omitted'),
     input: z.string().optional().describe('User question / prompt that produced the output (improves accuracy for helpfulness/safety)'),
     expected: z.string().optional().describe('Reference answer (required for correctness template)'),
     source_material: z.string().optional().describe('Provided RAG sources (required for faithfulness template)'),
     trace_id: z.string().optional().describe('Link this evaluation to a stored trace (id from log_trace / get_traces); an unknown id is rejected BEFORE the judge is called'),
-    max_cost_usd: z.number().positive().optional().describe('Cost cap in USD; defaults to IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL or 0.25'),
+    max_cost_usd: z.number().positive().optional().describe(`Cost cap in USD for this call; defaults to ${JUDGE_COST_CAP_VAR} or ${JUDGE_DEFAULT_COST_CAP_USD}. The worst case (two attempts, full max_output_tokens) is computed before the call and refused if it exceeds the cap`),
     max_output_tokens: z.number().int().positive().max(4096).optional().describe('Judge output token cap; default 512'),
     temperature: z.number().min(0).max(2).optional().describe('Sampling temperature; default 0 (deterministic)'),
     timeout_ms: z.number().int().positive().optional().describe('Per-request timeout; default 60_000'),
 };
-function inferProvider(model) {
+/** The models the pricing table knows, so an unknown one can be refused with the valid list. */
+export function supportedModels() {
+    return MODEL_PRICING.map((m) => m.model);
+}
+export function inferProvider(model) {
     const pricing = findPricing(model);
     if (!pricing) {
-        throw new Error(`Unknown model "${model}". Provider cannot be inferred. Supported models are listed in src/eval/llm-judge/pricing.ts.`);
+        throw irisError('IRIS_JUDGE_UNKNOWN_MODEL', `Unknown model "${model}": its provider and price are not known, so the cost cap cannot be enforced.`, {
+            field: 'model',
+            valid: supportedModels(),
+            recovery: ['Pass one of the supported models (see valid).', 'Pass provider explicitly only for a model in the list whose id is ambiguous.'],
+        });
     }
     return pricing.provider;
 }
-function resolveApiKey(provider) {
-    if (provider === 'anthropic') {
-        const key = process.env.IRIS_ANTHROPIC_API_KEY;
-        if (!key) {
-            throw new Error('Anthropic judge requires IRIS_ANTHROPIC_API_KEY. Set it in the environment or use a different provider.');
-        }
-        return key;
-    }
-    const key = process.env.IRIS_OPENAI_API_KEY;
+/**
+ * The key for the provider, from this process's environment. Missing is
+ * IRIS_JUDGE_NOT_ENABLED with the enable steps as recovery — the fact
+ * users get wrong is that a shell export does not reach the process an
+ * MCP client spawns, and the steps say so.
+ */
+export function resolveApiKey(provider, toolName = 'evaluate_with_llm_judge') {
+    const variable = JUDGE_KEY_VARS[provider];
+    const key = process.env[variable];
     if (!key) {
-        throw new Error('OpenAI judge requires IRIS_OPENAI_API_KEY. Set it in the environment or use a different provider.');
+        throw irisError('IRIS_JUDGE_NOT_ENABLED', `${toolName} needs ${variable} in the environment of the process that runs Iris; no key for ${provider} reached this process. Nothing was spent.`, {
+            field: variable,
+            recovery: judgeRecovery(provider),
+            see: CAPABILITIES_RESOURCE_URI,
+        });
     }
     return key;
 }
 function resolveMaxCost(paramValue) {
     if (paramValue !== undefined)
         return paramValue;
-    const envRaw = process.env.IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL;
-    if (envRaw) {
-        const parsed = Number(envRaw);
-        if (Number.isFinite(parsed) && parsed > 0)
-            return parsed;
-    }
-    return 0.25;
+    return judgeCostCapUsd();
 }
+export const judgeOutputSchema = z.looseObject({
+    id: z.string().describe('the evaluation id; read it back at iris://evaluations/{id}'),
+    trace_id: z.string().optional().describe('the linked trace, when one was named'),
+    score: z.number().describe('0..1 from the judge'),
+    passed: z.boolean().describe('the judge\'s own pass verdict for the template'),
+    rationale: z.string().describe('the judge\'s reasoning, in its words'),
+    dimensions: z.record(z.string(), z.unknown()).describe('per-dimension sub-scores for the template'),
+    model: z.string().describe('the model that judged'),
+    provider: z.enum(['anthropic', 'openai']).describe('the provider called'),
+    template: z.string().describe('the template used'),
+    input_tokens: z.number().describe('tokens sent, across both attempts when a retry ran'),
+    output_tokens: z.number().describe('tokens received, across both attempts when a retry ran'),
+    cost_usd: z.number().nullable().describe('the exact spend from the pricing table'),
+    latency_ms: z.number().describe('wall time of the provider call(s)'),
+    raw_response_id: z.string().optional().describe('the provider\'s response id, for your own audit'),
+});
 export function registerEvaluateWithLLMJudgeTool(server, storage) {
     server.registerTool('evaluate_with_llm_judge', {
         title: 'Evaluate With LLM Judge',
-        description: [
-            'Score agent output using an LLM as the judge (Anthropic or OpenAI). Returns a 0..1 score with rationale, per-dimension breakdown, and exact cost. The judge\'s own accuracy is measurable on a key you supply and is not yet published — see https://iris-eval.com/proof.',
-            '',
-            'Sibling tools — evaluate_output runs heuristic rules (free, deterministic, no API key needed); this tool runs LLM-based semantic scoring (paid, requires an API key). verify_citations is a SPECIALIZED form of LLM judging that focuses on citation grounding only. log_trace / get_traces handle trace I/O; list_rules / deploy_rule / delete_rule manage heuristic-rule lifecycle. evaluate_with_llm_judge is the GENERAL semantic-scoring path.',
-            '',
-            'Behavior. Calls an external LLM API (Anthropic or OpenAI) — costs money per call, takes as long as the provider takes, respects an IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL cap. Non-deterministic at temperature > 0; default temperature=0 gives near-deterministic scores. Writes one eval_result row to Iris storage (linked to trace_id if provided) plus captures provider response id + latency + token counts + cost in the rule_results payload. Rate-limited to 20 req/min on HTTP MCP; your LLM provider also enforces its own rate limits (we transparently retry once on 429).',
-            '',
-            'Output shape. Returns JSON: `{ "id": "<uuid>", "score": 0..1, "passed": boolean, "rationale": string, "dimensions": {...}, "model": string, "provider": "anthropic"|"openai", "template": string, "input_tokens": number, "output_tokens": number, "cost_usd": number, "latency_ms": number }`. `dimensions` has per-dimension sub-scores (e.g., accuracy template returns `{factual_claims, citations, internal_consistency}`).',
-            '',
-            'Use when heuristic rules (via evaluate_output) are too coarse for the quality signal you need — semantic correctness, factual accuracy vs a reference, RAG faithfulness to sources, nuanced safety/helpfulness. Pick the template that matches: `accuracy` (hallucination detection), `helpfulness` (does it address the ask), `safety` (harm potential beyond regex PII), `correctness` (vs reference answer — pass `expected`), `faithfulness` (RAG grounding — pass `source_material`).',
-            '',
-            "Don't use for simple regex/length/keyword checks (use evaluate_output with heuristic rules — they're free, deterministic and in-process). Don't use without an API key set (IRIS_ANTHROPIC_API_KEY or IRIS_OPENAI_API_KEY). Don't use on very large outputs (>8K tokens) without raising max_cost_usd — the pre-check will refuse the call.",
-            '',
-            'Parameters. model is required (no default — pick consciously since cost varies 100x across models). provider is auto-detected from the model name; override only for ambiguous IDs. expected is REQUIRED when template="correctness" (the reference answer to compare against); ignored for other templates. source_material is REQUIRED when template="faithfulness" (the RAG sources to ground against); ignored otherwise. input is optional but improves scoring on helpfulness/safety templates (gives the judge the user prompt that produced the output). max_cost_usd defaults to env var IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL or $0.25 — the worst-case cost is computed BEFORE the call (input_tokens × prompt_price + max_output_tokens × completion_price, PLUS the same for the one retry that fires if the judge\'s first reply is not valid JSON); call refused upfront if that two-attempt worst case would exceed. When a retry does run, the reported input_tokens / output_tokens / cost_usd / latency_ms are totals across both attempts. max_output_tokens caps the judge response (default 512, max 4096); higher = more rationale detail + more cost. temperature default 0 (deterministic). timeout_ms default 60000. trace_id optional but recommended (links eval to trace in dashboard). Defaults: temperature=0, max_output_tokens=512, max_cost_usd=$0.25, timeout_ms=60000.',
-            '',
-            'Error modes. Throws when the required API key env var is missing. Throws when trace_id does not match a stored trace — checked before the provider call, so no money is spent and nothing is written. Throws when the estimated worst-case cost exceeds max_cost_usd (raise the cap or trim prompts). Throws LLMJudgeError on provider errors — kind=`auth` on 401/403, `rate_limit` on 429 (auto-retried once), `server_error` on 5xx, `timeout` on abort, `malformed_response` when the judge fails to emit valid JSON on both attempts. Throws "Unknown model" for unsupported model IDs — update src/eval/llm-judge/pricing.ts first.',
-        ].join('\n'),
+        description: describeTool({
+            summary: 'Score an output with an LLM judge on your own provider key: a 0..1 score, a rationale, per-dimension sub-scores and the exact spend.',
+            does: `Calls Anthropic or OpenAI directly with the key in this process's environment (${JUDGE_KEY_VARS.anthropic} or ${JUDGE_KEY_VARS.openai}); Iris never proxies. ` +
+                'template picks the question: accuracy, helpfulness, safety, correctness (needs expected) or faithfulness (needs source_material); input improves helpfulness and safety. model is required; provider is inferred from it. ' +
+                `The worst-case spend — both attempts, full max_output_tokens — is computed BEFORE the call and refused if it exceeds max_cost_usd (default ${JUDGE_COST_CAP_VAR} or ${JUDGE_DEFAULT_COST_CAP_USD}). ` +
+                'temperature defaults to 0; a rate-limited call is retried once. One evaluation row is stored with the provider response id, tokens, cost and latency, linked to trace_id when given. ' +
+                "The judge's own accuracy is measurable on a key you supply and is not yet published (see iris://proof).",
+            whenNot: 'For length, keyword, PII, injection or cost checks: evaluate_output is free and deterministic. Without a key: the call returns IRIS_JUDGE_NOT_ENABLED with the enable steps — do not search for them. On very large outputs without raising max_cost_usd: the pre-check refuses.',
+            returns: judgeOutputSchema,
+            errors: 'IRIS_JUDGE_NOT_ENABLED (no key for the provider reached this process; recovery carries the steps). IRIS_JUDGE_UNKNOWN_MODEL (valid lists the models). IRIS_UNKNOWN_TRACE, checked before any spend. ' +
+                'IRIS_BUDGET_EXCEEDED (nothing spent; the message carries both numbers). IRIS_PROVIDER_ERROR with kind auth, rate_limit, bad_request, server_error, timeout or malformed_response, and retryable set. ' +
+                ERROR_ENVELOPE_SENTENCE,
+            siblings: {
+                evaluate_output: 'the free deterministic path',
+                verify_citations: 'citation grounding, the narrower judge',
+                log_trace: 'record the execution first',
+            },
+        }),
         inputSchema: strictInput(inputSchema),
+        outputSchema: judgeOutputSchema,
         annotations: {
             readOnlyHint: false, // Writes eval_result; also spends money (external API cost)
             destructiveHint: false, // Creates data; doesn't overwrite or delete
             idempotentHint: false, // Temperature > 0 may vary; even at T=0 provider non-determinism is possible; cost also varies per call
             openWorldHint: true, // Calls external APIs (Anthropic / OpenAI) — touches the world beyond local process
         },
-    }, async (args) => {
+    }, guarded(async (args) => {
         const provider = args.provider ?? inferProvider(args.model);
         const apiKey = resolveApiKey(provider);
         const maxCostUsd = resolveMaxCost(args.max_cost_usd);
@@ -95,8 +124,8 @@ export function registerEvaluateWithLLMJudgeTool(server, storage) {
         const result = await evaluateWithLLMJudge({
             output: args.output,
             template: args.template,
-            provider,
             model: args.model,
+            provider,
             apiKey,
             input: args.input,
             expected: args.expected,
@@ -106,12 +135,12 @@ export function registerEvaluateWithLLMJudgeTool(server, storage) {
             temperature: args.temperature,
             timeoutMs: args.timeout_ms,
         });
+        // Persist to eval_results so the dashboard can surface it.
+        // eval_type='custom' — LLM judge scores span all 4 heuristic
+        // categories (accuracy, helpfulness, safety, faithfulness); 'custom'
+        // is the honest bucket. rule_results[0] captures per-dimension
+        // breakdown + provider metadata for audit.
         const evalId = generateEvalId();
-        // Persist as a normal eval_result so the dashboard picks it up
-        // alongside heuristic scores. eval_type is 'custom' because LLM
-        // judge doesn't fit completeness/relevance/safety/cost taxonomy
-        // cleanly — it spans all four. The rule_results payload carries
-        // the full judge provenance.
         await insertLinkedEvalResult(storage, LOCAL_TENANT, {
             id: evalId,
             trace_id: args.trace_id,
@@ -132,28 +161,26 @@ export function registerEvaluateWithLLMJudgeTool(server, storage) {
             rules_evaluated: 1,
             rules_skipped: 0,
             insufficient_data: false,
+            // What the evaluation itself cost — the description promised it was
+            // kept and the write path stored none of it (arc zero, G15).
+            eval_cost_usd: result.costUsd ?? undefined,
+            eval_tokens: result.inputTokens + result.outputTokens,
         });
-        return {
-            content: [
-                {
-                    type: 'text',
-                    text: JSON.stringify({
-                        id: evalId,
-                        score: result.score,
-                        passed: result.passed,
-                        rationale: result.rationale,
-                        dimensions: result.dimensions,
-                        model: result.model,
-                        provider: result.provider,
-                        template: result.template,
-                        input_tokens: result.inputTokens,
-                        output_tokens: result.outputTokens,
-                        cost_usd: result.costUsd,
-                        latency_ms: result.latencyMs,
-                        raw_response_id: result.rawResponseId,
-                    }),
-                },
-            ],
-        };
-    });
+        return respond(judgeOutputSchema, {
+            id: evalId,
+            ...(args.trace_id ? { trace_id: args.trace_id } : {}),
+            score: result.score,
+            passed: result.passed,
+            rationale: result.rationale,
+            dimensions: result.dimensions,
+            model: result.model,
+            provider: result.provider,
+            template: result.template,
+            input_tokens: result.inputTokens,
+            output_tokens: result.outputTokens,
+            cost_usd: result.costUsd,
+            latency_ms: result.latencyMs,
+            raw_response_id: result.rawResponseId,
+        }, evaluationLinks(evalId, args.trace_id));
+    }));
 }

package/dist/tools/get-traces.d.ts

@@ -27,4 +27,13 @@ export interface TraceRangeArgs {
  * bound the tool rejects is never one the HTTP query quietly accepts.
  */
 export declare function addTraceRangeIssues(args: TraceRangeArgs, ctx: z.RefinementCtx): void;
+export declare const getTracesOutputSchema: z.ZodObject<{
+    traces: z.ZodArray<z.ZodObject<{
+        trace_id: z.ZodString;
+    }, z.core.$loose>>;
+    total: z.ZodNumber;
+    limit: z.ZodNumber;
+    offset: z.ZodNumber;
+    summary: z.ZodOptional<z.ZodObject<{}, z.core.$loose>>;
+}, z.core.$loose>;
 export declare function registerGetTracesTool(server: McpServer, storage: IStorageAdapter): void;

package/dist/tools/get-traces.js

@@ -1,6 +1,8 @@
 import { z } from 'zod';
 import { LOCAL_TENANT } from '../types/tenant.js';
 import { strictInput } from './strict-input.js';
+import { describeTool, ERROR_ENVELOPE_SENTENCE } from './describe.js';
+import { guarded, respond } from './respond.js';
 /*
  * An ISO-8601 instant (2026-08-01T00:00:00Z, offsets allowed) or calendar
  * date (2026-08-01). Stored timestamps are ISO strings and the adapter
@@ -61,7 +63,7 @@ const inputSchema = {
     // Mirrors traceQuerySchema in dashboard/validation.ts — both capture paths
     // (MCP tool, HTTP query) enforce the same 1..1000 bound. Unclamped, limit:-1
     // meant "LIMIT -1" in SQLite, i.e. every row (#332).
-    limit: z.number().int().min(1).max(1000).default(50).describe('Results per page (default 50, max 1000 — values >1000 return 400)'),
+    limit: z.number().int().min(1).max(1000).default(50).describe('Results per page (default 50, max 1000 — values above are rejected)'),
     offset: z.number().int().min(0).default(0).describe('Zero-based pagination offset — skip first N results (non-negative integer)'),
     sort_by: z.enum(['timestamp', 'latency_ms', 'cost_usd']).default('timestamp').describe('Sort by timestamp | latency_ms | cost_usd (default timestamp)'),
     sort_order: z.enum(['asc', 'desc']).default('desc').describe('Sort order: asc | desc (default desc — most recent / highest first)'),
@@ -69,34 +71,40 @@ const inputSchema = {
 };
 // Cross-field range checks — see addTraceRangeIssues above.
 const inputSchemaWithRanges = strictInput(inputSchema).superRefine(addTraceRangeIssues);
+export const getTracesOutputSchema = z.looseObject({
+    traces: z.array(z.looseObject({ trace_id: z.string() })).describe('the page of traces: trace_id, agent_name, framework, input, output, tool_calls, latency_ms, token_usage, cost_usd, metadata, timestamp'),
+    total: z.number().int().describe('how many traces match the filters, across every page'),
+    limit: z.number().int().describe('the page size applied'),
+    offset: z.number().int().describe('the offset applied'),
+    summary: z.looseObject({}).optional().describe('the dashboard aggregates for the last hour, when include_summary was true'),
+});
 export function registerGetTracesTool(server, storage) {
     server.registerTool('get_traces', {
         title: 'Get Traces',
-        description: [
-            'Query stored agent-execution traces with filters, pagination, and optional dashboard summary.',
-            '',
-            'Sibling tools — log_trace creates traces, delete_trace removes a single trace, evaluate_output / evaluate_with_llm_judge / verify_citations score them, list_rules / deploy_rule / delete_rule manage the custom-rule lifecycle. get_traces is the READ path for historical agent executions — never mutates anything.',
-            '',
-            'Behavior. Read-only: never mutates storage, never calls external services. Idempotent: repeated calls with the same args return consistent results (new traces logged after the call obviously show up on subsequent calls). Tenant-scoped: queries only the caller\'s tenant rows (LOCAL_TENANT in OSS). Paginates results (default limit 50, max 1000). Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio.',
-            '',
-            'Output shape. Returns JSON: `{ "traces": [{...traceRow}], "total": number, "limit": number, "offset": number, "summary"?: { total_traces, avg_latency_ms, total_cost_usd, error_rate, eval_pass_rate, traces_per_hour, top_agents } }`. Each trace row includes trace_id, agent_name, framework, input, output, tool_calls, latency_ms, token_usage, cost_usd, metadata, timestamp. `summary` only included when `include_summary: true`.',
-            '',
-            'Use when you need historical data: investigating a past failure, computing quality trends, comparing agents, or feeding an analytics job. Set `agent_name` / `framework` / `since` / `until` to narrow the query. Set `min_score` / `max_score` to surface outliers. Set `sort_by: "cost_usd"` + `sort_order: "desc"` to find the most expensive traces. Set `include_summary: true` when you want dashboard-style aggregates in one round-trip.',
-            '',
-            'Don\'t use to score a trace (use evaluate_output). Don\'t use to create a trace (use log_trace). Don\'t use as a live event stream — it\'s a query, not a subscription, and Iris has no event-stream endpoint; poll with exponential backoff.',
-            '',
-            'Parameters. limit defaults to 50, max 1000 (anything higher returns 400). offset is zero-based pagination (non-negative integer). since / until must be ISO 8601 timestamps or dates — `since` is inclusive (timestamp >= since), `until` is inclusive (timestamp <= until), and `since` may not be later than `until`. min_score / max_score are 0..1 and filter on the LATEST eval per trace, not all evals (so a trace with one failing + one passing eval may or may not match depending on which landed last); min_score may not exceed max_score. Combining since + sort_by="latency_ms" + sort_order="desc" is the canonical "find slow recent traces" query. include_summary returns dashboard-style aggregates in the SAME response (saves a round-trip; use true for dashboard ingest, false for analytics queries that don\'t need them). agent_name and framework are exact-match (no wildcards). Defaults: limit=50, offset=0, sort_by="timestamp", sort_order="desc", include_summary=false.',
-            '',
-            'Error modes. Returns 400 on invalid sort_by / sort_order (Zod enum). Returns 400 if limit > 1000 or offset < 0. Returns 400 — naming both values — on an empty range: min_score > max_score, since later than until, a score outside 0..1, or a since/until that is not an ISO 8601 timestamp or date (an unparseable bound is refused, never silently ignored). Returns 429 when HTTP rate limit exceeded. Storage failures propagate as 500. Empty result with `total: 0` on no matches (not an error).',
-        ].join('\n'),
+        description: describeTool({
+            summary: 'Query stored traces with filters, pagination and sorting; optionally include the dashboard summary in the same response.',
+            does: 'Read-only, local storage only. Filters are exact-match (agent_name, framework), inclusive time bounds (since, until — an ISO 8601 timestamp or date) and a score range applied to the LATEST evaluation of each trace (min_score, max_score, 0..1). ' +
+                'limit is 1..1000 (default 50), offset counts from 0, sort_by is timestamp, latency_ms or cost_usd, sort_order asc or desc (default: newest first). include_summary adds the one-hour dashboard aggregates. ' +
+                'A crossed range (min above max, since after until) is refused naming both values rather than returning an empty page that reads as "no such traces".',
+            whenNot: 'To score a trace (evaluate_output). To create one (log_trace). As a live stream: this is a query, and Iris has no event stream — poll with backoff.',
+            returns: getTracesOutputSchema,
+            errors: 'IRIS_STORAGE_ERROR when the database cannot be read. An out-of-range or crossed bound is refused before the handler runs, naming the values. An empty result is total 0, not an error. ' +
+                ERROR_ENVELOPE_SENTENCE,
+            siblings: {
+                log_trace: 'record an execution',
+                evaluate_output: 'score one output',
+                delete_trace: 'remove one trace',
+            },
+        }),
         inputSchema: inputSchemaWithRanges,
+        outputSchema: getTracesOutputSchema,
         annotations: {
             readOnlyHint: true, // Pure query: never writes, never deletes
             destructiveHint: false, // Inverse of readOnly — trivially false
             idempotentHint: true, // Same args → same result (modulo new traces that may have landed since)
             openWorldHint: false, // Queries local storage only; no external network
         },
-    }, async (args) => {
+    }, guarded(async (args) => {
         // OSS single-tenant: MCP caller is the local user.
         const result = await storage.queryTraces(LOCAL_TENANT, {
             filter: {
@@ -121,13 +129,6 @@ export function registerGetTracesTool(server, storage) {
         if (args.include_summary) {
             response.summary = await storage.getDashboardSummary(LOCAL_TENANT);
         }
-        return {
-            content: [
-                {
-                    type: 'text',
-                    text: JSON.stringify(response),
-                },
-            ],
-        };
-    });
+        return respond(getTracesOutputSchema, response);
+    }));
 }

package/dist/tools/index.d.ts

@@ -2,4 +2,12 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { IStorageAdapter } from '../types/query.js';
 import type { EvalEngine } from '../eval/engine.js';
 import type { CustomRuleStore } from '../custom-rule-store.js';
+/**
+ * Every tool this server registers, by name. The capabilities object
+ * lists it, the docs contract checks prose against it, and a test asserts
+ * it equals what tools/list returns — so a tool added below without a
+ * name here (or the reverse) fails before it ships.
+ */
+export declare const TOOL_NAMES: readonly ["log_trace", "evaluate_output", "get_traces", "list_rules", "deploy_rule", "delete_rule", "delete_trace", "evaluate_with_llm_judge", "verify_citations"];
+export type ToolName = (typeof TOOL_NAMES)[number];
 export declare function registerAllTools(server: McpServer, storage: IStorageAdapter, evalEngine: EvalEngine, customRuleStore: CustomRuleStore): void;

package/dist/tools/index.js

@@ -7,9 +7,30 @@ import { registerDeleteRuleTool } from './delete-rule.js';
 import { registerDeleteTraceTool } from './delete-trace.js';
 import { registerEvaluateWithLLMJudgeTool } from './evaluate-with-llm-judge.js';
 import { registerVerifyCitationsTool } from './verify-citations.js';
+import { dormantRulesFrom } from '../eval/dormant.js';
+import { LOCAL_TENANT } from '../types/tenant.js';
+/**
+ * Every tool this server registers, by name. The capabilities object
+ * lists it, the docs contract checks prose against it, and a test asserts
+ * it equals what tools/list returns — so a tool added below without a
+ * name here (or the reverse) fails before it ships.
+ */
+export const TOOL_NAMES = [
+    'log_trace',
+    'evaluate_output',
+    'get_traces',
+    'list_rules',
+    'deploy_rule',
+    'delete_rule',
+    'delete_trace',
+    'evaluate_with_llm_judge',
+    'verify_citations',
+];
 export function registerAllTools(server, storage, evalEngine, customRuleStore) {
     registerLogTraceTool(server, storage);
-    registerEvaluateOutputTool(server, storage, evalEngine);
+    registerEvaluateOutputTool(server, storage, evalEngine, {
+        dormant: () => dormantRulesFrom(customRuleStore.quarantined(LOCAL_TENANT)),
+    });
     registerGetTracesTool(server, storage);
     registerListRulesTool(server, customRuleStore, evalEngine);
     registerDeployRuleTool(server, customRuleStore, evalEngine);