@iris-eval/mcp-server 0.8.2 → 0.10.0

package/dist/tools/deploy-rule.js

@@ -14,11 +14,14 @@ import { z } from 'zod';
 import { createCustomRule } from '../eval/rules/custom.js';
 import { LOCAL_TENANT } from '../types/tenant.js';
 import { strictInput, strictNested } from './strict-input.js';
+import { describeTool, ERROR_ENVELOPE_SENTENCE } from './describe.js';
+import { guarded, respond } from './respond.js';
 const EvalTypeSchema = z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom']);
 /**
  * A rule with this name is already deployed and the caller did not ask to
  * replace it. Carries the existing rule(s) so an HTTP surface can answer
- * 409 with them beside the same message the MCP tool throws.
+ * 409 with them beside the same message the MCP tool returns as
+ * IRIS_DUPLICATE_RULE (src/tools/errors.ts maps it by name).
  */
 export class DuplicateRuleNameError extends Error {
     existing;
@@ -145,34 +148,41 @@ const inputSchemaWithAliases = strictInput(inputSchema).superRefine((args, ctx)
         ctx.addIssue({ code: 'custom', path: ['sourceMomentId'], message: 'pass either source_moment_id or sourceMomentId, not both' });
     }
 });
+export const deployRuleOutputSchema = z.looseObject({
+    rule: z.looseObject({ id: z.string(), name: z.string() }).describe('the rule as persisted: id (rule-<hex>, keep it for delete_rule), name, description, evalType, severity, definition, enabled, createdAt, updatedAt, version, sourceMomentId'),
+    replaced: z.array(z.looseObject({ id: z.string() })).optional().describe('with replace: true, the earlier rule(s) of the same name that were retired'),
+    warning: z.string().optional().describe('with replace: true, one sentence naming what was retired'),
+});
 export function registerDeployRuleTool(server, customRuleStore, evalEngine) {
     server.registerTool('deploy_rule', {
         title: 'Deploy Custom Rule',
-        description: [
-            'Deploy a new custom evaluation rule that will fire on every future evaluate_output call of its eval category.',
-            '',
-            'Sibling tools — list_rules enumerates deployed rules, delete_rule removes them (or disables/re-enables them with its `enabled` argument), evaluate_output runs them. log_trace / get_traces / delete_trace handle the trace lifecycle separately; evaluate_with_llm_judge / verify_citations run semantic scoring (not heuristic-rule-driven). deploy_rule is the WRITE path that grows the custom-rule library.',
-            '',
-            'Behavior. Writes a row to ~/.iris/custom-rules.json (atomic write via temp file + rename) and appends a `rule.deploy` entry to the audit log (~/.iris/audit.log). The rule activates immediately for the running process and persists across restarts. Each call mints a fresh rule_id. Rule names are unique among deployed rules: deploying a name that is already deployed is REJECTED unless `replace: true`, in which case the existing same-named rule(s) are deleted (audit `rule.delete` rows written, unregistered from the live engine) and the new rule takes their place under a new id — the response lists what was replaced. Rules are owned by the local tenant. Rate-limited to 20 req/min on HTTP MCP.',
-            '',
-            'Output shape. Returns JSON: `{ "rule": { "id": "rule-XXXX", "name", "description", "evalType", "severity", "definition", "enabled": true, "createdAt", "updatedAt", "version": 1, "sourceMomentId?" }, "replaced?": [{ "id", "evalType", "severity" }], "warning?": string }`. The returned rule is the canonical persisted form; save the `id` if you plan to disable or delete later. `replaced` and `warning` appear only when `replace: true` removed an earlier rule of the same name.',
-            '',
-            "Use when an agent observes a recurring failure pattern and decides to enforce it as a standing rule. The `source_moment_id` field preserves provenance — downstream audit can trace the rule back to the moment that inspired it. Combine with evaluate_output + get_traces: 1) evaluate_output surfaces failures; 2) get_traces filters to the failure set; 3) analyze the pattern; 4) deploy_rule bakes it into the default eval path.",
-            '',
-            "Don't use to VALIDATE a rule before committing — deploy writes immediately. Use the dashboard's preview endpoint (POST /api/v1/rules/custom/preview) to replay a definition against recent stored traces first. To UPDATE a rule: call deploy_rule with the same name and `replace: true` (the old rule is deleted, the new one gets a fresh id), or delete_rule then deploy_rule. To pause a rule without losing it: delete_rule with `enabled: false`.",
-            '',
-            'Parameters. Argument names are snake_case (eval_type, source_moment_id) — the camelCase spellings evalType / sourceMomentId are accepted as aliases for compatibility, but pass only one spelling of each. name is 1-80 chars (Zod-enforced min/max — the same cap the persisted store applies); appears in eval_result rule_results (alongside the rule id as `ruleId`) so make it human-readable. description is optional, max 500 chars (used in dashboard tooltips). eval_type determines WHEN the rule fires: a deployed rule runs ONLY on evaluate_output calls whose eval_type equals the rule\'s eval_type, plus eval_type="all" (which runs every bundle) — a "completeness" rule does NOT fire on eval_type="safety" or on eval_type="custom"; eval_type="custom" runs rules deployed under "custom" (and the call\'s inline custom_rules) and nothing else. severity decides what a FAILURE of the rule does: low/medium failures only lower the weighted score (and drive dashboard sort + audit alerts); high/critical failures HARD-FAIL the evaluation — the overall `passed` is forced to false regardless of the weighted score, and the rule is listed in the response\'s `critical_failures`. Severity never changes the numeric score itself (that uses the rule\'s weight). definition.type and definition.config must match (e.g., regex_match needs config.pattern; cost_threshold needs config.max_cost; min_length needs config.min_length; max_length needs config.max_length; contains_keywords/excludes_keywords need config.keywords). definition.name is optional and, if given, overwritten by the top-level name. Invalid configs are REJECTED at deploy time with the offending field named, instead of deploying and then failing every evaluation. source_moment_id is optional but recommended (preserves workflow-inversion provenance from Make-This-A-Rule composer). replace defaults to false. Defaults: severity="medium", replace=false.',
-            '',
-            "Error modes. Throws 400 on invalid definition (Zod rejects — e.g., regex that fails safe-regex2 ReDoS check, or length > 1000 chars), on an unknown key inside `definition` (the valid keys are listed; config keys are free-form), on empty `name` or `name` over 80 chars, on a non-positive weight, and when both spellings of an aliased argument are passed. Throws when a rule with the same name is already deployed and `replace` is not true — the message names the existing rule id so you can delete it, replace it, or pick another name. Any eval_type/definition.type combination is valid (a regex_match rule can enforce a safety policy; a max_length rule can express completeness) — there is no category/type mismatch error. Returns 429 when HTTP rate limit exceeded. File-write failures (disk full, read-only fs) propagate as 500; the audit log is best-effort and does not block deploy.",
-        ].join('\n'),
+        description: describeTool({
+            summary: 'Deploy a custom rule that fires on every future evaluate_output call of its bundle — persisted, active immediately, audited.',
+            does: 'Writes the rule to ~/.iris/custom-rules.json, appends a rule.deploy audit entry and registers it with the running engine, so it fires on the very next call and survives restarts. ' +
+                'eval_type says WHEN it fires (that bundle, and eval_type="all"); severity says what a failure DOES: low and medium only lower the weighted score, high and critical force passed to false and list the rule in critical_failures. ' +
+                'definition.type picks the check (regex_match, regex_no_match, min_length, max_length, contains_keywords, excludes_keywords, json_schema, cost_threshold) and definition.config carries its keys (pattern; min_length; max_length; keywords; max_cost). ' +
+                'Any bundle and type combine. Names are unique: a taken name is refused unless replace is true, which retires the earlier rule(s) first and reports them. Argument names are snake_case; the camelCase aliases evalType and sourceMomentId are accepted — pass one spelling of each.',
+            whenNot: 'To try a rule first: POST /api/v1/rules/custom/preview on the dashboard replays a definition against stored traces without deploying. For a one-off check on one call: the custom_rules argument of evaluate_output. To pause a rule: delete_rule with enabled: false.',
+            returns: deployRuleOutputSchema,
+            errors: 'IRIS_DUPLICATE_RULE when the name is deployed and replace is false (the message names the existing id). ' +
+                'IRIS_INVALID_RULE_CONFIG when the definition is rejected — a regex that fails the ReDoS check or exceeds 1000 characters, a missing config key — naming the field; nothing is deployed. ' +
+                'IRIS_STORAGE_ERROR when the store cannot be written. An unknown key in definition, a name over 80 characters, a non-positive weight or both spellings of an alias are refused before the handler runs. ' +
+                ERROR_ENVELOPE_SENTENCE,
+            siblings: {
+                list_rules: 'see what is deployed and the built-in roster',
+                delete_rule: 'remove, disable or re-enable',
+                evaluate_output: 'where the rule fires',
+            },
+        }),
         inputSchema: inputSchemaWithAliases,
+        outputSchema: deployRuleOutputSchema,
         annotations: {
             readOnlyHint: false,
             destructiveHint: false,
             idempotentHint: false,
             openWorldHint: false,
         },
-    }, async (args) => {
+    }, guarded(async (args) => {
         const evalType = (args.eval_type ?? args.evalType);
         const sourceMomentId = args.source_moment_id ?? args.sourceMomentId;
         // Server overrides the inner definition's `name` so it always matches
@@ -186,10 +196,12 @@ export function registerDeployRuleTool(server, customRuleStore, evalEngine) {
         };
         // Same-name redeploy (#373) — shared with the dashboard's deploy
         // route; see retireSameNamedRules above. Throws (nothing deployed)
-        // when the name is taken and replace is false.
+        // when the name is taken and replace is false; `guarded` turns that
+        // into IRIS_DUPLICATE_RULE.
         const replaced = retireSameNamedRules(customRuleStore, evalEngine, LOCAL_TENANT, args.name, args.replace, 'mcp');
         // OSS: MCP tools operate under LOCAL_TENANT. See list-rules.ts for context.
         const rule = customRuleStore.deploy(LOCAL_TENANT, {
+            replaces: replaced.map((r) => r.id),
             name: args.name,
             description: args.description,
             evalType,
@@ -205,18 +217,9 @@ export function registerDeployRuleTool(server, customRuleStore, evalEngine) {
         // Registered under its rule id so delete_rule can hot-remove it.
         // Severity rides along: high/critical makes the rule hard-failing.
         evalEngine.registerRule(rule.evalType, createCustomRule(rule.definition, rule.severity), rule.id);
-        return {
-            content: [
-                {
-                    type: 'text',
-                    text: JSON.stringify({
-                        rule,
-                        ...(replaced.length > 0
-                            ? { replaced, warning: replacedRulesWarning(args.name, replaced) }
-                            : {}),
-                    }),
-                },
-            ],
-        };
-    });
+        return respond(deployRuleOutputSchema, {
+            rule,
+            ...(replaced.length > 0 ? { replaced, warning: replacedRulesWarning(args.name, replaced) } : {}),
+        });
+    }));
 }

package/dist/tools/describe.d.ts

@@ -0,0 +1,20 @@
+import type { z } from 'zod';
+export declare const DESCRIPTION_HEADINGS: readonly ["What it does.", "When not to use it.", "Returns.", "Errors.", "Siblings."];
+export declare const DESCRIPTION_WORD_CAP = 450;
+export interface ToolDescriptionSpec {
+    /** One sentence: what calling this does. */
+    summary: string;
+    does: string;
+    whenNot: string;
+    /** The tool's output schema; every top-level field must carry a description. */
+    returns: z.ZodObject<z.ZodRawShape>;
+    errors: string;
+    /** Sibling tool → one clause on when it is the better call. */
+    siblings: Record<string, string>;
+}
+/** `field` (its description); … — from the schema, never retyped. */
+export declare function returnsFrom(schema: z.ZodObject<z.ZodRawShape>): string;
+export declare function wordCount(text: string): number;
+export declare function describeTool(spec: ToolDescriptionSpec): string;
+/** The sentence every "Errors" heading ends with, so the envelope is stated once. */
+export declare const ERROR_ENVELOPE_SENTENCE = "Every failure returns {\"error\":{\"code\",\"message\",\"recovery\":[]}} with isError true; follow recovery before retrying.";

package/dist/tools/describe.js

@@ -0,0 +1,36 @@
+export const DESCRIPTION_HEADINGS = ['What it does.', 'When not to use it.', 'Returns.', 'Errors.', 'Siblings.'];
+export const DESCRIPTION_WORD_CAP = 450;
+/** `field` (its description); … — from the schema, never retyped. */
+export function returnsFrom(schema) {
+    const parts = [];
+    for (const [key, field] of Object.entries(schema.shape)) {
+        const description = field.description;
+        if (!description)
+            throw new Error(`output field "${key}" has no description; add .describe() on the schema`);
+        parts.push(`\`${key}\` (${description})`);
+    }
+    return `JSON with ${parts.join('; ')}.`;
+}
+export function wordCount(text) {
+    return text.split(/\s+/).filter(Boolean).length;
+}
+export function describeTool(spec) {
+    const siblings = Object.entries(spec.siblings)
+        .map(([name, when]) => `${name} — ${when}`)
+        .join('; ');
+    const text = [
+        spec.summary,
+        `${DESCRIPTION_HEADINGS[0]} ${spec.does}`,
+        `${DESCRIPTION_HEADINGS[1]} ${spec.whenNot}`,
+        `${DESCRIPTION_HEADINGS[2]} ${returnsFrom(spec.returns)}`,
+        `${DESCRIPTION_HEADINGS[3]} ${spec.errors}`,
+        `${DESCRIPTION_HEADINGS[4]} ${siblings}.`,
+    ].join('\n\n');
+    const words = wordCount(text);
+    if (words > DESCRIPTION_WORD_CAP) {
+        throw new Error(`tool description is ${words} words; the cap is ${DESCRIPTION_WORD_CAP}`);
+    }
+    return text;
+}
+/** The sentence every "Errors" heading ends with, so the envelope is stated once. */
+export const ERROR_ENVELOPE_SENTENCE = 'Every failure returns {"error":{"code","message","recovery":[]}} with isError true; follow recovery before retrying.';

package/dist/tools/errors.d.ts

@@ -0,0 +1,36 @@
+import { LLMJudgeError } from '../eval/llm-judge/client.js';
+export declare const ERROR_CODE_CATALOGUE: readonly ["IRIS_INVALID_ARGUMENT", "IRIS_UNKNOWN_TRACE", "IRIS_DUPLICATE_RULE", "IRIS_INVALID_RULE_CONFIG", "IRIS_JUDGE_NOT_ENABLED", "IRIS_JUDGE_UNKNOWN_MODEL", "IRIS_BUDGET_EXCEEDED", "IRIS_PROVIDER_ERROR", "IRIS_JUDGE_FAILED", "IRIS_STORAGE_ERROR", "IRIS_INTERNAL_ERROR"];
+export type IrisErrorCode = (typeof ERROR_CODE_CATALOGUE)[number];
+export type ProviderErrorKind = LLMJudgeError['kind'];
+export interface IrisErrorEnvelope {
+    code: IrisErrorCode;
+    /** One sentence a person can act on. */
+    message: string;
+    /** What to do, in order, before retrying. */
+    recovery: string[];
+    /** True when the same call can succeed later without a change from the caller. */
+    retryable: boolean;
+    /** The argument or environment variable at fault, when the failure is about one. */
+    field?: string;
+    /** The values the field accepts, when there is a closed list. */
+    valid?: string[];
+    /** A resource that explains the limit or the state. */
+    see?: string;
+    /** IRIS_PROVIDER_ERROR only: the provider failure class. */
+    kind?: ProviderErrorKind;
+    /** When the provider said how long to wait. */
+    retryAfterMs?: number;
+}
+export declare class IrisError extends Error {
+    readonly envelope: IrisErrorEnvelope;
+    constructor(envelope: Omit<IrisErrorEnvelope, 'recovery' | 'retryable'> & Partial<Pick<IrisErrorEnvelope, 'recovery' | 'retryable'>>);
+}
+export declare function irisError(code: IrisErrorCode, message: string, extra?: Partial<Omit<IrisErrorEnvelope, 'code' | 'message'>>): IrisError;
+/**
+ * Every error a handler can throw, mapped to its code. Typed errors are
+ * matched by class or by name (the name check avoids importing the tool
+ * that defines the class, which would be a cycle); anything unrecognised
+ * is an internal error — reported as such, never dressed up as a caller
+ * mistake.
+ */
+export declare function toIrisError(err: unknown): IrisError;

package/dist/tools/errors.js

@@ -0,0 +1,134 @@
+/*
+ * Structured tool errors.
+ *
+ * A thrown Error reaches an MCP client as one line of text with isError
+ * set (the SDK flattens it), so the only thing an agent could do with
+ * "Anthropic judge requires IRIS_ANTHROPIC_API_KEY" was guess. Every
+ * failure a tool can produce is now a code from ONE catalogue with a
+ * message, the steps that clear it, whether a retry can help, and — when
+ * the failure is about one argument — the field and the valid values.
+ * Tools return these (src/tools/respond.ts wraps the handler); they never
+ * throw past the handler.
+ *
+ * The catalogue is exhaustive on purpose: tests/unit/tools/error-codes.test.ts
+ * provokes every code over a real transport and asserts the provoked set
+ * EQUALS this list, so a code nothing can raise cannot linger here and a
+ * new throw site cannot ship without a code.
+ *
+ * Two errors are not this module's: an argument the input schema rejects
+ * is refused by the SDK before the handler runs (the text names
+ * IRIS_INVALID_ARGUMENT and the valid arguments — see strict-input.ts), and
+ * an HTTP rate limit answers 429 at the transport before any tool runs.
+ */
+import { ZodError } from 'zod';
+import { LLMJudgeError } from '../eval/llm-judge/client.js';
+import { CAPABILITIES_RESOURCE_URI } from '../resources/uris.js';
+export const ERROR_CODE_CATALOGUE = [
+    'IRIS_INVALID_ARGUMENT',
+    'IRIS_UNKNOWN_TRACE',
+    'IRIS_DUPLICATE_RULE',
+    'IRIS_INVALID_RULE_CONFIG',
+    'IRIS_JUDGE_NOT_ENABLED',
+    'IRIS_JUDGE_UNKNOWN_MODEL',
+    'IRIS_BUDGET_EXCEEDED',
+    'IRIS_PROVIDER_ERROR',
+    'IRIS_JUDGE_FAILED',
+    'IRIS_STORAGE_ERROR',
+    'IRIS_INTERNAL_ERROR',
+];
+export class IrisError extends Error {
+    envelope;
+    constructor(envelope) {
+        super(envelope.message);
+        this.name = 'IrisError';
+        this.envelope = { retryable: false, recovery: [], ...envelope };
+    }
+}
+export function irisError(code, message, extra = {}) {
+    return new IrisError({ code, message, ...extra });
+}
+const CAPABILITIES = CAPABILITIES_RESOURCE_URI;
+function isSqliteError(err) {
+    const code = err.code;
+    if (typeof code === 'string' && code.startsWith('SQLITE_'))
+        return true;
+    const message = err instanceof Error ? err.message : '';
+    return /SQLITE|database is locked|disk I\/O error|readonly database/i.test(message);
+}
+/**
+ * Every error a handler can throw, mapped to its code. Typed errors are
+ * matched by class or by name (the name check avoids importing the tool
+ * that defines the class, which would be a cycle); anything unrecognised
+ * is an internal error — reported as such, never dressed up as a caller
+ * mistake.
+ */
+export function toIrisError(err) {
+    if (err instanceof IrisError)
+        return err;
+    const name = err?.name;
+    const message = err instanceof Error ? err.message : String(err);
+    if (name === 'DuplicateRuleNameError') {
+        return irisError('IRIS_DUPLICATE_RULE', message, {
+            field: 'name',
+            recovery: [
+                'Pass replace: true to retire the rule(s) with this name and deploy this one in their place.',
+                'Or call delete_rule with the existing rule id first.',
+                'Or choose a different name.',
+            ],
+        });
+    }
+    if (name === 'CostCapError') {
+        const e = err;
+        return irisError('IRIS_BUDGET_EXCEEDED', message, {
+            field: 'max_cost_usd',
+            recovery: [
+                `Raise max_cost_usd on the call (the worst case was ${e.estimatedUsd?.toFixed(4) ?? '?'} USD against a cap of ${e.capUsd?.toFixed(4) ?? '?'} USD).`,
+                'Or trim the output, the input or max_output_tokens so the worst case fits.',
+                'Nothing was spent.',
+            ],
+            see: CAPABILITIES,
+        });
+    }
+    if (err instanceof LLMJudgeError) {
+        const retryable = err.kind === 'rate_limit' || err.kind === 'timeout' || err.kind === 'server_error';
+        const recovery = {
+            auth: ['The provider refused the key. Check that the key in the env block of your MCP config is valid and live, then restart the session.'],
+            rate_limit: ['The provider rate-limited the call. Wait and retry; Iris already retried once.'],
+            bad_request: ['The provider rejected the request. Check the model name and the template inputs.'],
+            server_error: ['The provider failed on its side. Retry later.'],
+            timeout: ['The provider did not answer within timeout_ms. Retry, or raise timeout_ms.'],
+            malformed_response: ['The judge did not return valid JSON on two attempts. Retry; if it recurs, pick another model.'],
+            unknown: ['Retry once; if it recurs, report the message with the provider and model.'],
+        };
+        return irisError('IRIS_PROVIDER_ERROR', message, {
+            kind: err.kind,
+            retryable,
+            recovery: recovery[err.kind],
+            ...(err.retryAfterSeconds !== undefined ? { retryAfterMs: err.retryAfterSeconds * 1000 } : {}),
+        });
+    }
+    if (err instanceof ZodError) {
+        // The SDK validates tool input before the handler runs, so a ZodError
+        // inside a handler is the rule store rejecting a definition.
+        const first = err.issues[0];
+        const field = first?.path?.length ? first.path.map(String).join('.') : undefined;
+        return irisError('IRIS_INVALID_RULE_CONFIG', `The rule definition was rejected: ${err.issues.map((i) => `${i.path.map(String).join('.') || 'definition'}: ${i.message}`).join('; ')}`, {
+            ...(field ? { field } : {}),
+            recovery: ['Fix the named field and deploy again. Nothing was deployed.'],
+        });
+    }
+    if (isSqliteError(err)) {
+        const code = err.code ?? '';
+        return irisError('IRIS_STORAGE_ERROR', `Iris storage failed: ${message}`, {
+            retryable: code === 'SQLITE_BUSY' || code === 'SQLITE_LOCKED',
+            recovery: [
+                'Retry once. If it recurs, check that the database path is writable and the disk is not full.',
+                'Run `npx @iris-eval/mcp-server --self-test` to probe the configured home and database.',
+            ],
+        });
+    }
+    return irisError('IRIS_INTERNAL_ERROR', message || 'Iris hit an unexpected error.', {
+        recovery: ['Retry once. If it recurs, report the message with the Iris version from iris://capabilities.'],
+        see: CAPABILITIES,
+    });
+}

package/dist/tools/evaluate-output.d.ts

@@ -1,4 +1,11 @@
+import type { DormantRule } from '../eval/dormant.js';
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { IStorageAdapter } from '../types/query.js';
 import type { EvalEngine } from '../eval/engine.js';
-export declare function registerEvaluateOutputTool(server: McpServer, storage: IStorageAdapter, evalEngine: EvalEngine): void;
+/** The most inline custom rules one call may carry (see the argument description). */
+export declare const MAX_INLINE_CUSTOM_RULES = 10;
+export interface EvaluateOutputOptions {
+    /** The quarantined gating rules on this server, for coverage.dormant. */
+    dormant?: () => DormantRule[];
+}
+export declare function registerEvaluateOutputTool(server: McpServer, storage: IStorageAdapter, evalEngine: EvalEngine, options?: EvaluateOutputOptions): void;

package/dist/tools/evaluate-output.js

@@ -1,10 +1,16 @@
 import { z } from 'zod';
+import { toEvaluationResponse } from '../eval/response.js';
+import { evaluateOutputResponseSchema } from '../eval/response-schema.js';
 import { DEFAULT_EVAL_TYPE, DEFAULT_EVAL_TYPE_NOTE } from '../eval/engine.js';
 import { INJECTION_SCOPE_SENTENCE } from '../eval/rules/safety.js';
 import { LOCAL_TENANT } from '../types/tenant.js';
 import { strictInput, strictNested } from './strict-input.js';
 import { toolCallSchema } from './log-trace.js';
 import { getTraceOrThrow, insertLinkedEvalResult } from './trace-link.js';
+import { describeTool, ERROR_ENVELOPE_SENTENCE } from './describe.js';
+import { evaluationLinks, guarded, respond } from './respond.js';
+/** The most inline custom rules one call may carry (see the argument description). */
+export const MAX_INLINE_CUSTOM_RULES = 10;
 /*
  * Strict one level down (#376): `{ name, type, config, wieght: 5 }` used to
  * parse with `wieght` silently discarded, so the rule ran at weight 1 and
@@ -31,13 +37,13 @@ const inputSchema = {
     eval_type: z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom', 'all']).optional().describe('Rule bundle to apply: completeness | relevance | safety | cost | custom | all — picks which built-in rules fire. "all" runs every bundle in one call and adds a per-category breakdown. Defaults to "all" when omitted — every bundle runs, safety included, and the response carries a note saying the default ran'),
     expected: z.string().optional().describe('Expected output for comparison — consulted only by the completeness bundle\'s expected_coverage rule; NOT used by relevance (the relevance rules compare the output against `input`)'),
     input: z.string().optional().describe('Original input for context (the ask + any source material the agent was given) — REQUIRED when eval_type="relevance" (keyword_overlap and topic_consistency compare the output against it and skip without it); also grounds the safety bundle\'s hallucination signals'),
-    trace_id: z.string().optional().describe('Link evaluation to a trace — surfaces this eval in the dashboard\'s trace drill-through. Must be the id of a stored trace (from log_trace / get_traces); an unknown id is rejected before anything is evaluated'),
+    trace_id: z.string().optional().describe('Link evaluation to a trace — surfaces this eval in the dashboard\'s trace drill-through and lets the tool reuse the trace\'s stored tool_calls. Must be the id of a stored trace (from log_trace / get_traces); an unknown id is rejected before anything is evaluated'),
     // .max(10): inline rules skip the deploy-time probe, and the engine runs
     // rules synchronously — without a cap, one request carrying N sandbox-
     // defeating regex rules stalls the server linearly in N (measured 9.3s at
     // N=50). Ten is ample for per-call rules; persistent sets belong in
     // deploy_rule, where deploy-time validation probes each pattern.
-    custom_rules: z.array(CustomRuleSchema).max(10).optional().describe('Custom evaluation rules, max 10 per call (deploy persistent rule sets via deploy_rule instead) — fires REGARDLESS of eval_type; pass eval_type="custom" if you want ONLY these. Each entry accepts exactly name, type, config, weight — an unknown key (e.g. a misspelled weight) is rejected'),
+    custom_rules: z.array(CustomRuleSchema).max(MAX_INLINE_CUSTOM_RULES).optional().describe('Custom evaluation rules, max 10 per call (deploy persistent rule sets via deploy_rule instead) — fires REGARDLESS of eval_type; pass eval_type="custom" if you want ONLY these. Each entry accepts exactly name, type, config, weight — an unknown key (e.g. a misspelled weight) is rejected'),
     cost_usd: z.number().optional().describe('Cost in USD — consulted by the cost bundle (eval_type="cost" or "all") AND by any cost_threshold custom rule regardless of eval_type; omit it and such a rule skips rather than passes (a critical one is listed in critical_skipped)'),
     token_usage: z.object({
         prompt_tokens: z.number().optional(),
@@ -49,38 +55,38 @@ const inputSchema = {
     // is how that field goes missing on one path and not the other.
     tool_calls: z.array(toolCallSchema).optional().describe('What the agent DID — the tool calls it made, in order, each { tool_name, input?, output?, latency_ms?, error? } exactly as log_trace records them. Read by the trajectory rules — the rules that judge what the agent DID rather than what it wrote. Omit it and those rules SKIP rather than pass — an evaluation with no trajectory data reports "not judged", never "clean". When trace_id names a stored trace and this argument is omitted, the tool_calls stored on that trace are loaded and used, so a caller who already logged them need not resend them'),
 };
-export function registerEvaluateOutputTool(server, storage, evalEngine) {
+export function registerEvaluateOutputTool(server, storage, evalEngine, options) {
     server.registerTool('evaluate_output', {
         title: 'Evaluate Output',
-        description: [
-            'Score agent output against configurable eval rules and return a 0..1 score + per-rule breakdown.',
-            '',
-            'Sibling tools — evaluate_with_llm_judge runs semantic LLM-based scoring (slower, costs money; this tool is heuristic, free, deterministic), verify_citations checks citation grounding specifically, log_trace records executions, get_traces queries them, list_rules / deploy_rule / delete_rule manage the custom-rule lifecycle. evaluate_output is the FAST PATH for length / keyword / PII / injection / cost-threshold checks where rules are sufficient.',
-            '',
-            'Behavior. Deterministic, in-process scoring — same inputs always produce the same result. Writes one eval_result row to Iris storage (linked to trace_id if provided; unlinked otherwise). No external network calls — semantic scoring is a separate tool (evaluate_with_llm_judge) that needs an API key you supply. Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio. Runs in-process; no provider is called.',
-            '',
-            'Output shape. Returns JSON: `{ "id": "<uuid>", "eval_type": "<bundle that ran>", "score": 0..1, "passed": boolean, "critical_failures?": string[], "critical_skipped?": string[], "rule_results": [{ "ruleName", "ruleId?", "category?", "critical", "criticalSource", "passed", "score", "message", "skipped?", "skipReason?", "budgetExceeded?", "configInvalid?" }], "suggestions": string[], "rules_evaluated": number, "rules_skipped": number, "insufficient_data": boolean, "categories?": { "<bundle>": { "score": number|null, "passed": boolean|null, "rules_evaluated", "rules_skipped", "insufficient_data", "critical_failures?", "critical_skipped?" } }, "note?": string }`. `ruleId` is present on results produced by a deployed rule (rule-XXXX) so two rules sharing a name stay distinguishable. `categories` appears only for eval_type="all" and carries one entry per bundle that had rules, each with the same threshold + critical-veto semantics as a single-bundle run; `category` on each rule result says which bundle it came from. `insufficient_data=true` means no applicable rules fired (e.g., safety eval with only cost data). Inside `categories`, a bundle that evaluated no rule (every rule skipped for missing context — cost without `cost_usd`, relevance without `input`) reports `passed: null` and `score: null` with `insufficient_data: true`: it was not judged, so it is neither passing nor failing, and it does not count toward the overall verdict. The top-level `passed` stays a boolean and is false when NOTHING was evaluated — a gate keyed on it fails closed; read `insufficient_data` to tell "failed" from "not judged". `note` appears only when eval_type was omitted, saying that the default ran every bundle.',
-            '',
-            'What `passed` means. `score` and `passed` answer different questions. `score` is the weighted average across the rules that ran — a 0..1 quality gradient. `passed` is the ship/no-ship verdict: true only when the score clears the pass threshold (default 0.7, configurable via config `eval.defaultThreshold`) AND no critical rule failed. Critical rules HARD-FAIL: if one fails, `passed` is false regardless of the weighted score, and the culprits are listed in `critical_failures`. The critical rules are the genuine safety violations — `no_pii`, `no_injection_patterns`, `no_blocklist_words` — plus any deployed custom rule with severity high/critical. Which built-in rules are critical is CONFIGURABLE per deployment (`eval.criticalRules` / `eval.nonCriticalRules`), so do not infer it from this list: every rule result carries `critical` (the effective value) and `criticalSource` (`default` when the declaration on the rule itself decided it, `config` when this server promoted or demoted it), and `list_rules` reports the same for the whole built-in roster. A leaked SSN can never be averaged away by other rules passing. For eval_type="all" the veto spans every bundle: one critical failure anywhere forces the overall `passed` to false. One caveat, stated because it is reachable on purpose: a critical rule that SKIPPED did not judge the output and therefore cannot veto — a regex rule whose match blew the 100ms sandbox budget on crafted output skips, so `passed` can be true with no `critical_failures`. Every such rule is named in `critical_skipped`. If your gate must fail closed, treat a non-empty `critical_skipped` as UNKNOWN, not clean.',
-            '',
-            'Use when you want a quality score on a specific output — typically after log_trace records the execution. Pass `eval_type` to route to the right rule bundle: `completeness` (length, non-empty output, sentence count, coverage of `expected`), `relevance` (keyword overlap and topic consistency against `input`), `safety` (PII leak, prompt injection, hallucination markers, stub-output detection — pass `input` so the hallucination signals can cross-check the output against the material the agent was given), `cost` (budget threshold), `custom` (bring your own rules via `custom_rules`), or `all` (every bundle above in one call — completeness, relevance, safety, cost, plus rules deployed under "custom" and any inline custom_rules — with per-category scores in `categories` and one overall verdict; rules whose context is missing, such as relevance without `input` or cost without `cost_usd`, skip and are excluded from the score exactly as in a single-bundle run).',
-            '',
-            'Injection scope. ' + INJECTION_SCOPE_SENTENCE + ' Screening what reaches the agent is a different control, outside this tool.',
-            '',
-            'Don\'t use when the output is empty or has no applicable rules — the eval_type decides which rules apply, and invalid combinations return score=0 + insufficient_data=true (not an error, but not actionable). Don\'t use to VALIDATE JSON schemas directly (use your language\'s JSON Schema validator — Iris\'s `json_schema` custom rule type is for output-shape assertions, not arbitrary validation).',
-            '',
-            'Parameters. input is REQUIRED when eval_type="relevance" (keyword_overlap and topic_consistency compare the output against it; without it both rules skip and the response reports insufficient_data=true) AND grounds the safety bundle\'s hallucination signals (without it those signals stay silent rather than guess); ignored otherwise. expected is consulted only by the completeness bundle\'s expected_coverage rule; ignored for other eval_types — it is NOT the relevance target. cost_usd is consulted by the cost bundle AND by any cost_threshold custom rule regardless of eval_type — omit it and such a rule skips rather than passes (a critical one is listed in critical_skipped); token_usage is ONLY consulted by the cost bundle. tool_calls is what the agent DID (the trajectory) and is read only by the trajectory rules; omit it and those rules SKIP rather than pass, so an evaluation with no trajectory data reports "not judged" instead of "clean", and when trace_id names a stored trace the tool_calls stored on it are used unless this argument overrides them. custom_rules ALWAYS fires regardless of eval_type — pass eval_type="custom" if you want ONLY your rules to run (otherwise both your rules AND the eval_type bundle run together); each entry takes exactly name, type, config and weight. trace_id is optional but recommended (linking the eval to its trace surfaces it in the dashboard\'s drill-through) and must name a stored trace. Defaults: eval_type="all" — every bundle runs, safety included, and when you rely on that default the response carries a `note` saying so; pass a single bundle name to narrow the run.',
-            '',
-            'Error modes. Throws on unknown argument names (strict schema — a misspelled argument is rejected with the valid argument list, never silently dropped), and likewise on an unknown key inside a custom_rules entry (e.g. `wieght`) or inside a tool_calls entry (e.g. `err` for `error`) — the valid keys are listed; a rule\'s `config` keys are free-form and are not checked here. Throws on malformed custom_rules (Zod rejects the shape: missing name/type, unknown type, non-object config, non-positive weight) and on more than 10 custom_rules in one call (use deploy_rule for persistent rule sets). Throws when trace_id does not match a stored trace — checked BEFORE evaluating, so nothing is scored or written; the message names the trace_id. An inline rule whose CONFIG is unusable — a regex that fails the safe-regex2 ReDoS check or exceeds the 1000-char limit, a missing or non-string config.pattern, non-string keywords — does NOT error: that rule reports skipped with configInvalid=true and a skipReason naming the field, and the other rules still run (deploy_rule rejects the same configs with a 400 at deploy time). Returns 429 when HTTP rate limit exceeded. Storage failures propagate as 500. The eval itself never throws — failing rules report `passed: false` with a message, they don\'t bubble exceptions. A regex that exceeds the 100ms sandbox matching budget on a given output reports skipped with budgetExceeded=true instead of hanging the server (fail-open per rule — gate on that flag if you must fail closed).',
-        ].join('\n'),
+        description: describeTool({
+            summary: 'Score an agent output against the deterministic rule bundles: the ship verdict with its basis, every rule result with evidence and uncertainty, and what was not judged.',
+            does: 'In-process, no network, no key. eval_type picks one bundle (completeness, relevance, safety, cost, custom) or all (the default): every bundle plus deployed and inline custom rules, with a per-bundle breakdown in categories. ' +
+                'Inputs decide what can be judged: input is REQUIRED when eval_type="relevance" (keyword_overlap and topic_consistency compare the output against it and skip without it) and grounds the hallucination signals; ' +
+                'tool_calls, or a trace_id whose stored tool_calls are reused, feed the trajectory rules; cost_usd and token_usage feed the cost rules; expected feeds only expected_coverage. ' +
+                'A rule without its input SKIPS, is named, and never counts as a pass. custom_rules always fire. One row is stored, linked to trace_id when given.',
+            whenNot: 'To validate arbitrary JSON Schema (the json_schema custom type asserts an output\'s shape only). ' +
+                `To screen inputs before they reach an agent: ${INJECTION_SCOPE_SENTENCE} ` +
+                'For semantic judgment, evaluate_with_llm_judge and verify_citations need a key you supply.',
+            returns: evaluateOutputResponseSchema,
+            errors: 'IRIS_UNKNOWN_TRACE when trace_id names no stored trace — checked first, nothing scored or written. IRIS_STORAGE_ERROR when the row cannot be written. ' +
+                'Unknown arguments or keys are refused before the handler runs, naming the valid ones; a regex rule over its budget or with a broken config reports skipped, not an error. ' +
+                ERROR_ENVELOPE_SENTENCE,
+            siblings: {
+                log_trace: 'record the execution first',
+                evaluate_with_llm_judge: 'semantic scoring on your key',
+                verify_citations: 'citation grounding on your key',
+                list_rules: 'the roster, needs and published accuracy',
+            },
+        }),
         inputSchema: strictInput(inputSchema),
+        outputSchema: evaluateOutputResponseSchema,
         annotations: {
             readOnlyHint: false, // Writes an eval_result row
             destructiveHint: false, // Creates new data; doesn't overwrite or delete
             idempotentHint: true, // Deterministic: same inputs → same score (each call writes a distinct result row, but the SCORE is stable)
             openWorldHint: false, // No external network in heuristic mode; LLM-as-judge has its own tool with openWorldHint:true
         },
-    }, async (args) => {
+    }, guarded(async (args) => {
         // Refuse an unknown trace_id up front (#376): the old path ran the
         // evaluation and then surfaced SQLite's "FOREIGN KEY constraint
         // failed", which names neither the field nor the fix.
@@ -110,45 +116,18 @@ export function registerEvaluateOutputTool(server, storage, evalEngine) {
         };
         const customRules = args.custom_rules;
         const result = evalType === 'all'
-            ? evalEngine.evaluateAll(context, customRules)
-            : evalEngine.evaluate(evalType, context, customRules);
+            ? await evalEngine.evaluateAll(context, customRules)
+            : await evalEngine.evaluate(evalType, context, customRules);
         if (args.trace_id) {
             result.trace_id = args.trace_id;
         }
         // OSS single-tenant: MCP tool callers are the local user. Cloud
         // will derive tenant from the authenticated MCP session.
         await insertLinkedEvalResult(storage, LOCAL_TENANT, result);
-        return {
-            content: [
-                {
-                    type: 'text',
-                    text: JSON.stringify({
-                        id: result.id,
-                        // Echo which bundle actually ran. Without this, a caller who
-                        // omitted eval_type could not tell a "safety pass" from a
-                        // completeness eval that never ran a single safety rule.
-                        eval_type: result.eval_type,
-                        score: result.score,
-                        passed: result.passed,
-                        ...(result.critical_failures ? { critical_failures: result.critical_failures } : {}),
-                        // The other half of the veto contract. The engine names every
-                        // critical rule that SKIPPED (budget-killed regex, missing cost
-                        // data) so a fail-closed gate can treat the eval as unknown;
-                        // this response used to drop the field, so the gate the
-                        // description tells users to write keyed on something that
-                        // never arrived and read passed:true as clean.
-                        ...(result.critical_skipped ? { critical_skipped: result.critical_skipped } : {}),
-                        rule_results: result.rule_results,
-                        suggestions: result.suggestions,
-                        rules_evaluated: result.rules_evaluated,
-                        rules_skipped: result.rules_skipped,
-                        insufficient_data: result.insufficient_data,
-                        // Per-bundle breakdown — eval_type="all" only.
-                        ...(result.categories ? { categories: result.categories } : {}),
-                        ...(evalTypeOmitted ? { note: DEFAULT_EVAL_TYPE_NOTE } : {}),
-                    }),
-                },
-            ],
-        };
-    });
+        // One serializer for every evaluation surface (src/eval/response.ts):
+        // the tool, the HTTP ingest route, the resources and the drift-lock
+        // all read the same object, so a field added there reaches every
+        // reader at once.
+        return respond(evaluateOutputResponseSchema, toEvaluationResponse(result, { traceId: args.trace_id, dormant: options?.dormant?.(), ...(evalTypeOmitted ? { note: DEFAULT_EVAL_TYPE_NOTE } : {}) }), evaluationLinks(result.id, args.trace_id));
+    }));
 }

package/dist/tools/evaluate-with-llm-judge.d.ts

@@ -1,3 +1,37 @@
+import { z } from 'zod';
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { IStorageAdapter } from '../types/query.js';
+import type { LLMProvider } from '../eval/llm-judge/client.js';
+/** The models the pricing table knows, so an unknown one can be refused with the valid list. */
+export declare function supportedModels(): string[];
+export declare function inferProvider(model: string): LLMProvider;
+/**
+ * The key for the provider, from this process's environment. Missing is
+ * IRIS_JUDGE_NOT_ENABLED with the enable steps as recovery — the fact
+ * users get wrong is that a shell export does not reach the process an
+ * MCP client spawns, and the steps say so.
+ */
+export declare function resolveApiKey(provider: LLMProvider, toolName?: string): string;
+export declare const judgeOutputSchema: z.ZodObject<{
+    id: z.ZodString;
+    trace_id: z.ZodOptional<z.ZodString>;
+    score: z.ZodNumber;
+    passed: z.ZodBoolean;
+    pass_threshold: z.ZodNumber;
+    self_reported_pass: z.ZodOptional<z.ZodBoolean>;
+    disagreement: z.ZodOptional<z.ZodBoolean>;
+    rationale: z.ZodString;
+    dimensions: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    model: z.ZodString;
+    provider: z.ZodEnum<{
+        anthropic: "anthropic";
+        openai: "openai";
+    }>;
+    template: z.ZodString;
+    input_tokens: z.ZodNumber;
+    output_tokens: z.ZodNumber;
+    cost_usd: z.ZodNullable<z.ZodNumber>;
+    latency_ms: z.ZodNumber;
+    raw_response_id: z.ZodOptional<z.ZodString>;
+}, z.core.$loose>;
 export declare function registerEvaluateWithLLMJudgeTool(server: McpServer, storage: IStorageAdapter): void;