@iris-eval/mcp-server 0.8.1 → 0.9.0

package/dist/tools/delete-trace.js

@@ -12,48 +12,45 @@
 import { z } from 'zod';
 import { LOCAL_TENANT } from '../types/tenant.js';
 import { strictInput } from './strict-input.js';
+import { describeTool, ERROR_ENVELOPE_SENTENCE } from './describe.js';
+import { guarded, respond } from './respond.js';
 const inputSchema = {
     trace_id: z
         .string()
         .regex(/^[a-f0-9]{32}$/)
         .describe('Trace id to delete (32-hex lowercase; obtained from log_trace response or get_traces)'),
 };
+export const deleteTraceOutputSchema = z.looseObject({
+    deleted: z.boolean().describe('true when a trace row was removed; false when no trace with that id existed for this tenant'),
+    trace_id: z.string().describe('the id that was asked for'),
+});
 export function registerDeleteTraceTool(server, storage) {
     server.registerTool('delete_trace', {
         title: 'Delete Trace',
-        description: [
-            'Remove a single trace by id. Cascades to spans; eval_results keep the score history with trace_id NULLed.',
-            '',
-            'Sibling tools — log_trace creates traces, get_traces queries them, evaluate_output / evaluate_with_llm_judge / verify_citations score them. delete_rule handles custom-rule deletion (separate concern); list_rules / deploy_rule manage the custom-rule lifecycle. delete_trace is the DESTRUCTIVE single-row remove for traces; it does NOT touch eval_results (preserved for audit + drift analytics), spans cascade automatically.',
-            '',
-            'Behavior. DESTRUCTIVE — SQL DELETE scoped to the caller\'s tenant_id. Cascades: spans belonging to this trace are deleted (FK ON DELETE CASCADE); eval_results that referenced this trace have their trace_id set to NULL (FK ON DELETE SET NULL) so aggregate dashboards + historical scores remain valid even after the trace is gone. Not idempotent: deleting an already-deleted trace returns `deleted: false`. Does not emit an audit log entry in v0.4 — traces are user-scope data, not policy changes. Rate-limited to 20 req/min on HTTP MCP.',
-            '',
-            'Output shape. Returns JSON: `{ "deleted": boolean, "trace_id": string }`. `deleted=true` if a row was removed; `deleted=false` if no trace with that id existed (or it belonged to a different tenant — cross-tenant deletes silently fail).',
-            '',
-            "Use when a trace was captured in error, contains sensitive data that must be removed for compliance (e.g., a customer exercises GDPR right-to-erasure), or when cleaning up test data. Combine with get_traces to find candidates: query with filters → review → delete_trace(id) per target. For bulk time-window deletion, use `deleteTracesOlderThan` via the CLI / retention config — delete_trace is the single-row surgical path.",
-            '',
-            "Don't use to clean up OLD data in bulk (use retention config with --retention-days). Don't use to PAUSE a trace — traces are immutable once stored; there's nothing to pause. Don't use to delete eval_results — eval_results survive their trace's deletion intentionally (for audit + drift analysis); they're pruned only by retention.",
-            '',
-            'Parameters. trace_id is the only parameter; must match 32-char lowercase hex (Zod regex). The trace_id you pass is exactly what log_trace returned in its response, or what get_traces returned per row. Format mismatch fails Zod with 400 BEFORE the storage layer is touched. Cross-tenant trace_ids return `deleted: false` silently — they\'re invisible to the caller\'s tenant (prevents enumeration attacks; matches delete_rule\'s tenant-isolation contract).',
-            '',
-            "Error modes. Throws 400 on malformed trace_id (wrong format: not 32-char lowercase hex). Returns `{deleted: false}` when the id doesn't exist in the caller's tenant (not an error — the trace may simply have been deleted already). Returns 429 on HTTP rate limit. Storage failures propagate as 500.",
-        ].join('\n'),
+        description: describeTool({
+            summary: 'Remove one stored trace by id; its spans go with it, and every evaluation linked to it keeps its verdict and loses its text.',
+            does: "Deletes the trace row for the caller's tenant. Spans cascade. Evaluations linked to it keep their verdict, scores, criticality and evidence offsets; their output text, expected text, suggestions and rule messages are erased in the same transaction and erased_at is stamped, so no text from the trace survives in any evaluation. " +
+                "deleted is false when no trace has that id — already removed, or not this tenant's — and that is not an error. No audit entry is written: traces are user data, not policy.",
+            whenNot: 'To expire old data in bulk (retention.days; the sweep runs at boot and every retention.sweepIntervalHours). To delete evaluations: they are not deleted per row; retention and --purge cover them. To pause anything: traces are immutable, there is nothing to pause.',
+            returns: deleteTraceOutputSchema,
+            errors: 'IRIS_STORAGE_ERROR when the delete cannot run. A malformed trace_id (not 32 lowercase hex) is refused before the handler runs. ' +
+                ERROR_ENVELOPE_SENTENCE,
+            siblings: {
+                log_trace: 'store a trace',
+                get_traces: 'find the trace to delete',
+                delete_rule: 'the equivalent for custom rules',
+            },
+        }),
         inputSchema: strictInput(inputSchema),
+        outputSchema: deleteTraceOutputSchema,
         annotations: {
             readOnlyHint: false,
             destructiveHint: true,
             idempotentHint: false,
             openWorldHint: false,
         },
-    }, async (args) => {
+    }, guarded(async (args) => {
         const deleted = await storage.deleteTrace(LOCAL_TENANT, args.trace_id);
-        return {
-            content: [
-                {
-                    type: 'text',
-                    text: JSON.stringify({ deleted, trace_id: args.trace_id }),
-                },
-            ],
-        };
-    });
+        return respond(deleteTraceOutputSchema, { deleted, trace_id: args.trace_id });
+    }));
 }

package/dist/tools/deploy-rule.d.ts

@@ -1,3 +1,4 @@
+import { z } from 'zod';
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { CustomRuleStore } from '../custom-rule-store.js';
 import type { EvalEngine } from '../eval/engine.js';
@@ -6,7 +7,8 @@ import { type TenantId } from '../types/tenant.js';
 /**
  * A rule with this name is already deployed and the caller did not ask to
  * replace it. Carries the existing rule(s) so an HTTP surface can answer
- * 409 with them beside the same message the MCP tool throws.
+ * 409 with them beside the same message the MCP tool returns as
+ * IRIS_DUPLICATE_RULE (src/tools/errors.ts maps it by name).
  */
 export declare class DuplicateRuleNameError extends Error {
     readonly existing: DeployedCustomRule[];
@@ -34,4 +36,14 @@ export interface ReplacedRule {
 export declare function retireSameNamedRules(store: CustomRuleStore, engine: EvalEngine, tenantId: TenantId, name: string, replace: boolean, user: string): ReplacedRule[];
 /** The `warning` both deploy surfaces attach when a replace retired rules. */
 export declare function replacedRulesWarning(name: string, replaced: ReplacedRule[]): string;
+export declare const deployRuleOutputSchema: z.ZodObject<{
+    rule: z.ZodObject<{
+        id: z.ZodString;
+        name: z.ZodString;
+    }, z.core.$loose>;
+    replaced: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+    }, z.core.$loose>>>;
+    warning: z.ZodOptional<z.ZodString>;
+}, z.core.$loose>;
 export declare function registerDeployRuleTool(server: McpServer, customRuleStore: CustomRuleStore, evalEngine: EvalEngine): void;

package/dist/tools/deploy-rule.js

@@ -14,11 +14,14 @@ import { z } from 'zod';
 import { createCustomRule } from '../eval/rules/custom.js';
 import { LOCAL_TENANT } from '../types/tenant.js';
 import { strictInput, strictNested } from './strict-input.js';
+import { describeTool, ERROR_ENVELOPE_SENTENCE } from './describe.js';
+import { guarded, respond } from './respond.js';
 const EvalTypeSchema = z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom']);
 /**
  * A rule with this name is already deployed and the caller did not ask to
  * replace it. Carries the existing rule(s) so an HTTP surface can answer
- * 409 with them beside the same message the MCP tool throws.
+ * 409 with them beside the same message the MCP tool returns as
+ * IRIS_DUPLICATE_RULE (src/tools/errors.ts maps it by name).
  */
 export class DuplicateRuleNameError extends Error {
     existing;
@@ -145,34 +148,41 @@ const inputSchemaWithAliases = strictInput(inputSchema).superRefine((args, ctx)
         ctx.addIssue({ code: 'custom', path: ['sourceMomentId'], message: 'pass either source_moment_id or sourceMomentId, not both' });
     }
 });
+export const deployRuleOutputSchema = z.looseObject({
+    rule: z.looseObject({ id: z.string(), name: z.string() }).describe('the rule as persisted: id (rule-<hex>, keep it for delete_rule), name, description, evalType, severity, definition, enabled, createdAt, updatedAt, version, sourceMomentId'),
+    replaced: z.array(z.looseObject({ id: z.string() })).optional().describe('with replace: true, the earlier rule(s) of the same name that were retired'),
+    warning: z.string().optional().describe('with replace: true, one sentence naming what was retired'),
+});
 export function registerDeployRuleTool(server, customRuleStore, evalEngine) {
     server.registerTool('deploy_rule', {
         title: 'Deploy Custom Rule',
-        description: [
-            'Deploy a new custom evaluation rule that will fire on every future evaluate_output call of its eval category.',
-            '',
-            'Sibling tools — list_rules enumerates deployed rules, delete_rule removes them (or disables/re-enables them with its `enabled` argument), evaluate_output runs them. log_trace / get_traces / delete_trace handle the trace lifecycle separately; evaluate_with_llm_judge / verify_citations run semantic scoring (not heuristic-rule-driven). deploy_rule is the WRITE path that grows the custom-rule library.',
-            '',
-            'Behavior. Writes a row to ~/.iris/custom-rules.json (atomic write via temp file + rename) and appends a `rule.deploy` entry to the audit log (~/.iris/audit.log). The rule activates immediately for the running process and persists across restarts. Each call mints a fresh rule_id. Rule names are unique among deployed rules: deploying a name that is already deployed is REJECTED unless `replace: true`, in which case the existing same-named rule(s) are deleted (audit `rule.delete` rows written, unregistered from the live engine) and the new rule takes their place under a new id — the response lists what was replaced. Tenant-scoped in Cloud tier; OSS rules are owned by LOCAL_TENANT. Rate-limited to 20 req/min on HTTP MCP.',
-            '',
-            'Output shape. Returns JSON: `{ "rule": { "id": "rule-XXXX", "name", "description", "evalType", "severity", "definition", "enabled": true, "createdAt", "updatedAt", "version": 1, "sourceMomentId?" }, "replaced?": [{ "id", "evalType", "severity" }], "warning?": string }`. The returned rule is the canonical persisted form; save the `id` if you plan to disable or delete later. `replaced` and `warning` appear only when `replace: true` removed an earlier rule of the same name.',
-            '',
-            "Use when an agent observes a recurring failure pattern and decides to enforce it as a standing rule. The `source_moment_id` field preserves provenance — downstream audit can trace the rule back to the moment that inspired it. Combine with evaluate_output + get_traces: 1) evaluate_output surfaces failures; 2) get_traces filters to the failure set; 3) analyze the pattern; 4) deploy_rule bakes it into the default eval path.",
-            '',
-            "Don't use to VALIDATE a rule before committing — deploy writes immediately. Use the dashboard's preview endpoint (POST /api/v1/rules/custom/preview) to replay a definition against recent stored traces first. To UPDATE a rule: call deploy_rule with the same name and `replace: true` (the old rule is deleted, the new one gets a fresh id), or delete_rule then deploy_rule. To pause a rule without losing it: delete_rule with `enabled: false`.",
-            '',
-            'Parameters. Argument names are snake_case (eval_type, source_moment_id) — the camelCase spellings evalType / sourceMomentId are accepted as aliases for compatibility, but pass only one spelling of each. name is 1-80 chars (Zod-enforced min/max — the same cap the persisted store applies); appears in eval_result rule_results (alongside the rule id as `ruleId`) so make it human-readable. description is optional, max 500 chars (used in dashboard tooltips). eval_type determines WHEN the rule fires: a deployed rule runs ONLY on evaluate_output calls whose eval_type equals the rule\'s eval_type, plus eval_type="all" (which runs every bundle) — a "completeness" rule does NOT fire on eval_type="safety" or on eval_type="custom"; eval_type="custom" runs rules deployed under "custom" (and the call\'s inline custom_rules) and nothing else. severity decides what a FAILURE of the rule does: low/medium failures only lower the weighted score (and drive dashboard sort + audit alerts); high/critical failures HARD-FAIL the evaluation — the overall `passed` is forced to false regardless of the weighted score, and the rule is listed in the response\'s `critical_failures`. Severity never changes the numeric score itself (that uses the rule\'s weight). definition.type and definition.config must match (e.g., regex_match needs config.pattern; cost_threshold needs config.max_cost; min_length needs config.min_length; max_length needs config.max_length; contains_keywords/excludes_keywords need config.keywords). definition.name is optional and, if given, overwritten by the top-level name. Invalid configs are REJECTED at deploy time with the offending field named, instead of deploying and then failing every evaluation. source_moment_id is optional but recommended (preserves workflow-inversion provenance from Make-This-A-Rule composer). replace defaults to false. Defaults: severity="medium", replace=false.',
-            '',
-            "Error modes. Throws 400 on invalid definition (Zod rejects — e.g., regex that fails safe-regex2 ReDoS check, or length > 1000 chars), on an unknown key inside `definition` (the valid keys are listed; config keys are free-form), on empty `name` or `name` over 80 chars, on a non-positive weight, and when both spellings of an aliased argument are passed. Throws when a rule with the same name is already deployed and `replace` is not true — the message names the existing rule id so you can delete it, replace it, or pick another name. Any eval_type/definition.type combination is valid (a regex_match rule can enforce a safety policy; a max_length rule can express completeness) — there is no category/type mismatch error. Returns 429 when HTTP rate limit exceeded. File-write failures (disk full, read-only fs) propagate as 500; the audit log is best-effort and does not block deploy.",
-        ].join('\n'),
+        description: describeTool({
+            summary: 'Deploy a custom rule that fires on every future evaluate_output call of its bundle — persisted, active immediately, audited.',
+            does: 'Writes the rule to ~/.iris/custom-rules.json, appends a rule.deploy audit entry and registers it with the running engine, so it fires on the very next call and survives restarts. ' +
+                'eval_type says WHEN it fires (that bundle, and eval_type="all"); severity says what a failure DOES: low and medium only lower the weighted score, high and critical force passed to false and list the rule in critical_failures. ' +
+                'definition.type picks the check (regex_match, regex_no_match, min_length, max_length, contains_keywords, excludes_keywords, json_schema, cost_threshold) and definition.config carries its keys (pattern; min_length; max_length; keywords; max_cost). ' +
+                'Any bundle and type combine. Names are unique: a taken name is refused unless replace is true, which retires the earlier rule(s) first and reports them. Argument names are snake_case; the camelCase aliases evalType and sourceMomentId are accepted — pass one spelling of each.',
+            whenNot: 'To try a rule first: POST /api/v1/rules/custom/preview on the dashboard replays a definition against stored traces without deploying. For a one-off check on one call: the custom_rules argument of evaluate_output. To pause a rule: delete_rule with enabled: false.',
+            returns: deployRuleOutputSchema,
+            errors: 'IRIS_DUPLICATE_RULE when the name is deployed and replace is false (the message names the existing id). ' +
+                'IRIS_INVALID_RULE_CONFIG when the definition is rejected — a regex that fails the ReDoS check or exceeds 1000 characters, a missing config key — naming the field; nothing is deployed. ' +
+                'IRIS_STORAGE_ERROR when the store cannot be written. An unknown key in definition, a name over 80 characters, a non-positive weight or both spellings of an alias are refused before the handler runs. ' +
+                ERROR_ENVELOPE_SENTENCE,
+            siblings: {
+                list_rules: 'see what is deployed and the built-in roster',
+                delete_rule: 'remove, disable or re-enable',
+                evaluate_output: 'where the rule fires',
+            },
+        }),
         inputSchema: inputSchemaWithAliases,
+        outputSchema: deployRuleOutputSchema,
         annotations: {
             readOnlyHint: false,
             destructiveHint: false,
             idempotentHint: false,
             openWorldHint: false,
         },
-    }, async (args) => {
+    }, guarded(async (args) => {
         const evalType = (args.eval_type ?? args.evalType);
         const sourceMomentId = args.source_moment_id ?? args.sourceMomentId;
         // Server overrides the inner definition's `name` so it always matches
@@ -186,10 +196,12 @@ export function registerDeployRuleTool(server, customRuleStore, evalEngine) {
         };
         // Same-name redeploy (#373) — shared with the dashboard's deploy
         // route; see retireSameNamedRules above. Throws (nothing deployed)
-        // when the name is taken and replace is false.
+        // when the name is taken and replace is false; `guarded` turns that
+        // into IRIS_DUPLICATE_RULE.
         const replaced = retireSameNamedRules(customRuleStore, evalEngine, LOCAL_TENANT, args.name, args.replace, 'mcp');
         // OSS: MCP tools operate under LOCAL_TENANT. See list-rules.ts for context.
         const rule = customRuleStore.deploy(LOCAL_TENANT, {
+            replaces: replaced.map((r) => r.id),
             name: args.name,
             description: args.description,
             evalType,
@@ -205,18 +217,9 @@ export function registerDeployRuleTool(server, customRuleStore, evalEngine) {
         // Registered under its rule id so delete_rule can hot-remove it.
         // Severity rides along: high/critical makes the rule hard-failing.
         evalEngine.registerRule(rule.evalType, createCustomRule(rule.definition, rule.severity), rule.id);
-        return {
-            content: [
-                {
-                    type: 'text',
-                    text: JSON.stringify({
-                        rule,
-                        ...(replaced.length > 0
-                            ? { replaced, warning: replacedRulesWarning(args.name, replaced) }
-                            : {}),
-                    }),
-                },
-            ],
-        };
-    });
+        return respond(deployRuleOutputSchema, {
+            rule,
+            ...(replaced.length > 0 ? { replaced, warning: replacedRulesWarning(args.name, replaced) } : {}),
+        });
+    }));
 }

package/dist/tools/describe.d.ts

@@ -0,0 +1,20 @@
+import type { z } from 'zod';
+export declare const DESCRIPTION_HEADINGS: readonly ["What it does.", "When not to use it.", "Returns.", "Errors.", "Siblings."];
+export declare const DESCRIPTION_WORD_CAP = 450;
+export interface ToolDescriptionSpec {
+    /** One sentence: what calling this does. */
+    summary: string;
+    does: string;
+    whenNot: string;
+    /** The tool's output schema; every top-level field must carry a description. */
+    returns: z.ZodObject<z.ZodRawShape>;
+    errors: string;
+    /** Sibling tool → one clause on when it is the better call. */
+    siblings: Record<string, string>;
+}
+/** `field` (its description); … — from the schema, never retyped. */
+export declare function returnsFrom(schema: z.ZodObject<z.ZodRawShape>): string;
+export declare function wordCount(text: string): number;
+export declare function describeTool(spec: ToolDescriptionSpec): string;
+/** The sentence every "Errors" heading ends with, so the envelope is stated once. */
+export declare const ERROR_ENVELOPE_SENTENCE = "Every failure returns {\"error\":{\"code\",\"message\",\"recovery\":[]}} with isError true; follow recovery before retrying.";

package/dist/tools/describe.js

@@ -0,0 +1,36 @@
+export const DESCRIPTION_HEADINGS = ['What it does.', 'When not to use it.', 'Returns.', 'Errors.', 'Siblings.'];
+export const DESCRIPTION_WORD_CAP = 450;
+/** `field` (its description); … — from the schema, never retyped. */
+export function returnsFrom(schema) {
+    const parts = [];
+    for (const [key, field] of Object.entries(schema.shape)) {
+        const description = field.description;
+        if (!description)
+            throw new Error(`output field "${key}" has no description; add .describe() on the schema`);
+        parts.push(`\`${key}\` (${description})`);
+    }
+    return `JSON with ${parts.join('; ')}.`;
+}
+export function wordCount(text) {
+    return text.split(/\s+/).filter(Boolean).length;
+}
+export function describeTool(spec) {
+    const siblings = Object.entries(spec.siblings)
+        .map(([name, when]) => `${name} — ${when}`)
+        .join('; ');
+    const text = [
+        spec.summary,
+        `${DESCRIPTION_HEADINGS[0]} ${spec.does}`,
+        `${DESCRIPTION_HEADINGS[1]} ${spec.whenNot}`,
+        `${DESCRIPTION_HEADINGS[2]} ${returnsFrom(spec.returns)}`,
+        `${DESCRIPTION_HEADINGS[3]} ${spec.errors}`,
+        `${DESCRIPTION_HEADINGS[4]} ${siblings}.`,
+    ].join('\n\n');
+    const words = wordCount(text);
+    if (words > DESCRIPTION_WORD_CAP) {
+        throw new Error(`tool description is ${words} words; the cap is ${DESCRIPTION_WORD_CAP}`);
+    }
+    return text;
+}
+/** The sentence every "Errors" heading ends with, so the envelope is stated once. */
+export const ERROR_ENVELOPE_SENTENCE = 'Every failure returns {"error":{"code","message","recovery":[]}} with isError true; follow recovery before retrying.';

package/dist/tools/errors.d.ts

@@ -0,0 +1,36 @@
+import { LLMJudgeError } from '../eval/llm-judge/client.js';
+export declare const ERROR_CODE_CATALOGUE: readonly ["IRIS_INVALID_ARGUMENT", "IRIS_UNKNOWN_TRACE", "IRIS_DUPLICATE_RULE", "IRIS_INVALID_RULE_CONFIG", "IRIS_JUDGE_NOT_ENABLED", "IRIS_JUDGE_UNKNOWN_MODEL", "IRIS_BUDGET_EXCEEDED", "IRIS_PROVIDER_ERROR", "IRIS_JUDGE_FAILED", "IRIS_STORAGE_ERROR", "IRIS_INTERNAL_ERROR"];
+export type IrisErrorCode = (typeof ERROR_CODE_CATALOGUE)[number];
+export type ProviderErrorKind = LLMJudgeError['kind'];
+export interface IrisErrorEnvelope {
+    code: IrisErrorCode;
+    /** One sentence a person can act on. */
+    message: string;
+    /** What to do, in order, before retrying. */
+    recovery: string[];
+    /** True when the same call can succeed later without a change from the caller. */
+    retryable: boolean;
+    /** The argument or environment variable at fault, when the failure is about one. */
+    field?: string;
+    /** The values the field accepts, when there is a closed list. */
+    valid?: string[];
+    /** A resource that explains the limit or the state. */
+    see?: string;
+    /** IRIS_PROVIDER_ERROR only: the provider failure class. */
+    kind?: ProviderErrorKind;
+    /** When the provider said how long to wait. */
+    retryAfterMs?: number;
+}
+export declare class IrisError extends Error {
+    readonly envelope: IrisErrorEnvelope;
+    constructor(envelope: Omit<IrisErrorEnvelope, 'recovery' | 'retryable'> & Partial<Pick<IrisErrorEnvelope, 'recovery' | 'retryable'>>);
+}
+export declare function irisError(code: IrisErrorCode, message: string, extra?: Partial<Omit<IrisErrorEnvelope, 'code' | 'message'>>): IrisError;
+/**
+ * Every error a handler can throw, mapped to its code. Typed errors are
+ * matched by class or by name (the name check avoids importing the tool
+ * that defines the class, which would be a cycle); anything unrecognised
+ * is an internal error — reported as such, never dressed up as a caller
+ * mistake.
+ */
+export declare function toIrisError(err: unknown): IrisError;

package/dist/tools/errors.js

@@ -0,0 +1,134 @@
+/*
+ * Structured tool errors.
+ *
+ * A thrown Error reaches an MCP client as one line of text with isError
+ * set (the SDK flattens it), so the only thing an agent could do with
+ * "Anthropic judge requires IRIS_ANTHROPIC_API_KEY" was guess. Every
+ * failure a tool can produce is now a code from ONE catalogue with a
+ * message, the steps that clear it, whether a retry can help, and — when
+ * the failure is about one argument — the field and the valid values.
+ * Tools return these (src/tools/respond.ts wraps the handler); they never
+ * throw past the handler.
+ *
+ * The catalogue is exhaustive on purpose: tests/unit/tools/error-codes.test.ts
+ * provokes every code over a real transport and asserts the provoked set
+ * EQUALS this list, so a code nothing can raise cannot linger here and a
+ * new throw site cannot ship without a code.
+ *
+ * Two errors are not this module's: an argument the input schema rejects
+ * is refused by the SDK before the handler runs (the text names
+ * IRIS_INVALID_ARGUMENT and the valid arguments — see strict-input.ts), and
+ * an HTTP rate limit answers 429 at the transport before any tool runs.
+ */
+import { ZodError } from 'zod';
+import { LLMJudgeError } from '../eval/llm-judge/client.js';
+import { CAPABILITIES_RESOURCE_URI } from '../resources/uris.js';
+export const ERROR_CODE_CATALOGUE = [
+    'IRIS_INVALID_ARGUMENT',
+    'IRIS_UNKNOWN_TRACE',
+    'IRIS_DUPLICATE_RULE',
+    'IRIS_INVALID_RULE_CONFIG',
+    'IRIS_JUDGE_NOT_ENABLED',
+    'IRIS_JUDGE_UNKNOWN_MODEL',
+    'IRIS_BUDGET_EXCEEDED',
+    'IRIS_PROVIDER_ERROR',
+    'IRIS_JUDGE_FAILED',
+    'IRIS_STORAGE_ERROR',
+    'IRIS_INTERNAL_ERROR',
+];
+export class IrisError extends Error {
+    envelope;
+    constructor(envelope) {
+        super(envelope.message);
+        this.name = 'IrisError';
+        this.envelope = { retryable: false, recovery: [], ...envelope };
+    }
+}
+export function irisError(code, message, extra = {}) {
+    return new IrisError({ code, message, ...extra });
+}
+const CAPABILITIES = CAPABILITIES_RESOURCE_URI;
+function isSqliteError(err) {
+    const code = err.code;
+    if (typeof code === 'string' && code.startsWith('SQLITE_'))
+        return true;
+    const message = err instanceof Error ? err.message : '';
+    return /SQLITE|database is locked|disk I\/O error|readonly database/i.test(message);
+}
+/**
+ * Every error a handler can throw, mapped to its code. Typed errors are
+ * matched by class or by name (the name check avoids importing the tool
+ * that defines the class, which would be a cycle); anything unrecognised
+ * is an internal error — reported as such, never dressed up as a caller
+ * mistake.
+ */
+export function toIrisError(err) {
+    if (err instanceof IrisError)
+        return err;
+    const name = err?.name;
+    const message = err instanceof Error ? err.message : String(err);
+    if (name === 'DuplicateRuleNameError') {
+        return irisError('IRIS_DUPLICATE_RULE', message, {
+            field: 'name',
+            recovery: [
+                'Pass replace: true to retire the rule(s) with this name and deploy this one in their place.',
+                'Or call delete_rule with the existing rule id first.',
+                'Or choose a different name.',
+            ],
+        });
+    }
+    if (name === 'CostCapError') {
+        const e = err;
+        return irisError('IRIS_BUDGET_EXCEEDED', message, {
+            field: 'max_cost_usd',
+            recovery: [
+                `Raise max_cost_usd on the call (the worst case was ${e.estimatedUsd?.toFixed(4) ?? '?'} USD against a cap of ${e.capUsd?.toFixed(4) ?? '?'} USD).`,
+                'Or trim the output, the input or max_output_tokens so the worst case fits.',
+                'Nothing was spent.',
+            ],
+            see: CAPABILITIES,
+        });
+    }
+    if (err instanceof LLMJudgeError) {
+        const retryable = err.kind === 'rate_limit' || err.kind === 'timeout' || err.kind === 'server_error';
+        const recovery = {
+            auth: ['The provider refused the key. Check that the key in the env block of your MCP config is valid and live, then restart the session.'],
+            rate_limit: ['The provider rate-limited the call. Wait and retry; Iris already retried once.'],
+            bad_request: ['The provider rejected the request. Check the model name and the template inputs.'],
+            server_error: ['The provider failed on its side. Retry later.'],
+            timeout: ['The provider did not answer within timeout_ms. Retry, or raise timeout_ms.'],
+            malformed_response: ['The judge did not return valid JSON on two attempts. Retry; if it recurs, pick another model.'],
+            unknown: ['Retry once; if it recurs, report the message with the provider and model.'],
+        };
+        return irisError('IRIS_PROVIDER_ERROR', message, {
+            kind: err.kind,
+            retryable,
+            recovery: recovery[err.kind],
+            ...(err.retryAfterSeconds !== undefined ? { retryAfterMs: err.retryAfterSeconds * 1000 } : {}),
+        });
+    }
+    if (err instanceof ZodError) {
+        // The SDK validates tool input before the handler runs, so a ZodError
+        // inside a handler is the rule store rejecting a definition.
+        const first = err.issues[0];
+        const field = first?.path?.length ? first.path.map(String).join('.') : undefined;
+        return irisError('IRIS_INVALID_RULE_CONFIG', `The rule definition was rejected: ${err.issues.map((i) => `${i.path.map(String).join('.') || 'definition'}: ${i.message}`).join('; ')}`, {
+            ...(field ? { field } : {}),
+            recovery: ['Fix the named field and deploy again. Nothing was deployed.'],
+        });
+    }
+    if (isSqliteError(err)) {
+        const code = err.code ?? '';
+        return irisError('IRIS_STORAGE_ERROR', `Iris storage failed: ${message}`, {
+            retryable: code === 'SQLITE_BUSY' || code === 'SQLITE_LOCKED',
+            recovery: [
+                'Retry once. If it recurs, check that the database path is writable and the disk is not full.',
+                'Run `npx @iris-eval/mcp-server --self-test` to probe the configured home and database.',
+            ],
+        });
+    }
+    return irisError('IRIS_INTERNAL_ERROR', message || 'Iris hit an unexpected error.', {
+        recovery: ['Retry once. If it recurs, report the message with the Iris version from iris://capabilities.'],
+        see: CAPABILITIES,
+    });
+}

package/dist/tools/evaluate-output.d.ts

@@ -1,4 +1,11 @@
+import type { DormantRule } from '../eval/dormant.js';
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { IStorageAdapter } from '../types/query.js';
 import type { EvalEngine } from '../eval/engine.js';
-export declare function registerEvaluateOutputTool(server: McpServer, storage: IStorageAdapter, evalEngine: EvalEngine): void;
+/** The most inline custom rules one call may carry (see the argument description). */
+export declare const MAX_INLINE_CUSTOM_RULES = 10;
+export interface EvaluateOutputOptions {
+    /** The quarantined gating rules on this server, for coverage.dormant. */
+    dormant?: () => DormantRule[];
+}
+export declare function registerEvaluateOutputTool(server: McpServer, storage: IStorageAdapter, evalEngine: EvalEngine, options?: EvaluateOutputOptions): void;