@iris-eval/mcp-server 0.3.0 → 0.4.0-rc.1

package/dist/tools/evaluate-output.js

@@ -1,4 +1,5 @@
 import { z } from 'zod';
+import { LOCAL_TENANT } from '../types/tenant.js';
 const CustomRuleSchema = z.object({
     name: z.string(),
     type: z.enum([
@@ -25,8 +26,26 @@ const inputSchema = {
 export function registerEvaluateOutputTool(server, storage, evalEngine) {
     server.registerTool('evaluate_output', {
         title: 'Evaluate Output',
-        description: 'Evaluate agent output quality using configurable rules',
+        description: [
+            'Score agent output against configurable eval rules and return a 0..1 score + per-rule breakdown.',
+            '',
+            'Behavior. Deterministic, in-process scoring — same inputs always produce the same result. Writes one eval_result row to Iris storage (linked to trace_id if provided; unlinked otherwise). No external network calls in heuristic mode (v0.4 adds an llm_as_judge eval_type that DOES call LLM APIs; see the separate evaluate_with_llm_judge tool for that). Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio. Runs in ~5-50ms for rule-based evaluation.',
+            '',
+            'Output shape. Returns JSON: `{ "id": "<uuid>", "score": 0..1, "passed": boolean, "rule_results": [{ "ruleName", "passed", "score", "message", "skipped?" }], "suggestions": string[], "rules_evaluated": number, "rules_skipped": number, "insufficient_data": boolean }`. `insufficient_data=true` means no applicable rules fired (e.g., safety eval with only cost data).',
+            '',
+            'Use when you want a quality score on a specific output — typically after log_trace records the execution. Pass `eval_type` to route to the right rule bundle: `completeness` (length, sentence count, relevance to input), `relevance` (keyword overlap, topic consistency), `safety` (PII leak, prompt injection, hallucination markers, stub-output detection), `cost` (budget threshold), or `custom` (bring your own rules via `custom_rules`).',
+            '',
+            'Don\'t use when the output is empty or has no applicable rules — the eval_type decides which rules apply, and invalid combinations return score=0 + insufficient_data=true (not an error, but not actionable). Don\'t use to VALIDATE JSON schemas directly (use your language\'s JSON Schema validator — Iris\'s `json_schema` custom rule type is for output-shape assertions, not arbitrary validation).',
+            '',
+            'Error modes. Throws on malformed custom_rules (Zod rejects). Returns 400 on regex patterns that fail safe-regex2 ReDoS check or exceed 1000-char limit. Returns 429 when HTTP rate limit exceeded. Storage failures propagate as 500. The eval itself never throws — failing rules report `passed: false` with a message, they don\'t bubble exceptions.',
+        ].join('\n'),
         inputSchema,
+        annotations: {
+            readOnlyHint: false, // Writes an eval_result row
+            destructiveHint: false, // Creates new data; doesn't overwrite or delete
+            idempotentHint: true, // Deterministic: same inputs → same score (each call writes a distinct result row, but the SCORE is stable)
+            openWorldHint: false, // No external network in heuristic mode; LLM-as-judge has its own tool with openWorldHint:true
+        },
     }, async (args) => {
         const evalType = args.eval_type;
         const result = evalEngine.evaluate(evalType, {
@@ -39,7 +58,9 @@ export function registerEvaluateOutputTool(server, storage, evalEngine) {
         if (args.trace_id) {
             result.trace_id = args.trace_id;
         }
-        await storage.insertEvalResult(result);
+        // OSS single-tenant: MCP tool callers are the local user. Cloud
+        // will derive tenant from the authenticated MCP session.
+        await storage.insertEvalResult(LOCAL_TENANT, result);
         return {
             content: [
                 {

package/dist/tools/evaluate-with-llm-judge.d.ts

@@ -0,0 +1,3 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { IStorageAdapter } from '../types/query.js';
+export declare function registerEvaluateWithLLMJudgeTool(server: McpServer, storage: IStorageAdapter): void;

package/dist/tools/evaluate-with-llm-judge.js

@@ -0,0 +1,147 @@
+import { z } from 'zod';
+import { LOCAL_TENANT } from '../types/tenant.js';
+import { evaluateWithLLMJudge } from '../eval/llm-judge/evaluator.js';
+import { findPricing } from '../eval/llm-judge/pricing.js';
+import { generateEvalId } from '../utils/ids.js';
+const inputSchema = {
+    output: z.string().min(1).describe('The agent output text to evaluate'),
+    template: z
+        .enum(['accuracy', 'helpfulness', 'safety', 'correctness', 'faithfulness'])
+        .describe('Judge dimension: accuracy (factual correctness), helpfulness (does it address the ask), safety (harm potential), correctness (vs reference answer — requires `expected`), faithfulness (RAG grounding — requires `source_material`).'),
+    model: z
+        .string()
+        .describe('Model ID. Supported: anthropic = claude-opus-4-7 | claude-sonnet-4-6 | claude-haiku-4-5 | claude-haiku-4-5-20251001; openai = gpt-4o | gpt-4o-mini | o1-mini.'),
+    provider: z.enum(['anthropic', 'openai']).optional().describe('Auto-detected from model when omitted'),
+    input: z.string().optional().describe('User question / prompt that produced the output (improves accuracy for helpfulness/safety)'),
+    expected: z.string().optional().describe('Reference answer (required for correctness template)'),
+    source_material: z.string().optional().describe('Provided RAG sources (required for faithfulness template)'),
+    trace_id: z.string().optional().describe('Link this evaluation to a trace'),
+    max_cost_usd: z.number().positive().optional().describe('Cost cap in USD; defaults to IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL or 0.25'),
+    max_output_tokens: z.number().int().positive().max(4096).optional().describe('Judge output token cap; default 512'),
+    temperature: z.number().min(0).max(2).optional().describe('Sampling temperature; default 0 (deterministic)'),
+    timeout_ms: z.number().int().positive().optional().describe('Per-request timeout; default 60_000'),
+};
+function inferProvider(model) {
+    const pricing = findPricing(model);
+    if (!pricing) {
+        throw new Error(`Unknown model "${model}". Provider cannot be inferred. Supported models are listed in src/eval/llm-judge/pricing.ts.`);
+    }
+    return pricing.provider;
+}
+function resolveApiKey(provider) {
+    if (provider === 'anthropic') {
+        const key = process.env.IRIS_ANTHROPIC_API_KEY;
+        if (!key) {
+            throw new Error('Anthropic judge requires IRIS_ANTHROPIC_API_KEY. Set it in the environment or use a different provider.');
+        }
+        return key;
+    }
+    const key = process.env.IRIS_OPENAI_API_KEY;
+    if (!key) {
+        throw new Error('OpenAI judge requires IRIS_OPENAI_API_KEY. Set it in the environment or use a different provider.');
+    }
+    return key;
+}
+function resolveMaxCost(paramValue) {
+    if (paramValue !== undefined)
+        return paramValue;
+    const envRaw = process.env.IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL;
+    if (envRaw) {
+        const parsed = Number(envRaw);
+        if (Number.isFinite(parsed) && parsed > 0)
+            return parsed;
+    }
+    return 0.25;
+}
+export function registerEvaluateWithLLMJudgeTool(server, storage) {
+    server.registerTool('evaluate_with_llm_judge', {
+        title: 'Evaluate With LLM Judge',
+        description: [
+            'Score agent output using an LLM as the judge (Anthropic or OpenAI). Returns a calibrated 0..1 score with rationale, per-dimension breakdown, and exact cost.',
+            '',
+            'Behavior. Calls an external LLM API (Anthropic or OpenAI) — costs money per call, takes 1-10 seconds, respects an IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL cap. Non-deterministic at temperature > 0; default temperature=0 gives near-deterministic scores. Writes one eval_result row to Iris storage (linked to trace_id if provided) plus captures provider response id + latency + token counts + cost in the rule_results payload. Rate-limited to 20 req/min on HTTP MCP; your LLM provider also enforces its own rate limits (we transparently retry once on 429).',
+            '',
+            'Output shape. Returns JSON: `{ "id": "<uuid>", "score": 0..1, "passed": boolean, "rationale": string, "dimensions": {...}, "model": string, "provider": "anthropic"|"openai", "template": string, "input_tokens": number, "output_tokens": number, "cost_usd": number, "latency_ms": number }`. `dimensions` has per-dimension sub-scores (e.g., accuracy template returns `{factual_claims, citations, internal_consistency}`).',
+            '',
+            'Use when heuristic rules (via evaluate_output) are too coarse for the quality signal you need — semantic correctness, factual accuracy vs a reference, RAG faithfulness to sources, nuanced safety/helpfulness. Pick the template that matches: `accuracy` (hallucination detection), `helpfulness` (does it address the ask), `safety` (harm potential beyond regex PII), `correctness` (vs reference answer — pass `expected`), `faithfulness` (RAG grounding — pass `source_material`).',
+            '',
+            "Don't use for simple regex/length/keyword checks (use evaluate_output with heuristic rules — they're free, deterministic, 1000x faster). Don't use without an API key set (IRIS_ANTHROPIC_API_KEY or IRIS_OPENAI_API_KEY). Don't use on very large outputs (>8K tokens) without raising max_cost_usd — the pre-check will refuse the call.",
+            '',
+            'Error modes. Throws when the required API key env var is missing. Throws when the estimated worst-case cost exceeds max_cost_usd (raise the cap or trim prompts). Throws LLMJudgeError on provider errors — kind=`auth` on 401/403, `rate_limit` on 429 (auto-retried once), `server_error` on 5xx, `timeout` on abort, `malformed_response` when the judge fails to emit valid JSON on both attempts. Throws "Unknown model" for unsupported model IDs — update src/eval/llm-judge/pricing.ts first.',
+        ].join('\n'),
+        inputSchema,
+        annotations: {
+            readOnlyHint: false, // Writes eval_result; also spends money (external API cost)
+            destructiveHint: false, // Creates data; doesn't overwrite or delete
+            idempotentHint: false, // Temperature > 0 may vary; even at T=0 provider non-determinism is possible; cost also varies per call
+            openWorldHint: true, // Calls external APIs (Anthropic / OpenAI) — touches the world beyond local process
+        },
+    }, async (args) => {
+        const provider = args.provider ?? inferProvider(args.model);
+        const apiKey = resolveApiKey(provider);
+        const maxCostUsd = resolveMaxCost(args.max_cost_usd);
+        const result = await evaluateWithLLMJudge({
+            output: args.output,
+            template: args.template,
+            provider,
+            model: args.model,
+            apiKey,
+            input: args.input,
+            expected: args.expected,
+            sourceMaterial: args.source_material,
+            maxCostUsdPerEval: maxCostUsd,
+            maxOutputTokens: args.max_output_tokens,
+            temperature: args.temperature,
+            timeoutMs: args.timeout_ms,
+        });
+        const evalId = generateEvalId();
+        // Persist as a normal eval_result so the dashboard picks it up
+        // alongside heuristic scores. eval_type is 'custom' because LLM
+        // judge doesn't fit completeness/relevance/safety/cost taxonomy
+        // cleanly — it spans all four. The rule_results payload carries
+        // the full judge provenance.
+        await storage.insertEvalResult(LOCAL_TENANT, {
+            id: evalId,
+            trace_id: args.trace_id,
+            eval_type: 'custom',
+            output_text: args.output,
+            expected_text: args.expected,
+            score: result.score,
+            passed: result.passed,
+            rule_results: [
+                {
+                    ruleName: `llm_judge:${result.template}:${result.provider}/${result.model}`,
+                    passed: result.passed,
+                    score: result.score,
+                    message: result.rationale || 'LLM judge evaluation',
+                },
+            ],
+            suggestions: result.passed ? [] : [result.rationale],
+            rules_evaluated: 1,
+            rules_skipped: 0,
+            insufficient_data: false,
+        });
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: JSON.stringify({
+                        id: evalId,
+                        score: result.score,
+                        passed: result.passed,
+                        rationale: result.rationale,
+                        dimensions: result.dimensions,
+                        model: result.model,
+                        provider: result.provider,
+                        template: result.template,
+                        input_tokens: result.inputTokens,
+                        output_tokens: result.outputTokens,
+                        cost_usd: result.costUsd,
+                        latency_ms: result.latencyMs,
+                        raw_response_id: result.rawResponseId,
+                    }),
+                },
+            ],
+        };
+    });
+}

package/dist/tools/get-traces.js

@@ -1,4 +1,5 @@
 import { z } from 'zod';
+import { LOCAL_TENANT } from '../types/tenant.js';
 const inputSchema = {
     agent_name: z.string().optional().describe('Filter by agent name'),
     framework: z.string().optional().describe('Filter by framework'),
@@ -15,10 +16,29 @@ const inputSchema = {
 export function registerGetTracesTool(server, storage) {
     server.registerTool('get_traces', {
         title: 'Get Traces',
-        description: 'Query stored traces with filters, pagination, and optional summary stats',
+        description: [
+            'Query stored agent-execution traces with filters, pagination, and optional dashboard summary.',
+            '',
+            'Behavior. Read-only: never mutates storage, never calls external services. Idempotent: repeated calls with the same args return consistent results (new traces logged after the call obviously show up on subsequent calls). Tenant-scoped: queries only the caller\'s tenant rows (LOCAL_TENANT in OSS). Paginates results (default limit 50, max 1000). Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio.',
+            '',
+            'Output shape. Returns JSON: `{ "traces": [{...traceRow}], "total": number, "limit": number, "offset": number, "summary"?: { total_traces, avg_latency_ms, total_cost_usd, error_rate, eval_pass_rate, traces_per_hour, top_agents } }`. Each trace row includes trace_id, agent_name, framework, input, output, tool_calls, latency_ms, token_usage, cost_usd, metadata, timestamp. `summary` only included when `include_summary: true`.',
+            '',
+            'Use when you need historical data: investigating a past failure, computing quality trends, comparing agents, or feeding an analytics job. Set `agent_name` / `framework` / `since` / `until` to narrow the query. Set `min_score` / `max_score` to surface outliers. Set `sort_by: "cost_usd"` + `sort_order: "desc"` to find the most expensive traces. Set `include_summary: true` when you want dashboard-style aggregates in one round-trip.',
+            '',
+            'Don\'t use to score a trace (use evaluate_output). Don\'t use to create a trace (use log_trace). Don\'t use as a live event stream — it\'s a query, not a subscription; poll with exponential backoff or use the dashboard\'s SSE endpoint for real-time.',
+            '',
+            'Error modes. Returns 400 on invalid sort_by / sort_order (Zod enum). Returns 400 if limit > 1000. Returns 429 when HTTP rate limit exceeded. Storage failures propagate as 500. Empty result with `total: 0` on no matches (not an error).',
+        ].join('\n'),
         inputSchema,
+        annotations: {
+            readOnlyHint: true, // Pure query: never writes, never deletes
+            destructiveHint: false, // Inverse of readOnly — trivially false
+            idempotentHint: true, // Same args → same result (modulo new traces that may have landed since)
+            openWorldHint: false, // Queries local storage only; no external network
+        },
     }, async (args) => {
-        const result = await storage.queryTraces({
+        // OSS single-tenant: MCP caller is the local user.
+        const result = await storage.queryTraces(LOCAL_TENANT, {
             filter: {
                 agent_name: args.agent_name,
                 framework: args.framework,
@@ -39,7 +59,7 @@ export function registerGetTracesTool(server, storage) {
             offset: result.offset,
         };
         if (args.include_summary) {
-            response.summary = await storage.getDashboardSummary();
+            response.summary = await storage.getDashboardSummary(LOCAL_TENANT);
         }
         return {
             content: [

package/dist/tools/index.d.ts

@@ -1,4 +1,5 @@
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { IStorageAdapter } from '../types/query.js';
 import type { EvalEngine } from '../eval/engine.js';
-export declare function registerAllTools(server: McpServer, storage: IStorageAdapter, evalEngine: EvalEngine): void;
+import type { CustomRuleStore } from '../custom-rule-store.js';
+export declare function registerAllTools(server: McpServer, storage: IStorageAdapter, evalEngine: EvalEngine, customRuleStore: CustomRuleStore): void;

package/dist/tools/index.js

@@ -1,8 +1,20 @@
 import { registerLogTraceTool } from './log-trace.js';
 import { registerEvaluateOutputTool } from './evaluate-output.js';
 import { registerGetTracesTool } from './get-traces.js';
-export function registerAllTools(server, storage, evalEngine) {
+import { registerListRulesTool } from './list-rules.js';
+import { registerDeployRuleTool } from './deploy-rule.js';
+import { registerDeleteRuleTool } from './delete-rule.js';
+import { registerDeleteTraceTool } from './delete-trace.js';
+import { registerEvaluateWithLLMJudgeTool } from './evaluate-with-llm-judge.js';
+import { registerVerifyCitationsTool } from './verify-citations.js';
+export function registerAllTools(server, storage, evalEngine, customRuleStore) {
     registerLogTraceTool(server, storage);
     registerEvaluateOutputTool(server, storage, evalEngine);
     registerGetTracesTool(server, storage);
+    registerListRulesTool(server, customRuleStore);
+    registerDeployRuleTool(server, customRuleStore);
+    registerDeleteRuleTool(server, customRuleStore);
+    registerDeleteTraceTool(server, storage);
+    registerEvaluateWithLLMJudgeTool(server, storage);
+    registerVerifyCitationsTool(server, storage);
 }

package/dist/tools/list-rules.d.ts

@@ -0,0 +1,3 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { CustomRuleStore } from '../custom-rule-store.js';
+export declare function registerListRulesTool(server: McpServer, customRuleStore: CustomRuleStore): void;

package/dist/tools/list-rules.js

@@ -0,0 +1,66 @@
+/*
+ * list_rules MCP tool — enumerate deployed custom rules.
+ *
+ * Read-only view into the custom-rule store (~/.iris/custom-rules.json).
+ * Lets agents discover what rules are deployed, what each one evaluates,
+ * and which are enabled — so an agent can decide whether to call
+ * evaluate_output at all, and which eval_type to route through.
+ *
+ * Companion to deploy_rule / delete_rule. Together these replace the
+ * dashboard-only Make-This-A-Rule composer when an agent (not a human)
+ * needs to manage the rule set programmatically.
+ */
+import { z } from 'zod';
+const inputSchema = {
+    eval_type: z
+        .enum(['completeness', 'relevance', 'safety', 'cost', 'custom'])
+        .optional()
+        .describe('Filter to rules of a specific eval category'),
+    enabled_only: z
+        .boolean()
+        .default(false)
+        .describe('Return only enabled rules (excludes disabled ones)'),
+};
+export function registerListRulesTool(server, customRuleStore) {
+    server.registerTool('list_rules', {
+        title: 'List Custom Rules',
+        description: [
+            'Enumerate deployed custom evaluation rules from the local rule store.',
+            '',
+            'Behavior. Pure read of ~/.iris/custom-rules.json (in-memory cached; no disk read per call after server boot). No mutation, no external network. Tenant-scoped in Cloud tier; OSS returns all rules for the single local tenant. Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio. Returns in <5ms.',
+            '',
+            'Output shape. Returns JSON: `{ "rules": [{ "id": "rule-XXXX", "name", "description?", "evalType", "severity", "definition": { type, config, weight? }, "enabled": boolean, "deployedAt": ISO timestamp, "sourceMomentId?": string }], "total": number, "enabled_count": number }`. Empty array + total=0 when no rules deployed.',
+            '',
+            'Use when you need to know what custom rules are currently live (before calling evaluate_output, before deploying a similar rule to avoid duplicates, or when building a dashboard view). Filter with `eval_type` to scope to a specific category, or `enabled_only: true` to exclude disabled rules. Use get_traces to see trace data; use evaluate_output to run scoring; use list_rules only when you need the RULE INVENTORY.',
+            '',
+            "Don't use to count traces or evals (that's get_traces). Don't use to inspect built-in (non-custom) rules — those ship with the iris binary and are listed in docs/api-reference.md, not in the rule store. Don't use to deploy a rule (use deploy_rule); don't use to remove one (use delete_rule).",
+            '',
+            "Error modes. Returns empty list if the rule store file doesn't exist (first run). Returns 429 if HTTP rate limit exceeded. Never throws on valid input.",
+        ].join('\n'),
+        inputSchema,
+        annotations: {
+            readOnlyHint: true,
+            destructiveHint: false,
+            idempotentHint: true,
+            openWorldHint: false,
+        },
+    }, async (args) => {
+        let rules = customRuleStore.list();
+        if (args.eval_type) {
+            rules = rules.filter((r) => r.evalType === args.eval_type);
+        }
+        if (args.enabled_only) {
+            rules = rules.filter((r) => r.enabled);
+        }
+        const total = rules.length;
+        const enabled_count = rules.filter((r) => r.enabled).length;
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: JSON.stringify({ rules, total, enabled_count }),
+                },
+            ],
+        };
+    });
+}

package/dist/tools/log-trace.js

@@ -1,5 +1,7 @@
 import { z } from 'zod';
 import { generateTraceId, generateSpanId } from '../utils/ids.js';
+import { LOCAL_TENANT } from '../types/tenant.js';
+import { bestEffortExport } from '../otel/lazy.js';
 const ToolCallSchema = z.object({
     tool_name: z.string(),
     input: z.unknown().optional(),
@@ -44,8 +46,26 @@ const inputSchema = {
 export function registerLogTraceTool(server, storage) {
     server.registerTool('log_trace', {
         title: 'Log Trace',
-        description: 'Log an agent execution trace with spans, tool calls, and metrics',
+        description: [
+            'Persist a single agent execution trace (input, output, spans, tool calls, cost, latency, token usage).',
+            '',
+            'Behavior. Writes one row to Iris storage (SQLite by default; Postgres in Cloud tier). When IRIS_OTEL_ENDPOINT is set, ALSO fires a best-effort async export to the configured OTLP/HTTP collector (Jaeger, Tempo, Datadog OTLP, OTEL Collector). The OTel export is fire-and-forget — its success does not affect the tool response; failures are logged but the trace is still stored locally. No authentication in stdio mode; HTTP mode requires Bearer token. Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio. Not idempotent: each call mints a fresh trace_id, so resubmitting the same payload creates a duplicate trace.',
+            '',
+            'Output shape. Returns a JSON string: `{ "trace_id": "<32-hex>", "status": "stored" }`. The trace_id is the key you pass to evaluate_output or get_traces afterwards.',
+            '',
+            'Use when you want to record an agent execution for later evaluation, analysis, or audit. Call it AFTER the agent has produced output; call evaluate_output afterwards to score it; call get_traces to query historical traces. Store rich context: spans (span tree), tool_calls (which tools were invoked with latency/errors), token_usage, cost_usd, metadata (arbitrary key-value). All optional except agent_name.',
+            '',
+            'Don\'t use when you only need a transient log (use console logging). Don\'t use to update an existing trace — there is no update path in v0.4 (traces are immutable once stored).',
+            '',
+            'Error modes. Throws on missing agent_name. Throws on malformed span or tool_call objects (Zod rejects). Returns 500 on storage failure (disk full, DB locked). Never blocks on the agent — returns within ~50ms for typical payloads.',
+        ].join('\n'),
         inputSchema,
+        annotations: {
+            readOnlyHint: false, // Writes a row to storage
+            destructiveHint: false, // Creates new data; doesn't overwrite or delete
+            idempotentHint: false, // Each call mints a fresh trace_id; duplicate payloads produce distinct traces
+            openWorldHint: false, // Local storage first. When IRIS_OTEL_ENDPOINT is set a best-effort async OTel export runs but is non-blocking (tool succeeds even if export fails).
+        },
     }, async (args) => {
         const traceId = generateTraceId();
         const timestamp = args.timestamp ?? new Date().toISOString();
@@ -67,7 +87,15 @@ export function registerLogTraceTool(server, storage) {
                 trace_id: traceId,
             })),
         };
-        await storage.insertTrace(trace);
+        await storage.insertTrace(LOCAL_TENANT, trace);
+        // Best-effort async OTel export (fire-and-forget). No-op when
+        // IRIS_OTEL_ENDPOINT isn't configured. Errors are logged via the
+        // server logger but never affect the tool response — if the OTel
+        // collector is down we still want to store traces locally.
+        bestEffortExport(trace, (err) => {
+            // eslint-disable-next-line no-console
+            console.warn(`[iris.otel] ${err.message}`);
+        });
         return {
             content: [
                 {

package/dist/tools/verify-citations.d.ts

@@ -0,0 +1,3 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { IStorageAdapter } from '../types/query.js';
+export declare function registerVerifyCitationsTool(server: McpServer, storage: IStorageAdapter): void;

package/dist/tools/verify-citations.js

@@ -0,0 +1,157 @@
+import { z } from 'zod';
+import { LOCAL_TENANT } from '../types/tenant.js';
+import { verifyCitations } from '../eval/citation-verify/verifier.js';
+import { findPricing } from '../eval/llm-judge/pricing.js';
+import { generateEvalId } from '../utils/ids.js';
+const inputSchema = {
+    output: z.string().min(1).describe('The agent output containing citations to verify'),
+    model: z
+        .string()
+        .describe('Judge model for per-citation verification. Supported: anthropic = claude-opus-4-7 | claude-sonnet-4-6 | claude-haiku-4-5-20251001; openai = gpt-4o | gpt-4o-mini | o1-mini.'),
+    provider: z.enum(['anthropic', 'openai']).optional().describe('Auto-detected from model when omitted'),
+    allow_fetch: z.boolean().optional().describe('Permit outbound HTTP to resolve URLs/DOIs. Defaults to IRIS_CITATION_ALLOW_FETCH=1; false otherwise. SSRF-guarded regardless.'),
+    domain_allowlist: z
+        .array(z.string())
+        .optional()
+        .describe('Restrict fetches to hostnames in this list (suffix match allowed). Merged with IRIS_CITATION_DOMAINS env.'),
+    max_cost_usd_total: z.number().positive().optional().describe('Cap TOTAL judge cost across all citations in this call; default $1.00'),
+    max_citations: z.number().int().positive().max(50).optional().describe('Max citations to verify (extras skipped); default 20'),
+    per_source_timeout_ms: z.number().int().positive().optional().describe('Per-URL fetch timeout; default 10_000'),
+    per_source_max_bytes: z.number().int().positive().optional().describe('Per-URL body cap; default 5MB'),
+    trace_id: z.string().optional().describe('Link verification result to a trace'),
+};
+function inferProvider(model) {
+    const pricing = findPricing(model);
+    if (!pricing) {
+        throw new Error(`Unknown model "${model}". Provider cannot be inferred. Supported models: src/eval/llm-judge/pricing.ts.`);
+    }
+    return pricing.provider;
+}
+function resolveApiKey(provider) {
+    const key = provider === 'anthropic' ? process.env.IRIS_ANTHROPIC_API_KEY : process.env.IRIS_OPENAI_API_KEY;
+    if (!key) {
+        throw new Error(`${provider === 'anthropic' ? 'Anthropic' : 'OpenAI'} judge requires IRIS_${provider === 'anthropic' ? 'ANTHROPIC' : 'OPENAI'}_API_KEY for verify_citations.`);
+    }
+    return key;
+}
+function resolveAllowFetch(paramValue) {
+    if (paramValue !== undefined)
+        return paramValue;
+    return process.env.IRIS_CITATION_ALLOW_FETCH === '1';
+}
+function resolveDomainAllowlist(paramValue) {
+    const envRaw = process.env.IRIS_CITATION_DOMAINS;
+    const fromEnv = envRaw ? envRaw.split(',').map((s) => s.trim()).filter(Boolean) : [];
+    if (paramValue && paramValue.length > 0) {
+        return [...new Set([...fromEnv, ...paramValue])];
+    }
+    return fromEnv.length > 0 ? fromEnv : undefined;
+}
+export function registerVerifyCitationsTool(server, storage) {
+    server.registerTool('verify_citations', {
+        title: 'Verify Citations',
+        description: [
+            'Extract citations from agent output, fetch the cited sources, and use an LLM judge to check whether each source supports the claim in context. Returns per-citation verdicts + an overall support ratio.',
+            '',
+            'Behavior. Three-phase pipeline: (1) regex extraction of [N] numbered refs, (Author, Year) parentheticals, bare URLs, and DOIs (in-process, no network); (2) SSRF-guarded fetch of URL + DOI citations, with scheme allowlist, private/link-local/cloud-metadata IP blocking, optional domain allowlist (IRIS_CITATION_DOMAINS), 10s timeout, 5MB body cap, manual redirect chase (max 3, re-checked), in-process LRU cache; (3) per-citation LLM judge call asking "does this source support this claim?" with a 256-token verdict. Opt-in via allow_fetch=true or IRIS_CITATION_ALLOW_FETCH=1 — Iris refuses outbound HTTP by default. Cost-capped across the entire call by max_cost_usd_total (default $1.00) — the pipeline stops when the cap would be exceeded. Rate-limited to 20 req/min on HTTP MCP. Writes one eval_result row tagged with per-citation provenance.',
+            '',
+            'Output shape. Returns JSON: `{ "id": "<uuid>", "overall_score": 0..1|null, "passed": boolean, "total_citations_found": number, "total_resolved": number, "total_supported": number, "total_cost_usd": number, "citations": [{ "citation": { "raw", "kind", "identifier", "offset_start", "offset_end" }, "resolve_status": "ok"|"skipped"|"error", "resolve_error"?, "source"?: { "url", "status", "content_type", "bytes_fetched", "truncated" }, "judge"?: { "supported", "confidence", "rationale", "cost_usd", "latency_ms", "input_tokens", "output_tokens" } }] }`. `overall_score = supported / resolved`; `null` when nothing resolvable was found.',
+            '',
+            'Use when the output makes factual claims backed by [1]-style references, DOIs, or URLs and you want to separate "cited correctly" from "cited and wrong" from "cited but unresolvable". Particularly useful for research/legal/medical agents where fabricated citations are the dominant failure mode.',
+            "",
+            "Don't use when the agent output has no citations at all (overall_score will be null; the tool degrades gracefully but a heuristic rule is cheaper). Don't use without allow_fetch=true or IRIS_CITATION_ALLOW_FETCH=1 — the tool refuses outbound HTTP unless explicitly enabled. Don't use with an open allowlist + untrusted output on the public internet; you are effectively running a user-directed fetcher. For stricter safety set IRIS_CITATION_DOMAINS to a curated list.",
+            '',
+            'Error modes. Throws when the API key env var is missing. Throws "Unknown model" on unsupported model IDs. Per-citation errors are collected (resolve_error.kind = bad_scheme / ssrf / not_allowed_domain / timeout / too_large / bad_status / redirect_loop / not_text / fetch_disabled / malformed_judge_response / cost_cap_reached / unresolvable_kind) and returned in the response rather than thrown. An empty output or output with zero extractable citations returns overall_score=null + passed=true (nothing to fail).',
+        ].join('\n'),
+        inputSchema,
+        annotations: {
+            readOnlyHint: false, // Writes eval_result + spends money
+            destructiveHint: false, // Creates data; doesn't overwrite/delete
+            idempotentHint: false, // External fetches + provider non-determinism
+            openWorldHint: true, // Outbound HTTP to citation URLs + LLM provider API
+        },
+    }, async (args) => {
+        const provider = args.provider ?? inferProvider(args.model);
+        const apiKey = resolveApiKey(provider);
+        const allowFetch = resolveAllowFetch(args.allow_fetch);
+        const domainAllowlist = resolveDomainAllowlist(args.domain_allowlist);
+        const result = await verifyCitations({
+            output: args.output,
+            provider,
+            model: args.model,
+            apiKey,
+            allowFetch,
+            domainAllowlist,
+            maxCostUsdTotal: args.max_cost_usd_total,
+            maxCitations: args.max_citations,
+            perSourceTimeoutMs: args.per_source_timeout_ms,
+            perSourceMaxBytes: args.per_source_max_bytes,
+        });
+        const evalId = generateEvalId();
+        const score = result.overallScore ?? 0;
+        // Persist so dashboard can surface. eval_type='custom' — same
+        // rationale as evaluate_with_llm_judge (spans all 4 heuristic
+        // categories). rule_results[0] carries per-citation summary.
+        await storage.insertEvalResult(LOCAL_TENANT, {
+            id: evalId,
+            trace_id: args.trace_id,
+            eval_type: 'custom',
+            output_text: args.output,
+            score,
+            passed: result.passed,
+            rule_results: [
+                {
+                    ruleName: `semantic_citation_verify:${provider}/${args.model}`,
+                    passed: result.passed,
+                    score,
+                    message: result.overallScore === null
+                        ? `No resolvable citations (found ${result.totalCitationsFound}, resolved ${result.totalResolved})`
+                        : `${result.totalSupported}/${result.totalResolved} cited sources supported the output`,
+                },
+            ],
+            suggestions: result.passed ? [] : [`Only ${result.totalSupported}/${result.totalResolved} cited sources actually supported the claim.`],
+            rules_evaluated: 1,
+            rules_skipped: 0,
+            insufficient_data: result.overallScore === null,
+        });
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: JSON.stringify({
+                        id: evalId,
+                        overall_score: result.overallScore,
+                        passed: result.passed,
+                        total_citations_found: result.totalCitationsFound,
+                        total_resolved: result.totalResolved,
+                        total_supported: result.totalSupported,
+                        total_cost_usd: result.totalCostUsd,
+                        citations: result.citations.map((c) => ({
+                            citation: {
+                                raw: c.citation.raw,
+                                kind: c.citation.kind,
+                                identifier: c.citation.identifier,
+                                offset_start: c.citation.offsetStart,
+                                offset_end: c.citation.offsetEnd,
+                            },
+                            resolve_status: c.resolveStatus,
+                            resolve_error: c.resolveError,
+                            source: c.source,
+                            judge: c.judge
+                                ? {
+                                    supported: c.judge.supported,
+                                    confidence: c.judge.confidence,
+                                    rationale: c.judge.rationale,
+                                    cost_usd: c.judge.costUsd,
+                                    latency_ms: c.judge.latencyMs,
+                                    input_tokens: c.judge.inputTokens,
+                                    output_tokens: c.judge.outputTokens,
+                                }
+                                : undefined,
+                        })),
+                    }),
+                },
+            ],
+        };
+    });
+}

package/dist/types/custom-rule.d.ts

@@ -0,0 +1,70 @@
+import type { CustomRuleDefinition, EvalType } from './eval.js';
+export type RuleSeverity = 'low' | 'medium' | 'high' | 'critical';
+export interface DeployedCustomRule {
+    /** Stable id (e.g., "rule-<8-char-hex>"). Generated server-side on deploy. */
+    id: string;
+    /** User-readable name. Becomes the rule name in eval results. */
+    name: string;
+    /** Human-readable description (what the rule checks for, why it matters). */
+    description: string;
+    /** Eval category this rule belongs to. Determines when it fires. */
+    evalType: EvalType;
+    /** Severity used to sort rules in the dashboard + audit alerts. */
+    severity: RuleSeverity;
+    /** The check definition (regex pattern, length threshold, etc.). */
+    definition: CustomRuleDefinition;
+    /** Whether this rule is currently active. Disabled rules don't fire but are kept for audit. */
+    enabled: boolean;
+    /** ISO timestamp of deploy. */
+    createdAt: string;
+    /** ISO timestamp of most recent edit. */
+    updatedAt: string;
+    /** Optional moment ID the rule was extracted from (workflow inversion provenance). */
+    sourceMomentId?: string;
+    /** Version counter — incremented on edit. Starts at 1. */
+    version: number;
+}
+export interface CustomRulesFile {
+    /** File schema version — bump when shape changes. */
+    version: 1;
+    rules: DeployedCustomRule[];
+}
+export interface AuditLogEntry {
+    ts: string;
+    /**
+     * Which tenant the action belongs to. OSS installs always emit 'local'.
+     * Cloud installs emit the tenant resolved from the authenticated session.
+     *
+     * Optional for backward compatibility: entries written before v0.4.0
+     * don't have this field. Readers MUST treat missing `tenantId` as
+     * 'local' so old audit logs remain queryable on upgrade.
+     */
+    tenantId?: string;
+    /** Action taken — currently rule.deploy / rule.delete / rule.toggle. */
+    action: 'rule.deploy' | 'rule.delete' | 'rule.toggle' | 'rule.update';
+    /** Who initiated. v0.4 is single-user local — always "local". v0.5+ adds users. */
+    user: string;
+    ruleId: string;
+    ruleName?: string;
+    /** Optional detail: source moment id, prior version, etc. */
+    details?: Record<string, unknown>;
+}
+export interface RulePreviewResult {
+    /** Number of historical traces evaluated. */
+    tracesEvaluated: number;
+    /** How many traces would have FAILED this proposed rule. */
+    wouldFail: number;
+    /** How many would have PASSED. */
+    wouldPass: number;
+    /** How many would have skipped (rule not applicable to context). */
+    wouldSkip: number;
+    /** First 5 example traces that would fail (with brief output preview). */
+    examples: Array<{
+        traceId: string;
+        agentName: string;
+        timestamp: string;
+        outputPreview: string;
+    }>;
+    /** Time window covered. */
+    windowSinceIso: string;
+}

package/dist/types/custom-rule.js

@@ -0,0 +1 @@
+export {};