@iris-eval/mcp-server 0.3.1 → 0.4.0

package/dist/tools/delete-trace.js

@@ -0,0 +1,54 @@
+/*
+ * delete_trace MCP tool — remove a single trace by id.
+ *
+ * Destructive. Scoped to tenant — an agent cannot delete a trace
+ * belonging to another tenant (cross-tenant deletes look like
+ * "id not found").
+ *
+ * Cascades to spans via FK ON DELETE CASCADE. eval_results for the
+ * trace have their trace_id set to NULL (score history survives
+ * even after the underlying trace is gone).
+ */
+import { z } from 'zod';
+import { LOCAL_TENANT } from '../types/tenant.js';
+const inputSchema = {
+    trace_id: z
+        .string()
+        .regex(/^[a-f0-9]{32}$/)
+        .describe('Trace id to delete (32-hex lowercase; obtained from log_trace response or get_traces)'),
+};
+export function registerDeleteTraceTool(server, storage) {
+    server.registerTool('delete_trace', {
+        title: 'Delete Trace',
+        description: [
+            'Remove a single trace by id. Cascades to spans; eval_results keep the score history with trace_id NULLed.',
+            '',
+            'Behavior. DESTRUCTIVE — SQL DELETE scoped to the caller\'s tenant_id. Cascades: spans belonging to this trace are deleted (FK ON DELETE CASCADE); eval_results that referenced this trace have their trace_id set to NULL (FK ON DELETE SET NULL) so aggregate dashboards + historical scores remain valid even after the trace is gone. Not idempotent: deleting an already-deleted trace returns `deleted: false`. Does not emit an audit log entry in v0.4 — traces are user-scope data, not policy changes. Rate-limited to 20 req/min on HTTP MCP.',
+            '',
+            'Output shape. Returns JSON: `{ "deleted": boolean, "trace_id": string }`. `deleted=true` if a row was removed; `deleted=false` if no trace with that id existed (or it belonged to a different tenant — cross-tenant deletes silently fail).',
+            '',
+            "Use when a trace was captured in error, contains sensitive data that must be removed for compliance (e.g., a customer exercises GDPR right-to-erasure), or when cleaning up test data. Combine with get_traces to find candidates: query with filters → review → delete_trace(id) per target. For bulk time-window deletion, use `deleteTracesOlderThan` via the CLI / retention config — delete_trace is the single-row surgical path.",
+            '',
+            "Don't use to clean up OLD data in bulk (use retention config with --retention-days). Don't use to PAUSE a trace — traces are immutable once stored; there's nothing to pause. Don't use to delete eval_results — eval_results survive their trace's deletion intentionally (for audit + drift analysis); they're pruned only by retention.",
+            '',
+            "Error modes. Throws 400 on malformed trace_id (wrong format: not 32-char lowercase hex). Returns `{deleted: false}` when the id doesn't exist in the caller's tenant (not an error — the trace may simply have been deleted already). Returns 429 on HTTP rate limit. Storage failures propagate as 500.",
+        ].join('\n'),
+        inputSchema,
+        annotations: {
+            readOnlyHint: false,
+            destructiveHint: true,
+            idempotentHint: false,
+            openWorldHint: false,
+        },
+    }, async (args) => {
+        const deleted = await storage.deleteTrace(LOCAL_TENANT, args.trace_id);
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: JSON.stringify({ deleted, trace_id: args.trace_id }),
+                },
+            ],
+        };
+    });
+}

package/dist/tools/deploy-rule.d.ts

@@ -0,0 +1,3 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { CustomRuleStore } from '../custom-rule-store.js';
+export declare function registerDeployRuleTool(server: McpServer, customRuleStore: CustomRuleStore): void;

package/dist/tools/deploy-rule.js

@@ -0,0 +1,91 @@
+/*
+ * deploy_rule MCP tool — programmatically add a custom eval rule.
+ *
+ * Mirror of the dashboard's Make-This-A-Rule composer, but callable
+ * from an agent. An agent that observes a failure pattern can deploy
+ * a rule without a human in the loop.
+ *
+ * Writes to ~/.iris/custom-rules.json (single source of truth) and
+ * appends to the audit log. Persisted rules auto-load on server boot
+ * and fire on every future evaluate_output call of the matching
+ * eval_type.
+ */
+import { z } from 'zod';
+const CustomRuleDefinitionSchema = z.object({
+    name: z.string(),
+    type: z.enum([
+        'regex_match',
+        'regex_no_match',
+        'min_length',
+        'max_length',
+        'contains_keywords',
+        'excludes_keywords',
+        'json_schema',
+        'cost_threshold',
+    ]),
+    config: z.record(z.unknown()),
+    weight: z.number().optional(),
+});
+const inputSchema = {
+    name: z.string().min(1).max(120).describe('Human-readable rule name (used in eval results)'),
+    description: z
+        .string()
+        .max(500)
+        .optional()
+        .describe('What this rule checks for and why it matters'),
+    evalType: z
+        .enum(['completeness', 'relevance', 'safety', 'cost', 'custom'])
+        .describe('Eval category this rule belongs to; determines when it fires'),
+    severity: z
+        .enum(['low', 'medium', 'high', 'critical'])
+        .default('medium')
+        .describe('Severity used for dashboard sort + audit alerts'),
+    definition: CustomRuleDefinitionSchema.describe('Check definition (regex, length, keyword, cost, or schema)'),
+    sourceMomentId: z
+        .string()
+        .optional()
+        .describe('Optional Decision Moment ID the rule was derived from (preserves workflow-inversion provenance)'),
+};
+export function registerDeployRuleTool(server, customRuleStore) {
+    server.registerTool('deploy_rule', {
+        title: 'Deploy Custom Rule',
+        description: [
+            'Deploy a new custom evaluation rule that will fire on every future evaluate_output call of its eval category.',
+            '',
+            'Behavior. Writes a row to ~/.iris/custom-rules.json (atomic write via temp file + rename) and appends a `rule.deploy` entry to the audit log (~/.iris/audit.log). The rule activates immediately for the running process and persists across restarts. Each call mints a fresh rule_id; not idempotent (deploying twice creates two rules). Tenant-scoped in Cloud tier; OSS rules are owned by LOCAL_TENANT. Rate-limited to 20 req/min on HTTP MCP.',
+            '',
+            'Output shape. Returns JSON: `{ "rule": { "id": "rule-XXXX", "name", "description", "evalType", "severity", "definition", "enabled": true, "createdAt", "updatedAt", "version": 1, "sourceMomentId?" } }`. The returned rule is the canonical persisted form; save the `id` if you plan to update or delete later.',
+            '',
+            "Use when an agent observes a recurring failure pattern and decides to enforce it as a standing rule. The `sourceMomentId` field preserves provenance — downstream audit can trace the rule back to the moment that inspired it. Combine with evaluate_output + get_traces: 1) evaluate_output surfaces failures; 2) get_traces filters to the failure set; 3) analyze the pattern; 4) deploy_rule bakes it into the default eval path.",
+            '',
+            "Don't use to VALIDATE a rule before committing — deploy writes immediately. Use the dashboard's preview endpoint (POST /api/v1/rules/custom/preview) for dry-run validation against sample output. Don't use to EDIT an existing rule — this call only creates; edits require a dedicated flow (coming in v0.5). To update a rule today: delete_rule then deploy_rule with the new definition.",
+            '',
+            "Error modes. Throws 400 on invalid definition (Zod rejects — e.g., regex that fails safe-regex2 ReDoS check, or length > 1000 chars). Throws 400 on empty `name`. Throws 400 if the eval category mismatches the definition type. Returns 429 when HTTP rate limit exceeded. File-write failures (disk full, read-only fs) propagate as 500; the audit log is best-effort and does not block deploy.",
+        ].join('\n'),
+        inputSchema,
+        annotations: {
+            readOnlyHint: false,
+            destructiveHint: false,
+            idempotentHint: false,
+            openWorldHint: false,
+        },
+    }, async (args) => {
+        const rule = customRuleStore.deploy({
+            name: args.name,
+            description: args.description,
+            evalType: args.evalType,
+            severity: args.severity,
+            definition: args.definition,
+            sourceMomentId: args.sourceMomentId,
+            user: 'mcp',
+        });
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: JSON.stringify({ rule }),
+                },
+            ],
+        };
+    });
+}

package/dist/tools/evaluate-output.js

@@ -1,4 +1,5 @@
 import { z } from 'zod';
+import { LOCAL_TENANT } from '../types/tenant.js';
 const CustomRuleSchema = z.object({
     name: z.string(),
     type: z.enum([
@@ -25,8 +26,26 @@ const inputSchema = {
 export function registerEvaluateOutputTool(server, storage, evalEngine) {
     server.registerTool('evaluate_output', {
         title: 'Evaluate Output',
-        description: 'Evaluate agent output quality using configurable rules',
+        description: [
+            'Score agent output against configurable eval rules and return a 0..1 score + per-rule breakdown.',
+            '',
+            'Behavior. Deterministic, in-process scoring — same inputs always produce the same result. Writes one eval_result row to Iris storage (linked to trace_id if provided; unlinked otherwise). No external network calls in heuristic mode (v0.4 adds an llm_as_judge eval_type that DOES call LLM APIs; see the separate evaluate_with_llm_judge tool for that). Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio. Runs in ~5-50ms for rule-based evaluation.',
+            '',
+            'Output shape. Returns JSON: `{ "id": "<uuid>", "score": 0..1, "passed": boolean, "rule_results": [{ "ruleName", "passed", "score", "message", "skipped?" }], "suggestions": string[], "rules_evaluated": number, "rules_skipped": number, "insufficient_data": boolean }`. `insufficient_data=true` means no applicable rules fired (e.g., safety eval with only cost data).',
+            '',
+            'Use when you want a quality score on a specific output — typically after log_trace records the execution. Pass `eval_type` to route to the right rule bundle: `completeness` (length, sentence count, relevance to input), `relevance` (keyword overlap, topic consistency), `safety` (PII leak, prompt injection, hallucination markers, stub-output detection), `cost` (budget threshold), or `custom` (bring your own rules via `custom_rules`).',
+            '',
+            'Don\'t use when the output is empty or has no applicable rules — the eval_type decides which rules apply, and invalid combinations return score=0 + insufficient_data=true (not an error, but not actionable). Don\'t use to VALIDATE JSON schemas directly (use your language\'s JSON Schema validator — Iris\'s `json_schema` custom rule type is for output-shape assertions, not arbitrary validation).',
+            '',
+            'Error modes. Throws on malformed custom_rules (Zod rejects). Returns 400 on regex patterns that fail safe-regex2 ReDoS check or exceed 1000-char limit. Returns 429 when HTTP rate limit exceeded. Storage failures propagate as 500. The eval itself never throws — failing rules report `passed: false` with a message, they don\'t bubble exceptions.',
+        ].join('\n'),
         inputSchema,
+        annotations: {
+            readOnlyHint: false, // Writes an eval_result row
+            destructiveHint: false, // Creates new data; doesn't overwrite or delete
+            idempotentHint: true, // Deterministic: same inputs → same score (each call writes a distinct result row, but the SCORE is stable)
+            openWorldHint: false, // No external network in heuristic mode; LLM-as-judge has its own tool with openWorldHint:true
+        },
     }, async (args) => {
         const evalType = args.eval_type;
         const result = evalEngine.evaluate(evalType, {
@@ -39,7 +58,9 @@ export function registerEvaluateOutputTool(server, storage, evalEngine) {
         if (args.trace_id) {
             result.trace_id = args.trace_id;
         }
-        await storage.insertEvalResult(result);
+        // OSS single-tenant: MCP tool callers are the local user. Cloud
+        // will derive tenant from the authenticated MCP session.
+        await storage.insertEvalResult(LOCAL_TENANT, result);
         return {
             content: [
                 {

package/dist/tools/evaluate-with-llm-judge.d.ts

@@ -0,0 +1,3 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { IStorageAdapter } from '../types/query.js';
+export declare function registerEvaluateWithLLMJudgeTool(server: McpServer, storage: IStorageAdapter): void;

package/dist/tools/evaluate-with-llm-judge.js

@@ -0,0 +1,147 @@
+import { z } from 'zod';
+import { LOCAL_TENANT } from '../types/tenant.js';
+import { evaluateWithLLMJudge } from '../eval/llm-judge/evaluator.js';
+import { findPricing } from '../eval/llm-judge/pricing.js';
+import { generateEvalId } from '../utils/ids.js';
+const inputSchema = {
+    output: z.string().min(1).describe('The agent output text to evaluate'),
+    template: z
+        .enum(['accuracy', 'helpfulness', 'safety', 'correctness', 'faithfulness'])
+        .describe('Judge dimension: accuracy (factual correctness), helpfulness (does it address the ask), safety (harm potential), correctness (vs reference answer — requires `expected`), faithfulness (RAG grounding — requires `source_material`).'),
+    model: z
+        .string()
+        .describe('Model ID. Supported: anthropic = claude-opus-4-7 | claude-sonnet-4-6 | claude-haiku-4-5 | claude-haiku-4-5-20251001; openai = gpt-4o | gpt-4o-mini | o1-mini.'),
+    provider: z.enum(['anthropic', 'openai']).optional().describe('Auto-detected from model when omitted'),
+    input: z.string().optional().describe('User question / prompt that produced the output (improves accuracy for helpfulness/safety)'),
+    expected: z.string().optional().describe('Reference answer (required for correctness template)'),
+    source_material: z.string().optional().describe('Provided RAG sources (required for faithfulness template)'),
+    trace_id: z.string().optional().describe('Link this evaluation to a trace'),
+    max_cost_usd: z.number().positive().optional().describe('Cost cap in USD; defaults to IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL or 0.25'),
+    max_output_tokens: z.number().int().positive().max(4096).optional().describe('Judge output token cap; default 512'),
+    temperature: z.number().min(0).max(2).optional().describe('Sampling temperature; default 0 (deterministic)'),
+    timeout_ms: z.number().int().positive().optional().describe('Per-request timeout; default 60_000'),
+};
+function inferProvider(model) {
+    const pricing = findPricing(model);
+    if (!pricing) {
+        throw new Error(`Unknown model "${model}". Provider cannot be inferred. Supported models are listed in src/eval/llm-judge/pricing.ts.`);
+    }
+    return pricing.provider;
+}
+function resolveApiKey(provider) {
+    if (provider === 'anthropic') {
+        const key = process.env.IRIS_ANTHROPIC_API_KEY;
+        if (!key) {
+            throw new Error('Anthropic judge requires IRIS_ANTHROPIC_API_KEY. Set it in the environment or use a different provider.');
+        }
+        return key;
+    }
+    const key = process.env.IRIS_OPENAI_API_KEY;
+    if (!key) {
+        throw new Error('OpenAI judge requires IRIS_OPENAI_API_KEY. Set it in the environment or use a different provider.');
+    }
+    return key;
+}
+function resolveMaxCost(paramValue) {
+    if (paramValue !== undefined)
+        return paramValue;
+    const envRaw = process.env.IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL;
+    if (envRaw) {
+        const parsed = Number(envRaw);
+        if (Number.isFinite(parsed) && parsed > 0)
+            return parsed;
+    }
+    return 0.25;
+}
+export function registerEvaluateWithLLMJudgeTool(server, storage) {
+    server.registerTool('evaluate_with_llm_judge', {
+        title: 'Evaluate With LLM Judge',
+        description: [
+            'Score agent output using an LLM as the judge (Anthropic or OpenAI). Returns a calibrated 0..1 score with rationale, per-dimension breakdown, and exact cost.',
+            '',
+            'Behavior. Calls an external LLM API (Anthropic or OpenAI) — costs money per call, takes 1-10 seconds, respects an IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL cap. Non-deterministic at temperature > 0; default temperature=0 gives near-deterministic scores. Writes one eval_result row to Iris storage (linked to trace_id if provided) plus captures provider response id + latency + token counts + cost in the rule_results payload. Rate-limited to 20 req/min on HTTP MCP; your LLM provider also enforces its own rate limits (we transparently retry once on 429).',
+            '',
+            'Output shape. Returns JSON: `{ "id": "<uuid>", "score": 0..1, "passed": boolean, "rationale": string, "dimensions": {...}, "model": string, "provider": "anthropic"|"openai", "template": string, "input_tokens": number, "output_tokens": number, "cost_usd": number, "latency_ms": number }`. `dimensions` has per-dimension sub-scores (e.g., accuracy template returns `{factual_claims, citations, internal_consistency}`).',
+            '',
+            'Use when heuristic rules (via evaluate_output) are too coarse for the quality signal you need — semantic correctness, factual accuracy vs a reference, RAG faithfulness to sources, nuanced safety/helpfulness. Pick the template that matches: `accuracy` (hallucination detection), `helpfulness` (does it address the ask), `safety` (harm potential beyond regex PII), `correctness` (vs reference answer — pass `expected`), `faithfulness` (RAG grounding — pass `source_material`).',
+            '',
+            "Don't use for simple regex/length/keyword checks (use evaluate_output with heuristic rules — they're free, deterministic, 1000x faster). Don't use without an API key set (IRIS_ANTHROPIC_API_KEY or IRIS_OPENAI_API_KEY). Don't use on very large outputs (>8K tokens) without raising max_cost_usd — the pre-check will refuse the call.",
+            '',
+            'Error modes. Throws when the required API key env var is missing. Throws when the estimated worst-case cost exceeds max_cost_usd (raise the cap or trim prompts). Throws LLMJudgeError on provider errors — kind=`auth` on 401/403, `rate_limit` on 429 (auto-retried once), `server_error` on 5xx, `timeout` on abort, `malformed_response` when the judge fails to emit valid JSON on both attempts. Throws "Unknown model" for unsupported model IDs — update src/eval/llm-judge/pricing.ts first.',
+        ].join('\n'),
+        inputSchema,
+        annotations: {
+            readOnlyHint: false, // Writes eval_result; also spends money (external API cost)
+            destructiveHint: false, // Creates data; doesn't overwrite or delete
+            idempotentHint: false, // Temperature > 0 may vary; even at T=0 provider non-determinism is possible; cost also varies per call
+            openWorldHint: true, // Calls external APIs (Anthropic / OpenAI) — touches the world beyond local process
+        },
+    }, async (args) => {
+        const provider = args.provider ?? inferProvider(args.model);
+        const apiKey = resolveApiKey(provider);
+        const maxCostUsd = resolveMaxCost(args.max_cost_usd);
+        const result = await evaluateWithLLMJudge({
+            output: args.output,
+            template: args.template,
+            provider,
+            model: args.model,
+            apiKey,
+            input: args.input,
+            expected: args.expected,
+            sourceMaterial: args.source_material,
+            maxCostUsdPerEval: maxCostUsd,
+            maxOutputTokens: args.max_output_tokens,
+            temperature: args.temperature,
+            timeoutMs: args.timeout_ms,
+        });
+        const evalId = generateEvalId();
+        // Persist as a normal eval_result so the dashboard picks it up
+        // alongside heuristic scores. eval_type is 'custom' because LLM
+        // judge doesn't fit completeness/relevance/safety/cost taxonomy
+        // cleanly — it spans all four. The rule_results payload carries
+        // the full judge provenance.
+        await storage.insertEvalResult(LOCAL_TENANT, {
+            id: evalId,
+            trace_id: args.trace_id,
+            eval_type: 'custom',
+            output_text: args.output,
+            expected_text: args.expected,
+            score: result.score,
+            passed: result.passed,
+            rule_results: [
+                {
+                    ruleName: `llm_judge:${result.template}:${result.provider}/${result.model}`,
+                    passed: result.passed,
+                    score: result.score,
+                    message: result.rationale || 'LLM judge evaluation',
+                },
+            ],
+            suggestions: result.passed ? [] : [result.rationale],
+            rules_evaluated: 1,
+            rules_skipped: 0,
+            insufficient_data: false,
+        });
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: JSON.stringify({
+                        id: evalId,
+                        score: result.score,
+                        passed: result.passed,
+                        rationale: result.rationale,
+                        dimensions: result.dimensions,
+                        model: result.model,
+                        provider: result.provider,
+                        template: result.template,
+                        input_tokens: result.inputTokens,
+                        output_tokens: result.outputTokens,
+                        cost_usd: result.costUsd,
+                        latency_ms: result.latencyMs,
+                        raw_response_id: result.rawResponseId,
+                    }),
+                },
+            ],
+        };
+    });
+}

package/dist/tools/get-traces.js

@@ -1,4 +1,5 @@
 import { z } from 'zod';
+import { LOCAL_TENANT } from '../types/tenant.js';
 const inputSchema = {
     agent_name: z.string().optional().describe('Filter by agent name'),
     framework: z.string().optional().describe('Filter by framework'),
@@ -15,10 +16,29 @@ const inputSchema = {
 export function registerGetTracesTool(server, storage) {
     server.registerTool('get_traces', {
         title: 'Get Traces',
-        description: 'Query stored traces with filters, pagination, and optional summary stats',
+        description: [
+            'Query stored agent-execution traces with filters, pagination, and optional dashboard summary.',
+            '',
+            'Behavior. Read-only: never mutates storage, never calls external services. Idempotent: repeated calls with the same args return consistent results (new traces logged after the call obviously show up on subsequent calls). Tenant-scoped: queries only the caller\'s tenant rows (LOCAL_TENANT in OSS). Paginates results (default limit 50, max 1000). Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio.',
+            '',
+            'Output shape. Returns JSON: `{ "traces": [{...traceRow}], "total": number, "limit": number, "offset": number, "summary"?: { total_traces, avg_latency_ms, total_cost_usd, error_rate, eval_pass_rate, traces_per_hour, top_agents } }`. Each trace row includes trace_id, agent_name, framework, input, output, tool_calls, latency_ms, token_usage, cost_usd, metadata, timestamp. `summary` only included when `include_summary: true`.',
+            '',
+            'Use when you need historical data: investigating a past failure, computing quality trends, comparing agents, or feeding an analytics job. Set `agent_name` / `framework` / `since` / `until` to narrow the query. Set `min_score` / `max_score` to surface outliers. Set `sort_by: "cost_usd"` + `sort_order: "desc"` to find the most expensive traces. Set `include_summary: true` when you want dashboard-style aggregates in one round-trip.',
+            '',
+            'Don\'t use to score a trace (use evaluate_output). Don\'t use to create a trace (use log_trace). Don\'t use as a live event stream — it\'s a query, not a subscription; poll with exponential backoff or use the dashboard\'s SSE endpoint for real-time.',
+            '',
+            'Error modes. Returns 400 on invalid sort_by / sort_order (Zod enum). Returns 400 if limit > 1000. Returns 429 when HTTP rate limit exceeded. Storage failures propagate as 500. Empty result with `total: 0` on no matches (not an error).',
+        ].join('\n'),
         inputSchema,
+        annotations: {
+            readOnlyHint: true, // Pure query: never writes, never deletes
+            destructiveHint: false, // Inverse of readOnly — trivially false
+            idempotentHint: true, // Same args → same result (modulo new traces that may have landed since)
+            openWorldHint: false, // Queries local storage only; no external network
+        },
     }, async (args) => {
-        const result = await storage.queryTraces({
+        // OSS single-tenant: MCP caller is the local user.
+        const result = await storage.queryTraces(LOCAL_TENANT, {
             filter: {
                 agent_name: args.agent_name,
                 framework: args.framework,
@@ -39,7 +59,7 @@ export function registerGetTracesTool(server, storage) {
             offset: result.offset,
         };
         if (args.include_summary) {
-            response.summary = await storage.getDashboardSummary();
+            response.summary = await storage.getDashboardSummary(LOCAL_TENANT);
         }
         return {
             content: [

package/dist/tools/index.d.ts

@@ -1,4 +1,5 @@
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { IStorageAdapter } from '../types/query.js';
 import type { EvalEngine } from '../eval/engine.js';
-export declare function registerAllTools(server: McpServer, storage: IStorageAdapter, evalEngine: EvalEngine): void;
+import type { CustomRuleStore } from '../custom-rule-store.js';
+export declare function registerAllTools(server: McpServer, storage: IStorageAdapter, evalEngine: EvalEngine, customRuleStore: CustomRuleStore): void;

package/dist/tools/index.js

@@ -1,8 +1,20 @@
 import { registerLogTraceTool } from './log-trace.js';
 import { registerEvaluateOutputTool } from './evaluate-output.js';
 import { registerGetTracesTool } from './get-traces.js';
-export function registerAllTools(server, storage, evalEngine) {
+import { registerListRulesTool } from './list-rules.js';
+import { registerDeployRuleTool } from './deploy-rule.js';
+import { registerDeleteRuleTool } from './delete-rule.js';
+import { registerDeleteTraceTool } from './delete-trace.js';
+import { registerEvaluateWithLLMJudgeTool } from './evaluate-with-llm-judge.js';
+import { registerVerifyCitationsTool } from './verify-citations.js';
+export function registerAllTools(server, storage, evalEngine, customRuleStore) {
     registerLogTraceTool(server, storage);
     registerEvaluateOutputTool(server, storage, evalEngine);
     registerGetTracesTool(server, storage);
+    registerListRulesTool(server, customRuleStore);
+    registerDeployRuleTool(server, customRuleStore);
+    registerDeleteRuleTool(server, customRuleStore);
+    registerDeleteTraceTool(server, storage);
+    registerEvaluateWithLLMJudgeTool(server, storage);
+    registerVerifyCitationsTool(server, storage);
 }

package/dist/tools/list-rules.d.ts

@@ -0,0 +1,3 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { CustomRuleStore } from '../custom-rule-store.js';
+export declare function registerListRulesTool(server: McpServer, customRuleStore: CustomRuleStore): void;

package/dist/tools/list-rules.js

@@ -0,0 +1,66 @@
+/*
+ * list_rules MCP tool — enumerate deployed custom rules.
+ *
+ * Read-only view into the custom-rule store (~/.iris/custom-rules.json).
+ * Lets agents discover what rules are deployed, what each one evaluates,
+ * and which are enabled — so an agent can decide whether to call
+ * evaluate_output at all, and which eval_type to route through.
+ *
+ * Companion to deploy_rule / delete_rule. Together these replace the
+ * dashboard-only Make-This-A-Rule composer when an agent (not a human)
+ * needs to manage the rule set programmatically.
+ */
+import { z } from 'zod';
+const inputSchema = {
+    eval_type: z
+        .enum(['completeness', 'relevance', 'safety', 'cost', 'custom'])
+        .optional()
+        .describe('Filter to rules of a specific eval category'),
+    enabled_only: z
+        .boolean()
+        .default(false)
+        .describe('Return only enabled rules (excludes disabled ones)'),
+};
+export function registerListRulesTool(server, customRuleStore) {
+    server.registerTool('list_rules', {
+        title: 'List Custom Rules',
+        description: [
+            'Enumerate deployed custom evaluation rules from the local rule store.',
+            '',
+            'Behavior. Pure read of ~/.iris/custom-rules.json (in-memory cached; no disk read per call after server boot). No mutation, no external network. Tenant-scoped in Cloud tier; OSS returns all rules for the single local tenant. Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio. Returns in <5ms.',
+            '',
+            'Output shape. Returns JSON: `{ "rules": [{ "id": "rule-XXXX", "name", "description?", "evalType", "severity", "definition": { type, config, weight? }, "enabled": boolean, "deployedAt": ISO timestamp, "sourceMomentId?": string }], "total": number, "enabled_count": number }`. Empty array + total=0 when no rules deployed.',
+            '',
+            'Use when you need to know what custom rules are currently live (before calling evaluate_output, before deploying a similar rule to avoid duplicates, or when building a dashboard view). Filter with `eval_type` to scope to a specific category, or `enabled_only: true` to exclude disabled rules. Use get_traces to see trace data; use evaluate_output to run scoring; use list_rules only when you need the RULE INVENTORY.',
+            '',
+            "Don't use to count traces or evals (that's get_traces). Don't use to inspect built-in (non-custom) rules — those ship with the iris binary and are listed in docs/api-reference.md, not in the rule store. Don't use to deploy a rule (use deploy_rule); don't use to remove one (use delete_rule).",
+            '',
+            "Error modes. Returns empty list if the rule store file doesn't exist (first run). Returns 429 if HTTP rate limit exceeded. Never throws on valid input.",
+        ].join('\n'),
+        inputSchema,
+        annotations: {
+            readOnlyHint: true,
+            destructiveHint: false,
+            idempotentHint: true,
+            openWorldHint: false,
+        },
+    }, async (args) => {
+        let rules = customRuleStore.list();
+        if (args.eval_type) {
+            rules = rules.filter((r) => r.evalType === args.eval_type);
+        }
+        if (args.enabled_only) {
+            rules = rules.filter((r) => r.enabled);
+        }
+        const total = rules.length;
+        const enabled_count = rules.filter((r) => r.enabled).length;
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: JSON.stringify({ rules, total, enabled_count }),
+                },
+            ],
+        };
+    });
+}

package/dist/tools/log-trace.js

@@ -1,5 +1,7 @@
 import { z } from 'zod';
 import { generateTraceId, generateSpanId } from '../utils/ids.js';
+import { LOCAL_TENANT } from '../types/tenant.js';
+import { bestEffortExport } from '../otel/lazy.js';
 const ToolCallSchema = z.object({
     tool_name: z.string(),
     input: z.unknown().optional(),
@@ -44,8 +46,26 @@ const inputSchema = {
 export function registerLogTraceTool(server, storage) {
     server.registerTool('log_trace', {
         title: 'Log Trace',
-        description: 'Log an agent execution trace with spans, tool calls, and metrics',
+        description: [
+            'Persist a single agent execution trace (input, output, spans, tool calls, cost, latency, token usage).',
+            '',
+            'Behavior. Writes one row to Iris storage (SQLite by default; Postgres in Cloud tier). When IRIS_OTEL_ENDPOINT is set, ALSO fires a best-effort async export to the configured OTLP/HTTP collector (Jaeger, Tempo, Datadog OTLP, OTEL Collector). The OTel export is fire-and-forget — its success does not affect the tool response; failures are logged but the trace is still stored locally. No authentication in stdio mode; HTTP mode requires Bearer token. Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio. Not idempotent: each call mints a fresh trace_id, so resubmitting the same payload creates a duplicate trace.',
+            '',
+            'Output shape. Returns a JSON string: `{ "trace_id": "<32-hex>", "status": "stored" }`. The trace_id is the key you pass to evaluate_output or get_traces afterwards.',
+            '',
+            'Use when you want to record an agent execution for later evaluation, analysis, or audit. Call it AFTER the agent has produced output; call evaluate_output afterwards to score it; call get_traces to query historical traces. Store rich context: spans (span tree), tool_calls (which tools were invoked with latency/errors), token_usage, cost_usd, metadata (arbitrary key-value). All optional except agent_name.',
+            '',
+            'Don\'t use when you only need a transient log (use console logging). Don\'t use to update an existing trace — there is no update path in v0.4 (traces are immutable once stored).',
+            '',
+            'Error modes. Throws on missing agent_name. Throws on malformed span or tool_call objects (Zod rejects). Returns 500 on storage failure (disk full, DB locked). Never blocks on the agent — returns within ~50ms for typical payloads.',
+        ].join('\n'),
         inputSchema,
+        annotations: {
+            readOnlyHint: false, // Writes a row to storage
+            destructiveHint: false, // Creates new data; doesn't overwrite or delete
+            idempotentHint: false, // Each call mints a fresh trace_id; duplicate payloads produce distinct traces
+            openWorldHint: false, // Local storage first. When IRIS_OTEL_ENDPOINT is set a best-effort async OTel export runs but is non-blocking (tool succeeds even if export fails).
+        },
     }, async (args) => {
         const traceId = generateTraceId();
         const timestamp = args.timestamp ?? new Date().toISOString();
@@ -67,7 +87,15 @@ export function registerLogTraceTool(server, storage) {
                 trace_id: traceId,
             })),
         };
-        await storage.insertTrace(trace);
+        await storage.insertTrace(LOCAL_TENANT, trace);
+        // Best-effort async OTel export (fire-and-forget). No-op when
+        // IRIS_OTEL_ENDPOINT isn't configured. Errors are logged via the
+        // server logger but never affect the tool response — if the OTel
+        // collector is down we still want to store traces locally.
+        bestEffortExport(trace, (err) => {
+            // eslint-disable-next-line no-console
+            console.warn(`[iris.otel] ${err.message}`);
+        });
         return {
             content: [
                 {

package/dist/tools/verify-citations.d.ts

@@ -0,0 +1,3 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { IStorageAdapter } from '../types/query.js';
+export declare function registerVerifyCitationsTool(server: McpServer, storage: IStorageAdapter): void;