@iris-eval/mcp-server 0.4.0 → 0.4.1

package/dist/tools/list-rules.js

@@ -11,6 +11,7 @@
  * needs to manage the rule set programmatically.
  */
 import { z } from 'zod';
+import { LOCAL_TENANT } from '../types/tenant.js';
 const inputSchema = {
     eval_type: z
         .enum(['completeness', 'relevance', 'safety', 'cost', 'custom'])
@@ -27,6 +28,8 @@ export function registerListRulesTool(server, customRuleStore) {
         description: [
             'Enumerate deployed custom evaluation rules from the local rule store.',
             '',
+            'Sibling tools — deploy_rule adds custom rules, delete_rule removes them, evaluate_output runs them against agent output. log_trace / get_traces / delete_trace handle the trace lifecycle separately. list_rules is the READ path for the custom-rule store; nothing else exposes the inventory.',
+            '',
             'Behavior. Pure read of ~/.iris/custom-rules.json (in-memory cached; no disk read per call after server boot). No mutation, no external network. Tenant-scoped in Cloud tier; OSS returns all rules for the single local tenant. Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio. Returns in <5ms.',
             '',
             'Output shape. Returns JSON: `{ "rules": [{ "id": "rule-XXXX", "name", "description?", "evalType", "severity", "definition": { type, config, weight? }, "enabled": boolean, "deployedAt": ISO timestamp, "sourceMomentId?": string }], "total": number, "enabled_count": number }`. Empty array + total=0 when no rules deployed.',
@@ -35,6 +38,8 @@ export function registerListRulesTool(server, customRuleStore) {
             '',
             "Don't use to count traces or evals (that's get_traces). Don't use to inspect built-in (non-custom) rules — those ship with the iris binary and are listed in docs/api-reference.md, not in the rule store. Don't use to deploy a rule (use deploy_rule); don't use to remove one (use delete_rule).",
             '',
+            'Parameters. eval_type filter is exact-match against each rule\'s evalType field (no wildcards). enabled_only excludes rules that are deployed-but-disabled (toggled via the dashboard\'s rule-list affordance — there\'s no MCP toggle tool in v0.4). Both filters are AND-combined when both are set. Both are optional; with no filter, all rules return. Defaults: eval_type=undefined (no filter), enabled_only=false (returns all rules including disabled).',
+            '',
             "Error modes. Returns empty list if the rule store file doesn't exist (first run). Returns 429 if HTTP rate limit exceeded. Never throws on valid input.",
         ].join('\n'),
         inputSchema,
@@ -45,7 +50,10 @@ export function registerListRulesTool(server, customRuleStore) {
             openWorldHint: false,
         },
     }, async (args) => {
-        let rules = customRuleStore.list();
+        // OSS: MCP tools operate under LOCAL_TENANT. Cloud multi-tenant
+        // exposure is a v0.5 architectural item (MCP SDK doesn't pass
+        // session/tenant context to tool handlers).
+        let rules = customRuleStore.list(LOCAL_TENANT);
         if (args.eval_type) {
             rules = rules.filter((r) => r.evalType === args.eval_type);
         }

package/dist/tools/log-trace.js

@@ -31,17 +31,17 @@ const TokenUsageSchema = z.object({
     total_tokens: z.number().optional(),
 });
 const inputSchema = {
-    agent_name: z.string().describe('Name of the agent'),
-    framework: z.string().optional().describe('Agent framework name'),
-    input: z.string().optional().describe('Agent input text'),
-    output: z.string().optional().describe('Agent output text'),
-    tool_calls: z.array(ToolCallSchema).optional().describe('Tool calls made during execution'),
-    latency_ms: z.number().optional().describe('Total execution time in milliseconds'),
-    token_usage: TokenUsageSchema.optional().describe('Token usage breakdown'),
-    cost_usd: z.number().optional().describe('Total cost in USD'),
-    metadata: z.record(z.unknown()).optional().describe('Arbitrary metadata'),
-    spans: z.array(SpanSchema).optional().describe('Detailed execution spans'),
-    timestamp: z.string().optional().describe('Trace timestamp (ISO 8601)'),
+    agent_name: z.string().describe('Agent name — used for filtering in get_traces (e.g., "customer-support-bot")'),
+    framework: z.string().optional().describe('Agent framework identifier (e.g., langchain, autogen, custom)'),
+    input: z.string().optional().describe('Agent input text — the user prompt or upstream input that produced this output'),
+    output: z.string().optional().describe('Agent output text — what the agent produced (pass to evaluate_output for scoring)'),
+    tool_calls: z.array(ToolCallSchema).optional().describe('Tool calls made during execution (per-call latency, errors, input/output)'),
+    latency_ms: z.number().optional().describe('Total execution time in milliseconds (end-to-end agent latency)'),
+    token_usage: TokenUsageSchema.optional().describe('Token usage breakdown (prompt/completion/total — used for cost analysis)'),
+    cost_usd: z.number().optional().describe('Total cost in USD — overrides per-span aggregation when provided (treated as authoritative)'),
+    metadata: z.record(z.unknown()).optional().describe('Opaque key-value tags (e.g. {requestId, userId, env}) — queryable in dashboard, not via get_traces filters'),
+    spans: z.array(SpanSchema).optional().describe('Detailed execution spans (hierarchical span tree with timings, attributes, events)'),
+    timestamp: z.string().optional().describe('Trace timestamp (ISO 8601); defaults to now() when omitted'),
 };
 export function registerLogTraceTool(server, storage) {
     server.registerTool('log_trace', {
@@ -49,6 +49,8 @@ export function registerLogTraceTool(server, storage) {
         description: [
             'Persist a single agent execution trace (input, output, spans, tool calls, cost, latency, token usage).',
             '',
+            'Sibling tools — evaluate_output runs heuristic scoring on the trace; evaluate_with_llm_judge runs semantic LLM-based scoring; verify_citations checks citation grounding; get_traces queries stored traces; delete_trace removes a single trace; list_rules / deploy_rule / delete_rule manage custom evaluation rules. log_trace is the WRITE path that records executions; everything else reads, scores, or manages around it.',
+            '',
             'Behavior. Writes one row to Iris storage (SQLite by default; Postgres in Cloud tier). When IRIS_OTEL_ENDPOINT is set, ALSO fires a best-effort async export to the configured OTLP/HTTP collector (Jaeger, Tempo, Datadog OTLP, OTEL Collector). The OTel export is fire-and-forget — its success does not affect the tool response; failures are logged but the trace is still stored locally. No authentication in stdio mode; HTTP mode requires Bearer token. Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio. Not idempotent: each call mints a fresh trace_id, so resubmitting the same payload creates a duplicate trace.',
             '',
             'Output shape. Returns a JSON string: `{ "trace_id": "<32-hex>", "status": "stored" }`. The trace_id is the key you pass to evaluate_output or get_traces afterwards.',
@@ -57,6 +59,8 @@ export function registerLogTraceTool(server, storage) {
             '',
             'Don\'t use when you only need a transient log (use console logging). Don\'t use to update an existing trace — there is no update path in v0.4 (traces are immutable once stored).',
             '',
+            'Parameters. agent_name is required; everything else is optional. token_usage and cost_usd are summary fields — if you ALSO pass spans with per-tool-call costs, the summary fields are treated as authoritative (no auto-aggregation). spans without an explicit start_time fall back to the trace timestamp; spans with an end_time get a duration_ms derived. metadata is opaque key-value (queryable in the dashboard, not via get_traces filters). tool_calls record per-tool latency + errors; missing latency_ms means "not reported," not "zero." Defaults: span.kind="INTERNAL", span.status_code="UNSET", timestamp=now() if omitted.',
+            '',
             'Error modes. Throws on missing agent_name. Throws on malformed span or tool_call objects (Zod rejects). Returns 500 on storage failure (disk full, DB locked). Never blocks on the agent — returns within ~50ms for typical payloads.',
         ].join('\n'),
         inputSchema,

package/dist/tools/verify-citations.js

@@ -53,6 +53,8 @@ export function registerVerifyCitationsTool(server, storage) {
         description: [
             'Extract citations from agent output, fetch the cited sources, and use an LLM judge to check whether each source supports the claim in context. Returns per-citation verdicts + an overall support ratio.',
             '',
+            'Sibling tools — evaluate_with_llm_judge runs general semantic scoring (accuracy, helpfulness, correctness, faithfulness); this tool is specifically for citation grounding (does the cited source actually support the claim). evaluate_output\'s no_hallucination_markers heuristic detects FABRICATED-looking citations cheaply (free, no fetch); this tool resolves and verifies them (paid, opt-in fetch, SSRF-guarded). log_trace / get_traces handle trace I/O. verify_citations is the GROUNDING-CHECK path — narrowest in scope, deepest in rigor.',
+            '',
             'Behavior. Three-phase pipeline: (1) regex extraction of [N] numbered refs, (Author, Year) parentheticals, bare URLs, and DOIs (in-process, no network); (2) SSRF-guarded fetch of URL + DOI citations, with scheme allowlist, private/link-local/cloud-metadata IP blocking, optional domain allowlist (IRIS_CITATION_DOMAINS), 10s timeout, 5MB body cap, manual redirect chase (max 3, re-checked), in-process LRU cache; (3) per-citation LLM judge call asking "does this source support this claim?" with a 256-token verdict. Opt-in via allow_fetch=true or IRIS_CITATION_ALLOW_FETCH=1 — Iris refuses outbound HTTP by default. Cost-capped across the entire call by max_cost_usd_total (default $1.00) — the pipeline stops when the cap would be exceeded. Rate-limited to 20 req/min on HTTP MCP. Writes one eval_result row tagged with per-citation provenance.',
             '',
             'Output shape. Returns JSON: `{ "id": "<uuid>", "overall_score": 0..1|null, "passed": boolean, "total_citations_found": number, "total_resolved": number, "total_supported": number, "total_cost_usd": number, "citations": [{ "citation": { "raw", "kind", "identifier", "offset_start", "offset_end" }, "resolve_status": "ok"|"skipped"|"error", "resolve_error"?, "source"?: { "url", "status", "content_type", "bytes_fetched", "truncated" }, "judge"?: { "supported", "confidence", "rationale", "cost_usd", "latency_ms", "input_tokens", "output_tokens" } }] }`. `overall_score = supported / resolved`; `null` when nothing resolvable was found.',
@@ -61,6 +63,8 @@ export function registerVerifyCitationsTool(server, storage) {
             "",
             "Don't use when the agent output has no citations at all (overall_score will be null; the tool degrades gracefully but a heuristic rule is cheaper). Don't use without allow_fetch=true or IRIS_CITATION_ALLOW_FETCH=1 — the tool refuses outbound HTTP unless explicitly enabled. Don't use with an open allowlist + untrusted output on the public internet; you are effectively running a user-directed fetcher. For stricter safety set IRIS_CITATION_DOMAINS to a curated list.",
             '',
+            'Parameters. model is required; provider auto-detected from model name (override only for ambiguous IDs). allow_fetch=false by default — outbound HTTP is REFUSED unless explicitly true OR IRIS_CITATION_ALLOW_FETCH=1 env. domain_allowlist suffix-matches hostnames (e.g., "wikipedia.org" allows en.wikipedia.org); merged with IRIS_CITATION_DOMAINS env (UNION — either source permits). max_citations defaults 20, hard cap 50 (extras are skipped silently, NOT errored — check total_citations_found in the response if precise). max_cost_usd_total defaults $1.00 — the pipeline stops mid-citation when the next judge call would exceed the cap (returns partial verdicts). per_source_timeout_ms defaults 10000 (10s); per_source_max_bytes defaults 5MB (truncates at boundary, judges still run on truncated content). trace_id optional but recommended. Defaults: max_citations=20, max_cost_usd_total=$1.00, per_source_timeout_ms=10000, per_source_max_bytes=5242880, allow_fetch=false.',
+            '',
             'Error modes. Throws when the API key env var is missing. Throws "Unknown model" on unsupported model IDs. Per-citation errors are collected (resolve_error.kind = bad_scheme / ssrf / not_allowed_domain / timeout / too_large / bad_status / redirect_loop / not_text / fetch_disabled / malformed_judge_response / cost_cap_reached / unresolvable_kind) and returned in the response rather than thrown. An empty output or output with zero extractable citations returns overall_score=null + passed=true (nothing to fail).',
         ].join('\n'),
         inputSchema,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@iris-eval/mcp-server",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "description": "The agent eval standard for MCP. Score every agent output for quality, safety, and cost.",
   "mcpName": "io.github.iris-eval/mcp-server",
   "type": "module",
@@ -25,6 +25,10 @@
     "test:e2e:ui": "playwright test --ui",
     "version:check": "bash scripts/check-version.sh",
     "version:sync": "node scripts/sync-versions.mjs",
+    "claims:capture-tests": "node scripts/claims/capture-tests.mjs",
+    "claims:generate": "node scripts/claims/generate.mjs",
+    "claims:check": "node scripts/claims/generate.mjs --check",
+    "claims:check-hardcoded": "node scripts/claims/check-no-hardcoded.mjs",
     "clean": "rm -rf dist coverage",
     "seed:demo": "tsx scripts/seed-demo-data.ts",
     "demo": "tsx scripts/demo.ts",

package/server.json

@@ -6,12 +6,12 @@
     "url": "https://github.com/iris-eval/mcp-server",
     "source": "github"
   },
-  "version": "0.4.0",
+  "version": "0.4.1",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "@iris-eval/mcp-server",
-      "version": "0.4.0",
+      "version": "0.4.1",
       "transport": {
         "type": "stdio"
       },