@iris-eval/mcp-server 0.4.6 → 0.5.0

package/dist/tools/verify-citations.js

@@ -3,6 +3,7 @@ import { LOCAL_TENANT } from '../types/tenant.js';
 import { verifyCitations } from '../eval/citation-verify/verifier.js';
 import { findPricing } from '../eval/llm-judge/pricing.js';
 import { generateEvalId } from '../utils/ids.js';
+import { strictInput } from './strict-input.js';
 const inputSchema = {
     output: z.string().min(1).describe('The agent output containing citations to verify'),
     model: z
@@ -57,7 +58,7 @@ export function registerVerifyCitationsTool(server, storage) {
             '',
             'Behavior. Three-phase pipeline: (1) regex extraction of [N] numbered refs, (Author, Year) parentheticals, bare URLs, and DOIs (in-process, no network); (2) SSRF-guarded fetch of URL + DOI citations, with scheme allowlist, private/link-local/cloud-metadata IP blocking, optional domain allowlist (IRIS_CITATION_DOMAINS), 10s timeout, 5MB body cap, manual redirect chase (max 3, re-checked), in-process LRU cache; (3) per-citation LLM judge call asking "does this source support this claim?" with a 256-token verdict. Opt-in via allow_fetch=true or IRIS_CITATION_ALLOW_FETCH=1 — Iris refuses outbound HTTP by default. Cost-capped across the entire call by max_cost_usd_total (default $1.00) — the pipeline stops when the cap would be exceeded. Rate-limited to 20 req/min on HTTP MCP. Writes one eval_result row tagged with per-citation provenance.',
             '',
-            'Output shape. Returns JSON: `{ "id": "<uuid>", "overall_score": 0..1|null, "passed": boolean, "total_citations_found": number, "total_resolved": number, "total_supported": number, "total_cost_usd": number, "citations": [{ "citation": { "raw", "kind", "identifier", "offset_start", "offset_end" }, "resolve_status": "ok"|"skipped"|"error", "resolve_error"?, "source"?: { "url", "status", "content_type", "bytes_fetched", "truncated" }, "judge"?: { "supported", "confidence", "rationale", "cost_usd", "latency_ms", "input_tokens", "output_tokens" } }] }`. `overall_score = supported / resolved`; `null` when nothing resolvable was found.',
+            'Output shape. Returns JSON: `{ "id": "<uuid>", "overall_score": 0..1|null, "passed": boolean, "total_citations_found": number, "total_resolved": number, "total_judged": number, "total_supported": number, "total_cost_usd": number, "citations": [{ "citation": { "raw", "kind", "identifier", "offset_start", "offset_end" }, "resolve_status": "ok"|"skipped"|"error", "resolve_error"?, "source"?: { "url", "status", "content_type", "bytes_fetched", "truncated" }, "judge"?: { "supported", "confidence", "rationale", "cost_usd", "latency_ms", "input_tokens", "output_tokens" } }] }`. `overall_score = supported / judged`; `null` when nothing was judged (no resolvable citations, or every judge call failed). Infrastructure failures (cost cap, judge timeout/error, malformed verdict) leave a citation resolved-but-unjudged — reported per-citation via resolve_error, never scored as unsupported.',
             '',
             'Use when the output makes factual claims backed by [1]-style references, DOIs, or URLs and you want to separate "cited correctly" from "cited and wrong" from "cited but unresolvable". Particularly useful for research/legal/medical agents where fabricated citations are the dominant failure mode.',
             "",
@@ -67,7 +68,7 @@ export function registerVerifyCitationsTool(server, storage) {
             '',
             'Error modes. Throws when the API key env var is missing. Throws "Unknown model" on unsupported model IDs. Per-citation errors are collected (resolve_error.kind = bad_scheme / ssrf / not_allowed_domain / timeout / too_large / bad_status / redirect_loop / not_text / fetch_disabled / malformed_judge_response / cost_cap_reached / unresolvable_kind) and returned in the response rather than thrown. An empty output or output with zero extractable citations returns overall_score=null + passed=true (nothing to fail).',
         ].join('\n'),
-        inputSchema,
+        inputSchema: strictInput(inputSchema),
         annotations: {
             readOnlyHint: false, // Writes eval_result + spends money
             destructiveHint: false, // Creates data; doesn't overwrite/delete
@@ -109,11 +110,11 @@ export function registerVerifyCitationsTool(server, storage) {
                     passed: result.passed,
                     score,
                     message: result.overallScore === null
-                        ? `No resolvable citations (found ${result.totalCitationsFound}, resolved ${result.totalResolved})`
-                        : `${result.totalSupported}/${result.totalResolved} cited sources supported the output`,
+                        ? `No citations judged (found ${result.totalCitationsFound}, resolved ${result.totalResolved}, judged 0)`
+                        : `${result.totalSupported}/${result.totalJudged} judged sources supported the output`,
                 },
             ],
-            suggestions: result.passed ? [] : [`Only ${result.totalSupported}/${result.totalResolved} cited sources actually supported the claim.`],
+            suggestions: result.passed ? [] : [`Only ${result.totalSupported}/${result.totalJudged} judged sources actually supported the claim.`],
             rules_evaluated: 1,
             rules_skipped: 0,
             insufficient_data: result.overallScore === null,
@@ -128,6 +129,7 @@ export function registerVerifyCitationsTool(server, storage) {
                         passed: result.passed,
                         total_citations_found: result.totalCitationsFound,
                         total_resolved: result.totalResolved,
+                        total_judged: result.totalJudged,
                         total_supported: result.totalSupported,
                         total_cost_usd: result.totalCostUsd,
                         citations: result.citations.map((c) => ({

package/dist/transport/http.js

@@ -56,9 +56,31 @@ export async function createHttpTransport(mcpServer, config, logger) {
      * `127.0.0.1:0` and reject every real request with a 403 that looks
      * exactly like an attack. Routes are registered immediately after, and
      * the port is not discoverable by any client until this function returns.
+     *
+     * The callback MUST inspect its error argument. Express 5 wires the
+     * listen callback as `server.once('error', done)` as well as the
+     * listening callback — so on EADDRINUSE it is invoked WITH the error.
+     * Ignoring that argument resolved this promise on a server that never
+     * bound: the caller then logged "HTTP transport listening on <port>"
+     * while another process owned the port, and the process idled forever.
+     * A CI health poll got 200 from the OTHER instance and shipped
+     * evaluations to a stranger's database. A bind failure must reject,
+     * name the port, and take the process down nonzero.
      */
-    const httpServer = await new Promise((resolve) => {
-        const server = app.listen(config.transport.port, config.transport.host, () => resolve(server));
+    const httpServer = await new Promise((resolve, reject) => {
+        const server = app.listen(config.transport.port, config.transport.host, (err) => {
+            if (err) {
+                const bind = `${config.transport.host}:${config.transport.port}`;
+                const code = err.code;
+                reject(code === 'EADDRINUSE'
+                    ? new Error(`HTTP transport failed to start: port ${config.transport.port} is already in use ` +
+                        `(EADDRINUSE on ${bind}). Another process — possibly another iris instance — owns it. ` +
+                        `Pass --port <other> (or set IRIS_PORT) or stop the other process.`)
+                    : new Error(`HTTP transport failed to bind ${bind}: ${err.message}`));
+                return;
+            }
+            resolve(server);
+        });
     });
     const address = httpServer.address();
     const port = typeof address === 'object' && address ? address.port : config.transport.port;

package/dist/types/decision-moment.d.ts

@@ -120,3 +120,15 @@ export interface MomentQueryResult {
     limit: number;
     offset: number;
 }
+export interface RankedFailure extends DecisionMoment {
+    /** Severity × recency-decay blend, 0-1. Higher = shown first. */
+    rankScore: number;
+}
+export interface FailureQueryResult {
+    failures: RankedFailure[];
+    /** How many recent traces were scanned to build the list. */
+    scanned: number;
+    /** Total traces matching the filter (pre-scan-cap). */
+    total: number;
+    limit: number;
+}

package/dist/types/eval.d.ts

@@ -4,6 +4,18 @@ export interface EvalRule {
     description: string;
     evalType: EvalType;
     weight: number;
+    /**
+     * Hard-fail marker. When a critical rule FAILS (and was not skipped), the
+     * overall eval reports passed=false regardless of the weighted score.
+     *
+     * Exists because the weighted average routinely outvotes a genuine
+     * violation: an output leaking a real SSN failed no_pii while the other
+     * safety rules passed, scoring ~0.765 — above the 0.7 threshold — so the
+     * one field every CI gate reads said passed:true about the product's
+     * flagship failure scenario. The score stays a quality gradient; `passed`
+     * is the verdict, and a critical violation must never be averaged away.
+     */
+    critical?: boolean;
     evaluate(context: EvalContext): EvalRuleResult;
 }
 export interface EvalContext {
@@ -23,6 +35,17 @@ export interface EvalContext {
     costUsd?: number;
     metadata?: Record<string, unknown>;
     customConfig?: Record<string, unknown>;
+    /**
+     * Per-evaluation regex circuit breaker, initialized by the engine (never
+     * by callers). Each sandbox budget breach increments `breaches`; once it
+     * reaches the cap, remaining regex rules in the SAME evaluation skip
+     * without running. Bounds how long a single hostile output can stall a
+     * request: without it, N regex rules × (budget + worker respawn) of
+     * main-thread stall scale linearly with N.
+     */
+    regexBudget?: {
+        breaches: number;
+    };
 }
 export interface EvalRuleResult {
     ruleName: string;
@@ -31,6 +54,8 @@ export interface EvalRuleResult {
     message: string;
     skipped?: boolean;
     skipReason?: string;
+    configInvalid?: boolean;
+    budgetExceeded?: boolean;
 }
 export interface EvalResult {
     id: string;
@@ -46,6 +71,13 @@ export interface EvalResult {
     rules_evaluated?: number;
     rules_skipped?: number;
     insufficient_data?: boolean;
+    /**
+     * Names of critical rules that failed (present only when non-empty).
+     * Any entry here forces passed=false regardless of the weighted score —
+     * this field is how a caller tells "failed the quality bar" apart from
+     * "committed a hard violation".
+     */
+    critical_failures?: string[];
 }
 export type CustomRuleType = 'regex_match' | 'regex_no_match' | 'min_length' | 'max_length' | 'contains_keywords' | 'excludes_keywords' | 'json_schema' | 'cost_threshold';
 export interface CustomRuleDefinition {

package/dist/types/query.d.ts

@@ -23,7 +23,7 @@ export interface TraceQueryResult {
     limit: number;
     offset: number;
 }
-export type EvalStatsPeriod = '24h' | '7d' | '30d' | 'all';
+export type EvalStatsPeriod = '24h' | '2d' | '7d' | '14d' | '30d' | '60d' | '90d' | '180d' | 'all';
 export interface EvalStats {
     passRate: number;
     avgScore: number;

package/dist/utils/write-atomic.d.ts

@@ -1 +1,3 @@
+export declare const OWNER_ONLY_FILE_MODE = 384;
+export declare function ensureOwnerOnly(...paths: string[]): void;
 export declare function writeAtomic(targetPath: string, contents: string): void;

package/dist/utils/write-atomic.js

@@ -1,4 +1,4 @@
-import { mkdirSync, writeFileSync, renameSync, unlinkSync } from 'node:fs';
+import { mkdirSync, writeFileSync, renameSync, unlinkSync, chmodSync, existsSync } from 'node:fs';
 import { dirname } from 'node:path';
 import { randomBytes } from 'node:crypto';
 /*
@@ -28,6 +28,38 @@ import { randomBytes } from 'node:crypto';
  */
 const TRANSIENT_RENAME_ERRORS = new Set(['EPERM', 'EACCES', 'EBUSY']);
 const MAX_ATTEMPTS = 5;
+/*
+ * Owner-only (0600). These files hold agent inputs and outputs verbatim —
+ * and a tool whose job is detecting PII necessarily stores the PII it found.
+ * Node's default is 0666 before umask, so a typical umask leaves them 0644:
+ * world-readable to every local account on a shared POSIX host. The mode is
+ * a no-op on Windows (ACL inheritance governs there), which is exactly why
+ * local testing never surfaces it.
+ *
+ * Set at creation on the temp file, so the bytes are never briefly readable
+ * between write and chmod; rename preserves the mode.
+ */
+export const OWNER_ONLY_FILE_MODE = 0o600;
+/*
+ * Narrow an EXISTING file to owner-only. Two cases need this, because a mode
+ * passed at write time only applies when the write creates the file:
+ *   - files written before this change (every install that predates it),
+ *   - files a library creates for us (better-sqlite3 opens iris.db, and WAL
+ *     mode adds iris.db-wal / iris.db-shm on first write).
+ * Best-effort by design: a missing file, a read-only mount, or a
+ * non-POSIX filesystem must never take the server down over permissions.
+ */
+export function ensureOwnerOnly(...paths) {
+    for (const p of paths) {
+        try {
+            if (existsSync(p))
+                chmodSync(p, OWNER_ONLY_FILE_MODE);
+        }
+        catch {
+            // Best effort — see above.
+        }
+    }
+}
 function sleepSync(ms) {
     // Synchronous by necessity — writeAtomic is sync, and making it async
     // would ripple through every caller for a Windows-only edge case.
@@ -36,7 +68,7 @@ function sleepSync(ms) {
 export function writeAtomic(targetPath, contents) {
     mkdirSync(dirname(targetPath), { recursive: true });
     const tmp = `${targetPath}.tmp.${process.pid}.${randomBytes(6).toString('hex')}`;
-    writeFileSync(tmp, contents, 'utf-8');
+    writeFileSync(tmp, contents, { encoding: 'utf-8', mode: OWNER_ONLY_FILE_MODE });
     let lastError;
     for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
         try {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@iris-eval/mcp-server",
-  "version": "0.4.6",
-  "description": "The agent eval standard for MCP. Score every agent output for quality, safety, and cost.",
+  "version": "0.5.0",
+  "description": "Stop shipping agents on vibes. Score every agent output for quality, safety, and cost.",
   "mcpName": "io.github.iris-eval/mcp-server",
   "type": "module",
   "main": "dist/index.js",
@@ -22,6 +22,7 @@
     "test:coverage": "vitest run --coverage",
     "test:integration": "vitest run tests/integration/",
     "test:e2e": "playwright test",
+    "test:uat": "node tests/uat/run-uat.mjs",
     "test:e2e:ui": "playwright test --ui",
     "version:check": "bash scripts/check-version.sh",
     "version:sync": "node scripts/sync-versions.mjs",

package/server.json

@@ -1,17 +1,17 @@
 {
   "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
   "name": "io.github.iris-eval/mcp-server",
-  "description": "The agent eval standard for MCP. Score every agent output for quality, safety, and cost.",
+  "description": "Stop shipping agents on vibes. Score every agent output for quality, safety, and cost.",
   "repository": {
     "url": "https://github.com/iris-eval/mcp-server",
     "source": "github"
   },
-  "version": "0.4.6",
+  "version": "0.5.0",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "@iris-eval/mcp-server",
-      "version": "0.4.6",
+      "version": "0.5.0",
       "transport": {
         "type": "stdio"
       },

package/dist/dashboard/assets/index-B4Aw6ozt.css

@@ -1 +0,0 @@
-@import "https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@500..700&family=Manrope:wght@400..700&family=JetBrains+Mono:wght@400..700&display=swap";:root{--lightningcss-light:initial;--lightningcss-dark: ;color-scheme:light dark;--iris-50:#f0fdfa;--iris-100:#ccfbf1;--iris-200:#99f6e4;--iris-300:#5eead4;--iris-400:#2dd4bf;--iris-500:#14b8a6;--iris-600:#0d9488;--iris-700:#0f766e;--iris-800:#115e59;--iris-900:#134e4a;--iris-950:#042f2e;--eval-pass:#22c55e;--eval-warn:#eab308;--eval-fail:#ef4444;--eval-tool:#3b82f6;--eval-llm:#a855f7;--eval-skipped:#71717a}@media (prefers-color-scheme:dark){:root{--lightningcss-light: ;--lightningcss-dark:initial}}:root,[data-theme=dark]{--bg-base:#050508;--bg-raised:#08080e;--bg-surface:#0d0d15;--bg-card:#101018;--bg-card-hover:#16161f;--border-subtle:#ffffff0d;--border-default:#ffffff14;--border-strong:#ffffff24;--border-glow:#14b8a680;--text-primary:#f0f0f5;--text-secondary:#9494a8;--text-muted:#5e5e72;--text-accent:var(--iris-400);--glow-primary:#14b8a61f;--glow-strong:#14b8a640;--shadow-sm:0 1px 2px #0000004d;--shadow-md:0 4px 6px #0006;--shadow-lg:0 10px 15px #00000080}[data-theme=light]{--bg-base:#fafcfc;--bg-raised:#f1f5f5;--bg-surface:#e8eded;--bg-card:#fff;--bg-card-hover:#f4f8f8;--border-subtle:#0000000a;--border-default:#00000014;--border-strong:#00000024;--border-glow:#0d948859;--text-primary:#0a0f0e;--text-secondary:#3d5250;--text-muted:#7a908e;--text-accent:var(--iris-700);--glow-primary:#0d94880f;--glow-strong:#0d94881f;--shadow-sm:0 1px 2px #0000000f;--shadow-md:0 4px 6px #00000014;--shadow-lg:0 10px 15px #0000001a}:root{--font-display:"Space Grotesk", -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;--font-body:"Manrope", -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;--font-mono:"JetBrains Mono", "Fira Code", ui-monospace, monospace;--font-sans:var(--font-body);--text-caption-xs:11px;--text-caption:12px;--text-body-sm:13px;--text-body:14px;--text-body-lg:15px;--text-heading-sm:16px;--text-heading:20px;--text-display-sm:28px;--text-display:40px;--font-size-xs:var(--text-caption);--font-size-sm:var(--text-body-sm);--font-size-base:var(--text-body);--font-size-lg:var(--text-body-lg);--font-size-xl:var(--text-heading-sm);--font-size-2xl:var(--text-heading);--font-size-3xl:var(--text-display-sm);--leading-body:1.5;--leading-heading:1.2;--leading-display:1.1;--leading-mono:1.4;--space-0_5:2px;--space-1:4px;--space-1_5:6px;--space-2:8px;--space-2_5:10px;--space-3:12px;--space-4:16px;--space-5:20px;--space-6:24px;--space-8:32px;--space-10:40px;--space-12:48px;--space-16:64px;--space-20:80px;--space-24:96px}:root,[data-density=compact]{--density-row:32px;--density-padding:var(--space-3);--density-body:var(--text-body-sm)}[data-density=comfortable]{--density-row:44px;--density-padding:var(--space-4);--density-body:var(--text-body)}:root{--sidebar-width-expanded:256px;--sidebar-width-collapsed:64px;--header-height:56px;--page-toolbar-height:40px;--radius-xs:4px;--radius-sm:6px;--radius:8px;--radius-lg:12px;--radius-xl:16px;--radius-pill:999px;--border-radius:var(--radius);--border-radius-sm:var(--radius-xs);--border-radius-lg:var(--radius-lg);--transition-instant:.1s ease;--transition-fast:.15s ease;--transition-base:.2s ease;--transition-slow:.3s ease;--ease-iris:cubic-bezier(.25, .4, .25, 1);--bg-primary:var(--bg-base);--bg-secondary:var(--bg-raised);--bg-tertiary:var(--bg-surface);--bg-hover:var(--bg-card-hover);--border-color:var(--border-default);--accent-primary:var(--iris-500);--accent-primary-hover:var(--iris-400);--accent-success:var(--eval-pass);--accent-error:var(--eval-fail);--accent-warning:var(--eval-warn);--accent-tool:var(--eval-tool);--accent-llm:var(--eval-llm)}html{transition:background-color var(--transition-base), color var(--transition-base)}*,:before,:after{box-sizing:border-box;margin:0;padding:0}html,body,#root{width:100%;height:100%}body{font-family:var(--font-body);font-size:var(--text-body);color:var(--text-primary);background-color:var(--bg-base);line-height:var(--leading-body);-webkit-font-smoothing:antialiased;font-feature-settings:"cv11", "ss01"}h1,h2,h3,h4,.display{font-family:var(--font-display);letter-spacing:-.01em;font-weight:600}a{color:var(--text-accent);text-decoration:none}a:hover{color:var(--iris-300)}button{cursor:pointer;font-family:inherit}input,select{font-family:inherit;font-size:inherit}code,pre{font-family:var(--font-mono)}a:focus-visible,button:focus-visible,input:focus-visible,select:focus-visible,[role=button]:focus-visible{outline:2px solid var(--iris-500);outline-offset:2px;border-radius:var(--radius-xs)}.iris-sr-reveal{clip:rect(0, 0, 0, 0);white-space:nowrap;border:0;width:1px;height:1px;margin:-1px;padding:0;position:absolute;overflow:hidden}.iris-sr-reveal:focus-within{width:auto;height:auto;margin:var(--space-3) 0 0 0;padding:var(--space-3) var(--space-4);clip:auto;white-space:normal;background:var(--bg-card);color:var(--text-primary);border:1px solid var(--iris-500);border-radius:var(--radius-sm);position:static;overflow:visible}.iris-sr-reveal:focus-within>li{padding:var(--space-1) 0;list-style:none}::selection{background:var(--iris-600);color:#fff}::-webkit-scrollbar{width:8px;height:8px}::-webkit-scrollbar-track{background:var(--bg-base)}::-webkit-scrollbar-thumb{background:var(--border-strong);border-radius:var(--radius-xs)}::-webkit-scrollbar-thumb:hover{background:var(--text-muted)}html{scrollbar-color:var(--border-strong) transparent;scrollbar-width:thin}@keyframes pulse-ring{0%{opacity:.5;transform:scale(1)}to{opacity:0;transform:scale(2.5)}}.pulse-dot{position:relative}.pulse-dot:after{content:"";background:var(--iris-500);border-radius:50%;animation:2s ease-out infinite pulse-ring;position:absolute;inset:-2px}@media (prefers-reduced-motion:reduce){*,:before,:after{scroll-behavior:auto!important;transition-duration:.01ms!important;animation-duration:.01ms!important;animation-iteration-count:1!important}}@media (width<=767px){aside[aria-label=Main\ navigation]{width:160px}main{overflow-x:auto}}@media print{body{color:#000!important;background:#fff!important}aside[aria-label=Main\ navigation],header,[role=region][aria-label=Welcome],[role=region][aria-label=Bulk\ actions],[role=status],[role=dialog]{display:none!important}body,#root,main{height:auto!important;overflow:visible!important}main{padding:0!important}tr,pre,code{page-break-inside:avoid}h1,h2,h3{page-break-after:avoid}[aria-label*=violation],[aria-label*=spike],[aria-label*=collision],[aria-label*=Pass],[aria-label*=Fail]{border:1px solid #000!important}a{color:#000!important;text-decoration:underline!important}a[href^=http]:after{content:" (" attr(href) ")";color:#555;font-size:80%}}